Browse code

- added mb_atomic_* ops, equivalent to membar(); atomic_*, but more optimized (e.g. on x86 most atomic ops act also as memory barriers so mb_atomic_* will be equiv. to atomic_* )

- support for mips cpu which don't implement full mips isa2, but do support
ll and sc

Andrei Pelinescu-Onciul authored on 15/03/2006 19:16:39
Showing 2 changed files
... ...
@@ -136,27 +136,28 @@ int atomic_ops_init();
136 136
 
137 137
 #endif /* NOSMP */
138 138
 
139
+
139 140
 #define atomic_inc(var) \
140 141
 	asm volatile( \
141 142
 			__LOCK_PREF " incl %0 \n\t"  \
142
-			: "=m"((var)->val) : "m"((var)->val) : "cc" \
143
+			: "=m"((var)->val) : "m"((var)->val) : "cc", "memory" \
143 144
 			) 
144 145
 
145 146
 #define atomic_dec(var) \
146 147
 	asm volatile( \
147 148
 			__LOCK_PREF " decl %0 \n\t" \
148
-			: "=m"((var)->val) : "m"((var)->val) : "cc" \
149
+			: "=m"((var)->val) : "m"((var)->val) : "cc", "memory" \
149 150
 			) 
150 151
 
151 152
 #define atomic_and(var, i) \
152 153
 	asm volatile( \
153 154
 			__LOCK_PREF " andl %1, %0 \n\t" \
154
-			: "=m"((var)->val) : "r"((i)), "m"((var)->val) : "cc" \
155
+			: "=m"((var)->val) : "ri"((i)), "m"((var)->val) : "cc", "memory" \
155 156
 			)
156 157
 #define atomic_or(var, i) \
157 158
 	asm volatile( \
158 159
 			__LOCK_PREF " orl %1, %0 \n\t" \
159
-			: "=m"((var)->val) : "r"((i)), "m"((var)->val) : "cc" \
160
+			: "=m"((var)->val) : "ri"((i)), "m"((var)->val) : "cc", "memory" \
160 161
 			)
161 162
 
162 163
 
... ...
@@ -168,7 +169,7 @@ inline static long atomic_inc_and_test(atomic_t* var)
168 168
 	asm volatile(
169 169
 			__LOCK_PREF " incl %0 \n\t"
170 170
 			"setz  %1 \n\t"
171
-			: "=m"(var->val), "=qm"(ret) : "m" (var->val) : "cc"
171
+			: "=m"(var->val), "=qm"(ret) : "m" (var->val) : "cc", "memory"
172 172
 			);
173 173
 	return ret;
174 174
 }
... ...
@@ -182,28 +183,78 @@ inline static long atomic_dec_and_test(atomic_t* var)
182 182
 	asm volatile(
183 183
 			__LOCK_PREF " decl %0 \n\t"
184 184
 			"setz  %1 \n\t"
185
-			: "=m"(var->val), "=qm"(ret) : "m" (var->val) : "cc"
185
+			: "=m"(var->val), "=qm"(ret) : "m" (var->val) : "cc", "memory"
186
+			);
187
+	return ret;
188
+}
189
+
190
+#ifdef NOSMP
191
+
192
+#define mb_atomic_set(v, i) \
193
+	do{ \
194
+		membar(); \
195
+		atomic_set(v, i); \
196
+	}while(0)
197
+
198
+
199
+inline static long mb_atomic_get(atomic_t* v)
200
+{
201
+	membar();
202
+	return atomic_get(v);
203
+}
204
+#else /* NOSMP */
205
+
206
+
207
+inline static void mb_atomic_set(atomic_t* v, long i)
208
+{
209
+	asm volatile(
210
+			"xchgl %1, %0 \n\t"
211
+			: "+q"(i), "=m"(v->val) : "m"((v)->val) : "memory" 
212
+			);
213
+}
214
+
215
+
216
+
217
+inline static long mb_atomic_get(atomic_t* var)
218
+{
219
+	long ret;
220
+	
221
+	asm volatile(
222
+			__LOCK_PREF " cmpxchg %0, %1 \n\t"
223
+			: "=a"(ret)  : "m"(var->val) : "cc", "memory"
186 224
 			);
187 225
 	return ret;
188 226
 }
189 227
 
228
+#endif /* NOSMP */
229
+
230
+
231
+/* on x86 atomic intructions act also as barriers */
232
+#define mb_atomic_inc(v)	atomic_inc(v)
233
+#define mb_atomic_dec(v)	atomic_dec(v)
234
+#define mb_atomic_or(v, m)	atomic_or(v, m)
235
+#define mb_atomic_and(v, m)	atomic_and(v, m)
236
+#define mb_atomic_inc_and_test(v)	atomic_inc_and_test(v)
237
+#define mb_atomic_dec_and_test(v)	atomic_dec_and_test(v)
190 238
 
191 239
 
192
-#elif defined __CPU_mips2
240
+#elif defined __CPU_mips2 || ( defined __CPU_mips && defined MIPS_HAS_LLSC )
193 241
 
194 242
 #define HAVE_ASM_INLINE_ATOMIC_OPS
195 243
 
196 244
 #ifdef NOSMP
197
-#define membar() asm volatile ("" : : : memory) /* gcc do not cache barrier*/
245
+#define membar() asm volatile ("" : : : "memory") /* gcc do not cache barrier*/
198 246
 #define membar_read()  membar()
199 247
 #define membar_write() membar()
200 248
 #else
201 249
 
202 250
 #define membar() \
203 251
 	asm volatile( \
252
+			".set push \n\t" \
204 253
 			".set noreorder \n\t" \
254
+			".set mips2 \n\t" \
205 255
 			"    sync\n\t" \
206
-			".set reorder \n\t" \
256
+			".set pop \n\t" \
207 257
 			: : : "memory" \
208 258
 			) 
209 259
 
... ...
@@ -216,13 +267,15 @@ inline static long atomic_dec_and_test(atomic_t* var)
216 216
 
217 217
 /* main asm block */
218 218
 #define ATOMIC_ASM_OP(op) \
219
+			".set push \n\t" \
219 220
 			".set noreorder \n\t" \
221
+			".set mips2 \n\t" \
220 222
 			"1:   ll %1, %0 \n\t" \
221 223
 			"     " op "\n\t" \
222 224
 			"     sc %2, %0 \n\t" \
223 225
 			"     beqz %2, 1b \n\t" \
224 226
 			"     nop \n\t" \
225
-			".set reorder \n\t" 
227
+			".set pop \n\t" 
226 228
 
227 229
 
228 230
 #define ATOMIC_FUNC_DECL(NAME, OP, RET_TYPE, RET_EXPR) \
... ...
@@ -278,6 +331,62 @@ ATOMIC_FUNC_DECL_CT(dec_and_test, "subu %2, %1, %3", 1, long, (ret-1)==0 )
278 278
 ATOMIC_FUNC_DECL1(and, "and %2, %1, %3", void, /* no return */ )
279 279
 ATOMIC_FUNC_DECL1(or,  "or  %2, %1, %3", void,  /* no return */ )
280 280
 
281
+
282
+/* with integrated membar */
283
+
284
+#define mb_atomic_set(v, i) \
285
+	do{ \
286
+		membar(); \
287
+		atomic_set(v, i); \
288
+	}while(0)
289
+
290
+
291
+
292
+inline static long mb_atomic_get(atomic_t* v)
293
+{
294
+	membar();
295
+	return atomic_get(v);
296
+}
297
+
298
+
299
+#define mb_atomic_inc(v) \
300
+	do{ \
301
+		membar(); \
302
+		atomic_inc(v); \
303
+	}while(0)
304
+
305
+#define mb_atomic_dec(v) \
306
+	do{ \
307
+		membar(); \
308
+		atomic_dec(v); \
309
+	}while(0)
310
+
311
+#define mb_atomic_or(v, m) \
312
+	do{ \
313
+		membar(); \
314
+		atomic_or(v, m); \
315
+	}while(0)
316
+
317
+#define mb_atomic_and(v, m) \
318
+	do{ \
319
+		membar(); \
320
+		atomic_and(v, m); \
321
+	}while(0)
322
+
323
+inline static int mb_atomic_inc_and_test(atomic_t* v)
324
+{
325
+	membar();
326
+	return atomic_inc_and_test(v);
327
+}
328
+
329
+inline static int mb_atomic_dec_and_test(atomic_t* v)
330
+{
331
+	membar();
332
+	return atomic_dec_and_test(v);
333
+}
334
+
335
+
336
+
281 337
 #endif /* __CPU_xxx  => no known cpu */
282 338
 
283 339
 
... ...
@@ -121,11 +121,13 @@ inline static int tsl(fl_lock_t* lock)
121 121
 			: "r"(1), "b" (lock) :
122 122
 			"memory", "cc"
123 123
         );
124
-#elif defined __CPU_mips2
124
+#elif defined __CPU_mips2 || ( defined __CPU_mips && defined MIPS_HAS_LLSC )
125 125
 	long tmp;
126 126
 	
127 127
 	asm volatile(
128
+		".set push \n\t"
128 129
 		".set noreorder\n\t"
130
+		".set mips2 \n\t"
129 131
 		"1:  ll %1, %2   \n\t"
130 132
 		"    li %0, 1 \n\t"
131 133
 		"    sc %0, %2  \n\t"
... ...
@@ -134,7 +136,7 @@ inline static int tsl(fl_lock_t* lock)
134 134
 #ifndef NOSMP
135 135
 		"    sync \n\t"
136 136
 #endif
137
-		".set reorder\n\t"
137
+		".set pop\n\t"
138 138
 		: "=&r" (tmp), "=&r" (val), "=m" (*lock) 
139 139
 		: "m" (*lock) 
140 140
 		: "memory"
... ...
@@ -221,14 +223,16 @@ inline static void release_lock(fl_lock_t* lock)
221 221
 			: "r"(0), "b" (lock)
222 222
 			: "memory"
223 223
 	);
224
-#elif defined __CPU_mips2
224
+#elif defined __CPU_mips2 || ( defined __CPU_mips && defined MIPS_HAS_LLSC )
225 225
 	asm volatile(
226
+		".set push \n\t"
226 227
 		".set noreorder \n\t"
228
+		".set mips2 \n\t"
227 229
 #ifndef NOSMP
228 230
 		"    sync \n\t"
229 231
 #endif
230 232
 		"    sw $0, %0 \n\t"
231
-		".set reorder \n\t"
233
+		".set pop \n\t"
232 234
 		: "=m" (*lock)  : /* no input */ : "memory"
233 235
 	);
234 236
 #elif defined __CPU_alpha