Browse code

- added mb_atomic_* ops, equivalent to membar(); atomic_*, but more optimized (e.g. on x86 most atomic ops act also as memory barriers so mb_atomic_* will be equiv. to atomic_* )

- support for mips cpu which don't implement full mips isa2, but do support
ll and sc

Andrei Pelinescu-Onciul authored on 15/03/2006 19:16:39
Showing 2 changed files
... ...
@@ -136,27 +136,28 @@ int atomic_ops_init();
136 136
 
137 137
 #endif /* NOSMP */
138 138
 
139
+
139 140
 #define atomic_inc(var) \
140 141
 	asm volatile( \
141 142
 			__LOCK_PREF " incl %0 \n\t"  \
142
-			: "=m"((var)->val) : "m"((var)->val) : "cc" \
143
+			: "=m"((var)->val) : "m"((var)->val) : "cc", "memory" \
143 144
 			) 
144 145
 
145 146
 #define atomic_dec(var) \
146 147
 	asm volatile( \
147 148
 			__LOCK_PREF " decl %0 \n\t" \
148
-			: "=m"((var)->val) : "m"((var)->val) : "cc" \
149
+			: "=m"((var)->val) : "m"((var)->val) : "cc", "memory" \
149 150
 			) 
150 151
 
151 152
 #define atomic_and(var, i) \
152 153
 	asm volatile( \
153 154
 			__LOCK_PREF " andl %1, %0 \n\t" \
154
-			: "=m"((var)->val) : "r"((i)), "m"((var)->val) : "cc" \
155
+			: "=m"((var)->val) : "ri"((i)), "m"((var)->val) : "cc", "memory" \
155 156
 			)
156 157
 #define atomic_or(var, i) \
157 158
 	asm volatile( \
158 159
 			__LOCK_PREF " orl %1, %0 \n\t" \
159
-			: "=m"((var)->val) : "r"((i)), "m"((var)->val) : "cc" \
160
+			: "=m"((var)->val) : "ri"((i)), "m"((var)->val) : "cc", "memory" \
160 161
 			)
161 162
 
162 163
 
... ...
@@ -168,7 +169,7 @@ inline static long atomic_inc_and_test(atomic_t* var)
168 169
 	asm volatile(
169 170
 			__LOCK_PREF " incl %0 \n\t"
170 171
 			"setz  %1 \n\t"
171
-			: "=m"(var->val), "=qm"(ret) : "m" (var->val) : "cc"
172
+			: "=m"(var->val), "=qm"(ret) : "m" (var->val) : "cc", "memory"
172 173
 			);
173 174
 	return ret;
174 175
 }
... ...
@@ -182,28 +183,78 @@ inline static long atomic_dec_and_test(atomic_t* var)
182 183
 	asm volatile(
183 184
 			__LOCK_PREF " decl %0 \n\t"
184 185
 			"setz  %1 \n\t"
185
-			: "=m"(var->val), "=qm"(ret) : "m" (var->val) : "cc"
186
+			: "=m"(var->val), "=qm"(ret) : "m" (var->val) : "cc", "memory"
187
+			);
188
+	return ret;
189
+}
190
+
191
+#ifdef NOSMP
192
+
193
+#define mb_atomic_set(v, i) \
194
+	do{ \
195
+		membar(); \
196
+		atomic_set(v, i); \
197
+	}while(0)
198
+
199
+
200
+inline static long mb_atomic_get(atomic_t* v)
201
+{
202
+	membar();
203
+	return atomic_get(v);
204
+}
205
+#else /* NOSMP */
206
+
207
+
208
+inline static void mb_atomic_set(atomic_t* v, long i)
209
+{
210
+	asm volatile(
211
+			"xchgl %1, %0 \n\t"
212
+			: "+q"(i), "=m"(v->val) : "m"((v)->val) : "memory" 
213
+			);
214
+}
215
+
216
+
217
+
218
+inline static long mb_atomic_get(atomic_t* var)
219
+{
220
+	long ret;
221
+	
222
+	asm volatile(
223
+			__LOCK_PREF " cmpxchg %0, %1 \n\t"
224
+			: "=a"(ret)  : "m"(var->val) : "cc", "memory"
186 225
 			);
187 226
 	return ret;
188 227
 }
189 228
 
229
+#endif /* NOSMP */
230
+
231
+
232
+/* on x86 atomic intructions act also as barriers */
233
+#define mb_atomic_inc(v)	atomic_inc(v)
234
+#define mb_atomic_dec(v)	atomic_dec(v)
235
+#define mb_atomic_or(v, m)	atomic_or(v, m)
236
+#define mb_atomic_and(v, m)	atomic_and(v, m)
237
+#define mb_atomic_inc_and_test(v)	atomic_inc_and_test(v)
238
+#define mb_atomic_dec_and_test(v)	atomic_dec_and_test(v)
190 239
 
191 240
 
192
-#elif defined __CPU_mips2
241
+#elif defined __CPU_mips2 || ( defined __CPU_mips && defined MIPS_HAS_LLSC )
193 242
 
194 243
 #define HAVE_ASM_INLINE_ATOMIC_OPS
195 244
 
196 245
 #ifdef NOSMP
197
-#define membar() asm volatile ("" : : : memory) /* gcc do not cache barrier*/
246
+#define membar() asm volatile ("" : : : "memory") /* gcc do not cache barrier*/
198 247
 #define membar_read()  membar()
199 248
 #define membar_write() membar()
200 249
 #else
201 250
 
202 251
 #define membar() \
203 252
 	asm volatile( \
253
+			".set push \n\t" \
204 254
 			".set noreorder \n\t" \
255
+			".set mips2 \n\t" \
205 256
 			"    sync\n\t" \
206
-			".set reorder \n\t" \
257
+			".set pop \n\t" \
207 258
 			: : : "memory" \
208 259
 			) 
209 260
 
... ...
@@ -216,13 +267,15 @@ inline static long atomic_dec_and_test(atomic_t* var)
216 267
 
217 268
 /* main asm block */
218 269
 #define ATOMIC_ASM_OP(op) \
270
+			".set push \n\t" \
219 271
 			".set noreorder \n\t" \
272
+			".set mips2 \n\t" \
220 273
 			"1:   ll %1, %0 \n\t" \
221 274
 			"     " op "\n\t" \
222 275
 			"     sc %2, %0 \n\t" \
223 276
 			"     beqz %2, 1b \n\t" \
224 277
 			"     nop \n\t" \
225
-			".set reorder \n\t" 
278
+			".set pop \n\t" 
226 279
 
227 280
 
228 281
 #define ATOMIC_FUNC_DECL(NAME, OP, RET_TYPE, RET_EXPR) \
... ...
@@ -278,6 +331,62 @@ ATOMIC_FUNC_DECL_CT(dec_and_test, "subu %2, %1, %3", 1, long, (ret-1)==0 )
278 331
 ATOMIC_FUNC_DECL1(and, "and %2, %1, %3", void, /* no return */ )
279 332
 ATOMIC_FUNC_DECL1(or,  "or  %2, %1, %3", void,  /* no return */ )
280 333
 
334
+
335
+/* with integrated membar */
336
+
337
+#define mb_atomic_set(v, i) \
338
+	do{ \
339
+		membar(); \
340
+		atomic_set(v, i); \
341
+	}while(0)
342
+
343
+
344
+
345
+inline static long mb_atomic_get(atomic_t* v)
346
+{
347
+	membar();
348
+	return atomic_get(v);
349
+}
350
+
351
+
352
+#define mb_atomic_inc(v) \
353
+	do{ \
354
+		membar(); \
355
+		atomic_inc(v); \
356
+	}while(0)
357
+
358
+#define mb_atomic_dec(v) \
359
+	do{ \
360
+		membar(); \
361
+		atomic_dec(v); \
362
+	}while(0)
363
+
364
+#define mb_atomic_or(v, m) \
365
+	do{ \
366
+		membar(); \
367
+		atomic_or(v, m); \
368
+	}while(0)
369
+
370
+#define mb_atomic_and(v, m) \
371
+	do{ \
372
+		membar(); \
373
+		atomic_and(v, m); \
374
+	}while(0)
375
+
376
+inline static int mb_atomic_inc_and_test(atomic_t* v)
377
+{
378
+	membar();
379
+	return atomic_inc_and_test(v);
380
+}
381
+
382
+inline static int mb_atomic_dec_and_test(atomic_t* v)
383
+{
384
+	membar();
385
+	return atomic_dec_and_test(v);
386
+}
387
+
388
+
389
+
281 390
 #endif /* __CPU_xxx  => no known cpu */
282 391
 
283 392
 
... ...
@@ -121,11 +121,13 @@ inline static int tsl(fl_lock_t* lock)
121 121
 			: "r"(1), "b" (lock) :
122 122
 			"memory", "cc"
123 123
         );
124
-#elif defined __CPU_mips2
124
+#elif defined __CPU_mips2 || ( defined __CPU_mips && defined MIPS_HAS_LLSC )
125 125
 	long tmp;
126 126
 	
127 127
 	asm volatile(
128
+		".set push \n\t"
128 129
 		".set noreorder\n\t"
130
+		".set mips2 \n\t"
129 131
 		"1:  ll %1, %2   \n\t"
130 132
 		"    li %0, 1 \n\t"
131 133
 		"    sc %0, %2  \n\t"
... ...
@@ -134,7 +136,7 @@ inline static int tsl(fl_lock_t* lock)
134 136
 #ifndef NOSMP
135 137
 		"    sync \n\t"
136 138
 #endif
137
-		".set reorder\n\t"
139
+		".set pop\n\t"
138 140
 		: "=&r" (tmp), "=&r" (val), "=m" (*lock) 
139 141
 		: "m" (*lock) 
140 142
 		: "memory"
... ...
@@ -221,14 +223,16 @@ inline static void release_lock(fl_lock_t* lock)
221 223
 			: "r"(0), "b" (lock)
222 224
 			: "memory"
223 225
 	);
224
-#elif defined __CPU_mips2
226
+#elif defined __CPU_mips2 || ( defined __CPU_mips && defined MIPS_HAS_LLSC )
225 227
 	asm volatile(
228
+		".set push \n\t"
226 229
 		".set noreorder \n\t"
230
+		".set mips2 \n\t"
227 231
 #ifndef NOSMP
228 232
 		"    sync \n\t"
229 233
 #endif
230 234
 		"    sw $0, %0 \n\t"
231
-		".set reorder \n\t"
235
+		".set pop \n\t"
232 236
 		: "=m" (*lock)  : /* no input */ : "memory"
233 237
 	);
234 238
 #elif defined __CPU_alpha