Browse code

- added callbacks/hooks for dst_blacklist_add: one can register such a callback and deny adding a destination to the blacklist list, or it can change the destinations flags.

- autodetect if locks or lock_set are not limited and if so use
one lock per hash bucket or a lock_set of hash table size (should work
better on multiple cpus, less cacheline invalidations)

Andrei Pelinescu-Onciul authored on 30/05/2007 22:22:42
Showing 2 changed files
... ...
@@ -29,6 +29,7 @@
29 29
 /* History:
30 30
  * --------
31 31
  *  2006-07-29  created by andrei
32
+ *  2007-05-39  added hooks for add; more locks to reduce contention (andrei)
32 33
  */
33 34
 
34 35
 
... ...
@@ -43,6 +44,7 @@
43 43
 #include "ip_addr.h"
44 44
 #include "error.h"
45 45
 #include "rpc.h"
46
+#include "compiler_opt.h"
46 47
 
47 48
 
48 49
 
... ...
@@ -66,7 +68,54 @@ struct dst_blst_entry{
66 66
 #define DEFAULT_BLST_TIMER_INTERVAL		60 /* 1 min */
67 67
 
68 68
 
69
+/* lock method */
70
+#ifdef GEN_LOCK_T_UNLIMITED
71
+#define BLST_LOCK_PER_BUCKET
72
+#elif defined GEN_LOCK_SET_T_UNLIMITED
73
+#define BLST_LOCK_SET
74
+#else
75
+#define BLST_ONE_LOCK
76
+#endif
77
+
78
+
79
+#ifdef BLST_LOCK_PER_BUCKET
80
+/* lock included in the hash bucket */
81
+#define LOCK_BLST(h)		lock_get(&dst_blst_hash[(h)].lock)
82
+#define UNLOCK_BLST(h)		lock_release(&dst_blst_hash[(h)].lock)
83
+#elif defined BLST_LOCK_SET
84
+static gen_lock_set_t* blst_lock_set=0;
85
+#define LOCK_BLST(h)		lock_set_get(blst_lock_set, (h))
86
+#define UNLOCK_BLST(h)		lock_set_release(blst_lock_set, (h))
87
+#else
88
+/* use only one lock */
69 89
 static gen_lock_t* blst_lock=0;
90
+#define LOCK_BLST(h)		lock_get(blst_lock)
91
+#define UNLOCK_BLST(h)		lock_release(blst_lock)
92
+#endif
93
+
94
+
95
+
96
+
97
+#define BLST_HASH_STATS
98
+
99
+#ifdef BLST_HASH_STATS
100
+#define BLST_HASH_STATS_DEC(h) dst_blst_hash[(h)].entries--
101
+#define BLST_HASH_STATS_INC(h) dst_blst_hash[(h)].entries++
102
+#else
103
+#define BLST_HASH_STATS_DEC(h) do{}while(0)
104
+#define BLST_HASH_STATS_INC(h) do{}while(0)
105
+#endif
106
+
107
+struct dst_blst_lst_head{
108
+	struct dst_blst_entry* first;
109
+#ifdef BLST_LOCK_PER_BUCKET
110
+	gen_lock_t	lock;
111
+#endif
112
+#ifdef BLST_HASH_STATS
113
+	unsigned int entries;
114
+#endif
115
+};
116
+
70 117
 static struct timer_ln* blst_timer_h=0;
71 118
 
72 119
 static volatile unsigned int* blst_mem_used=0;
... ...
@@ -74,11 +123,96 @@ unsigned int  blst_max_mem=DEFAULT_BLST_MAX_MEM; /* maximum memory used
74 74
 													for the blacklist entries*/
75 75
 unsigned int blst_timeout=DEFAULT_BLST_TIMEOUT;
76 76
 unsigned int blst_timer_interval=DEFAULT_BLST_TIMER_INTERVAL;
77
-struct dst_blst_entry** dst_blst_hash=0;
77
+struct dst_blst_lst_head* dst_blst_hash=0;
78
+
79
+
80
+
81
+#ifdef DST_BLACKLIST_HOOKS
82
+
83
+#define MAX_BLST_HOOKS 1
84
+
85
+static struct blacklist_hook* blacklist_hooks;
86
+static unsigned int blacklist_max_hooks=MAX_BLST_HOOKS;
87
+static int last_blacklist_hook_idx=0;
88
+
89
+static int init_blacklist_hooks()
90
+{
91
+	blacklist_hooks=pkg_malloc(blacklist_max_hooks*
92
+								sizeof(struct blacklist_hook));
93
+	if (blacklist_hooks==0)
94
+		goto error;
95
+	memset(blacklist_hooks, 0,
96
+				blacklist_max_hooks*sizeof(struct blacklist_hook));
97
+	return 0;
98
+error:
99
+	LOG(L_ERR, "blacklist_hooks: memory allocation failure\n");
100
+	return -1;
101
+}
102
+
103
+
104
+static void destroy_blacklist_hooks()
105
+{
106
+	int r;
107
+
108
+	if (blacklist_hooks){
109
+		for (r=0; r<last_blacklist_hook_idx; r++){
110
+			if (blacklist_hooks[r].destroy)
111
+				blacklist_hooks[r].destroy();
112
+		}
113
+		pkg_free(blacklist_hooks);
114
+		blacklist_hooks=0;
115
+	}
116
+}
78 117
 
79 118
 
80
-#define LOCK_BLST()		lock_get(blst_lock)
81
-#define UNLOCK_BLST()	lock_release(blst_lock)
119
+/* allocates a new hook
120
+ * returns 0 on success and -1 on error
121
+ * must be called from mod init (from the main process, before forking)*/
122
+int register_blacklist_hook(struct blacklist_hook *h)
123
+{
124
+	struct blacklist_hook* tmp;
125
+	int new_max_hooks;
126
+	
127
+	if (blacklist_max_hooks==0)
128
+		goto error;
129
+	if (last_blacklist_hook_idx >= blacklist_max_hooks){
130
+		new_max_hooks=2*blacklist_max_hooks;
131
+		tmp=pkg_realloc(blacklist_hooks, 
132
+				new_max_hooks*sizeof(struct blacklist_hook));
133
+		if (tmp==0){
134
+			goto error;
135
+		}
136
+		blacklist_hooks=tmp;
137
+		/* init the new chunk */
138
+		memset(&blacklist_hooks[last_blacklist_hook_idx+1], 0, 
139
+					(new_max_hooks-blacklist_max_hooks-1)*
140
+						sizeof(struct blacklist_hook));
141
+		blacklist_max_hooks=new_max_hooks;
142
+	}
143
+	blacklist_hooks[last_blacklist_hook_idx]=*h;
144
+	last_blacklist_hook_idx++;
145
+	return 0;
146
+error:
147
+	return -1;
148
+}
149
+
150
+
151
+int blacklist_run_hooks(struct dest_info* si, unsigned char* flags)
152
+{
153
+	int r;
154
+	int ret;
155
+	
156
+	ret=DST_BLACKLIST_ACCEPT; /* default, if no hook installed accept 
157
+								blacklist operation */
158
+	for (r=0; r<last_blacklist_hook_idx; r++){
159
+		ret=blacklist_hooks[r].on_blst_add(si, flags);
160
+		if (ret!=DST_BLACKLIST_ACCEPT) break;
161
+	}
162
+	return ret;
163
+}
164
+
165
+
166
+#endif /* DST_BLACKLIST_HOOKS */
82 167
 
83 168
 
84 169
 inline static void blst_destroy_entry(struct dst_blst_entry* e)
... ...
@@ -133,14 +267,26 @@ void destroy_dst_blacklist()
133 133
 		timer_free(blst_timer_h);
134 134
 		blst_timer_h=0;
135 135
 	}
136
+#ifdef BLST_LOCK_PER_BUCKET
137
+		for(r=0; r<DST_BLST_HASH_SIZE; r++)
138
+			lock_destroy(&dst_blst_hash[r].lock);
139
+#elif defined BLST_LOCK_SET
140
+		if (blst_lock_set){
141
+			lock_set_destroy(blst_lock_set);
142
+			lock_set_dealloc(blst_lock_set);
143
+			blst_lock_set=0;
144
+		}
145
+#else
136 146
 	if (blst_lock){
137 147
 		lock_destroy(blst_lock);
138 148
 		lock_dealloc(blst_lock);
139 149
 		blst_lock=0;
140 150
 	}
151
+#endif
152
+	
141 153
 	if (dst_blst_hash){
142 154
 		for(r=0; r<DST_BLST_HASH_SIZE; r++){
143
-			for (crt=&dst_blst_hash[r], tmp=&(*crt)->next; *crt; 
155
+			for (crt=&dst_blst_hash[r].first, tmp=&(*crt)->next; *crt; 
144 156
 					crt=tmp, tmp=&(*crt)->next){
145 157
 			e=*crt;
146 158
 			*crt=(*crt)->next;
... ...
@@ -154,6 +300,9 @@ void destroy_dst_blacklist()
154 154
 		shm_free((void*)blst_mem_used);
155 155
 		blst_mem_used=0;
156 156
 	}
157
+#ifdef DST_BLACKLIST_HOOKS
158
+	destroy_blacklist_hooks();
159
+#endif
157 160
 }
158 161
 
159 162
 
... ...
@@ -161,22 +310,51 @@ void destroy_dst_blacklist()
161 161
 int init_dst_blacklist()
162 162
 {
163 163
 	int ret;
164
+#ifdef BLST_LOCK_PER_BUCKET
165
+	int r;
166
+#endif
164 167
 	
165 168
 	ret=-1;
169
+#ifdef DST_BLACKLIST_HOOKS
170
+	if (init_blacklist_hooks()!=0){
171
+		ret=E_OUT_OF_MEM;
172
+		goto error;
173
+	}
174
+#endif
166 175
 	blst_mem_used=shm_malloc(sizeof(*blst_mem_used));
167 176
 	if (blst_mem_used==0){
168 177
 		ret=E_OUT_OF_MEM;
169 178
 		goto error;
170 179
 	}
171 180
 	*blst_mem_used=0;
172
-	dst_blst_hash=shm_malloc(sizeof(struct dst_blst_entry*) *
181
+	dst_blst_hash=shm_malloc(sizeof(struct dst_blst_lst_head) *
173 182
 											DST_BLST_HASH_SIZE);
174 183
 	if (dst_blst_hash==0){
175 184
 		ret=E_OUT_OF_MEM;
176 185
 		goto error;
177 186
 	}
178
-	memset(dst_blst_hash, 0, sizeof(struct dst_blst_entry*) *
187
+	memset(dst_blst_hash, 0, sizeof(struct dst_blst_lst_head) *
179 188
 								DST_BLST_HASH_SIZE);
189
+#ifdef BLST_LOCK_PER_BUCKET
190
+	for (r=0; r<DST_BLST_HASH_SIZE; r++){
191
+		if (lock_init(&dst_blst_hash[r].lock)==0){
192
+			ret=-1;
193
+			goto error;
194
+		}
195
+	}
196
+#elif defined BLST_LOCK_SET
197
+	blst_lock_set=lock_set_alloc(DST_BLST_HASH_SIZE);
198
+	if (blst_lock_set==0){
199
+		ret=E_OUT_OF_MEM;
200
+		goto error;
201
+	}
202
+	if (lock_set_init(blst_lock_set)==0){
203
+		lock_set_dealloc(blst_lock_set);
204
+		blst_lock_set=0;
205
+		ret=-1;
206
+		goto error;
207
+	}
208
+#else /* BLST_ONE_LOCK */
180 209
 	blst_lock=lock_alloc();
181 210
 	if (blst_lock==0){
182 211
 		ret=E_OUT_OF_MEM;
... ...
@@ -188,6 +366,7 @@ int init_dst_blacklist()
188 188
 		ret=-1;
189 189
 		goto error;
190 190
 	}
191
+#endif /* BLST*LOCK*/
191 192
 	blst_timer_h=timer_alloc();
192 193
 	if (blst_timer_h==0){
193 194
 		ret=E_OUT_OF_MEM;
... ...
@@ -240,6 +419,7 @@ inline static struct dst_blst_entry* _dst_blacklist_lst_find(
240 240
 	type=(ip->af==AF_INET6)*BLST_IS_IPV6;
241 241
 	for (crt=head, tmp=&(*head)->next; *crt; crt=tmp, tmp=&(*crt)->next){
242 242
 		e=*crt;
243
+		prefetch_loc_r((*crt)->next, 1);
243 244
 		/* remove old expired entries */
244 245
 		if ((s_ticks_t)(now-(*crt)->expire)>=0){
245 246
 			*crt=(*crt)->next;
... ...
@@ -260,8 +440,6 @@ inline static struct dst_blst_entry* _dst_blacklist_lst_find(
260 260
 /* frees all the expired entries until either there are no more of them
261 261
  *  or the total memory used is <= target (to free all of them use -1 for 
262 262
  *  targer)
263
- *  It must be called with LOCK_BLST held (or call dst_blacklist_clean_expired
264
- *   instead).
265 263
  *  params:   target  - free expired entries until no more then taget memory 
266 264
  *                      is used  (use 0 to free all of them)
267 265
  *            delta   - consider an entry expired if it expires after delta
... ...
@@ -271,9 +449,9 @@ inline static struct dst_blst_entry* _dst_blacklist_lst_find(
271 271
  *  returns: number of deleted entries
272 272
  *  This function should be called periodically from a timer
273 273
  */
274
-inline static int _dst_blacklist_clean_expired_unsafe(unsigned int target,
275
-												ticks_t delta,
276
-												ticks_t timeout)
274
+inline static int dst_blacklist_clean_expired(unsigned int target,
275
+									  ticks_t delta,
276
+									  ticks_t timeout)
277 277
 {
278 278
 	static unsigned short start=0;
279 279
 	unsigned short h;
... ...
@@ -283,57 +461,41 @@ inline static int _dst_blacklist_clean_expired_unsafe(unsigned int target,
283 283
 	ticks_t start_time;
284 284
 	ticks_t now;
285 285
 	int no=0;
286
+	int i;
286 287
 	
287 288
 	now=start_time=get_ticks_raw();
288 289
 	for(h=start; h!=(start+DST_BLST_HASH_SIZE); h++){
289
-		for (crt=&dst_blst_hash[h%DST_BLST_HASH_SIZE], tmp=&(*crt)->next;
290
-				*crt; crt=tmp, tmp=&(*crt)->next){
291
-			e=*crt;
292
-			if ((s_ticks_t)(now+delta-(*crt)->expire)>=0){
293
-				*crt=(*crt)->next;
294
-				*blst_mem_used-=DST_BLST_ENTRY_SIZE(*e);
295
-				blst_destroy_entry(e);
296
-				no++;
297
-				if (*blst_mem_used<=target)
298
-					goto skip;
290
+		i=h%DST_BLST_HASH_SIZE;
291
+		if (dst_blst_hash[i].first){
292
+			LOCK_BLST(i);
293
+			for (crt=&dst_blst_hash[i].first, tmp=&(*crt)->next;
294
+					*crt; crt=tmp, tmp=&(*crt)->next){
295
+				e=*crt;
296
+				prefetch_loc_r((*crt)->next, 1);
297
+				if ((s_ticks_t)(now+delta-(*crt)->expire)>=0){
298
+					*crt=(*crt)->next;
299
+					*blst_mem_used-=DST_BLST_ENTRY_SIZE(*e);
300
+					blst_destroy_entry(e);
301
+					BLST_HASH_STATS_DEC(i);
302
+					no++;
303
+					if (*blst_mem_used<=target){
304
+						UNLOCK_BLST(i);
305
+						goto skip;
306
+					}
307
+				}
308
+			}
309
+			UNLOCK_BLST(i);
310
+			/* check for timeout only "between" hash cells */
311
+			now=get_ticks_raw();
312
+			if ((now-start_time)>=timeout){
313
+				DBG("_dst_blacklist_clean_expired_unsafe: timeout: %d > %d\n",
314
+						TICKS_TO_MS(now-start_time), TICKS_TO_MS(timeout));
315
+				goto skip;
299 316
 			}
300
-		}
301
-		/* check for timeout only "between" hash cells */
302
-		now=get_ticks_raw();
303
-		if ((now-start_time)>=timeout){
304
-			DBG("_dst_blacklist_clean_expired_unsafe: timeout: %d > %d\n",
305
-					TICKS_TO_MS(now-start_time), TICKS_TO_MS(timeout));
306
-			goto skip;
307 317
 		}
308 318
 	}
309 319
 skip:
310 320
 	start=h; /* next time we start where we left */
311
-	return no;
312
-}
313
-
314
-
315
-
316
-/* frees all the expired entries until either there are no more of them
317
- *  or the total memory used is <= target (to free all of them use -1 for 
318
- *  targer)
319
- *  params:   target  - free expired entries until no more then taget memory 
320
- *                      is used  (use 0 to free all of them)
321
- *            delta   - consider an entry expired if it expires after delta
322
- *                      ticks from now
323
- *            timeout - exit after timeout ticks
324
- *
325
- *  returns: number of deleted entries
326
- *  This function should be called periodically from a timer
327
- */
328
-inline static int dst_blacklist_clean_expired(unsigned int target,
329
-												ticks_t delta,
330
-												ticks_t timeout)
331
-{
332
-	int no;
333
-	
334
-	LOCK_BLST();
335
-		no=_dst_blacklist_clean_expired_unsafe(target, delta, timeout);
336
-	UNLOCK_BLST();
337 321
 	if (no){
338 322
 		DBG("dst_blacklist_clean_expired, %d entries removed\n", no);
339 323
 	}
... ...
@@ -376,24 +538,28 @@ inline static int dst_blacklist_add_ip(unsigned char err_flags,
376 376
 	now=get_ticks_raw();
377 377
 	hash=dst_blst_hash_no(proto, ip, port);
378 378
 	/* check if the entry already exists */
379
-	LOCK_BLST();
380
-		e=_dst_blacklist_lst_find(&dst_blst_hash[hash], ip, proto, port, now);
379
+	LOCK_BLST(hash);
380
+		e=_dst_blacklist_lst_find(&dst_blst_hash[hash].first, ip, proto,
381
+																port, now);
381 382
 		if (e){
382 383
 			e->flags|=err_flags;
383 384
 			e->expire=now+S_TO_TICKS(blst_timeout); /* update the timeout */
384 385
 		}else{
385
-			if ((*blst_mem_used+size)>=blst_max_mem){
386
+			if (unlikely((*blst_mem_used+size)>=blst_max_mem)){
387
+				UNLOCK_BLST(hash);
386 388
 				/* first try to free some memory  (~ 12%), but don't
387 389
 				 * spend more then 250 ms*/
388
-				_dst_blacklist_clean_expired_unsafe(*blst_mem_used/16*14, 0, 
390
+				dst_blacklist_clean_expired(*blst_mem_used/16*14, 0, 
389 391
 															MS_TO_TICKS(250));
390
-				if (*blst_mem_used+size>=blst_max_mem){
392
+				if (unlikely(*blst_mem_used+size>=blst_max_mem)){
391 393
 					ret=-1;
392 394
 					goto error;
393 395
 				}
396
+				LOCK_BLST(hash);
394 397
 			}
395 398
 			e=shm_malloc(size);
396 399
 			if (e==0){
400
+				UNLOCK_BLST(hash);
397 401
 				ret=E_OUT_OF_MEM;
398 402
 				goto error;
399 403
 			}
... ...
@@ -404,10 +570,11 @@ inline static int dst_blacklist_add_ip(unsigned char err_flags,
404 404
 			memcpy(e->ip, ip->u.addr, ip->len);
405 405
 			e->expire=now+S_TO_TICKS(blst_timeout); /* update the timeout */
406 406
 			e->next=0;
407
-			dst_blacklist_lst_add(&dst_blst_hash[hash], e);
407
+			dst_blacklist_lst_add(&dst_blst_hash[hash].first, e);
408
+			BLST_HASH_STATS_INC(hash);
408 409
 		}
410
+	UNLOCK_BLST(hash);
409 411
 error:
410
-	UNLOCK_BLST();
411 412
 	return ret;
412 413
 }
413 414
 
... ...
@@ -426,12 +593,13 @@ inline static int dst_is_blacklisted_ip(unsigned char proto,
426 426
 	ret=0;
427 427
 	now=get_ticks_raw();
428 428
 	hash=dst_blst_hash_no(proto, ip, port);
429
-	LOCK_BLST();
430
-		e=_dst_blacklist_lst_find(&dst_blst_hash[hash], ip, proto, port, now);
429
+	LOCK_BLST(hash);
430
+		e=_dst_blacklist_lst_find(&dst_blst_hash[hash].first, ip, proto,
431
+									port, now);
431 432
 		if (e){
432 433
 			ret=e->flags;
433 434
 		}
434
-	UNLOCK_BLST();
435
+	UNLOCK_BLST(hash);
435 436
 	return ret;
436 437
 }
437 438
 
... ...
@@ -440,6 +608,11 @@ inline static int dst_is_blacklisted_ip(unsigned char proto,
440 440
 int dst_blacklist_add(unsigned char err_flags,  struct dest_info* si)
441 441
 {
442 442
 	struct ip_addr ip;
443
+
444
+#ifdef DST_BLACKLIST_HOOKS
445
+	if (blacklist_run_hooks(si, &err_flags)!=DST_BLACKLIST_ACCEPT)
446
+		return 0;
447
+#endif
443 448
 	su2ip_addr(&ip, &si->to);
444 449
 	return dst_blacklist_add_ip(err_flags, si->proto, &ip,
445 450
 								su_getport(&si->to));
... ...
@@ -491,9 +664,9 @@ void dst_blst_debug(rpc_t* rpc, void* ctx)
491 491
 	struct ip_addr ip;
492 492
 	
493 493
 	now=get_ticks_raw();
494
-	LOCK_BLST();
495 494
 		for(h=0; h<DST_BLST_HASH_SIZE; h++){
496
-			for(e=dst_blst_hash[h]; e; e=e->next){
495
+			LOCK_BLST(h);
496
+			for(e=dst_blst_hash[h].first; e; e=e->next){
497 497
 				dst_blst_entry2ip(&ip, e);
498 498
 				rpc->add(ctx, "ssddd", get_proto_name(e->proto), 
499 499
 										ip_addr2a(&ip), e->port, 
... ...
@@ -502,9 +675,33 @@ void dst_blst_debug(rpc_t* rpc, void* ctx)
502 502
 										-TICKS_TO_S(now-e->expire) ,
503 503
 										e->flags);
504 504
 			}
505
+			UNLOCK_BLST(h);
506
+		}
507
+}
508
+
509
+/* only for debugging, it helds the lock too long for "production" use */
510
+void dst_blst_hash_stats(rpc_t* rpc, void* ctx)
511
+{
512
+	int h;
513
+	struct dst_blst_entry* e;
514
+#ifdef BLST_HASH_STATS
515
+	int n;
516
+	
517
+	n=0;
518
+#endif
519
+	
520
+		for(h=0; h<DST_BLST_HASH_SIZE; h++){
521
+#ifdef BLST_HASH_STATS
522
+			LOCK_BLST(h);
523
+			for(e=dst_blst_hash[h].first; e; e=e->next) n++;
524
+			UNLOCK_BLST(h);
525
+			rpc->add(ctx, "dd", h, n);
526
+#else
527
+			rpc->add(ctx, "dd", h, dst_blst_hash[h].entries);
528
+#endif
505 529
 		}
506
-	UNLOCK_BLST();
507 530
 }
508 531
 
532
+
509 533
 #endif /* USE_DST_BLACKLIST */
510 534
 
... ...
@@ -46,6 +46,22 @@
46 46
 #define BLST_ADM_PROHIBITED	(1<<6)	/* administratively prohibited */
47 47
 #define BLST_PERMANENT		(1<<7)  /* never deleted, never expires */
48 48
 
49
+
50
+/*#define DST_BLACKLIST_HOOKS*/
51
+
52
+#define DST_BLACKLIST_ACCEPT 0
53
+#define DST_BLACKLIST_DENY  -1
54
+
55
+#ifdef DST_BLACKLIST_HOOKS
56
+struct blacklist_hook{
57
+	int (*on_blst_add)(struct dest_info* si, unsigned char* err_flags);
58
+	/* called before ser shutdown */
59
+	void (*destroy)(void);
60
+};
61
+
62
+int register_blacklist_hook(struct blacklist_hook *h);
63
+#endif /* DST_BLACKLIST_HOOKS */
64
+
49 65
 int init_dst_blacklist();
50 66
 void destroy_dst_blacklist();
51 67