Browse code

- added callbacks/hooks for dst_blacklist_add: one can register such a callback and deny adding a destination to the blacklist list, or it can change the destinations flags.

- autodetect if locks or lock_set are not limited and if so use
one lock per hash bucket or a lock_set of hash table size (should work
better on multiple cpus, less cacheline invalidations)

Andrei Pelinescu-Onciul authored on 30/05/2007 22:22:42
Showing 2 changed files
... ...
@@ -29,6 +29,7 @@
29 29
 /* History:
30 30
  * --------
31 31
  *  2006-07-29  created by andrei
32
+ *  2007-05-39  added hooks for add; more locks to reduce contention (andrei)
32 33
  */
33 34
 
34 35
 
... ...
@@ -43,6 +44,7 @@
43 44
 #include "ip_addr.h"
44 45
 #include "error.h"
45 46
 #include "rpc.h"
47
+#include "compiler_opt.h"
46 48
 
47 49
 
48 50
 
... ...
@@ -66,7 +68,54 @@ struct dst_blst_entry{
66 68
 #define DEFAULT_BLST_TIMER_INTERVAL		60 /* 1 min */
67 69
 
68 70
 
71
+/* lock method */
72
+#ifdef GEN_LOCK_T_UNLIMITED
73
+#define BLST_LOCK_PER_BUCKET
74
+#elif defined GEN_LOCK_SET_T_UNLIMITED
75
+#define BLST_LOCK_SET
76
+#else
77
+#define BLST_ONE_LOCK
78
+#endif
79
+
80
+
81
+#ifdef BLST_LOCK_PER_BUCKET
82
+/* lock included in the hash bucket */
83
+#define LOCK_BLST(h)		lock_get(&dst_blst_hash[(h)].lock)
84
+#define UNLOCK_BLST(h)		lock_release(&dst_blst_hash[(h)].lock)
85
+#elif defined BLST_LOCK_SET
86
+static gen_lock_set_t* blst_lock_set=0;
87
+#define LOCK_BLST(h)		lock_set_get(blst_lock_set, (h))
88
+#define UNLOCK_BLST(h)		lock_set_release(blst_lock_set, (h))
89
+#else
90
+/* use only one lock */
69 91
 static gen_lock_t* blst_lock=0;
92
+#define LOCK_BLST(h)		lock_get(blst_lock)
93
+#define UNLOCK_BLST(h)		lock_release(blst_lock)
94
+#endif
95
+
96
+
97
+
98
+
99
+#define BLST_HASH_STATS
100
+
101
+#ifdef BLST_HASH_STATS
102
+#define BLST_HASH_STATS_DEC(h) dst_blst_hash[(h)].entries--
103
+#define BLST_HASH_STATS_INC(h) dst_blst_hash[(h)].entries++
104
+#else
105
+#define BLST_HASH_STATS_DEC(h) do{}while(0)
106
+#define BLST_HASH_STATS_INC(h) do{}while(0)
107
+#endif
108
+
109
+struct dst_blst_lst_head{
110
+	struct dst_blst_entry* first;
111
+#ifdef BLST_LOCK_PER_BUCKET
112
+	gen_lock_t	lock;
113
+#endif
114
+#ifdef BLST_HASH_STATS
115
+	unsigned int entries;
116
+#endif
117
+};
118
+
70 119
 static struct timer_ln* blst_timer_h=0;
71 120
 
72 121
 static volatile unsigned int* blst_mem_used=0;
... ...
@@ -74,11 +123,96 @@ unsigned int  blst_max_mem=DEFAULT_BLST_MAX_MEM; /* maximum memory used
74 123
 													for the blacklist entries*/
75 124
 unsigned int blst_timeout=DEFAULT_BLST_TIMEOUT;
76 125
 unsigned int blst_timer_interval=DEFAULT_BLST_TIMER_INTERVAL;
77
-struct dst_blst_entry** dst_blst_hash=0;
126
+struct dst_blst_lst_head* dst_blst_hash=0;
127
+
128
+
129
+
130
+#ifdef DST_BLACKLIST_HOOKS
131
+
132
+#define MAX_BLST_HOOKS 1
133
+
134
+static struct blacklist_hook* blacklist_hooks;
135
+static unsigned int blacklist_max_hooks=MAX_BLST_HOOKS;
136
+static int last_blacklist_hook_idx=0;
137
+
138
+static int init_blacklist_hooks()
139
+{
140
+	blacklist_hooks=pkg_malloc(blacklist_max_hooks*
141
+								sizeof(struct blacklist_hook));
142
+	if (blacklist_hooks==0)
143
+		goto error;
144
+	memset(blacklist_hooks, 0,
145
+				blacklist_max_hooks*sizeof(struct blacklist_hook));
146
+	return 0;
147
+error:
148
+	LOG(L_ERR, "blacklist_hooks: memory allocation failure\n");
149
+	return -1;
150
+}
151
+
152
+
153
+static void destroy_blacklist_hooks()
154
+{
155
+	int r;
156
+
157
+	if (blacklist_hooks){
158
+		for (r=0; r<last_blacklist_hook_idx; r++){
159
+			if (blacklist_hooks[r].destroy)
160
+				blacklist_hooks[r].destroy();
161
+		}
162
+		pkg_free(blacklist_hooks);
163
+		blacklist_hooks=0;
164
+	}
165
+}
78 166
 
79 167
 
80
-#define LOCK_BLST()		lock_get(blst_lock)
81
-#define UNLOCK_BLST()	lock_release(blst_lock)
168
+/* allocates a new hook
169
+ * returns 0 on success and -1 on error
170
+ * must be called from mod init (from the main process, before forking)*/
171
+int register_blacklist_hook(struct blacklist_hook *h)
172
+{
173
+	struct blacklist_hook* tmp;
174
+	int new_max_hooks;
175
+	
176
+	if (blacklist_max_hooks==0)
177
+		goto error;
178
+	if (last_blacklist_hook_idx >= blacklist_max_hooks){
179
+		new_max_hooks=2*blacklist_max_hooks;
180
+		tmp=pkg_realloc(blacklist_hooks, 
181
+				new_max_hooks*sizeof(struct blacklist_hook));
182
+		if (tmp==0){
183
+			goto error;
184
+		}
185
+		blacklist_hooks=tmp;
186
+		/* init the new chunk */
187
+		memset(&blacklist_hooks[last_blacklist_hook_idx+1], 0, 
188
+					(new_max_hooks-blacklist_max_hooks-1)*
189
+						sizeof(struct blacklist_hook));
190
+		blacklist_max_hooks=new_max_hooks;
191
+	}
192
+	blacklist_hooks[last_blacklist_hook_idx]=*h;
193
+	last_blacklist_hook_idx++;
194
+	return 0;
195
+error:
196
+	return -1;
197
+}
198
+
199
+
200
+int blacklist_run_hooks(struct dest_info* si, unsigned char* flags)
201
+{
202
+	int r;
203
+	int ret;
204
+	
205
+	ret=DST_BLACKLIST_ACCEPT; /* default, if no hook installed accept 
206
+								blacklist operation */
207
+	for (r=0; r<last_blacklist_hook_idx; r++){
208
+		ret=blacklist_hooks[r].on_blst_add(si, flags);
209
+		if (ret!=DST_BLACKLIST_ACCEPT) break;
210
+	}
211
+	return ret;
212
+}
213
+
214
+
215
+#endif /* DST_BLACKLIST_HOOKS */
82 216
 
83 217
 
84 218
 inline static void blst_destroy_entry(struct dst_blst_entry* e)
... ...
@@ -133,14 +267,26 @@ void destroy_dst_blacklist()
133 267
 		timer_free(blst_timer_h);
134 268
 		blst_timer_h=0;
135 269
 	}
270
+#ifdef BLST_LOCK_PER_BUCKET
271
+		for(r=0; r<DST_BLST_HASH_SIZE; r++)
272
+			lock_destroy(&dst_blst_hash[r].lock);
273
+#elif defined BLST_LOCK_SET
274
+		if (blst_lock_set){
275
+			lock_set_destroy(blst_lock_set);
276
+			lock_set_dealloc(blst_lock_set);
277
+			blst_lock_set=0;
278
+		}
279
+#else
136 280
 	if (blst_lock){
137 281
 		lock_destroy(blst_lock);
138 282
 		lock_dealloc(blst_lock);
139 283
 		blst_lock=0;
140 284
 	}
285
+#endif
286
+	
141 287
 	if (dst_blst_hash){
142 288
 		for(r=0; r<DST_BLST_HASH_SIZE; r++){
143
-			for (crt=&dst_blst_hash[r], tmp=&(*crt)->next; *crt; 
289
+			for (crt=&dst_blst_hash[r].first, tmp=&(*crt)->next; *crt; 
144 290
 					crt=tmp, tmp=&(*crt)->next){
145 291
 			e=*crt;
146 292
 			*crt=(*crt)->next;
... ...
@@ -154,6 +300,9 @@ void destroy_dst_blacklist()
154 300
 		shm_free((void*)blst_mem_used);
155 301
 		blst_mem_used=0;
156 302
 	}
303
+#ifdef DST_BLACKLIST_HOOKS
304
+	destroy_blacklist_hooks();
305
+#endif
157 306
 }
158 307
 
159 308
 
... ...
@@ -161,22 +310,51 @@ void destroy_dst_blacklist()
161 310
 int init_dst_blacklist()
162 311
 {
163 312
 	int ret;
313
+#ifdef BLST_LOCK_PER_BUCKET
314
+	int r;
315
+#endif
164 316
 	
165 317
 	ret=-1;
318
+#ifdef DST_BLACKLIST_HOOKS
319
+	if (init_blacklist_hooks()!=0){
320
+		ret=E_OUT_OF_MEM;
321
+		goto error;
322
+	}
323
+#endif
166 324
 	blst_mem_used=shm_malloc(sizeof(*blst_mem_used));
167 325
 	if (blst_mem_used==0){
168 326
 		ret=E_OUT_OF_MEM;
169 327
 		goto error;
170 328
 	}
171 329
 	*blst_mem_used=0;
172
-	dst_blst_hash=shm_malloc(sizeof(struct dst_blst_entry*) *
330
+	dst_blst_hash=shm_malloc(sizeof(struct dst_blst_lst_head) *
173 331
 											DST_BLST_HASH_SIZE);
174 332
 	if (dst_blst_hash==0){
175 333
 		ret=E_OUT_OF_MEM;
176 334
 		goto error;
177 335
 	}
178
-	memset(dst_blst_hash, 0, sizeof(struct dst_blst_entry*) *
336
+	memset(dst_blst_hash, 0, sizeof(struct dst_blst_lst_head) *
179 337
 								DST_BLST_HASH_SIZE);
338
+#ifdef BLST_LOCK_PER_BUCKET
339
+	for (r=0; r<DST_BLST_HASH_SIZE; r++){
340
+		if (lock_init(&dst_blst_hash[r].lock)==0){
341
+			ret=-1;
342
+			goto error;
343
+		}
344
+	}
345
+#elif defined BLST_LOCK_SET
346
+	blst_lock_set=lock_set_alloc(DST_BLST_HASH_SIZE);
347
+	if (blst_lock_set==0){
348
+		ret=E_OUT_OF_MEM;
349
+		goto error;
350
+	}
351
+	if (lock_set_init(blst_lock_set)==0){
352
+		lock_set_dealloc(blst_lock_set);
353
+		blst_lock_set=0;
354
+		ret=-1;
355
+		goto error;
356
+	}
357
+#else /* BLST_ONE_LOCK */
180 358
 	blst_lock=lock_alloc();
181 359
 	if (blst_lock==0){
182 360
 		ret=E_OUT_OF_MEM;
... ...
@@ -188,6 +366,7 @@ int init_dst_blacklist()
188 366
 		ret=-1;
189 367
 		goto error;
190 368
 	}
369
+#endif /* BLST*LOCK*/
191 370
 	blst_timer_h=timer_alloc();
192 371
 	if (blst_timer_h==0){
193 372
 		ret=E_OUT_OF_MEM;
... ...
@@ -240,6 +419,7 @@ inline static struct dst_blst_entry* _dst_blacklist_lst_find(
240 419
 	type=(ip->af==AF_INET6)*BLST_IS_IPV6;
241 420
 	for (crt=head, tmp=&(*head)->next; *crt; crt=tmp, tmp=&(*crt)->next){
242 421
 		e=*crt;
422
+		prefetch_loc_r((*crt)->next, 1);
243 423
 		/* remove old expired entries */
244 424
 		if ((s_ticks_t)(now-(*crt)->expire)>=0){
245 425
 			*crt=(*crt)->next;
... ...
@@ -260,8 +440,6 @@ inline static struct dst_blst_entry* _dst_blacklist_lst_find(
260 440
 /* frees all the expired entries until either there are no more of them
261 441
  *  or the total memory used is <= target (to free all of them use -1 for 
262 442
  *  targer)
263
- *  It must be called with LOCK_BLST held (or call dst_blacklist_clean_expired
264
- *   instead).
265 443
  *  params:   target  - free expired entries until no more then taget memory 
266 444
  *                      is used  (use 0 to free all of them)
267 445
  *            delta   - consider an entry expired if it expires after delta
... ...
@@ -271,9 +449,9 @@ inline static struct dst_blst_entry* _dst_blacklist_lst_find(
271 449
  *  returns: number of deleted entries
272 450
  *  This function should be called periodically from a timer
273 451
  */
274
-inline static int _dst_blacklist_clean_expired_unsafe(unsigned int target,
275
-												ticks_t delta,
276
-												ticks_t timeout)
452
+inline static int dst_blacklist_clean_expired(unsigned int target,
453
+									  ticks_t delta,
454
+									  ticks_t timeout)
277 455
 {
278 456
 	static unsigned short start=0;
279 457
 	unsigned short h;
... ...
@@ -283,57 +461,41 @@ inline static int _dst_blacklist_clean_expired_unsafe(unsigned int target,
283 461
 	ticks_t start_time;
284 462
 	ticks_t now;
285 463
 	int no=0;
464
+	int i;
286 465
 	
287 466
 	now=start_time=get_ticks_raw();
288 467
 	for(h=start; h!=(start+DST_BLST_HASH_SIZE); h++){
289
-		for (crt=&dst_blst_hash[h%DST_BLST_HASH_SIZE], tmp=&(*crt)->next;
290
-				*crt; crt=tmp, tmp=&(*crt)->next){
291
-			e=*crt;
292
-			if ((s_ticks_t)(now+delta-(*crt)->expire)>=0){
293
-				*crt=(*crt)->next;
294
-				*blst_mem_used-=DST_BLST_ENTRY_SIZE(*e);
295
-				blst_destroy_entry(e);
296
-				no++;
297
-				if (*blst_mem_used<=target)
298
-					goto skip;
468
+		i=h%DST_BLST_HASH_SIZE;
469
+		if (dst_blst_hash[i].first){
470
+			LOCK_BLST(i);
471
+			for (crt=&dst_blst_hash[i].first, tmp=&(*crt)->next;
472
+					*crt; crt=tmp, tmp=&(*crt)->next){
473
+				e=*crt;
474
+				prefetch_loc_r((*crt)->next, 1);
475
+				if ((s_ticks_t)(now+delta-(*crt)->expire)>=0){
476
+					*crt=(*crt)->next;
477
+					*blst_mem_used-=DST_BLST_ENTRY_SIZE(*e);
478
+					blst_destroy_entry(e);
479
+					BLST_HASH_STATS_DEC(i);
480
+					no++;
481
+					if (*blst_mem_used<=target){
482
+						UNLOCK_BLST(i);
483
+						goto skip;
484
+					}
485
+				}
486
+			}
487
+			UNLOCK_BLST(i);
488
+			/* check for timeout only "between" hash cells */
489
+			now=get_ticks_raw();
490
+			if ((now-start_time)>=timeout){
491
+				DBG("_dst_blacklist_clean_expired_unsafe: timeout: %d > %d\n",
492
+						TICKS_TO_MS(now-start_time), TICKS_TO_MS(timeout));
493
+				goto skip;
299 494
 			}
300
-		}
301
-		/* check for timeout only "between" hash cells */
302
-		now=get_ticks_raw();
303
-		if ((now-start_time)>=timeout){
304
-			DBG("_dst_blacklist_clean_expired_unsafe: timeout: %d > %d\n",
305
-					TICKS_TO_MS(now-start_time), TICKS_TO_MS(timeout));
306
-			goto skip;
307 495
 		}
308 496
 	}
309 497
 skip:
310 498
 	start=h; /* next time we start where we left */
311
-	return no;
312
-}
313
-
314
-
315
-
316
-/* frees all the expired entries until either there are no more of them
317
- *  or the total memory used is <= target (to free all of them use -1 for 
318
- *  targer)
319
- *  params:   target  - free expired entries until no more then taget memory 
320
- *                      is used  (use 0 to free all of them)
321
- *            delta   - consider an entry expired if it expires after delta
322
- *                      ticks from now
323
- *            timeout - exit after timeout ticks
324
- *
325
- *  returns: number of deleted entries
326
- *  This function should be called periodically from a timer
327
- */
328
-inline static int dst_blacklist_clean_expired(unsigned int target,
329
-												ticks_t delta,
330
-												ticks_t timeout)
331
-{
332
-	int no;
333
-	
334
-	LOCK_BLST();
335
-		no=_dst_blacklist_clean_expired_unsafe(target, delta, timeout);
336
-	UNLOCK_BLST();
337 499
 	if (no){
338 500
 		DBG("dst_blacklist_clean_expired, %d entries removed\n", no);
339 501
 	}
... ...
@@ -376,24 +538,28 @@ inline static int dst_blacklist_add_ip(unsigned char err_flags,
376 538
 	now=get_ticks_raw();
377 539
 	hash=dst_blst_hash_no(proto, ip, port);
378 540
 	/* check if the entry already exists */
379
-	LOCK_BLST();
380
-		e=_dst_blacklist_lst_find(&dst_blst_hash[hash], ip, proto, port, now);
541
+	LOCK_BLST(hash);
542
+		e=_dst_blacklist_lst_find(&dst_blst_hash[hash].first, ip, proto,
543
+																port, now);
381 544
 		if (e){
382 545
 			e->flags|=err_flags;
383 546
 			e->expire=now+S_TO_TICKS(blst_timeout); /* update the timeout */
384 547
 		}else{
385
-			if ((*blst_mem_used+size)>=blst_max_mem){
548
+			if (unlikely((*blst_mem_used+size)>=blst_max_mem)){
549
+				UNLOCK_BLST(hash);
386 550
 				/* first try to free some memory  (~ 12%), but don't
387 551
 				 * spend more then 250 ms*/
388
-				_dst_blacklist_clean_expired_unsafe(*blst_mem_used/16*14, 0, 
552
+				dst_blacklist_clean_expired(*blst_mem_used/16*14, 0, 
389 553
 															MS_TO_TICKS(250));
390
-				if (*blst_mem_used+size>=blst_max_mem){
554
+				if (unlikely(*blst_mem_used+size>=blst_max_mem)){
391 555
 					ret=-1;
392 556
 					goto error;
393 557
 				}
558
+				LOCK_BLST(hash);
394 559
 			}
395 560
 			e=shm_malloc(size);
396 561
 			if (e==0){
562
+				UNLOCK_BLST(hash);
397 563
 				ret=E_OUT_OF_MEM;
398 564
 				goto error;
399 565
 			}
... ...
@@ -404,10 +570,11 @@ inline static int dst_blacklist_add_ip(unsigned char err_flags,
404 570
 			memcpy(e->ip, ip->u.addr, ip->len);
405 571
 			e->expire=now+S_TO_TICKS(blst_timeout); /* update the timeout */
406 572
 			e->next=0;
407
-			dst_blacklist_lst_add(&dst_blst_hash[hash], e);
573
+			dst_blacklist_lst_add(&dst_blst_hash[hash].first, e);
574
+			BLST_HASH_STATS_INC(hash);
408 575
 		}
576
+	UNLOCK_BLST(hash);
409 577
 error:
410
-	UNLOCK_BLST();
411 578
 	return ret;
412 579
 }
413 580
 
... ...
@@ -426,12 +593,13 @@ inline static int dst_is_blacklisted_ip(unsigned char proto,
426 593
 	ret=0;
427 594
 	now=get_ticks_raw();
428 595
 	hash=dst_blst_hash_no(proto, ip, port);
429
-	LOCK_BLST();
430
-		e=_dst_blacklist_lst_find(&dst_blst_hash[hash], ip, proto, port, now);
596
+	LOCK_BLST(hash);
597
+		e=_dst_blacklist_lst_find(&dst_blst_hash[hash].first, ip, proto,
598
+									port, now);
431 599
 		if (e){
432 600
 			ret=e->flags;
433 601
 		}
434
-	UNLOCK_BLST();
602
+	UNLOCK_BLST(hash);
435 603
 	return ret;
436 604
 }
437 605
 
... ...
@@ -440,6 +608,11 @@ inline static int dst_is_blacklisted_ip(unsigned char proto,
440 608
 int dst_blacklist_add(unsigned char err_flags,  struct dest_info* si)
441 609
 {
442 610
 	struct ip_addr ip;
611
+
612
+#ifdef DST_BLACKLIST_HOOKS
613
+	if (blacklist_run_hooks(si, &err_flags)!=DST_BLACKLIST_ACCEPT)
614
+		return 0;
615
+#endif
443 616
 	su2ip_addr(&ip, &si->to);
444 617
 	return dst_blacklist_add_ip(err_flags, si->proto, &ip,
445 618
 								su_getport(&si->to));
... ...
@@ -491,9 +664,9 @@ void dst_blst_debug(rpc_t* rpc, void* ctx)
491 664
 	struct ip_addr ip;
492 665
 	
493 666
 	now=get_ticks_raw();
494
-	LOCK_BLST();
495 667
 		for(h=0; h<DST_BLST_HASH_SIZE; h++){
496
-			for(e=dst_blst_hash[h]; e; e=e->next){
668
+			LOCK_BLST(h);
669
+			for(e=dst_blst_hash[h].first; e; e=e->next){
497 670
 				dst_blst_entry2ip(&ip, e);
498 671
 				rpc->add(ctx, "ssddd", get_proto_name(e->proto), 
499 672
 										ip_addr2a(&ip), e->port, 
... ...
@@ -502,9 +675,33 @@ void dst_blst_debug(rpc_t* rpc, void* ctx)
502 675
 										-TICKS_TO_S(now-e->expire) ,
503 676
 										e->flags);
504 677
 			}
678
+			UNLOCK_BLST(h);
679
+		}
680
+}
681
+
682
+/* only for debugging, it helds the lock too long for "production" use */
683
+void dst_blst_hash_stats(rpc_t* rpc, void* ctx)
684
+{
685
+	int h;
686
+	struct dst_blst_entry* e;
687
+#ifdef BLST_HASH_STATS
688
+	int n;
689
+	
690
+	n=0;
691
+#endif
692
+	
693
+		for(h=0; h<DST_BLST_HASH_SIZE; h++){
694
+#ifdef BLST_HASH_STATS
695
+			LOCK_BLST(h);
696
+			for(e=dst_blst_hash[h].first; e; e=e->next) n++;
697
+			UNLOCK_BLST(h);
698
+			rpc->add(ctx, "dd", h, n);
699
+#else
700
+			rpc->add(ctx, "dd", h, dst_blst_hash[h].entries);
701
+#endif
505 702
 		}
506
-	UNLOCK_BLST();
507 703
 }
508 704
 
705
+
509 706
 #endif /* USE_DST_BLACKLIST */
510 707
 
... ...
@@ -46,6 +46,22 @@
46 46
 #define BLST_ADM_PROHIBITED	(1<<6)	/* administratively prohibited */
47 47
 #define BLST_PERMANENT		(1<<7)  /* never deleted, never expires */
48 48
 
49
+
50
+/*#define DST_BLACKLIST_HOOKS*/
51
+
52
+#define DST_BLACKLIST_ACCEPT 0
53
+#define DST_BLACKLIST_DENY  -1
54
+
55
+#ifdef DST_BLACKLIST_HOOKS
56
+struct blacklist_hook{
57
+	int (*on_blst_add)(struct dest_info* si, unsigned char* err_flags);
58
+	/* called before ser shutdown */
59
+	void (*destroy)(void);
60
+};
61
+
62
+int register_blacklist_hook(struct blacklist_hook *h);
63
+#endif /* DST_BLACKLIST_HOOKS */
64
+
49 65
 int init_dst_blacklist();
50 66
 void destroy_dst_blacklist();
51 67