Browse code

- added sf_malloc: a multi-process safe (internal locking), multi-pool f_malloc version. Should perform better on lots of CPU once properly tunned (experimental for now) - added ll_malloc: same as above but most operation are lockless (except for "big" fragment handling). For now needs tunning and it still keeps too many debugging statistics. (experimental for now)

Andrei Pelinescu-Onciul authored on 12/06/2007 18:07:12
Showing 10 changed files
... ...
@@ -75,7 +75,7 @@ MAIN_NAME=ser
75 75
 VERSION = 2
76 76
 PATCHLEVEL = 1
77 77
 SUBLEVEL =  0
78
-EXTRAVERSION = -dev7-tm
78
+EXTRAVERSION = -dev8-sf_malloc
79 79
 
80 80
 SER_VER = $(shell expr $(VERSION) \* 1000000 + $(PATCHLEVEL) \* 1000 + \
81 81
 			$(SUBLEVEL) )
... ...
@@ -318,6 +318,14 @@ endif
318 318
 #		an even faster malloc, not recommended for debugging
319 319
 # -DDL_MALLOC
320 320
 #		a malloc implementation based on Doug Lea's dl_malloc
321
+# -DSF_MALLOC 
322
+#		an experimental multi-CPU, pool based, multi-process safe version of 
323
+#		F_MALLOC. Should give better performance on machines with lots of CPUs
324
+#		after some tunning.
325
+# -DLL_MALLOC
326
+#		an experimental multi-CPU, pool based, multi-process safe, mostly
327
+#		lockless version of SF_MALLOC/F_MALLOC. Not for production use for
328
+#		now.
321 329
 # -DDBG_MALLOC
322 330
 #		issues additional debugging information if lock/unlock is called
323 331
 # -DFAST_LOCK
... ...
@@ -413,6 +421,8 @@ DEFS+= $(extra_defs) \
413 413
 	 -DUSE_DNS_FAILOVER \
414 414
 	 -DUSE_DST_BLACKLIST \
415 415
 	 -DDBG_QM_MALLOC \
416
+	 #-DLL_MALLOC \
417
+	 #-DSF_MALLOC \
416 418
 	 #-DDL_MALLOC \
417 419
 	 #-DF_MALLOC \
418 420
 	 #-DDBG_F_MALLOC \
... ...
@@ -453,10 +453,12 @@ char* pgid_file = 0;
453 453
 void cleanup(show_status)
454 454
 {
455 455
 	/*clean-up*/
456
+#ifndef SHM_SAFE_MALLOC
456 457
 	if (mem_lock)
457 458
 		shm_unlock(); /* hack: force-unlock the shared memory lock in case
458 459
 					 some process crashed and let it locked; this will
459 460
 					 allow an almost gracious shutdown */
461
+#endif
460 462
 	destroy_modules();
461 463
 #ifdef USE_DNS_CACHE
462 464
 	destroy_dns_cache();
463 465
new file mode 100644
... ...
@@ -0,0 +1,1095 @@
0
+/* $Id$
1
+ *
2
+ * shared memory, multi-process safe, pool based, mostly lockless version of 
3
+ *  f_malloc
4
+ *
5
+ * Copyright (C) 2007 iptelorg GmbH
6
+ *
7
+ * Permission to use, copy, modify, and distribute this software for any
8
+ * purpose with or without fee is hereby granted, provided that the above
9
+ * copyright notice and this permission notice appear in all copies.
10
+ *
11
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
12
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
14
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
16
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
17
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18
+ */
19
+/*
20
+ * History:
21
+ * --------
22
+ *              created by andrei
23
+ *  2003-07-06  added fm_realloc (andrei)
24
+ *  2004-07-19  fragments book keeping code and support for 64 bits
25
+ *               memory blocks (64 bits machine & size >=2^32) 
26
+ *              GET_HASH s/</<=/ (avoids waste of 1 hash cell)   (andrei)
27
+ *  2004-11-10  support for > 4Gb mem., switched to long (andrei)
28
+ *  2005-03-02  added fm_info() (andrei)
29
+ *  2005-12-12  fixed realloc shrink real_used accounting (andrei)
30
+ *              fixed initial size (andrei)
31
+ *  2006-02-03  fixed realloc out of mem. free bug (andrei)
32
+ *  2006-04-07  s/DBG/MDBG (andrei)
33
+ *  2007-02-23  added fm_available() (andrei)
34
+ *  2007-06-09  forked from the fm_maloc code (andrei)
35
+ *  2007-06-11  forked from the sfm_maloc code (andrei)
36
+ */
37
+
38
+
39
+#ifdef LL_MALLOC
40
+
41
+#include <string.h>
42
+#include <stdlib.h>
43
+
44
+#include "ll_malloc.h"
45
+#include "../dprint.h"
46
+#include "../globals.h"
47
+#include "memdbg.h"
48
+
49
+#define MAX_POOL_FRAGS 10000 /* max fragments per pool hash bucket */
50
+#define MIN_POOL_FRAGS 10    /* min fragments per pool hash bucket */
51
+
52
+/*useful macros*/
53
+
54
+#define FRAG_NEXT(f) \
55
+	((struct sfm_frag*)((char*)(f)+sizeof(struct sfm_frag)+(f)->size ))
56
+
57
+
58
+/* SF_ROUNDTO= 2^k so the following works */
59
+#define ROUNDTO_MASK	(~((unsigned long)SF_ROUNDTO-1))
60
+#define ROUNDUP(s)		(((s)+(SF_ROUNDTO-1))&ROUNDTO_MASK)
61
+#define ROUNDDOWN(s)	((s)&ROUNDTO_MASK)
62
+
63
+#define FRAG_OVERHEAD	(sizeof(struct sfm_frag))
64
+#define INIT_OVERHEAD	\
65
+	(ROUNDUP(sizeof(struct sfm_block))+sizeof(struct sfm_frag))
66
+
67
+
68
+
69
+/* finds hash if s <=SF_MALLOC_OPTIMIZE */
70
+#define GET_SMALL_HASH(s) (unsigned long)(s)/SF_ROUNDTO
71
+/* finds hash if s > SF_MALLOC_OPTIMIZE */
72
+#define GET_BIG_HASH(s) \
73
+	(SF_MALLOC_OPTIMIZE/SF_ROUNDTO+big_hash_idx((s))-SF_MALLOC_OPTIMIZE_FACTOR+1)
74
+
75
+/* finds the hash value for s, s=SF_ROUNDTO multiple*/
76
+#define GET_HASH(s)   ( ((unsigned long)(s)<=SF_MALLOC_OPTIMIZE)?\
77
+							GET_SMALL_HASH(s): GET_BIG_HASH(s) )
78
+
79
+
80
+#define UN_HASH_SMALL(h) ((unsigned long)(h)*SF_ROUNDTO)
81
+#define UN_HASH_BIG(h) (1UL<<((unsigned long)(h)-SF_MALLOC_OPTIMIZE/SF_ROUNDTO+\
82
+							SF_MALLOC_OPTIMIZE_FACTOR-1))
83
+
84
+#define UN_HASH(h)	( ((unsigned long)(h)<=(SF_MALLOC_OPTIMIZE/SF_ROUNDTO))?\
85
+						UN_HASH_SMALL(h): UN_HASH_BIG(h) )
86
+
87
+#define BITMAP_BITS (sizeof(((struct sfm_block*)0)->bitmap)*8)
88
+#define BITMAP_BLOCK_SIZE ((SF_MALLOC_OPTIMIZE/SF_ROUNDTO)/ BITMAP_BITS)
89
+/* only for "small" hashes (up to HASH(SF_MALLOC_OPTIMIZE) */
90
+#define HASH_BIT_POS(h) (((unsigned long)(h))/BITMAP_BLOCK_SIZE)
91
+#define HASH_TO_BITMAP(h) (1UL<<HASH_BIT_POS(h))
92
+#define BIT_TO_HASH(b) ((b)*BITMAP_BLOCK_SIZE)
93
+
94
+
95
+
96
+/* mark/test used/unused frags */
97
+#define FRAG_MARK_USED(f)
98
+#define FRAG_CLEAR_USED(f)
99
+#define FRAG_WAS_USED(f)   (1)
100
+
101
+/* other frag related defines:
102
+ * MEM_COALESCE_FRAGS 
103
+ * MEM_FRAG_AVOIDANCE
104
+ */
105
+#define MEM_FRAG_AVOIDANCE
106
+
107
+
108
+#define SFM_REALLOC_REMALLOC
109
+
110
+/* computes hash number for big buckets*/
111
+inline static unsigned long big_hash_idx(unsigned long s)
112
+{
113
+	unsigned long idx;
114
+	/* s is rounded => s = k*2^n (SF_ROUNDTO=2^n) 
115
+	 * index= i such that 2^i > s >= 2^(i-1)
116
+	 *
117
+	 * => index = number of the first non null bit in s*/
118
+	idx=sizeof(long)*8-1;
119
+	for (; !(s&(1UL<<(sizeof(long)*8-1))) ; s<<=1, idx--);
120
+	return idx;
121
+}
122
+
123
+
124
+#ifdef DBG_F_MALLOC
125
+#define ST_CHECK_PATTERN   0xf0f0f0f0
126
+#define END_CHECK_PATTERN1 0xc0c0c0c0
127
+#define END_CHECK_PATTERN2 0xabcdefed
128
+#endif
129
+
130
+
131
+#ifdef SFM_ONE_LOCK
132
+
133
+#define SFM_MAIN_HASH_LOCK(qm, hash) lock_get(&(qm)->lock)
134
+#define SFM_MAIN_HASH_UNLOCK(qm, hash) lock_release(&(qm)->lock)
135
+#define SFM_POOL_LOCK(p, hash) lock_get(&(p)->lock)
136
+#define SFM_POOL_UNLOCK(p, hash) lock_release(&(p)->lock)
137
+
138
+#warn "degraded performance, only one lock"
139
+
140
+#elif defined SFM_LOCK_PER_BUCKET
141
+
142
+#define SFM_MAIN_HASH_LOCK(qm, hash) \
143
+	lock_get(&(qm)->free_hash[(hash)].lock)
144
+#define SFM_MAIN_HASH_UNLOCK(qm, hash)  \
145
+	lock_release(&(qm)->free_hash[(hash)].lock)
146
+#define SFM_POOL_LOCK(p, hash) lock_get(&(p)->pool_hash[(hash)].lock)
147
+#define SFM_POOL_UNLOCK(p, hash) lock_release(&(p)->pool_hash[(hash)].lock)
148
+#else
149
+#error no locks defined
150
+#endif /* SFM_ONE_LOCK/SFM_LOCK_PER_BUCKET */
151
+
152
+#define SFM_BIG_GET_AND_SPLIT_LOCK(qm)   lock_get(&(qm)->get_and_split)
153
+#define SFM_BIG_GET_AND_SPLIT_UNLOCK(qm) lock_release(&(qm)->get_and_split)
154
+
155
+static unsigned long sfm_max_hash=0; /* maximum hash value (no point in
156
+										 searching further) */
157
+static unsigned long pool_id=(unsigned long)-1;
158
+
159
+
160
+/* call for each child */
161
+int sfm_pool_reset()
162
+{
163
+	pool_id=(unsigned long)-1;
164
+	return 0;
165
+}
166
+
167
+
168
+#define sfm_fix_pool_id(qm) \
169
+	do{ \
170
+		if (unlikely(pool_id>=SFM_POOLS_NO)) \
171
+			pool_id=((unsigned)atomic_add(&(qm)->crt_id, 1))%SFM_POOLS_NO; \
172
+	}while(0)
173
+
174
+
175
+
176
+static inline void frag_push(struct sfm_frag** head, struct sfm_frag* frag)
177
+{
178
+	register struct sfm_frag* old;
179
+	register struct sfm_frag* crt;
180
+	
181
+	crt=(void*)atomic_get_long(head);
182
+	do{
183
+		frag->u.nxt_free=crt;
184
+		old=crt;
185
+		membar_write_atomic_op();
186
+		crt=(void*)atomic_cmpxchg_long((void*)head, (long)old, (long)frag);
187
+	}while(crt!=old);
188
+}
189
+
190
+
191
+static inline struct sfm_frag* frag_pop(struct sfm_frag** head)
192
+{
193
+	register struct sfm_frag* old;
194
+	register struct sfm_frag* crt;
195
+	register struct sfm_frag* nxt;
196
+	
197
+	crt=(void*)atomic_get_long(head);
198
+	do{
199
+		/* if circular list, test not needed */
200
+		nxt=crt?crt->u.nxt_free:0;
201
+		old=crt;
202
+		membar_read_atomic_op();
203
+		crt=(void*)atomic_cmpxchg_long((void*)head, (long)old, (long)nxt);
204
+	}while(crt!=old);
205
+	return crt;
206
+}
207
+
208
+
209
+static inline void sfm_pool_insert (struct sfm_pool* pool, int hash,
210
+								struct sfm_frag* frag)
211
+{
212
+	unsigned long hash_bit;
213
+
214
+	frag_push(&pool->pool_hash[hash].first, frag);
215
+	atomic_inc_long((long*)&pool->pool_hash[hash].no);
216
+	/* set it only if not already set (avoids an expensive
217
+	 * cache trashing atomic write op) */
218
+	hash_bit=HASH_TO_BITMAP(hash);
219
+	if  (!(atomic_get_long((long*)&pool->bitmap) & hash_bit))
220
+		atomic_or_long((long*)&pool->bitmap, hash_bit);
221
+}
222
+
223
+
224
+
225
+/* returns 1 if it's ok to add a fragm. to pool p_id @ hash, 0 otherwise */
226
+static inline int sfm_check_pool(struct sfm_block* qm, unsigned long p_id,
227
+									int hash, int split)
228
+{
229
+	/* TODO: come up with something better
230
+	 * if fragment is some  split/rest from an allocation, that is
231
+	 *  >= requested size, accept it, else
232
+	 *  look at misses and current fragments and decide based on them */
233
+	return (p_id<SFM_POOLS_NO) && (split ||
234
+			( (qm->pool[p_id].pool_hash[hash].no < MIN_POOL_FRAGS) ||
235
+			  ((qm->pool[p_id].pool_hash[hash].misses > 
236
+				 qm->pool[p_id].pool_hash[hash].no) &&
237
+				(qm->pool[p_id].pool_hash[hash].no<MAX_POOL_FRAGS) ) ) );
238
+}
239
+
240
+
241
+/* choose on which pool to add a free'd packet
242
+ * return - pool idx or -1 if it should be added to main*/
243
+static inline unsigned long  sfm_choose_pool(struct sfm_block* qm,
244
+												struct sfm_frag* frag,
245
+												int hash, int split)
246
+{
247
+	/* check original pool first */
248
+	if (sfm_check_pool(qm, frag->id, hash, split))
249
+		return frag->id;
250
+	else{
251
+		/* check if our pool is properly set */
252
+		sfm_fix_pool_id(qm);
253
+		/* check if my pool needs some frags */
254
+		if ((pool_id!=frag->id) && (sfm_check_pool(qm,  pool_id, hash, 0))){
255
+			frag->id=pool_id;
256
+			return pool_id;
257
+		}
258
+	}
259
+	/* else add it back to main */
260
+	frag->id=(unsigned long)(-1);
261
+	return frag->id;
262
+}
263
+
264
+
265
+static inline void sfm_insert_free(struct sfm_block* qm, struct sfm_frag* frag,
266
+									int split)
267
+{
268
+	struct sfm_frag** f;
269
+	unsigned long p_id;
270
+	int hash;
271
+	unsigned long hash_bit;
272
+	
273
+	if (likely(frag->size<=SF_POOL_MAX_SIZE)){
274
+		hash=GET_SMALL_HASH(frag->size);
275
+		if (unlikely((p_id=sfm_choose_pool(qm, frag, hash, split))==
276
+					(unsigned long)-1)){
277
+			/* add it back to the "main" hash */
278
+				frag->id=(unsigned long)(-1); /* main hash marker */
279
+				/*insert it here*/
280
+				frag_push(&(qm->free_hash[hash].first), frag);
281
+				atomic_inc_long((long*)&qm->free_hash[hash].no);
282
+				/* set it only if not already set (avoids an expensive
283
+		 		* cache trashing atomic write op) */
284
+				hash_bit=HASH_TO_BITMAP(hash);
285
+				if  (!(atomic_get_long((long*)&qm->bitmap) & hash_bit))
286
+					atomic_or_long((long*)&qm->bitmap, hash_bit);
287
+		}else{
288
+			/* add it to one of the pools pool */
289
+			sfm_pool_insert(&qm->pool[p_id], hash, frag);
290
+		}
291
+	}else{
292
+		hash=GET_BIG_HASH(frag->size);
293
+		SFM_MAIN_HASH_LOCK(qm, hash);
294
+			f=&(qm->free_hash[hash].first);
295
+			for(; *f; f=&((*f)->u.nxt_free))
296
+				if (frag->size <= (*f)->size) break;
297
+			frag->id=(unsigned long)(-1); /* main hash marker */
298
+			/*insert it here*/
299
+			frag->u.nxt_free=*f;
300
+			*f=frag;
301
+			qm->free_hash[hash].no++;
302
+			/* inc. big hash free size ? */
303
+		SFM_MAIN_HASH_UNLOCK(qm, hash);
304
+	}
305
+	
306
+}
307
+
308
+
309
+
310
+ /* size should be already rounded-up */
311
+static inline
312
+#ifdef DBG_F_MALLOC 
313
+void sfm_split_frag(struct sfm_block* qm, struct sfm_frag* frag,
314
+					unsigned long size,
315
+					const char* file, const char* func, unsigned int line)
316
+#else
317
+void sfm_split_frag(struct sfm_block* qm, struct sfm_frag* frag,
318
+					unsigned long size)
319
+#endif
320
+{
321
+	unsigned long rest;
322
+	struct sfm_frag* n;
323
+	int bigger_rest;
324
+	
325
+	rest=frag->size-size;
326
+#ifdef MEM_FRAG_AVOIDANCE
327
+	if ((rest> (FRAG_OVERHEAD+SF_MALLOC_OPTIMIZE))||
328
+		(rest>=(FRAG_OVERHEAD+size))){ /* the residue fragm. is big enough*/
329
+		bigger_rest=1;
330
+#else
331
+	if (rest>(FRAG_OVERHEAD+SF_MIN_FRAG_SIZE)){
332
+		bigger_rest=rest>=(size+FRAG_OVERHEAD);
333
+#endif
334
+		frag->size=size;
335
+		/*split the fragment*/
336
+		n=FRAG_NEXT(frag);
337
+		n->size=rest-FRAG_OVERHEAD;
338
+		n->id=pool_id;
339
+		FRAG_CLEAR_USED(n); /* never used */
340
+#ifdef DBG_F_MALLOC
341
+		/* frag created by malloc, mark it*/
342
+		n->file=file;
343
+		n->func="frag. from sfm_malloc";
344
+		n->line=line;
345
+		n->check=ST_CHECK_PATTERN;
346
+#endif
347
+		/* reinsert n in free list*/
348
+		sfm_insert_free(qm, n, bigger_rest);
349
+	}else{
350
+		/* we cannot split this fragment any more => alloc all of it*/
351
+	}
352
+}
353
+
354
+
355
+
356
+/* init malloc and return a sfm_block*/
357
+struct sfm_block* sfm_malloc_init(char* address, unsigned long size)
358
+{
359
+	char* start;
360
+	char* end;
361
+	struct sfm_block* qm;
362
+	unsigned long init_overhead;
363
+	int r;
364
+#ifdef SFM_LOCK_PER_BUCKET
365
+	int i;
366
+#endif
367
+	
368
+	/* make address and size multiple of 8*/
369
+	start=(char*)ROUNDUP((unsigned long) address);
370
+	DBG("sfm_malloc_init: SF_OPTIMIZE=%lu, /SF_ROUNDTO=%lu\n",
371
+			SF_MALLOC_OPTIMIZE, SF_MALLOC_OPTIMIZE/SF_ROUNDTO);
372
+	DBG("sfm_malloc_init: SF_HASH_SIZE=%lu, sfm_block size=%lu\n",
373
+			SF_HASH_SIZE, (long)sizeof(struct sfm_block));
374
+	DBG("sfm_malloc_init(%p, %lu), start=%p\n", address, size, start);
375
+
376
+	if (size<start-address) return 0;
377
+	size-=(start-address);
378
+	if (size <(SF_MIN_FRAG_SIZE+FRAG_OVERHEAD)) return 0;
379
+	size=ROUNDDOWN(size);
380
+
381
+	init_overhead=INIT_OVERHEAD;
382
+	
383
+	
384
+	if (size < init_overhead)
385
+	{
386
+		/* not enough mem to create our control structures !!!*/
387
+		return 0;
388
+	}
389
+	end=start+size;
390
+	qm=(struct sfm_block*)start;
391
+	memset(qm, 0, sizeof(struct sfm_block));
392
+	qm->size=size;
393
+	size-=init_overhead;
394
+	
395
+	qm->first_frag=(struct sfm_frag*)(start+ROUNDUP(sizeof(struct sfm_block)));
396
+	qm->last_frag=(struct sfm_frag*)(end-sizeof(struct sfm_frag));
397
+	/* init initial fragment*/
398
+	qm->first_frag->size=size;
399
+	qm->first_frag->id=(unsigned long)-1; /* not in a pool */
400
+	qm->last_frag->size=0;
401
+	
402
+#ifdef DBG_F_MALLOC
403
+	qm->first_frag->check=ST_CHECK_PATTERN;
404
+	qm->last_frag->check=END_CHECK_PATTERN1;
405
+#endif
406
+	
407
+	/* link initial fragment into the free list*/
408
+	
409
+	sfm_insert_free(qm, qm->first_frag, 0);
410
+	sfm_max_hash=GET_HASH(size);
411
+	
412
+	/* init locks */
413
+	if (lock_init(&qm->get_and_split)==0)
414
+		goto error;
415
+#ifdef SFM_ONE_LOCK
416
+	if (lock_init(&qm->lock)==0){
417
+		lock_destroy(&qm->get_and_split);
418
+		goto error;
419
+	}
420
+	for (r=0; r<SFM_POOLS_NO; r++){
421
+		if (lock_init(&qm->pool[r].lock)==0){
422
+			for (;r>0; r--) lock_destroy(&qm->pool[r-1].lock);
423
+			lock_destroy(&qm->lock);
424
+			lock_destroy(&qm->get_and_split);
425
+			goto error;
426
+		}
427
+	}
428
+#elif defined(SFM_LOCK_PER_BUCKET)
429
+	for (r=0; r<SF_HASH_SIZE; r++)
430
+		if (lock_init(&qm->free_hash[r].lock)==0){
431
+			for(;r>0; r--) lock_destroy(&qm->free_hash[r-1].lock);
432
+			lock_destroy(&qm->get_and_split);
433
+			goto error;
434
+		}
435
+	for (i=0; i<SFM_POOLS_NO; i++){
436
+		for (r=0; r<SF_HASH_POOL_SIZE; r++)
437
+			if (lock_init(&qm->pool[i].pool_hash[r].lock)==0){
438
+				for(;r>0; r--) lock_destroy(&qm->pool[i].poo_hash[r].lock);
439
+				for(; i>0; i--){
440
+					for (r=0; r<SF_HASH_POOL_SIZE; r++)
441
+						lock_destroy(&qm->pool[i].pool_hash[r].lock);
442
+				}
443
+				for (r=0; r<SF_HASH_SIZE; r++)
444
+					lock_destroy(&qm->free_hash[r].lock);
445
+				lock_destroy(&qm->get_and_split);
446
+				goto error;
447
+			}
448
+	}
449
+#endif
450
+	qm->is_init=1;
451
+	return qm;
452
+error:
453
+	return 0;
454
+}
455
+
456
+
457
+
458
+/* cleanup */
459
+void sfm_malloc_destroy(struct sfm_block* qm)
460
+{
461
+	int r, i;
462
+	/* destroy all the locks */
463
+	if (!qm || !qm->is_init)
464
+		return; /* nothing to do */
465
+	lock_destroy(&qm->get_and_split);
466
+#ifdef SFM_ONE_LOCK
467
+	lock_destroy(&qm->lock);
468
+	for (r=0; r<SFM_POOLS_NO; r++){
469
+		lock_destroy(&qm->pool[r].lock);
470
+	}
471
+#elif defined(SFM_LOCK_PER_BUCKET)
472
+	for (r=0; r<SF_HASH_SIZE; r++)
473
+		lock_destroy(&qm->free_hash[r].lock);
474
+	for (i=0; i<SFM_POOLS_NO; i++){
475
+		for (r=0; r<SF_HASH_POOL_SIZE; r++)
476
+			lock_destroy(&qm->pool[i].pool_hash[r].lock);
477
+	}
478
+#endif
479
+	qm->is_init=0;
480
+
481
+}
482
+
483
+
484
+/* returns next set bit in bitmap, starts at b
485
+ * if b is set, returns b
486
+ * if not found returns BITMAP_BITS */
487
+static inline unsigned long _next_set_bit(unsigned long b,
488
+											unsigned long* bitmap)
489
+{
490
+	for (; !((1UL<<b)& *bitmap) && b<BITMAP_BITS; b++);
491
+	return b;
492
+}
493
+
494
+/* returns start of block b and sets *end
495
+ * (handles also the "rest" block at the end ) */
496
+static inline unsigned long _hash_range(unsigned long b, unsigned long* end)
497
+{
498
+	unsigned long s;
499
+	
500
+	if ((unlikely(b>=BITMAP_BITS))){
501
+		s=BIT_TO_HASH(BITMAP_BITS);
502
+		*end=SF_HASH_POOL_SIZE; /* last, possible rest block */
503
+	}else{
504
+		s=BIT_TO_HASH(b);
505
+		*end=s+BITMAP_BLOCK_SIZE;
506
+	}
507
+	return s;
508
+}
509
+
510
+
511
+#ifdef DBG_F_MALLOC
512
+static inline struct sfm_frag* pool_get_frag(struct sfm_block* qm,
513
+						struct sfm_pool*  pool, int hash, unisgned long size,
514
+						const char* file, const char* func, unsigned int line)
515
+#else
516
+static inline struct sfm_frag* pool_get_frag(struct sfm_block* qm,
517
+											struct sfm_pool*  pool,
518
+											int hash, unsigned long size)
519
+#endif
520
+{
521
+	int r;
522
+	int next_block;
523
+	struct sfm_frag* volatile* f;
524
+	struct sfm_frag* frag;
525
+	unsigned long b;
526
+	unsigned long eob;
527
+
528
+	/* special case for r=hash */
529
+	r=hash;
530
+	f=&pool->pool_hash[r].first;
531
+
532
+	/* detach it from the free list */
533
+	if ((frag=frag_pop((struct sfm_frag**)f))==0)
534
+		goto not_found;
535
+found:
536
+	atomic_dec_long((long*)&pool->pool_hash[r].no);
537
+	frag->u.nxt_free=0; /* mark it as 'taken' */
538
+	frag->id=pool_id;
539
+#ifdef DBG_F_MALLOC
540
+	sfm_split_frag(qm, frag, size, file, func, line);
541
+#else
542
+	sfm_split_frag(qm, frag, size);
543
+#endif
544
+	if (&qm->pool[pool_id]==pool)
545
+		atomic_inc_long((long*)&pool->hits);
546
+	return frag;
547
+	
548
+not_found:
549
+	atomic_inc_long((long*)&pool->pool_hash[r].misses);
550
+	r++;
551
+	b=HASH_BIT_POS(r);
552
+	
553
+	while(r<SF_HASH_POOL_SIZE){
554
+		b=_next_set_bit(b, &pool->bitmap);
555
+		next_block=_hash_range(b, &eob);
556
+		r=(r<next_block)?next_block:r;
557
+		for (; r<eob; r++){
558
+			f=&pool->pool_hash[r].first;
559
+			if ((frag=frag_pop((struct sfm_frag**)f))!=0)
560
+				goto found;
561
+			atomic_inc_long((long*)&pool->pool_hash[r].misses);
562
+		}
563
+		b++;
564
+	}
565
+	atomic_inc_long((long*)&pool->missed);
566
+	return 0;
567
+}
568
+
569
+
570
+
571
+#ifdef DBG_F_MALLOC
572
+static inline struct sfm_frag* main_get_frag(struct sfm_block* qm, int hash,
573
+						unsigned long size,
574
+						const char* file, const char* func, unsigned int line)
575
+#else
576
+static inline struct sfm_frag* main_get_frag(struct sfm_block* qm, int hash,
577
+												unsigned long size)
578
+#endif
579
+{
580
+	int r;
581
+	int next_block;
582
+	struct sfm_frag* volatile* f;
583
+	struct sfm_frag* frag;
584
+	unsigned long b;
585
+	unsigned long eob;
586
+
587
+	r=hash;
588
+	b=HASH_BIT_POS(r);
589
+	while(r<=SF_MALLOC_OPTIMIZE/SF_ROUNDTO){
590
+			b=_next_set_bit(b, &qm->bitmap);
591
+			next_block=_hash_range(b, &eob);
592
+			r=(r<next_block)?next_block:r;
593
+			for (; r<eob; r++){
594
+				f=&qm->free_hash[r].first;
595
+				if ((frag=frag_pop((struct sfm_frag**)f))!=0){
596
+					atomic_dec_long((long*)&qm->free_hash[r].no);
597
+					frag->u.nxt_free=0; /* mark it as 'taken' */
598
+					frag->id=pool_id;
599
+#ifdef DBG_F_MALLOC
600
+					sfm_split_frag(qm, frag, size, file, func, line);
601
+#else
602
+					sfm_split_frag(qm, frag, size);
603
+#endif
604
+					return frag;
605
+				}
606
+			}
607
+			b++;
608
+	}
609
+	/* big fragments */
610
+	SFM_BIG_GET_AND_SPLIT_LOCK(qm);
611
+	for (; r<= sfm_max_hash ; r++){
612
+		f=&qm->free_hash[r].first;
613
+		if (*f){
614
+			SFM_MAIN_HASH_LOCK(qm, r);
615
+			if (unlikely((*f)==0)){
616
+				/* not found */
617
+				SFM_MAIN_HASH_UNLOCK(qm, r);
618
+				continue; 
619
+			}
620
+			for(;(*f); f=&((*f)->u.nxt_free))
621
+				if ((*f)->size>=size){
622
+					/* found, detach it from the free list*/
623
+					frag=*f;
624
+					*f=frag->u.nxt_free;
625
+					frag->u.nxt_free=0; /* mark it as 'taken' */
626
+					qm->free_hash[r].no--;
627
+					SFM_MAIN_HASH_UNLOCK(qm, r);
628
+					frag->id=pool_id;
629
+#ifdef DBG_F_MALLOC
630
+					sfm_split_frag(qm, frag, size, file, func, line);
631
+#else
632
+					sfm_split_frag(qm, frag, size);
633
+#endif
634
+					SFM_BIG_GET_AND_SPLIT_UNLOCK(qm);
635
+					return frag;
636
+				};
637
+			SFM_MAIN_HASH_UNLOCK(qm, r);
638
+			/* try in a bigger bucket */
639
+		}
640
+	}
641
+	SFM_BIG_GET_AND_SPLIT_UNLOCK(qm);
642
+	return 0;
643
+}
644
+
645
+
646
+
647
+#ifdef DBG_F_MALLOC
648
+void* sfm_malloc(struct sfm_block* qm, unsigned long size,
649
+					const char* file, const char* func, unsigned int line)
650
+#else
651
+void* sfm_malloc(struct sfm_block* qm, unsigned long size)
652
+#endif
653
+{
654
+	struct sfm_frag* frag;
655
+	int hash;
656
+	unsigned int i;
657
+	
658
+#ifdef DBG_F_MALLOC
659
+	MDBG("sfm_malloc(%p, %lu) called from %s: %s(%d)\n", qm, size, file, func,
660
+			line);
661
+#endif
662
+	/*size must be a multiple of 8*/
663
+	size=ROUNDUP(size);
664
+/*	if (size>(qm->size-qm->real_used)) return 0; */
665
+
666
+	/* check if our pool id is set */
667
+	sfm_fix_pool_id(qm);
668
+	
669
+	/*search for a suitable free frag*/
670
+	if (likely(size<=SF_POOL_MAX_SIZE)){
671
+		hash=GET_SMALL_HASH(size);
672
+		/* try first in our pool */
673
+#ifdef DBG_F_MALLOC
674
+		if (likely((frag=pool_get_frag(qm, &qm->pool[pool_id], hash, size,
675
+										file, func, line))!=0))
676
+			goto found;
677
+		/* try in the "main" free hash, go through all the hash */
678
+		if (likely((frag=main_get_frag(qm, hash, size, file, func, line))!=0))
679
+			goto found;
680
+		/* really low mem , try in other pools */
681
+		for (i=(pool_id+1); i< (pool_id+SFM_POOLS_NO); i++){
682
+			if ((frag=pool_get_frag(qm, &qm->pool[i%SFM_POOLS_NO], hash, size,
683
+										file, func, line))!=0)
684
+				goto found;
685
+		}
686
+#else
687
+		if (likely((frag=pool_get_frag(qm, &qm->pool[pool_id], hash, size))
688
+					!=0 ))
689
+			goto found;
690
+		/* try in the "main" free hash, go through all the hash */
691
+		if (likely((frag=main_get_frag(qm, hash, size))!=0))
692
+			goto found;
693
+		/* really low mem , try in other pools */
694
+		for (i=(pool_id+1); i< (pool_id+SFM_POOLS_NO); i++){
695
+			if ((frag=pool_get_frag(qm, &qm->pool[i%SFM_POOLS_NO], hash, size))
696
+					!=0 )
697
+				goto found;
698
+		}
699
+#endif
700
+		/* not found, bad! */
701
+		return 0;
702
+	}else{
703
+		hash=GET_BIG_HASH(size);
704
+#ifdef DBG_F_MALLOC
705
+		if ((frag=main_get_frag(qm, hash, size, file, func, line))==0)
706
+			return 0; /* not found, bad! */
707
+#else
708
+		if ((frag=main_get_frag(qm, hash, size))==0)
709
+			return 0; /* not found, bad! */
710
+#endif
711
+	}
712
+
713
+found:
714
+	/* we found it!*/
715
+#ifdef DBG_F_MALLOC
716
+	frag->file=file;
717
+	frag->func=func;
718
+	frag->line=line;
719
+	frag->check=ST_CHECK_PATTERN;
720
+	MDBG("sfm_malloc(%p, %lu) returns address %p \n", qm, size,
721
+		(char*)frag+sizeof(struct sfm_frag));
722
+#endif
723
+	FRAG_MARK_USED(frag); /* mark it as used */
724
+	return (char*)frag+sizeof(struct sfm_frag);
725
+}
726
+
727
+
728
+
729
+#ifdef DBG_F_MALLOC
730
+void sfm_free(struct sfm_block* qm, void* p, const char* file,
731
+				const char* func, unsigned int line)
732
+#else
733
+void sfm_free(struct sfm_block* qm, void* p)
734
+#endif
735
+{
736
+	struct sfm_frag* f;
737
+
738
+#ifdef DBG_F_MALLOC
739
+	MDBG("sfm_free(%p, %p), called from %s: %s(%d)\n", qm, p, file, func,
740
+				line);
741
+	if (p>(void*)qm->last_frag || p<(void*)qm->first_frag){
742
+		LOG(L_CRIT, "BUG: sfm_free: bad pointer %p (out of memory block!) - "
743
+				"aborting\n", p);
744
+		abort();
745
+	}
746
+#endif
747
+	if (unlikely(p==0)) {
748
+		LOG(L_WARN, "WARNING: sfm_free: free(0) called\n");
749
+		return;
750
+	}
751
+	f=(struct sfm_frag*) ((char*)p-sizeof(struct sfm_frag));
752
+#ifdef DBG_F_MALLOC
753
+	MDBG("sfm_free: freeing block alloc'ed from %s: %s(%ld)\n",
754
+			f->file, f->func, f->line);
755
+#endif
756
+#ifdef DBG_F_MALLOC
757
+	f->file=file;
758
+	f->func=func;
759
+	f->line=line;
760
+#endif
761
+	sfm_insert_free(qm, f, 0);
762
+}
763
+
764
+
765
+#ifdef DBG_F_MALLOC
766
+void* sfm_realloc(struct sfm_block* qm, void* p, unsigned long size,
767
+					const char* file, const char* func, unsigned int line)
768
+#else
769
+void* sfm_realloc(struct sfm_block* qm, void* p, unsigned long size)
770
+#endif
771
+{
772
+	struct sfm_frag *f;
773
+	unsigned long orig_size;
774
+	void *ptr;
775
+#ifndef SFM_REALLOC_REMALLOC
776
+	struct sfm_frag *n;
777
+	struct sfm_frag **pf;
778
+	unsigned long diff;
779
+	unsigned long p_id;
780
+	int hash;
781
+	unsigned long n_size;
782
+	struct sfm_pool * pool;
783
+#endif
784
+	
785
+#ifdef DBG_F_MALLOC
786
+	MDBG("sfm_realloc(%p, %p, %lu) called from %s: %s(%d)\n", qm, p, size,
787
+			file, func, line);
788
+	if ((p)&&(p>(void*)qm->last_frag || p<(void*)qm->first_frag)){
789
+		LOG(L_CRIT, "BUG: sfm_free: bad pointer %p (out of memory block!) - "
790
+				"aborting\n", p);
791
+		abort();
792
+	}
793
+#endif
794
+	if (size==0) {
795
+		if (p)
796
+#ifdef DBG_F_MALLOC
797
+			sfm_free(qm, p, file, func, line);
798
+#else
799
+			sfm_free(qm, p);
800
+#endif
801
+		return 0;
802
+	}
803
+	if (p==0)
804
+#ifdef DBG_F_MALLOC
805
+		return sfm_malloc(qm, size, file, func, line);
806
+#else
807
+		return sfm_malloc(qm, size);
808
+#endif
809
+	f=(struct sfm_frag*) ((char*)p-sizeof(struct sfm_frag));
810
+#ifdef DBG_F_MALLOC
811
+	MDBG("sfm_realloc: realloc'ing frag %p alloc'ed from %s: %s(%ld)\n",
812
+			f, f->file, f->func, f->line);
813
+#endif
814
+	size=ROUNDUP(size);
815
+	orig_size=f->size;
816
+	if (f->size > size){
817
+		/* shrink */
818
+#ifdef DBG_F_MALLOC
819
+		MDBG("sfm_realloc: shrinking from %lu to %lu\n", f->size, size);
820
+		sfm_split_frag(qm, f, size, file, "frag. from sfm_realloc", line);
821
+#else
822
+		sfm_split_frag(qm, f, size);
823
+#endif
824
+	}else if (f->size<size){
825
+		/* grow */
826
+#ifdef DBG_F_MALLOC
827
+		MDBG("sfm_realloc: growing from %lu to %lu\n", f->size, size);
828
+#endif
829
+#ifndef SFM_REALLOC_REMALLOC
830
+/* should set a magic value in list head and in push/pop if magic value =>
831
+ * lock and wait */
832
+#error LL_MALLOC realloc not finished yet
833
+		diff=size-f->size;
834
+		n=FRAG_NEXT(f);
835
+		if (((char*)n < (char*)qm->last_frag) && 
836
+				(n->u.nxt_free)&&((n->size+FRAG_OVERHEAD)>=diff)){
837
+			/* join  */
838
+			/* detach n from the free list */
839
+try_again:
840
+			p_id=n->id;
841
+			n_size=n->size;
842
+			if ((unlikely(p_id >=SFM_POOLS_NO))){
843
+				hash=GET_HASH(n_size);
844
+				SFM_MAIN_HASH_LOCK(qm, hash);
845
+				if (unlikely((n->u.nxt_free==0) ||
846
+							((n->size+FRAG_OVERHEAD)<diff))){ 
847
+					SFM_MAIN_HASH_UNLOCK(qm, hash);
848
+					goto not_found;
849
+				}
850
+				if (unlikely((n->id!=p_id) || (n->size!=n_size))){
851
+					/* fragment still free, but changed, either 
852
+					 * moved to another pool or has a diff. size */
853
+					SFM_MAIN_HASH_UNLOCK(qm, hash);
854
+					goto try_again;
855
+				}
856
+				pf=&(qm->free_hash[hash].first);
857
+				/* find it */
858
+				for(;(*pf)&&(*pf!=n); pf=&((*pf)->u.nxt_free));/*FIXME slow */
859
+				if (*pf==0){
860
+					SFM_MAIN_HASH_UNLOCK(qm, hash);
861
+					/* not found, bad! */
862
+					LOG(L_WARN, "WARNING: sfm_realloc: could not find %p in "
863
+							    "free " "list (hash=%d)\n", n, hash);
864
+					/* somebody is in the process of changing it ? */
865
+					goto not_found;
866
+				}
867
+				/* detach */
868
+				*pf=n->u.nxt_free;
869
+				n->u.nxt_free=0; /* mark it immediately as detached */
870
+				qm->free_hash[hash].no--;
871
+				SFM_MAIN_HASH_UNLOCK(qm, hash);
872
+				/* join */
873
+				f->size+=n->size+FRAG_OVERHEAD;
874
+				/* split it if necessary */
875
+				if (f->size > size){
876
+			#ifdef DBG_F_MALLOC
877
+					sfm_split_frag(qm, f, size, file, "fragm. from "
878
+									"sfm_realloc", line);
879
+			#else
880
+					sfm_split_frag(qm, f, size);
881
+			#endif
882
+				}
883
+			}else{ /* p_id < SFM_POOLS_NO (=> in a pool )*/
884
+				hash=GET_SMALL_HASH(n_size);
885
+				pool=&qm->pool[p_id];
886
+				SFM_POOL_LOCK(pool, hash);
887
+				if (unlikely((n->u.nxt_free==0) ||
888
+							((n->size+FRAG_OVERHEAD)<diff))){
889
+					SFM_POOL_UNLOCK(pool, hash);
890
+					goto not_found;
891
+				}
892
+				if (unlikely((n->id!=p_id) || (n->size!=n_size))){
893
+					/* fragment still free, but changed, either 
894
+					 * moved to another pool or has a diff. size */
895
+					SFM_POOL_UNLOCK(pool, hash);
896
+					goto try_again;
897
+				}
898
+				pf=&(pool->pool_hash[hash].first);
899
+				/* find it */
900
+				for(;(*pf)&&(*pf!=n); pf=&((*pf)->u.nxt_free));/*FIXME slow */
901
+				if (*pf==0){
902
+					SFM_POOL_UNLOCK(pool, hash);
903
+					/* not found, bad! */
904
+					LOG(L_WARN, "WARNING: sfm_realloc: could not find %p in "
905
+							    "free " "list (hash=%d)\n", n, hash);
906
+					/* somebody is in the process of changing it ? */
907
+					goto not_found;
908
+				}
909
+				/* detach */
910
+				*pf=n->u.nxt_free;
911
+				n->u.nxt_free=0; /* mark it immediately as detached */
912
+				pool->pool_hash[hash].no--;
913
+				SFM_POOL_UNLOCK(pool, hash);
914
+				/* join */
915
+				f->size+=n->size+FRAG_OVERHEAD;
916
+				/* split it if necessary */
917
+				if (f->size > size){
918
+			#ifdef DBG_F_MALLOC
919
+					sfm_split_frag(qm, f, size, file, "fragm. from "
920
+									"sfm_realloc", line);
921
+			#else
922
+					sfm_split_frag(qm, f, size);
923
+			#endif
924
+				}
925
+			}
926
+		}else{
927
+not_found:
928
+			/* could not join => realloc */
929
+#else/* SFM_REALLOC_REMALLOC */ 
930
+		{
931
+#endif /* SFM_REALLOC_REMALLOC */
932
+	#ifdef DBG_F_MALLOC
933
+			ptr=sfm_malloc(qm, size, file, func, line);
934
+	#else
935
+			ptr=sfm_malloc(qm, size);
936
+	#endif
937
+			if (ptr){
938
+				/* copy, need by libssl */
939
+				memcpy(ptr, p, orig_size);
940
+	#ifdef DBG_F_MALLOC
941
+				sfm_free(qm, p, file, func, line);
942
+	#else
943
+				sfm_free(qm, p);
944
+	#endif
945
+			}
946
+			p=ptr;
947
+		}
948
+	}else{
949
+		/* do nothing */
950
+#ifdef DBG_F_MALLOC
951
+		MDBG("sfm_realloc: doing nothing, same size: %lu - %lu\n", 
952
+				f->size, size);
953
+#endif
954
+	}
955
+#ifdef DBG_F_MALLOC
956
+	MDBG("sfm_realloc: returning %p\n", p);
957
+#endif
958
+	return p;
959
+}
960
+
961
+
962
+
963
+void sfm_status(struct sfm_block* qm)
964
+{
965
+	struct sfm_frag* f;
966
+	int i,j;
967
+	int h;
968
+	int unused;
969
+	unsigned long size;
970
+	int k;
971
+
972
+#warning ll_status doesn't work (might crash if used)
973
+
974
+	LOG(memlog, "sfm_status (%p):\n", qm);
975
+	if (!qm) return;
976
+
977
+	LOG(memlog, " heap size= %ld\n", qm->size);
978
+	LOG(memlog, "dumping free list:\n");
979
+	for(h=0,i=0,size=0;h<=sfm_max_hash;h++){
980
+		SFM_MAIN_HASH_LOCK(qm, h);
981
+		unused=0;
982
+		for (f=qm->free_hash[h].first,j=0; f;
983
+				size+=f->size,f=f->u.nxt_free,i++,j++){
984
+			if (!FRAG_WAS_USED(f)){
985
+				unused++;
986
+#ifdef DBG_F_MALLOC
987
+				LOG(memlog, "unused fragm.: hash = %3d, fragment %p,"
988
+							" address %p size %lu, created from %s: %s(%ld)\n",
989
+						    h, f, (char*)f+sizeof(struct sfm_frag), f->size,
990
+							f->file, f->func, f->line);
991
+#endif
992
+			};
993
+		}
994
+		if (j) LOG(memlog, "hash = %3d fragments no.: %5d, unused: %5d\n\t\t"
995
+							" bucket size: %9lu - %9lu (first %9lu)\n",
996
+							h, j, unused, UN_HASH(h),
997
+						((h<=SF_MALLOC_OPTIMIZE/SF_ROUNDTO)?1:2)* UN_HASH(h),
998
+							qm->free_hash[h].first->size
999
+				);
1000
+		if (j!=qm->free_hash[h].no){
1001
+			LOG(L_CRIT, "BUG: sfm_status: different free frag. count: %d!=%ld"
1002
+					" for hash %3d\n", j, qm->free_hash[h].no, h);
1003
+		}
1004
+		SFM_MAIN_HASH_UNLOCK(qm, h);
1005
+	}
1006
+	for (k=0; k<SFM_POOLS_NO; k++){
1007
+		for(h=0;h<SF_HASH_POOL_SIZE;h++){
1008
+			SFM_POOL_LOCK(&qm->pool[k], h);
1009
+			unused=0;
1010
+			for (f=qm->pool[k].pool_hash[h].first,j=0; f;
1011
+					size+=f->size,f=f->u.nxt_free,i++,j++){
1012
+				if (!FRAG_WAS_USED(f)){
1013
+					unused++;
1014
+#ifdef DBG_F_MALLOC
1015
+					LOG(memlog, "[%2d] unused fragm.: hash = %3d, fragment %p,"
1016
+								" address %p size %lu, created from %s: "
1017
+								"%s(%ld)\n", k
1018
+								h, f, (char*)f+sizeof(struct sfm_frag),
1019
+								f->size, f->file, f->func, f->line);
1020
+#endif
1021
+				};
1022
+			}
1023
+			if (j) LOG(memlog, "[%2d] hash = %3d fragments no.: %5d, unused: "
1024
+								"%5d\n\t\t bucket size: %9lu - %9lu "
1025
+								"(first %9lu)\n",
1026
+								k, h, j, unused, UN_HASH(h),
1027
+							((h<=SF_MALLOC_OPTIMIZE/SF_ROUNDTO)?1:2) *
1028
+								UN_HASH(h),
1029
+								qm->pool[k].pool_hash[h].first->size
1030
+					);
1031
+			if (j!=qm->pool[k].pool_hash[h].no){
1032
+				LOG(L_CRIT, "BUG: sfm_status: [%d] different free frag."
1033
+							" count: %d!=%ld for hash %3d\n",
1034
+							k, j, qm->pool[k].pool_hash[h].no, h);
1035
+			}
1036
+			SFM_POOL_UNLOCK(&qm->pool[k], h);
1037
+		}
1038
+	}
1039
+	LOG(memlog, "TOTAL: %6d free fragments = %6lu free bytes\n", i, size);
1040
+	LOG(memlog, "-----------------------------\n");
1041
+}
1042
+
1043
+
1044
+
1045
+/* fills a malloc info structure with info about the block
1046
+ * if a parameter is not supported, it will be filled with 0 */
1047
+void sfm_info(struct sfm_block* qm, struct mem_info* info)
1048
+{
1049
+	int r, k;
1050
+	unsigned long total_frags;
1051
+	struct sfm_frag* f;
1052
+	
1053
+	memset(info,0, sizeof(*info));
1054
+	total_frags=0;
1055
+	info->total_size=qm->size;
1056
+	info->min_frag=SF_MIN_FRAG_SIZE;
1057
+	/* we'll have to compute it all */
1058
+	for (r=0; r<=SF_MALLOC_OPTIMIZE/SF_ROUNDTO; r++){
1059
+		info->free+=qm->free_hash[r].no*UN_HASH(r);
1060
+		total_frags+=qm->free_hash[r].no;
1061
+	}
1062
+	for(;r<=sfm_max_hash; r++){
1063
+		total_frags+=qm->free_hash[r].no;
1064
+		SFM_MAIN_HASH_LOCK(qm, r);
1065
+		for(f=qm->free_hash[r].first;f;f=f->u.nxt_free){
1066
+			info->free+=f->size;
1067
+		}
1068
+		SFM_MAIN_HASH_UNLOCK(qm, r);
1069
+	}
1070
+	for (k=0; k<SFM_POOLS_NO; k++){
1071
+		for (r=0; r<SF_HASH_POOL_SIZE; r++){
1072
+			info->free+=qm->pool[k].pool_hash[r].no*UN_HASH(r);
1073
+			total_frags+=qm->pool[k].pool_hash[r].no;
1074
+		}
1075
+	}
1076
+	info->real_used=info->total_size-info->free;
1077
+	info->used=info->real_used-total_frags*FRAG_OVERHEAD-INIT_OVERHEAD
1078
+				-FRAG_OVERHEAD;
1079
+	info->max_used=0; /* we don't really know */
1080
+	info->total_frags=total_frags;
1081
+}
1082
+
1083
+
1084
+
1085
+/* returns how much free memory is available
1086
+ * on error (not compiled with bookkeeping code) returns (unsigned long)(-1) */
1087
+unsigned long sfm_available(struct sfm_block* qm)
1088
+{
1089
+	/* we don't know how much free memory we have and it's to expensive
1090
+	 * to compute it */
1091
+	return ((unsigned long)-1);
1092
+}
1093
+
1094
+#endif
0 1095
new file mode 100644
... ...
@@ -0,0 +1,178 @@
0
+/* $Id$
1
+ *
2
+ * shared memory, multi-process safe, pool based, mostly lockless version of 
3
+ *  f_malloc
4
+ *
5
+ * Copyright (C) 2007 iptelorg GmbH
6
+ *
7
+ * Permission to use, copy, modify, and distribute this software for any
8
+ * purpose with or without fee is hereby granted, provided that the above
9
+ * copyright notice and this permission notice appear in all copies.
10
+ *
11
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
12
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
14
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
16
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
17
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18
+ */
19
+/*
20
+ * History:
21
+ * --------
22
+ *  2003-05-21  on sparc64 roundto 8 even in debugging mode (so malloc'ed
23
+ *               long longs will be 64 bit aligned) (andrei)
24
+ *  2004-07-19  support for 64 bit (2^64 mem. block) and more info
25
+ *               for the future de-fragmentation support (andrei)
26
+ *  2004-11-10  support for > 4Gb mem., switched to long (andrei)
27
+ *  2007-06-11  forked from the sf_malloc code (andrei)
28
+ */
29
+
30
+
31
+#if !defined(ll_malloc_h)  
32
+#define ll_malloc_h
33
+
34
+
35
+#include "meminfo.h"
36
+
37
+#include "../lock_ops.h"
38
+#include "../atomic_ops.h"
39
+#include "../compiler_opt.h"
40
+/* defs*/
41
+
42
+
43
+#ifdef GEN_LOCK_T_UNLIMITED
44
+#define SFM_LOCK_PER_BUCKET
45
+#else
46
+#define SFM_ONE_LOCK
47
+#endif
48
+
49
+#ifdef DBG_SF_MALLOC
50
+#if defined(__CPU_sparc64) || defined(__CPU_sparc)
51
+/* tricky, on sun in 32 bits mode long long must be 64 bits aligned
52
+ * but long can be 32 bits aligned => malloc should return long long
53
+ * aligned memory */
54
+	#define SF_ROUNDTO	sizeof(long long)
55
+#else
56
+	#define SF_ROUNDTO	sizeof(void*) /* size we round to, must be = 2^n, and
57
+                      sizeof(sfm_frag) must be multiple of SF_ROUNDTO !*/
58
+#endif
59
+#else /* DBG_SF_MALLOC */
60
+	#define SF_ROUNDTO 8UL
61
+#endif
62
+#define SF_MIN_FRAG_SIZE	SF_ROUNDTO
63
+
64
+#define SFM_POOLS_NO 4U /* the more the better, but higher initial
65
+                            mem. consumption */
66
+
67
+#define SF_MALLOC_OPTIMIZE_FACTOR 14UL /*used below */
68
+#define SF_MALLOC_OPTIMIZE  (1UL<<SF_MALLOC_OPTIMIZE_FACTOR)
69
+								/* size to optimize for,
70
+									(most allocs <= this size),
71
+									must be 2^k */
72
+
73
+#define SF_HASH_POOL_SIZE	(SF_MALLOC_OPTIMIZE/SF_ROUNDTO + 1)
74
+#define SF_POOL_MAX_SIZE	SF_MALLOC_OPTIMIZE
75
+
76
+#define SF_HASH_SIZE (SF_MALLOC_OPTIMIZE/SF_ROUNDTO + \
77
+		(sizeof(long)*8-SF_MALLOC_OPTIMIZE_FACTOR)+1)
78
+
79
+/* hash structure:
80
+ * 0 .... SF_MALLOC_OPTIMIZE/SF_ROUNDTO  - small buckets, size increases with
81
+ *                            SF_ROUNDTO from bucket to bucket
82
+ * +1 .... end -  size = 2^k, big buckets */
83
+
84
+struct sfm_frag{
85
+	union{
86
+		struct sfm_frag* nxt_free;
87
+		long reserved;
88
+	}u;
89
+	unsigned long size;
90
+	unsigned long id; /* TODO better optimize the size */
91
+	/* pad to SF_ROUNDTO multiple */
92
+	char _pad[((3*sizeof(long)+SF_ROUNDTO-1)&~(SF_ROUNDTO-1))-3*sizeof(long)];
93
+#ifdef DBG_SF_MALLOC
94
+	const char* file;
95
+	const char* func;
96
+	unsigned long line;
97
+	unsigned long check;
98
+#endif
99
+};
100
+
101
+struct sfm_frag_lnk{
102
+	struct sfm_frag* first;
103
+#ifdef SFM_LOCK_PER_BUCKET
104
+	gen_lock_t lock;
105
+#endif
106
+	unsigned long no;
107
+};
108
+
109
+struct sfm_pool_head{
110
+	struct sfm_frag* first;
111
+#ifdef SFM_LOCK_PER_BUCKET
112
+	gen_lock_t lock;
113
+#endif
114
+	unsigned long no;
115
+	unsigned long misses;
116
+};
117
+
118
+struct sfm_pool{
119
+#ifdef SFM_ONE_LOCK
120
+	gen_lock_t lock;
121
+#endif
122
+	unsigned long missed;
123
+	unsigned long hits; /* debugging only TODO: remove */
124
+	unsigned long bitmap;
125
+	struct sfm_pool_head pool_hash[SF_HASH_POOL_SIZE];
126
+};
127
+
128
+struct sfm_block{
129
+#ifdef SFM_ONE_LOCK
130
+	gen_lock_t lock;
131
+#endif
132
+	atomic_t crt_id; /* current pool */
133
+	unsigned long size; /* total size */
134
+	/* stats are kept now per bucket */
135
+	struct sfm_frag* first_frag;
136
+	struct sfm_frag* last_frag;
137
+	unsigned long bitmap; /* only up to SF_MALLOC_OPTIMIZE */
138
+	struct sfm_frag_lnk free_hash[SF_HASH_SIZE];
139
+	struct sfm_pool pool[SFM_POOLS_NO];
140
+	int is_init;
141
+	gen_lock_t get_and_split;
142
+	char _pad[256];
143
+};
144
+
145
+
146
+
147
+struct sfm_block* sfm_malloc_init(char* address, unsigned long size);
148
+void sfm_malloc_destroy(struct sfm_block* qm);
149
+int sfm_pool_reset();
150
+
151
+#ifdef DBG_SF_MALLOC
152
+void* sfm_malloc(struct sfm_block*, unsigned long size,
153
+					const char* file, const char* func, unsigned int line);
154
+#else
155
+void* sfm_malloc(struct sfm_block*, unsigned long size);
156
+#endif
157
+
158
+#ifdef DBG_SF_MALLOC
159
+void  sfm_free(struct sfm_block*, void* p, const char* file, const char* func, 
160
+				unsigned int line);
161
+#else
162
+void  sfm_free(struct sfm_block*, void* p);
163
+#endif
164
+
165
+#ifdef DBG_SF_MALLOC
166
+void*  sfm_realloc(struct sfm_block*, void* p, unsigned long size, 
167
+					const char* file, const char* func, unsigned int line);
168
+#else
169
+void*  sfm_realloc(struct sfm_block*, void* p, unsigned long size);
170
+#endif
171
+
172
+void  sfm_status(struct sfm_block*);
173
+void  sfm_info(struct sfm_block*, struct mem_info*);
174
+
175
+unsigned long sfm_available(struct sfm_block*);
176
+
177
+#endif
0 178
new file mode 100644
... ...
@@ -0,0 +1,1114 @@
0
+/* $Id$
1
+ *
2
+ * shared memory, multi-process safe, pool based version of f_malloc
3
+ *
4
+ * Copyright (C) 2007 iptelorg GmbH
5
+ *
6
+ * Permission to use, copy, modify, and distribute this software for any
7
+ * purpose with or without fee is hereby granted, provided that the above
8
+ * copyright notice and this permission notice appear in all copies.
9
+ *
10
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17
+ */
18
+/*
19
+ * History:
20
+ * --------
21
+ *              created by andrei
22
+ *  2003-07-06  added fm_realloc (andrei)
23
+ *  2004-07-19  fragments book keeping code and support for 64 bits
24
+ *               memory blocks (64 bits machine & size >=2^32) 
25
+ *              GET_HASH s/</<=/ (avoids waste of 1 hash cell)   (andrei)
26
+ *  2004-11-10  support for > 4Gb mem., switched to long (andrei)
27
+ *  2005-03-02  added fm_info() (andrei)
28
+ *  2005-12-12  fixed realloc shrink real_used accounting (andrei)
29
+ *              fixed initial size (andrei)
30
+ *  2006-02-03  fixed realloc out of mem. free bug (andrei)
31
+ *  2006-04-07  s/DBG/MDBG (andrei)
32
+ *  2007-02-23  added fm_available() (andrei)
33
+ *  2007-06-09  forked from the fm_maloc code (andrei)
34
+ */
35
+
36
+
37
+#ifdef SF_MALLOC
38
+
39
+#include <string.h>
40
+#include <stdlib.h>
41
+
42
+#include "sf_malloc.h"
43
+#include "../dprint.h"
44
+#include "../globals.h"
45
+#include "memdbg.h"
46
+
47
+#define MAX_POOL_FRAGS 10000 /* max fragments per pool hash bucket */
48
+#define MIN_POOL_FRAGS 10    /* min fragments per pool hash bucket */
49
+
50
+/*useful macros*/
51
+
52
+#define FRAG_NEXT(f) \
53
+	((struct sfm_frag*)((char*)(f)+sizeof(struct sfm_frag)+(f)->size ))
54
+
55
+
56
+/* SF_ROUNDTO= 2^k so the following works */
57
+#define ROUNDTO_MASK	(~((unsigned long)SF_ROUNDTO-1))
58
+#define ROUNDUP(s)		(((s)+(SF_ROUNDTO-1))&ROUNDTO_MASK)
59
+#define ROUNDDOWN(s)	((s)&ROUNDTO_MASK)
60
+
61
+#define FRAG_OVERHEAD	(sizeof(struct sfm_frag))
62
+#define INIT_OVERHEAD	\
63
+	(ROUNDUP(sizeof(struct sfm_block))+sizeof(struct sfm_frag))
64
+
65
+
66
+
67
+/* finds hash if s <=SF_MALLOC_OPTIMIZE */
68
+#define GET_SMALL_HASH(s) (unsigned long)(s)/SF_ROUNDTO
69
+/* finds hash if s > SF_MALLOC_OPTIMIZE */
70
+#define GET_BIG_HASH(s) \
71
+	(SF_MALLOC_OPTIMIZE/SF_ROUNDTO+big_hash_idx((s))-SF_MALLOC_OPTIMIZE_FACTOR+1)
72
+
73
+/* finds the hash value for s, s=SF_ROUNDTO multiple*/
74
+#define GET_HASH(s)   ( ((unsigned long)(s)<=SF_MALLOC_OPTIMIZE)?\
75
+							GET_SMALL_HASH(s): GET_BIG_HASH(s) )
76
+
77
+
78
+#define UN_HASH_SMALL(h) ((unsigned long)(h)*SF_ROUNDTO)
79
+#define UN_HASH_BIG(h) (1UL<<((unsigned long)(h)-SF_MALLOC_OPTIMIZE/SF_ROUNDTO+\
80
+							SF_MALLOC_OPTIMIZE_FACTOR-1))
81
+
82
+#define UN_HASH(h)	( ((unsigned long)(h)<=(SF_MALLOC_OPTIMIZE/SF_ROUNDTO))?\
83
+						UN_HASH_SMALL(h): UN_HASH_BIG(h) )
84
+
85
+#define BITMAP_BITS (sizeof(((struct sfm_block*)0)->bitmap)*8)
86
+#define BITMAP_BLOCK_SIZE ((SF_MALLOC_OPTIMIZE/SF_ROUNDTO)/ BITMAP_BITS)
87
+/* only for "small" hashes (up to HASH(SF_MALLOC_OPTIMIZE) */