Browse code

- added sf_malloc: a multi-process safe (internal locking), multi-pool f_malloc version. Should perform better on lots of CPU once properly tunned (experimental for now) - added ll_malloc: same as above but most operation are lockless (except for "big" fragment handling). For now needs tunning and it still keeps too many debugging statistics. (experimental for now)

Andrei Pelinescu-Onciul authored on 12/06/2007 18:07:12
Showing 10 changed files
... ...
@@ -75,7 +75,7 @@ MAIN_NAME=ser
75 75
 VERSION = 2
76 76
 PATCHLEVEL = 1
77 77
 SUBLEVEL =  0
78
-EXTRAVERSION = -dev7-tm
78
+EXTRAVERSION = -dev8-sf_malloc
79 79
 
80 80
 SER_VER = $(shell expr $(VERSION) \* 1000000 + $(PATCHLEVEL) \* 1000 + \
81 81
 			$(SUBLEVEL) )
... ...
@@ -318,6 +318,14 @@ endif
318 318
 #		an even faster malloc, not recommended for debugging
319 319
 # -DDL_MALLOC
320 320
 #		a malloc implementation based on Doug Lea's dl_malloc
321
+# -DSF_MALLOC 
322
+#		an experimental multi-CPU, pool based, multi-process safe version of 
323
+#		F_MALLOC. Should give better performance on machines with lots of CPUs
324
+#		after some tunning.
325
+# -DLL_MALLOC
326
+#		an experimental multi-CPU, pool based, multi-process safe, mostly
327
+#		lockless version of SF_MALLOC/F_MALLOC. Not for production use for
328
+#		now.
321 329
 # -DDBG_MALLOC
322 330
 #		issues additional debugging information if lock/unlock is called
323 331
 # -DFAST_LOCK
... ...
@@ -413,6 +421,8 @@ DEFS+= $(extra_defs) \
413 421
 	 -DUSE_DNS_FAILOVER \
414 422
 	 -DUSE_DST_BLACKLIST \
415 423
 	 -DDBG_QM_MALLOC \
424
+	 #-DLL_MALLOC \
425
+	 #-DSF_MALLOC \
416 426
 	 #-DDL_MALLOC \
417 427
 	 #-DF_MALLOC \
418 428
 	 #-DDBG_F_MALLOC \
... ...
@@ -453,10 +453,12 @@ char* pgid_file = 0;
453 453
 void cleanup(show_status)
454 454
 {
455 455
 	/*clean-up*/
456
+#ifndef SHM_SAFE_MALLOC
456 457
 	if (mem_lock)
457 458
 		shm_unlock(); /* hack: force-unlock the shared memory lock in case
458 459
 					 some process crashed and let it locked; this will
459 460
 					 allow an almost gracious shutdown */
461
+#endif
460 462
 	destroy_modules();
461 463
 #ifdef USE_DNS_CACHE
462 464
 	destroy_dns_cache();
463 465
new file mode 100644
... ...
@@ -0,0 +1,1095 @@
1
+/* $Id$
2
+ *
3
+ * shared memory, multi-process safe, pool based, mostly lockless version of 
4
+ *  f_malloc
5
+ *
6
+ * Copyright (C) 2007 iptelorg GmbH
7
+ *
8
+ * Permission to use, copy, modify, and distribute this software for any
9
+ * purpose with or without fee is hereby granted, provided that the above
10
+ * copyright notice and this permission notice appear in all copies.
11
+ *
12
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
+ */
20
+/*
21
+ * History:
22
+ * --------
23
+ *              created by andrei
24
+ *  2003-07-06  added fm_realloc (andrei)
25
+ *  2004-07-19  fragments book keeping code and support for 64 bits
26
+ *               memory blocks (64 bits machine & size >=2^32) 
27
+ *              GET_HASH s/</<=/ (avoids waste of 1 hash cell)   (andrei)
28
+ *  2004-11-10  support for > 4Gb mem., switched to long (andrei)
29
+ *  2005-03-02  added fm_info() (andrei)
30
+ *  2005-12-12  fixed realloc shrink real_used accounting (andrei)
31
+ *              fixed initial size (andrei)
32
+ *  2006-02-03  fixed realloc out of mem. free bug (andrei)
33
+ *  2006-04-07  s/DBG/MDBG (andrei)
34
+ *  2007-02-23  added fm_available() (andrei)
35
+ *  2007-06-09  forked from the fm_maloc code (andrei)
36
+ *  2007-06-11  forked from the sfm_maloc code (andrei)
37
+ */
38
+
39
+
40
+#ifdef LL_MALLOC
41
+
42
+#include <string.h>
43
+#include <stdlib.h>
44
+
45
+#include "ll_malloc.h"
46
+#include "../dprint.h"
47
+#include "../globals.h"
48
+#include "memdbg.h"
49
+
50
+#define MAX_POOL_FRAGS 10000 /* max fragments per pool hash bucket */
51
+#define MIN_POOL_FRAGS 10    /* min fragments per pool hash bucket */
52
+
53
+/*useful macros*/
54
+
55
+#define FRAG_NEXT(f) \
56
+	((struct sfm_frag*)((char*)(f)+sizeof(struct sfm_frag)+(f)->size ))
57
+
58
+
59
+/* SF_ROUNDTO= 2^k so the following works */
60
+#define ROUNDTO_MASK	(~((unsigned long)SF_ROUNDTO-1))
61
+#define ROUNDUP(s)		(((s)+(SF_ROUNDTO-1))&ROUNDTO_MASK)
62
+#define ROUNDDOWN(s)	((s)&ROUNDTO_MASK)
63
+
64
+#define FRAG_OVERHEAD	(sizeof(struct sfm_frag))
65
+#define INIT_OVERHEAD	\
66
+	(ROUNDUP(sizeof(struct sfm_block))+sizeof(struct sfm_frag))
67
+
68
+
69
+
70
+/* finds hash if s <=SF_MALLOC_OPTIMIZE */
71
+#define GET_SMALL_HASH(s) (unsigned long)(s)/SF_ROUNDTO
72
+/* finds hash if s > SF_MALLOC_OPTIMIZE */
73
+#define GET_BIG_HASH(s) \
74
+	(SF_MALLOC_OPTIMIZE/SF_ROUNDTO+big_hash_idx((s))-SF_MALLOC_OPTIMIZE_FACTOR+1)
75
+
76
+/* finds the hash value for s, s=SF_ROUNDTO multiple*/
77
+#define GET_HASH(s)   ( ((unsigned long)(s)<=SF_MALLOC_OPTIMIZE)?\
78
+							GET_SMALL_HASH(s): GET_BIG_HASH(s) )
79
+
80
+
81
+#define UN_HASH_SMALL(h) ((unsigned long)(h)*SF_ROUNDTO)
82
+#define UN_HASH_BIG(h) (1UL<<((unsigned long)(h)-SF_MALLOC_OPTIMIZE/SF_ROUNDTO+\
83
+							SF_MALLOC_OPTIMIZE_FACTOR-1))
84
+
85
+#define UN_HASH(h)	( ((unsigned long)(h)<=(SF_MALLOC_OPTIMIZE/SF_ROUNDTO))?\
86
+						UN_HASH_SMALL(h): UN_HASH_BIG(h) )
87
+
88
+#define BITMAP_BITS (sizeof(((struct sfm_block*)0)->bitmap)*8)
89
+#define BITMAP_BLOCK_SIZE ((SF_MALLOC_OPTIMIZE/SF_ROUNDTO)/ BITMAP_BITS)
90
+/* only for "small" hashes (up to HASH(SF_MALLOC_OPTIMIZE) */
91
+#define HASH_BIT_POS(h) (((unsigned long)(h))/BITMAP_BLOCK_SIZE)
92
+#define HASH_TO_BITMAP(h) (1UL<<HASH_BIT_POS(h))
93
+#define BIT_TO_HASH(b) ((b)*BITMAP_BLOCK_SIZE)
94
+
95
+
96
+
97
+/* mark/test used/unused frags */
98
+#define FRAG_MARK_USED(f)
99
+#define FRAG_CLEAR_USED(f)
100
+#define FRAG_WAS_USED(f)   (1)
101
+
102
+/* other frag related defines:
103
+ * MEM_COALESCE_FRAGS 
104
+ * MEM_FRAG_AVOIDANCE
105
+ */
106
+#define MEM_FRAG_AVOIDANCE
107
+
108
+
109
+#define SFM_REALLOC_REMALLOC
110
+
111
+/* computes hash number for big buckets*/
112
+inline static unsigned long big_hash_idx(unsigned long s)
113
+{
114
+	unsigned long idx;
115
+	/* s is rounded => s = k*2^n (SF_ROUNDTO=2^n) 
116
+	 * index= i such that 2^i > s >= 2^(i-1)
117
+	 *
118
+	 * => index = number of the first non null bit in s*/
119
+	idx=sizeof(long)*8-1;
120
+	for (; !(s&(1UL<<(sizeof(long)*8-1))) ; s<<=1, idx--);
121
+	return idx;
122
+}
123
+
124
+
125
+#ifdef DBG_F_MALLOC
126
+#define ST_CHECK_PATTERN   0xf0f0f0f0
127
+#define END_CHECK_PATTERN1 0xc0c0c0c0
128
+#define END_CHECK_PATTERN2 0xabcdefed
129
+#endif
130
+
131
+
132
+#ifdef SFM_ONE_LOCK
133
+
134
+#define SFM_MAIN_HASH_LOCK(qm, hash) lock_get(&(qm)->lock)
135
+#define SFM_MAIN_HASH_UNLOCK(qm, hash) lock_release(&(qm)->lock)
136
+#define SFM_POOL_LOCK(p, hash) lock_get(&(p)->lock)
137
+#define SFM_POOL_UNLOCK(p, hash) lock_release(&(p)->lock)
138
+
139
+#warn "degraded performance, only one lock"
140
+
141
+#elif defined SFM_LOCK_PER_BUCKET
142
+
143
+#define SFM_MAIN_HASH_LOCK(qm, hash) \
144
+	lock_get(&(qm)->free_hash[(hash)].lock)
145
+#define SFM_MAIN_HASH_UNLOCK(qm, hash)  \
146
+	lock_release(&(qm)->free_hash[(hash)].lock)
147
+#define SFM_POOL_LOCK(p, hash) lock_get(&(p)->pool_hash[(hash)].lock)
148
+#define SFM_POOL_UNLOCK(p, hash) lock_release(&(p)->pool_hash[(hash)].lock)
149
+#else
150
+#error no locks defined
151
+#endif /* SFM_ONE_LOCK/SFM_LOCK_PER_BUCKET */
152
+
153
+#define SFM_BIG_GET_AND_SPLIT_LOCK(qm)   lock_get(&(qm)->get_and_split)
154
+#define SFM_BIG_GET_AND_SPLIT_UNLOCK(qm) lock_release(&(qm)->get_and_split)
155
+
156
+static unsigned long sfm_max_hash=0; /* maximum hash value (no point in
157
+										 searching further) */
158
+static unsigned long pool_id=(unsigned long)-1;
159
+
160
+
161
+/* call for each child */
162
+int sfm_pool_reset()
163
+{
164
+	pool_id=(unsigned long)-1;
165
+	return 0;
166
+}
167
+
168
+
169
+#define sfm_fix_pool_id(qm) \
170
+	do{ \
171
+		if (unlikely(pool_id>=SFM_POOLS_NO)) \
172
+			pool_id=((unsigned)atomic_add(&(qm)->crt_id, 1))%SFM_POOLS_NO; \
173
+	}while(0)
174
+
175
+
176
+
177
+static inline void frag_push(struct sfm_frag** head, struct sfm_frag* frag)
178
+{
179
+	register struct sfm_frag* old;
180
+	register struct sfm_frag* crt;
181
+	
182
+	crt=(void*)atomic_get_long(head);
183
+	do{
184
+		frag->u.nxt_free=crt;
185
+		old=crt;
186
+		membar_write_atomic_op();
187
+		crt=(void*)atomic_cmpxchg_long((void*)head, (long)old, (long)frag);
188
+	}while(crt!=old);
189
+}
190
+
191
+
192
+static inline struct sfm_frag* frag_pop(struct sfm_frag** head)
193
+{
194
+	register struct sfm_frag* old;
195
+	register struct sfm_frag* crt;
196
+	register struct sfm_frag* nxt;
197
+	
198
+	crt=(void*)atomic_get_long(head);
199
+	do{
200
+		/* if circular list, test not needed */
201
+		nxt=crt?crt->u.nxt_free:0;
202
+		old=crt;
203
+		membar_read_atomic_op();
204
+		crt=(void*)atomic_cmpxchg_long((void*)head, (long)old, (long)nxt);
205
+	}while(crt!=old);
206
+	return crt;
207
+}
208
+
209
+
210
+static inline void sfm_pool_insert (struct sfm_pool* pool, int hash,
211
+								struct sfm_frag* frag)
212
+{
213
+	unsigned long hash_bit;
214
+
215
+	frag_push(&pool->pool_hash[hash].first, frag);
216
+	atomic_inc_long((long*)&pool->pool_hash[hash].no);
217
+	/* set it only if not already set (avoids an expensive
218
+	 * cache trashing atomic write op) */
219
+	hash_bit=HASH_TO_BITMAP(hash);
220
+	if  (!(atomic_get_long((long*)&pool->bitmap) & hash_bit))
221
+		atomic_or_long((long*)&pool->bitmap, hash_bit);
222
+}
223
+
224
+
225
+
226
+/* returns 1 if it's ok to add a fragm. to pool p_id @ hash, 0 otherwise */
227
+static inline int sfm_check_pool(struct sfm_block* qm, unsigned long p_id,
228
+									int hash, int split)
229
+{
230
+	/* TODO: come up with something better
231
+	 * if fragment is some  split/rest from an allocation, that is
232
+	 *  >= requested size, accept it, else
233
+	 *  look at misses and current fragments and decide based on them */
234
+	return (p_id<SFM_POOLS_NO) && (split ||
235
+			( (qm->pool[p_id].pool_hash[hash].no < MIN_POOL_FRAGS) ||
236
+			  ((qm->pool[p_id].pool_hash[hash].misses > 
237
+				 qm->pool[p_id].pool_hash[hash].no) &&
238
+				(qm->pool[p_id].pool_hash[hash].no<MAX_POOL_FRAGS) ) ) );
239
+}
240
+
241
+
242
+/* choose on which pool to add a free'd packet
243
+ * return - pool idx or -1 if it should be added to main*/
244
+static inline unsigned long  sfm_choose_pool(struct sfm_block* qm,
245
+												struct sfm_frag* frag,
246
+												int hash, int split)
247
+{
248
+	/* check original pool first */
249
+	if (sfm_check_pool(qm, frag->id, hash, split))
250
+		return frag->id;
251
+	else{
252
+		/* check if our pool is properly set */
253
+		sfm_fix_pool_id(qm);
254
+		/* check if my pool needs some frags */
255
+		if ((pool_id!=frag->id) && (sfm_check_pool(qm,  pool_id, hash, 0))){
256
+			frag->id=pool_id;
257
+			return pool_id;
258
+		}
259
+	}
260
+	/* else add it back to main */
261
+	frag->id=(unsigned long)(-1);
262
+	return frag->id;
263
+}
264
+
265
+
266
+static inline void sfm_insert_free(struct sfm_block* qm, struct sfm_frag* frag,
267
+									int split)
268
+{
269
+	struct sfm_frag** f;
270
+	unsigned long p_id;
271
+	int hash;
272
+	unsigned long hash_bit;
273
+	
274
+	if (likely(frag->size<=SF_POOL_MAX_SIZE)){
275
+		hash=GET_SMALL_HASH(frag->size);
276
+		if (unlikely((p_id=sfm_choose_pool(qm, frag, hash, split))==
277
+					(unsigned long)-1)){
278
+			/* add it back to the "main" hash */
279
+				frag->id=(unsigned long)(-1); /* main hash marker */
280
+				/*insert it here*/
281
+				frag_push(&(qm->free_hash[hash].first), frag);
282
+				atomic_inc_long((long*)&qm->free_hash[hash].no);
283
+				/* set it only if not already set (avoids an expensive
284
+		 		* cache trashing atomic write op) */
285
+				hash_bit=HASH_TO_BITMAP(hash);
286
+				if  (!(atomic_get_long((long*)&qm->bitmap) & hash_bit))
287
+					atomic_or_long((long*)&qm->bitmap, hash_bit);
288
+		}else{
289
+			/* add it to one of the pools pool */
290
+			sfm_pool_insert(&qm->pool[p_id], hash, frag);
291
+		}
292
+	}else{
293
+		hash=GET_BIG_HASH(frag->size);
294
+		SFM_MAIN_HASH_LOCK(qm, hash);
295
+			f=&(qm->free_hash[hash].first);
296
+			for(; *f; f=&((*f)->u.nxt_free))
297
+				if (frag->size <= (*f)->size) break;
298
+			frag->id=(unsigned long)(-1); /* main hash marker */
299
+			/*insert it here*/
300
+			frag->u.nxt_free=*f;
301
+			*f=frag;
302
+			qm->free_hash[hash].no++;
303
+			/* inc. big hash free size ? */
304
+		SFM_MAIN_HASH_UNLOCK(qm, hash);
305
+	}
306
+	
307
+}
308
+
309
+
310
+
311
+ /* size should be already rounded-up */
312
+static inline
313
+#ifdef DBG_F_MALLOC 
314
+void sfm_split_frag(struct sfm_block* qm, struct sfm_frag* frag,
315
+					unsigned long size,
316
+					const char* file, const char* func, unsigned int line)
317
+#else
318
+void sfm_split_frag(struct sfm_block* qm, struct sfm_frag* frag,
319
+					unsigned long size)
320
+#endif
321
+{
322
+	unsigned long rest;
323
+	struct sfm_frag* n;
324
+	int bigger_rest;
325
+	
326
+	rest=frag->size-size;
327
+#ifdef MEM_FRAG_AVOIDANCE
328
+	if ((rest> (FRAG_OVERHEAD+SF_MALLOC_OPTIMIZE))||
329
+		(rest>=(FRAG_OVERHEAD+size))){ /* the residue fragm. is big enough*/
330
+		bigger_rest=1;
331
+#else
332
+	if (rest>(FRAG_OVERHEAD+SF_MIN_FRAG_SIZE)){
333
+		bigger_rest=rest>=(size+FRAG_OVERHEAD);
334
+#endif
335
+		frag->size=size;
336
+		/*split the fragment*/
337
+		n=FRAG_NEXT(frag);
338
+		n->size=rest-FRAG_OVERHEAD;
339
+		n->id=pool_id;
340
+		FRAG_CLEAR_USED(n); /* never used */
341
+#ifdef DBG_F_MALLOC
342
+		/* frag created by malloc, mark it*/
343
+		n->file=file;
344
+		n->func="frag. from sfm_malloc";
345
+		n->line=line;
346
+		n->check=ST_CHECK_PATTERN;
347
+#endif
348
+		/* reinsert n in free list*/
349
+		sfm_insert_free(qm, n, bigger_rest);
350
+	}else{
351
+		/* we cannot split this fragment any more => alloc all of it*/
352
+	}
353
+}
354
+
355
+
356
+
357
+/* init malloc and return a sfm_block*/
358
+struct sfm_block* sfm_malloc_init(char* address, unsigned long size)
359
+{
360
+	char* start;
361
+	char* end;
362
+	struct sfm_block* qm;
363
+	unsigned long init_overhead;
364
+	int r;
365
+#ifdef SFM_LOCK_PER_BUCKET
366
+	int i;
367
+#endif
368
+	
369
+	/* make address and size multiple of 8*/
370
+	start=(char*)ROUNDUP((unsigned long) address);
371
+	DBG("sfm_malloc_init: SF_OPTIMIZE=%lu, /SF_ROUNDTO=%lu\n",
372
+			SF_MALLOC_OPTIMIZE, SF_MALLOC_OPTIMIZE/SF_ROUNDTO);
373
+	DBG("sfm_malloc_init: SF_HASH_SIZE=%lu, sfm_block size=%lu\n",
374
+			SF_HASH_SIZE, (long)sizeof(struct sfm_block));
375
+	DBG("sfm_malloc_init(%p, %lu), start=%p\n", address, size, start);
376
+
377
+	if (size<start-address) return 0;
378
+	size-=(start-address);
379
+	if (size <(SF_MIN_FRAG_SIZE+FRAG_OVERHEAD)) return 0;
380
+	size=ROUNDDOWN(size);
381
+
382
+	init_overhead=INIT_OVERHEAD;
383
+	
384
+	
385
+	if (size < init_overhead)
386
+	{
387
+		/* not enough mem to create our control structures !!!*/
388
+		return 0;
389
+	}
390
+	end=start+size;
391
+	qm=(struct sfm_block*)start;
392
+	memset(qm, 0, sizeof(struct sfm_block));
393
+	qm->size=size;
394
+	size-=init_overhead;
395
+	
396
+	qm->first_frag=(struct sfm_frag*)(start+ROUNDUP(sizeof(struct sfm_block)));
397
+	qm->last_frag=(struct sfm_frag*)(end-sizeof(struct sfm_frag));
398
+	/* init initial fragment*/
399
+	qm->first_frag->size=size;
400
+	qm->first_frag->id=(unsigned long)-1; /* not in a pool */
401
+	qm->last_frag->size=0;
402
+	
403
+#ifdef DBG_F_MALLOC
404
+	qm->first_frag->check=ST_CHECK_PATTERN;
405
+	qm->last_frag->check=END_CHECK_PATTERN1;
406
+#endif
407
+	
408
+	/* link initial fragment into the free list*/
409
+	
410
+	sfm_insert_free(qm, qm->first_frag, 0);
411
+	sfm_max_hash=GET_HASH(size);
412
+	
413
+	/* init locks */
414
+	if (lock_init(&qm->get_and_split)==0)
415
+		goto error;
416
+#ifdef SFM_ONE_LOCK
417
+	if (lock_init(&qm->lock)==0){
418
+		lock_destroy(&qm->get_and_split);
419
+		goto error;
420
+	}
421
+	for (r=0; r<SFM_POOLS_NO; r++){
422
+		if (lock_init(&qm->pool[r].lock)==0){
423
+			for (;r>0; r--) lock_destroy(&qm->pool[r-1].lock);
424
+			lock_destroy(&qm->lock);
425
+			lock_destroy(&qm->get_and_split);
426
+			goto error;
427
+		}
428
+	}
429
+#elif defined(SFM_LOCK_PER_BUCKET)
430
+	for (r=0; r<SF_HASH_SIZE; r++)
431
+		if (lock_init(&qm->free_hash[r].lock)==0){
432
+			for(;r>0; r--) lock_destroy(&qm->free_hash[r-1].lock);
433
+			lock_destroy(&qm->get_and_split);
434
+			goto error;
435
+		}
436
+	for (i=0; i<SFM_POOLS_NO; i++){
437
+		for (r=0; r<SF_HASH_POOL_SIZE; r++)
438
+			if (lock_init(&qm->pool[i].pool_hash[r].lock)==0){
439
+				for(;r>0; r--) lock_destroy(&qm->pool[i].poo_hash[r].lock);
440
+				for(; i>0; i--){
441
+					for (r=0; r<SF_HASH_POOL_SIZE; r++)
442
+						lock_destroy(&qm->pool[i].pool_hash[r].lock);
443
+				}
444
+				for (r=0; r<SF_HASH_SIZE; r++)
445
+					lock_destroy(&qm->free_hash[r].lock);
446
+				lock_destroy(&qm->get_and_split);
447
+				goto error;
448
+			}
449
+	}
450
+#endif
451
+	qm->is_init=1;
452
+	return qm;
453
+error:
454
+	return 0;
455
+}
456
+
457
+
458
+
459
+/* cleanup */
460
+void sfm_malloc_destroy(struct sfm_block* qm)
461
+{
462
+	int r, i;
463
+	/* destroy all the locks */
464
+	if (!qm || !qm->is_init)
465
+		return; /* nothing to do */
466
+	lock_destroy(&qm->get_and_split);
467
+#ifdef SFM_ONE_LOCK
468
+	lock_destroy(&qm->lock);
469
+	for (r=0; r<SFM_POOLS_NO; r++){
470
+		lock_destroy(&qm->pool[r].lock);
471
+	}
472
+#elif defined(SFM_LOCK_PER_BUCKET)
473
+	for (r=0; r<SF_HASH_SIZE; r++)
474
+		lock_destroy(&qm->free_hash[r].lock);
475
+	for (i=0; i<SFM_POOLS_NO; i++){
476
+		for (r=0; r<SF_HASH_POOL_SIZE; r++)
477
+			lock_destroy(&qm->pool[i].pool_hash[r].lock);
478
+	}
479
+#endif
480
+	qm->is_init=0;
481
+
482
+}
483
+
484
+
485
+/* returns next set bit in bitmap, starts at b
486
+ * if b is set, returns b
487
+ * if not found returns BITMAP_BITS */
488
+static inline unsigned long _next_set_bit(unsigned long b,
489
+											unsigned long* bitmap)
490
+{
491
+	for (; !((1UL<<b)& *bitmap) && b<BITMAP_BITS; b++);
492
+	return b;
493
+}
494
+
495
+/* returns start of block b and sets *end
496
+ * (handles also the "rest" block at the end ) */
497
+static inline unsigned long _hash_range(unsigned long b, unsigned long* end)
498
+{
499
+	unsigned long s;
500
+	
501
+	if ((unlikely(b>=BITMAP_BITS))){
502
+		s=BIT_TO_HASH(BITMAP_BITS);
503
+		*end=SF_HASH_POOL_SIZE; /* last, possible rest block */
504
+	}else{
505
+		s=BIT_TO_HASH(b);
506
+		*end=s+BITMAP_BLOCK_SIZE;
507
+	}
508
+	return s;
509
+}
510
+
511
+
512
+#ifdef DBG_F_MALLOC
513
+static inline struct sfm_frag* pool_get_frag(struct sfm_block* qm,
514
+						struct sfm_pool*  pool, int hash, unisgned long size,
515
+						const char* file, const char* func, unsigned int line)
516
+#else
517
+static inline struct sfm_frag* pool_get_frag(struct sfm_block* qm,
518
+											struct sfm_pool*  pool,
519
+											int hash, unsigned long size)
520
+#endif
521
+{
522
+	int r;
523
+	int next_block;
524
+	struct sfm_frag* volatile* f;
525
+	struct sfm_frag* frag;
526
+	unsigned long b;
527
+	unsigned long eob;
528
+
529
+	/* special case for r=hash */
530
+	r=hash;
531
+	f=&pool->pool_hash[r].first;
532
+
533
+	/* detach it from the free list */
534
+	if ((frag=frag_pop((struct sfm_frag**)f))==0)
535
+		goto not_found;
536
+found:
537
+	atomic_dec_long((long*)&pool->pool_hash[r].no);
538
+	frag->u.nxt_free=0; /* mark it as 'taken' */
539
+	frag->id=pool_id;
540
+#ifdef DBG_F_MALLOC
541
+	sfm_split_frag(qm, frag, size, file, func, line);
542
+#else
543
+	sfm_split_frag(qm, frag, size);
544
+#endif
545
+	if (&qm->pool[pool_id]==pool)
546
+		atomic_inc_long((long*)&pool->hits);
547
+	return frag;
548
+	
549
+not_found:
550
+	atomic_inc_long((long*)&pool->pool_hash[r].misses);
551
+	r++;
552
+	b=HASH_BIT_POS(r);
553
+	
554
+	while(r<SF_HASH_POOL_SIZE){
555
+		b=_next_set_bit(b, &pool->bitmap);
556
+		next_block=_hash_range(b, &eob);
557
+		r=(r<next_block)?next_block:r;
558
+		for (; r<eob; r++){
559
+			f=&pool->pool_hash[r].first;
560
+			if ((frag=frag_pop((struct sfm_frag**)f))!=0)
561
+				goto found;
562
+			atomic_inc_long((long*)&pool->pool_hash[r].misses);
563
+		}
564
+		b++;
565
+	}
566
+	atomic_inc_long((long*)&pool->missed);
567
+	return 0;
568
+}
569
+
570
+
571
+
572
+#ifdef DBG_F_MALLOC
573
+static inline struct sfm_frag* main_get_frag(struct sfm_block* qm, int hash,
574
+						unsigned long size,
575
+						const char* file, const char* func, unsigned int line)
576
+#else
577
+static inline struct sfm_frag* main_get_frag(struct sfm_block* qm, int hash,
578
+												unsigned long size)
579
+#endif
580
+{
581
+	int r;
582
+	int next_block;
583
+	struct sfm_frag* volatile* f;
584
+	struct sfm_frag* frag;
585
+	unsigned long b;
586
+	unsigned long eob;
587
+
588
+	r=hash;
589
+	b=HASH_BIT_POS(r);
590
+	while(r<=SF_MALLOC_OPTIMIZE/SF_ROUNDTO){
591
+			b=_next_set_bit(b, &qm->bitmap);
592
+			next_block=_hash_range(b, &eob);
593
+			r=(r<next_block)?next_block:r;
594
+			for (; r<eob; r++){
595
+				f=&qm->free_hash[r].first;
596
+				if ((frag=frag_pop((struct sfm_frag**)f))!=0){
597
+					atomic_dec_long((long*)&qm->free_hash[r].no);
598
+					frag->u.nxt_free=0; /* mark it as 'taken' */
599
+					frag->id=pool_id;
600
+#ifdef DBG_F_MALLOC
601
+					sfm_split_frag(qm, frag, size, file, func, line);
602
+#else
603
+					sfm_split_frag(qm, frag, size);
604
+#endif
605
+					return frag;
606
+				}
607
+			}
608
+			b++;
609
+	}
610
+	/* big fragments */
611
+	SFM_BIG_GET_AND_SPLIT_LOCK(qm);
612
+	for (; r<= sfm_max_hash ; r++){
613
+		f=&qm->free_hash[r].first;
614
+		if (*f){
615
+			SFM_MAIN_HASH_LOCK(qm, r);
616
+			if (unlikely((*f)==0)){
617
+				/* not found */
618
+				SFM_MAIN_HASH_UNLOCK(qm, r);
619
+				continue; 
620
+			}
621
+			for(;(*f); f=&((*f)->u.nxt_free))
622
+				if ((*f)->size>=size){
623
+					/* found, detach it from the free list*/
624
+					frag=*f;
625
+					*f=frag->u.nxt_free;
626
+					frag->u.nxt_free=0; /* mark it as 'taken' */
627
+					qm->free_hash[r].no--;
628
+					SFM_MAIN_HASH_UNLOCK(qm, r);
629
+					frag->id=pool_id;
630
+#ifdef DBG_F_MALLOC
631
+					sfm_split_frag(qm, frag, size, file, func, line);
632
+#else
633
+					sfm_split_frag(qm, frag, size);
634
+#endif
635
+					SFM_BIG_GET_AND_SPLIT_UNLOCK(qm);
636
+					return frag;
637
+				};
638
+			SFM_MAIN_HASH_UNLOCK(qm, r);
639
+			/* try in a bigger bucket */
640
+		}
641
+	}
642
+	SFM_BIG_GET_AND_SPLIT_UNLOCK(qm);
643
+	return 0;
644
+}
645
+
646
+
647
+
648
+#ifdef DBG_F_MALLOC
649
+void* sfm_malloc(struct sfm_block* qm, unsigned long size,
650
+					const char* file, const char* func, unsigned int line)
651
+#else
652
+void* sfm_malloc(struct sfm_block* qm, unsigned long size)
653
+#endif
654
+{
655
+	struct sfm_frag* frag;
656
+	int hash;
657
+	unsigned int i;
658
+	
659
+#ifdef DBG_F_MALLOC
660
+	MDBG("sfm_malloc(%p, %lu) called from %s: %s(%d)\n", qm, size, file, func,
661
+			line);
662
+#endif
663
+	/*size must be a multiple of 8*/
664
+	size=ROUNDUP(size);
665
+/*	if (size>(qm->size-qm->real_used)) return 0; */
666
+
667
+	/* check if our pool id is set */
668
+	sfm_fix_pool_id(qm);
669
+	
670
+	/*search for a suitable free frag*/
671
+	if (likely(size<=SF_POOL_MAX_SIZE)){
672
+		hash=GET_SMALL_HASH(size);
673
+		/* try first in our pool */
674
+#ifdef DBG_F_MALLOC
675
+		if (likely((frag=pool_get_frag(qm, &qm->pool[pool_id], hash, size,
676
+										file, func, line))!=0))
677
+			goto found;
678
+		/* try in the "main" free hash, go through all the hash */
679
+		if (likely((frag=main_get_frag(qm, hash, size, file, func, line))!=0))
680
+			goto found;
681
+		/* really low mem , try in other pools */
682
+		for (i=(pool_id+1); i< (pool_id+SFM_POOLS_NO); i++){
683
+			if ((frag=pool_get_frag(qm, &qm->pool[i%SFM_POOLS_NO], hash, size,
684
+										file, func, line))!=0)
685
+				goto found;
686
+		}
687
+#else
688
+		if (likely((frag=pool_get_frag(qm, &qm->pool[pool_id], hash, size))
689
+					!=0 ))
690
+			goto found;
691
+		/* try in the "main" free hash, go through all the hash */
692
+		if (likely((frag=main_get_frag(qm, hash, size))!=0))
693
+			goto found;
694
+		/* really low mem , try in other pools */
695
+		for (i=(pool_id+1); i< (pool_id+SFM_POOLS_NO); i++){
696
+			if ((frag=pool_get_frag(qm, &qm->pool[i%SFM_POOLS_NO], hash, size))
697
+					!=0 )
698
+				goto found;
699
+		}
700
+#endif
701
+		/* not found, bad! */
702
+		return 0;
703
+	}else{
704
+		hash=GET_BIG_HASH(size);
705
+#ifdef DBG_F_MALLOC
706
+		if ((frag=main_get_frag(qm, hash, size, file, func, line))==0)
707
+			return 0; /* not found, bad! */
708
+#else
709
+		if ((frag=main_get_frag(qm, hash, size))==0)
710
+			return 0; /* not found, bad! */
711
+#endif
712
+	}
713
+
714
+found:
715
+	/* we found it!*/
716
+#ifdef DBG_F_MALLOC
717
+	frag->file=file;
718
+	frag->func=func;
719
+	frag->line=line;
720
+	frag->check=ST_CHECK_PATTERN;
721
+	MDBG("sfm_malloc(%p, %lu) returns address %p \n", qm, size,
722
+		(char*)frag+sizeof(struct sfm_frag));
723
+#endif
724
+	FRAG_MARK_USED(frag); /* mark it as used */
725
+	return (char*)frag+sizeof(struct sfm_frag);
726
+}
727
+
728
+
729
+
730
+#ifdef DBG_F_MALLOC
731
+void sfm_free(struct sfm_block* qm, void* p, const char* file,
732
+				const char* func, unsigned int line)
733
+#else
734
+void sfm_free(struct sfm_block* qm, void* p)
735
+#endif
736
+{
737
+	struct sfm_frag* f;
738
+
739
+#ifdef DBG_F_MALLOC
740
+	MDBG("sfm_free(%p, %p), called from %s: %s(%d)\n", qm, p, file, func,
741
+				line);
742
+	if (p>(void*)qm->last_frag || p<(void*)qm->first_frag){
743
+		LOG(L_CRIT, "BUG: sfm_free: bad pointer %p (out of memory block!) - "
744
+				"aborting\n", p);
745
+		abort();
746
+	}
747
+#endif
748
+	if (unlikely(p==0)) {
749
+		LOG(L_WARN, "WARNING: sfm_free: free(0) called\n");
750
+		return;
751
+	}
752
+	f=(struct sfm_frag*) ((char*)p-sizeof(struct sfm_frag));
753
+#ifdef DBG_F_MALLOC
754
+	MDBG("sfm_free: freeing block alloc'ed from %s: %s(%ld)\n",
755
+			f->file, f->func, f->line);
756
+#endif
757
+#ifdef DBG_F_MALLOC
758
+	f->file=file;
759
+	f->func=func;
760
+	f->line=line;
761
+#endif
762
+	sfm_insert_free(qm, f, 0);
763
+}
764
+
765
+
766
+#ifdef DBG_F_MALLOC
767
+void* sfm_realloc(struct sfm_block* qm, void* p, unsigned long size,
768
+					const char* file, const char* func, unsigned int line)
769
+#else
770
+void* sfm_realloc(struct sfm_block* qm, void* p, unsigned long size)
771
+#endif
772
+{
773
+	struct sfm_frag *f;
774
+	unsigned long orig_size;
775
+	void *ptr;
776
+#ifndef SFM_REALLOC_REMALLOC
777
+	struct sfm_frag *n;
778
+	struct sfm_frag **pf;
779
+	unsigned long diff;
780
+	unsigned long p_id;
781
+	int hash;
782
+	unsigned long n_size;
783
+	struct sfm_pool * pool;
784
+#endif
785
+	
786
+#ifdef DBG_F_MALLOC
787
+	MDBG("sfm_realloc(%p, %p, %lu) called from %s: %s(%d)\n", qm, p, size,
788
+			file, func, line);
789
+	if ((p)&&(p>(void*)qm->last_frag || p<(void*)qm->first_frag)){
790
+		LOG(L_CRIT, "BUG: sfm_free: bad pointer %p (out of memory block!) - "
791
+				"aborting\n", p);
792
+		abort();
793
+	}
794
+#endif
795
+	if (size==0) {
796
+		if (p)
797
+#ifdef DBG_F_MALLOC
798
+			sfm_free(qm, p, file, func, line);
799
+#else
800
+			sfm_free(qm, p);
801
+#endif
802
+		return 0;
803
+	}
804
+	if (p==0)
805
+#ifdef DBG_F_MALLOC
806
+		return sfm_malloc(qm, size, file, func, line);
807
+#else
808
+		return sfm_malloc(qm, size);
809
+#endif
810
+	f=(struct sfm_frag*) ((char*)p-sizeof(struct sfm_frag));
811
+#ifdef DBG_F_MALLOC
812
+	MDBG("sfm_realloc: realloc'ing frag %p alloc'ed from %s: %s(%ld)\n",
813
+			f, f->file, f->func, f->line);
814
+#endif
815
+	size=ROUNDUP(size);
816
+	orig_size=f->size;
817
+	if (f->size > size){
818
+		/* shrink */
819
+#ifdef DBG_F_MALLOC
820
+		MDBG("sfm_realloc: shrinking from %lu to %lu\n", f->size, size);
821
+		sfm_split_frag(qm, f, size, file, "frag. from sfm_realloc", line);
822
+#else
823
+		sfm_split_frag(qm, f, size);
824
+#endif
825
+	}else if (f->size<size){
826
+		/* grow */
827
+#ifdef DBG_F_MALLOC
828
+		MDBG("sfm_realloc: growing from %lu to %lu\n", f->size, size);
829
+#endif
830
+#ifndef SFM_REALLOC_REMALLOC
831
+/* should set a magic value in list head and in push/pop if magic value =>
832
+ * lock and wait */
833
+#error LL_MALLOC realloc not finished yet
834
+		diff=size-f->size;
835
+		n=FRAG_NEXT(f);
836
+		if (((char*)n < (char*)qm->last_frag) && 
837
+				(n->u.nxt_free)&&((n->size+FRAG_OVERHEAD)>=diff)){
838
+			/* join  */
839
+			/* detach n from the free list */
840
+try_again:
841
+			p_id=n->id;
842
+			n_size=n->size;
843
+			if ((unlikely(p_id >=SFM_POOLS_NO))){
844
+				hash=GET_HASH(n_size);
845
+				SFM_MAIN_HASH_LOCK(qm, hash);
846
+				if (unlikely((n->u.nxt_free==0) ||
847
+							((n->size+FRAG_OVERHEAD)<diff))){ 
848
+					SFM_MAIN_HASH_UNLOCK(qm, hash);
849
+					goto not_found;
850
+				}
851
+				if (unlikely((n->id!=p_id) || (n->size!=n_size))){
852
+					/* fragment still free, but changed, either 
853
+					 * moved to another pool or has a diff. size */
854
+					SFM_MAIN_HASH_UNLOCK(qm, hash);
855
+					goto try_again;
856
+				}
857
+				pf=&(qm->free_hash[hash].first);
858
+				/* find it */
859
+				for(;(*pf)&&(*pf!=n); pf=&((*pf)->u.nxt_free));/*FIXME slow */
860
+				if (*pf==0){
861
+					SFM_MAIN_HASH_UNLOCK(qm, hash);
862
+					/* not found, bad! */
863
+					LOG(L_WARN, "WARNING: sfm_realloc: could not find %p in "
864
+							    "free " "list (hash=%d)\n", n, hash);
865
+					/* somebody is in the process of changing it ? */
866
+					goto not_found;
867
+				}
868
+				/* detach */
869
+				*pf=n->u.nxt_free;
870
+				n->u.nxt_free=0; /* mark it immediately as detached */
871
+				qm->free_hash[hash].no--;
872
+				SFM_MAIN_HASH_UNLOCK(qm, hash);
873
+				/* join */
874
+				f->size+=n->size+FRAG_OVERHEAD;
875
+				/* split it if necessary */
876
+				if (f->size > size){
877
+			#ifdef DBG_F_MALLOC
878
+					sfm_split_frag(qm, f, size, file, "fragm. from "
879
+									"sfm_realloc", line);
880
+			#else
881
+					sfm_split_frag(qm, f, size);
882
+			#endif
883
+				}
884
+			}else{ /* p_id < SFM_POOLS_NO (=> in a pool )*/
885
+				hash=GET_SMALL_HASH(n_size);
886
+				pool=&qm->pool[p_id];
887
+				SFM_POOL_LOCK(pool, hash);
888
+				if (unlikely((n->u.nxt_free==0) ||
889
+							((n->size+FRAG_OVERHEAD)<diff))){
890
+					SFM_POOL_UNLOCK(pool, hash);
891
+					goto not_found;
892
+				}
893
+				if (unlikely((n->id!=p_id) || (n->size!=n_size))){
894
+					/* fragment still free, but changed, either 
895
+					 * moved to another pool or has a diff. size */
896
+					SFM_POOL_UNLOCK(pool, hash);
897
+					goto try_again;
898
+				}
899
+				pf=&(pool->pool_hash[hash].first);
900
+				/* find it */
901
+				for(;(*pf)&&(*pf!=n); pf=&((*pf)->u.nxt_free));/*FIXME slow */
902
+				if (*pf==0){
903
+					SFM_POOL_UNLOCK(pool, hash);
904
+					/* not found, bad! */
905
+					LOG(L_WARN, "WARNING: sfm_realloc: could not find %p in "
906
+							    "free " "list (hash=%d)\n", n, hash);
907
+					/* somebody is in the process of changing it ? */
908
+					goto not_found;
909
+				}
910
+				/* detach */
911
+				*pf=n->u.nxt_free;
912
+				n->u.nxt_free=0; /* mark it immediately as detached */
913
+				pool->pool_hash[hash].no--;
914
+				SFM_POOL_UNLOCK(pool, hash);
915
+				/* join */
916
+				f->size+=n->size+FRAG_OVERHEAD;
917
+				/* split it if necessary */
918
+				if (f->size > size){
919
+			#ifdef DBG_F_MALLOC
920
+					sfm_split_frag(qm, f, size, file, "fragm. from "
921
+									"sfm_realloc", line);
922
+			#else
923
+					sfm_split_frag(qm, f, size);
924
+			#endif
925
+				}
926
+			}
927
+		}else{
928
+not_found:
929
+			/* could not join => realloc */
930
+#else/* SFM_REALLOC_REMALLOC */ 
931
+		{
932
+#endif /* SFM_REALLOC_REMALLOC */
933
+	#ifdef DBG_F_MALLOC
934
+			ptr=sfm_malloc(qm, size, file, func, line);
935
+	#else
936
+			ptr=sfm_malloc(qm, size);
937
+	#endif
938
+			if (ptr){
939
+				/* copy, need by libssl */
940
+				memcpy(ptr, p, orig_size);
941
+	#ifdef DBG_F_MALLOC
942
+				sfm_free(qm, p, file, func, line);
943
+	#else
944
+				sfm_free(qm, p);
945
+	#endif
946
+			}
947
+			p=ptr;
948
+		}
949
+	}else{
950
+		/* do nothing */
951
+#ifdef DBG_F_MALLOC
952
+		MDBG("sfm_realloc: doing nothing, same size: %lu - %lu\n", 
953
+				f->size, size);
954
+#endif
955
+	}
956
+#ifdef DBG_F_MALLOC
957
+	MDBG("sfm_realloc: returning %p\n", p);
958
+#endif
959
+	return p;
960
+}
961
+
962
+
963
+
964
+void sfm_status(struct sfm_block* qm)
965
+{
966
+	struct sfm_frag* f;
967
+	int i,j;
968
+	int h;
969
+	int unused;
970
+	unsigned long size;
971
+	int k;
972
+
973
+#warning ll_status doesn't work (might crash if used)
974
+
975
+	LOG(memlog, "sfm_status (%p):\n", qm);
976
+	if (!qm) return;
977
+
978
+	LOG(memlog, " heap size= %ld\n", qm->size);
979
+	LOG(memlog, "dumping free list:\n");
980
+	for(h=0,i=0,size=0;h<=sfm_max_hash;h++){
981
+		SFM_MAIN_HASH_LOCK(qm, h);
982
+		unused=0;
983
+		for (f=qm->free_hash[h].first,j=0; f;
984
+				size+=f->size,f=f->u.nxt_free,i++,j++){
985
+			if (!FRAG_WAS_USED(f)){
986
+				unused++;
987
+#ifdef DBG_F_MALLOC
988
+				LOG(memlog, "unused fragm.: hash = %3d, fragment %p,"
989
+							" address %p size %lu, created from %s: %s(%ld)\n",
990
+						    h, f, (char*)f+sizeof(struct sfm_frag), f->size,
991
+							f->file, f->func, f->line);
992
+#endif
993
+			};
994
+		}
995
+		if (j) LOG(memlog, "hash = %3d fragments no.: %5d, unused: %5d\n\t\t"
996
+							" bucket size: %9lu - %9lu (first %9lu)\n",
997
+							h, j, unused, UN_HASH(h),
998
+						((h<=SF_MALLOC_OPTIMIZE/SF_ROUNDTO)?1:2)* UN_HASH(h),
999
+							qm->free_hash[h].first->size
1000
+				);
1001
+		if (j!=qm->free_hash[h].no){
1002
+			LOG(L_CRIT, "BUG: sfm_status: different free frag. count: %d!=%ld"
1003
+					" for hash %3d\n", j, qm->free_hash[h].no, h);
1004
+		}
1005
+		SFM_MAIN_HASH_UNLOCK(qm, h);
1006
+	}
1007
+	for (k=0; k<SFM_POOLS_NO; k++){
1008
+		for(h=0;h<SF_HASH_POOL_SIZE;h++){
1009
+			SFM_POOL_LOCK(&qm->pool[k], h);
1010
+			unused=0;
1011
+			for (f=qm->pool[k].pool_hash[h].first,j=0; f;
1012
+					size+=f->size,f=f->u.nxt_free,i++,j++){
1013
+				if (!FRAG_WAS_USED(f)){
1014
+					unused++;
1015
+#ifdef DBG_F_MALLOC
1016
+					LOG(memlog, "[%2d] unused fragm.: hash = %3d, fragment %p,"
1017
+								" address %p size %lu, created from %s: "
1018
+								"%s(%ld)\n", k
1019
+								h, f, (char*)f+sizeof(struct sfm_frag),
1020
+								f->size, f->file, f->func, f->line);
1021
+#endif
1022
+				};
1023
+			}
1024
+			if (j) LOG(memlog, "[%2d] hash = %3d fragments no.: %5d, unused: "
1025
+								"%5d\n\t\t bucket size: %9lu - %9lu "
1026
+								"(first %9lu)\n",
1027
+								k, h, j, unused, UN_HASH(h),
1028
+							((h<=SF_MALLOC_OPTIMIZE/SF_ROUNDTO)?1:2) *
1029
+								UN_HASH(h),
1030
+								qm->pool[k].pool_hash[h].first->size
1031
+					);
1032
+			if (j!=qm->pool[k].pool_hash[h].no){
1033
+				LOG(L_CRIT, "BUG: sfm_status: [%d] different free frag."
1034
+							" count: %d!=%ld for hash %3d\n",
1035
+							k, j, qm->pool[k].pool_hash[h].no, h);
1036
+			}
1037
+			SFM_POOL_UNLOCK(&qm->pool[k], h);
1038
+		}
1039
+	}
1040
+	LOG(memlog, "TOTAL: %6d free fragments = %6lu free bytes\n", i, size);
1041
+	LOG(memlog, "-----------------------------\n");
1042
+}
1043
+
1044
+
1045
+
1046
+/* fills a malloc info structure with info about the block
1047
+ * if a parameter is not supported, it will be filled with 0 */
1048
+void sfm_info(struct sfm_block* qm, struct mem_info* info)
1049
+{
1050
+	int r, k;
1051
+	unsigned long total_frags;
1052
+	struct sfm_frag* f;
1053
+	
1054
+	memset(info,0, sizeof(*info));
1055
+	total_frags=0;
1056
+	info->total_size=qm->size;
1057
+	info->min_frag=SF_MIN_FRAG_SIZE;
1058
+	/* we'll have to compute it all */
1059
+	for (r=0; r<=SF_MALLOC_OPTIMIZE/SF_ROUNDTO; r++){
1060
+		info->free+=qm->free_hash[r].no*UN_HASH(r);
1061
+		total_frags+=qm->free_hash[r].no;
1062
+	}
1063
+	for(;r<=sfm_max_hash; r++){
1064
+		total_frags+=qm->free_hash[r].no;
1065
+		SFM_MAIN_HASH_LOCK(qm, r);
1066
+		for(f=qm->free_hash[r].first;f;f=f->u.nxt_free){
1067
+			info->free+=f->size;
1068
+		}
1069
+		SFM_MAIN_HASH_UNLOCK(qm, r);
1070
+	}
1071
+	for (k=0; k<SFM_POOLS_NO; k++){
1072
+		for (r=0; r<SF_HASH_POOL_SIZE; r++){
1073
+			info->free+=qm->pool[k].pool_hash[r].no*UN_HASH(r);
1074
+			total_frags+=qm->pool[k].pool_hash[r].no;
1075
+		}
1076
+	}
1077
+	info->real_used=info->total_size-info->free;
1078
+	info->used=info->real_used-total_frags*FRAG_OVERHEAD-INIT_OVERHEAD
1079
+				-FRAG_OVERHEAD;
1080
+	info->max_used=0; /* we don't really know */
1081
+	info->total_frags=total_frags;
1082
+}
1083
+
1084
+
1085
+
1086
+/* returns how much free memory is available
1087
+ * on error (not compiled with bookkeeping code) returns (unsigned long)(-1) */
1088
+unsigned long sfm_available(struct sfm_block* qm)
1089
+{
1090
+	/* we don't know how much free memory we have and it's to expensive
1091
+	 * to compute it */
1092
+	return ((unsigned long)-1);
1093
+}
1094
+
1095
+#endif
0 1096
new file mode 100644
... ...
@@ -0,0 +1,178 @@
1
+/* $Id$
2
+ *
3
+ * shared memory, multi-process safe, pool based, mostly lockless version of 
4
+ *  f_malloc
5
+ *
6
+ * Copyright (C) 2007 iptelorg GmbH
7
+ *
8
+ * Permission to use, copy, modify, and distribute this software for any
9
+ * purpose with or without fee is hereby granted, provided that the above
10
+ * copyright notice and this permission notice appear in all copies.
11
+ *
12
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19
+ */
20
+/*
21
+ * History:
22
+ * --------
23
+ *  2003-05-21  on sparc64 roundto 8 even in debugging mode (so malloc'ed
24
+ *               long longs will be 64 bit aligned) (andrei)
25
+ *  2004-07-19  support for 64 bit (2^64 mem. block) and more info
26
+ *               for the future de-fragmentation support (andrei)
27
+ *  2004-11-10  support for > 4Gb mem., switched to long (andrei)
28
+ *  2007-06-11  forked from the sf_malloc code (andrei)
29
+ */
30
+
31
+
32
+#if !defined(ll_malloc_h)  
33
+#define ll_malloc_h
34
+
35
+
36
+#include "meminfo.h"
37
+
38
+#include "../lock_ops.h"
39
+#include "../atomic_ops.h"
40
+#include "../compiler_opt.h"
41
+/* defs*/
42
+
43
+
44
+#ifdef GEN_LOCK_T_UNLIMITED
45
+#define SFM_LOCK_PER_BUCKET
46
+#else
47
+#define SFM_ONE_LOCK
48
+#endif
49
+
50
+#ifdef DBG_SF_MALLOC
51
+#if defined(__CPU_sparc64) || defined(__CPU_sparc)
52
+/* tricky, on sun in 32 bits mode long long must be 64 bits aligned
53
+ * but long can be 32 bits aligned => malloc should return long long
54
+ * aligned memory */
55
+	#define SF_ROUNDTO	sizeof(long long)
56
+#else
57
+	#define SF_ROUNDTO	sizeof(void*) /* size we round to, must be = 2^n, and
58
+                      sizeof(sfm_frag) must be multiple of SF_ROUNDTO !*/
59
+#endif
60
+#else /* DBG_SF_MALLOC */
61
+	#define SF_ROUNDTO 8UL
62
+#endif
63
+#define SF_MIN_FRAG_SIZE	SF_ROUNDTO
64
+
65
+#define SFM_POOLS_NO 4U /* the more the better, but higher initial
66
+                            mem. consumption */
67
+
68
+#define SF_MALLOC_OPTIMIZE_FACTOR 14UL /*used below */
69
+#define SF_MALLOC_OPTIMIZE  (1UL<<SF_MALLOC_OPTIMIZE_FACTOR)
70
+								/* size to optimize for,
71
+									(most allocs <= this size),
72
+									must be 2^k */
73
+
74
+#define SF_HASH_POOL_SIZE	(SF_MALLOC_OPTIMIZE/SF_ROUNDTO + 1)
75
+#define SF_POOL_MAX_SIZE	SF_MALLOC_OPTIMIZE
76
+
77
+#define SF_HASH_SIZE (SF_MALLOC_OPTIMIZE/SF_ROUNDTO + \
78
+		(sizeof(long)*8-SF_MALLOC_OPTIMIZE_FACTOR)+1)
79
+
80
+/* hash structure:
81
+ * 0 .... SF_MALLOC_OPTIMIZE/SF_ROUNDTO  - small buckets, size increases with
82
+ *                            SF_ROUNDTO from bucket to bucket
83
+ * +1 .... end -  size = 2^k, big buckets */
84
+
85
+struct sfm_frag{
86
+	union{
87
+		struct sfm_frag* nxt_free;
88
+		long reserved;
89
+	}u;
90
+	unsigned long size;
91
+	unsigned long id; /* TODO better optimize the size */
92
+	/* pad to SF_ROUNDTO multiple */
93
+	char _pad[((3*sizeof(long)+SF_ROUNDTO-1)&~(SF_ROUNDTO-1))-3*sizeof(long)];
94
+#ifdef DBG_SF_MALLOC
95
+	const char* file;
96
+	const char* func;
97
+	unsigned long line;
98
+	unsigned long check;
99
+#endif
100
+};
101
+
102
+struct sfm_frag_lnk{
103
+	struct sfm_frag* first;
104
+#ifdef SFM_LOCK_PER_BUCKET
105
+	gen_lock_t lock;
106
+#endif
107
+	unsigned long no;
108
+};
109
+
110
+struct sfm_pool_head{
111
+	struct sfm_frag* first;
112
+#ifdef SFM_LOCK_PER_BUCKET
113
+	gen_lock_t lock;
114
+#endif
115
+	unsigned long no;
116
+	unsigned long misses;
117
+};
118
+
119
+struct sfm_pool{
120
+#ifdef SFM_ONE_LOCK
121
+	gen_lock_t lock;
122
+#endif
123
+	unsigned long missed;
124
+	unsigned long hits; /* debugging only TODO: remove */
125
+	unsigned long bitmap;
126
+	struct sfm_pool_head pool_hash[SF_HASH_POOL_SIZE];
127
+};
128
+
129
+struct sfm_block{
130
+#ifdef SFM_ONE_LOCK
131
+	gen_lock_t lock;
132
+#endif
133
+	atomic_t crt_id; /* current pool */
134
+	unsigned long size; /* total size */
135
+	/* stats are kept now per bucket */
136
+	struct sfm_frag* first_frag;
137
+	struct sfm_frag* last_frag;
138
+	unsigned long bitmap; /* only up to SF_MALLOC_OPTIMIZE */
139
+	struct sfm_frag_lnk free_hash[SF_HASH_SIZE];
140
+	struct sfm_pool pool[SFM_POOLS_NO];
141
+	int is_init;
142
+	gen_lock_t get_and_split;
143
+	char _pad[256];
144
+};
145
+
146
+
147
+
148
+struct sfm_block* sfm_malloc_init(char* address, unsigned long size);
149
+void sfm_malloc_destroy(struct sfm_block* qm);
150
+int sfm_pool_reset();
151
+
152
+#ifdef DBG_SF_MALLOC
153
+void* sfm_malloc(struct sfm_block*, unsigned long size,
154
+					const char* file, const char* func, unsigned int line);
155
+#else
156
+void* sfm_malloc(struct sfm_block*, unsigned long size);
157
+#endif
158
+
159
+#ifdef DBG_SF_MALLOC
160
+void  sfm_free(struct sfm_block*, void* p, const char* file, const char* func, 
161
+				unsigned int line);
162
+#else
163
+void  sfm_free(struct sfm_block*, void* p);
164
+#endif
165
+
166
+#ifdef DBG_SF_MALLOC
167
+void*  sfm_realloc(struct sfm_block*, void* p, unsigned long size, 
168
+					const char* file, const char* func, unsigned int line);
169
+#else
170
+void*  sfm_realloc(struct sfm_block*, void* p, unsigned long size);
171
+#endif
172
+
173
+void  sfm_status(struct sfm_block*);
174
+void  sfm_info(struct sfm_block*, struct mem_info*);
175
+
176
+unsigned long sfm_available(struct sfm_block*);
177
+
178
+#endif
0 179
new file mode 100644
... ...
@@ -0,0 +1,1114 @@
1
+/* $Id$
2
+ *
3
+ * shared memory, multi-process safe, pool based version of f_malloc
4
+ *
5
+ * Copyright (C) 2007 iptelorg GmbH
6
+ *
7
+ * Permission to use, copy, modify, and distribute this software for any
8
+ * purpose with or without fee is hereby granted, provided that the above
9
+ * copyright notice and this permission notice appear in all copies.
10
+ *
11
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
12
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
14
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
16
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
17
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18
+ */
19
+/*
20
+ * History:
21
+ * --------
22
+ *              created by andrei
23
+ *  2003-07-06  added fm_realloc (andrei)
24
+ *  2004-07-19  fragments book keeping code and support for 64 bits
25
+ *               memory blocks (64 bits machine & size >=2^32) 
26
+ *              GET_HASH s/</<=/ (avoids waste of 1 hash cell)   (andrei)
27
+ *  2004-11-10  support for > 4Gb mem., switched to long (andrei)
28
+ *  2005-03-02  added fm_info() (andrei)
29
+ *  2005-12-12  fixed realloc shrink real_used accounting (andrei)
30
+ *              fixed initial size (andrei)
31
+ *  2006-02-03  fixed realloc out of mem. free bug (andrei)
32
+ *  2006-04-07  s/DBG/MDBG (andrei)
33
+ *  2007-02-23  added fm_available() (andrei)
34
+ *  2007-06-09  forked from the fm_maloc code (andrei)
35
+ */
36
+
37
+
38
+#ifdef SF_MALLOC
39
+
40
+#include <string.h>
41
+#include <stdlib.h>
42
+
43
+#include "sf_malloc.h"
44
+#include "../dprint.h"
45
+#include "../globals.h"
46
+#include "memdbg.h"
47
+
48
+#define MAX_POOL_FRAGS 10000 /* max fragments per pool hash bucket */
49
+#define MIN_POOL_FRAGS 10    /* min fragments per pool hash bucket */
50
+
51
+/*useful macros*/
52
+
53
+#define FRAG_NEXT(f) \
54
+	((struct sfm_frag*)((char*)(f)+sizeof(struct sfm_frag)+(f)->size ))
55
+
56
+
57
+/* SF_ROUNDTO= 2^k so the following works */
58
+#define ROUNDTO_MASK	(~((unsigned long)SF_ROUNDTO-1))
59
+#define ROUNDUP(s)		(((s)+(SF_ROUNDTO-1))&ROUNDTO_MASK)
60
+#define ROUNDDOWN(s)	((s)&ROUNDTO_MASK)
61
+
62
+#define FRAG_OVERHEAD	(sizeof(struct sfm_frag))
63
+#define INIT_OVERHEAD	\
64
+	(ROUNDUP(sizeof(struct sfm_block))+sizeof(struct sfm_frag))
65
+
66
+
67
+
68
+/* finds hash if s <=SF_MALLOC_OPTIMIZE */
69
+#define GET_SMALL_HASH(s) (unsigned long)(s)/SF_ROUNDTO
70
+/* finds hash if s > SF_MALLOC_OPTIMIZE */
71
+#define GET_BIG_HASH(s) \
72
+	(SF_MALLOC_OPTIMIZE/SF_ROUNDTO+big_hash_idx((s))-SF_MALLOC_OPTIMIZE_FACTOR+1)
73
+
74
+/* finds the hash value for s, s=SF_ROUNDTO multiple*/
75
+#define GET_HASH(s)   ( ((unsigned long)(s)<=SF_MALLOC_OPTIMIZE)?\
76
+							GET_SMALL_HASH(s): GET_BIG_HASH(s) )
77
+
78
+
79
+#define UN_HASH_SMALL(h) ((unsigned long)(h)*SF_ROUNDTO)
80
+#define UN_HASH_BIG(h) (1UL<<((unsigned long)(h)-SF_MALLOC_OPTIMIZE/SF_ROUNDTO+\
81
+							SF_MALLOC_OPTIMIZE_FACTOR-1))
82
+
83
+#define UN_HASH(h)	( ((unsigned long)(h)<=(SF_MALLOC_OPTIMIZE/SF_ROUNDTO))?\
84
+						UN_HASH_SMALL(h): UN_HASH_BIG(h) )
85
+
86
+#define BITMAP_BITS (sizeof(((struct sfm_block*)0)->bitmap)*8)
87
+#define BITMAP_BLOCK_SIZE ((SF_MALLOC_OPTIMIZE/SF_ROUNDTO)/ BITMAP_BITS)
88
+/* only for "small" hashes (up to HASH(SF_MALLOC_OPTIMIZE) */