Browse code

mem: introducing TLSF malloc

TLSF is a real-time oriented dynamic allocator, with O(1) malloc() and free() costs

Camille Oudot authored on 22/04/2015 08:10:24
Showing 4 changed files
1 1
new file mode 100644
... ...
@@ -0,0 +1,102 @@
1
+Two Level Segregated Fit memory allocator implementation.
2
+Written by Matthew Conte (matt@baisoku.org).
3
+Public Domain, no restrictions.
4
+
5
+	http://tlsf.baisoku.org
6
+
7
+Features
8
+--------
9
+* O(1) cost for malloc, free, realloc, memalign
10
+* Extremely low overhead per allocation (4 bytes)
11
+* Low overhead per TLSF management of pools (~3kB)
12
+* Low fragmentation
13
+* Compiles to only a few kB of code and data
14
+* Support for adding and removing memory pool regions on the fly
15
+
16
+Caveats
17
+-------
18
+* Currently, assumes architecture can make 4-byte aligned accesses
19
+* Not designed to be thread safe; the user must provide this
20
+
21
+Notes
22
+-----
23
+This code was based on the TLSF 1.4 spec and documentation found at:
24
+
25
+	http://rtportal.upv.es/rtmalloc/allocators/tlsf/index.shtml
26
+
27
+It also leverages the TLSF 2.0 improvement to shrink the per-block overhead
28
+from 8 to 4 bytes.
29
+
30
+Known Issues
31
+------------
32
+* Due to the internal block structure size and the implementation
33
+details of tlsf_memalign, there is worst-case behavior when requesting
34
+small (<16 byte) blocks aligned to 8-byte boundaries. Overuse of memalign
35
+will generally increase fragmentation, but this particular case will leave
36
+lots of unusable "holes" in the pool. The solution would be to internally
37
+align all blocks to 8 bytes, but this will require significantl changes
38
+to the implementation. Contact me if you are interested.
39
+
40
+History
41
+-------
42
+2014/02/08 - v3.0
43
+   * This version is based on improvements from 3DInteractive GmbH
44
+   * Interface changed to allow more than one memory pool
45
+   * Separated pool handling from control structure (adding, removing, debugging)
46
+   * Control structure and pools can still be constructed in the same memory block
47
+   * Memory blocks for control structure and pools are checked for alignment
48
+   * Added functions to retrieve control structure size, alignment size, min and
49
+     max block size, overhead of pool structure, and overhead of a single allocation
50
+   * Minimal Pool size is tlsf_block_size_min() + tlsf_pool_overhead()
51
+   * Pool must be empty when it is removed, in order to allow O(1) removal
52
+
53
+2011/10/20 - v2.0
54
+   * 64-bit support
55
+   * More compiler intrinsics for ffs/fls
56
+   * ffs/fls verification during TLSF creation in debug builds
57
+
58
+2008/04/04 - v1.9
59
+   * Add tlsf_heap_check, a heap integrity check
60
+   * Support a predefined tlsf_assert macro
61
+   * Fix realloc case where block should shrink; if adjacent block is
62
+     in use, execution would go down the slow path
63
+
64
+2007/02/08 - v1.8
65
+   * Fix for unnecessary reallocation in tlsf_realloc
66
+
67
+2007/02/03 - v1.7
68
+   * tlsf_heap_walk takes a callback
69
+   * tlsf_realloc now returns NULL on failure
70
+   * tlsf_memalign optimization for 4-byte alignment
71
+   * Usage of size_t where appropriate
72
+
73
+2006/11/21 - v1.6
74
+   * ffs/fls broken out into tlsfbits.h
75
+   * tlsf_overhead queries per-pool overhead
76
+
77
+2006/11/07 - v1.5
78
+   * Smart realloc implementation
79
+   * Smart memalign implementation
80
+
81
+2006/10/11 - v1.4
82
+   * Add some ffs/fls implementations
83
+   * Minor code footprint reduction
84
+
85
+2006/09/14 - v1.3
86
+   * Profiling indicates heavy use of blocks of
87
+     size 1-128, so implement small block handling
88
+   * Reduce pool overhead by about 1kb
89
+   * Reduce minimum block size from 32 to 12 bytes
90
+   * Realloc bug fix
91
+
92
+2006/09/09 - v1.2
93
+   * Add tlsf_block_size
94
+   * Static assertion mechanism for invariants
95
+   * Minor bugfixes 
96
+
97
+2006/09/01 - v1.1
98
+   * Add tlsf_realloc
99
+   * Add tlsf_walk_heap
100
+
101
+2006/08/25 - v1.0
102
+   * First release
0 103
new file mode 100644
... ...
@@ -0,0 +1,1051 @@
1
+#include <assert.h>
2
+#include <limits.h>
3
+#include <stddef.h>
4
+#include <stdio.h>
5
+#include <stdlib.h>
6
+#include <string.h>
7
+
8
+#include "tlsf.h"
9
+#include "tlsfbits.h"
10
+
11
+/*
12
+** Constants.
13
+*/
14
+
15
+/* Public constants: may be modified. */
16
+enum tlsf_public
17
+{
18
+	/* log2 of number of linear subdivisions of block sizes. */
19
+	SL_INDEX_COUNT_LOG2 = 5,
20
+};
21
+
22
+/* Private constants: do not modify. */
23
+enum tlsf_private
24
+{
25
+#if defined (TLSF_64BIT)
26
+	/* All allocation sizes and addresses are aligned to 8 bytes. */
27
+	ALIGN_SIZE_LOG2 = 3,
28
+#else
29
+	/* All allocation sizes and addresses are aligned to 4 bytes. */
30
+	ALIGN_SIZE_LOG2 = 2,
31
+#endif
32
+	ALIGN_SIZE = (1 << ALIGN_SIZE_LOG2),
33
+
34
+	/*
35
+	** We support allocations of sizes up to (1 << FL_INDEX_MAX) bits.
36
+	** However, because we linearly subdivide the second-level lists, and
37
+	** our minimum size granularity is 4 bytes, it doesn't make sense to
38
+	** create first-level lists for sizes smaller than SL_INDEX_COUNT * 4,
39
+	** or (1 << (SL_INDEX_COUNT_LOG2 + 2)) bytes, as there we will be
40
+	** trying to split size ranges into more slots than we have available.
41
+	** Instead, we calculate the minimum threshold size, and place all
42
+	** blocks below that size into the 0th first-level list.
43
+	*/
44
+
45
+#if defined (TLSF_64BIT)
46
+	/*
47
+	** TODO: We can increase this to support larger sizes, at the expense
48
+	** of more overhead in the TLSF structure.
49
+	*/
50
+	FL_INDEX_MAX = 32,
51
+#else
52
+	FL_INDEX_MAX = 30,
53
+#endif
54
+	SL_INDEX_COUNT = (1 << SL_INDEX_COUNT_LOG2),
55
+	FL_INDEX_SHIFT = (SL_INDEX_COUNT_LOG2 + ALIGN_SIZE_LOG2),
56
+	FL_INDEX_COUNT = (FL_INDEX_MAX - FL_INDEX_SHIFT + 1),
57
+
58
+	SMALL_BLOCK_SIZE = (1 << FL_INDEX_SHIFT),
59
+};
60
+
61
+/*
62
+** Cast and min/max macros.
63
+*/
64
+
65
+#define tlsf_cast(t, exp)	((t) (exp))
66
+#define tlsf_min(a, b)		((a) < (b) ? (a) : (b))
67
+#define tlsf_max(a, b)		((a) > (b) ? (a) : (b))
68
+
69
+/*
70
+** Set assert macro, if it has not been provided by the user.
71
+*/
72
+#if !defined (tlsf_assert)
73
+#define tlsf_assert assert
74
+#endif
75
+
76
+/*
77
+** Static assertion mechanism.
78
+*/
79
+
80
+#define _tlsf_glue2(x, y) x ## y
81
+#define _tlsf_glue(x, y) _tlsf_glue2(x, y)
82
+#define tlsf_static_assert(exp) \
83
+	typedef char _tlsf_glue(static_assert, __LINE__) [(exp) ? 1 : -1]
84
+
85
+/* This code has been tested on 32- and 64-bit (LP/LLP) architectures. */
86
+tlsf_static_assert(sizeof(int) * CHAR_BIT == 32);
87
+tlsf_static_assert(sizeof(size_t) * CHAR_BIT >= 32);
88
+tlsf_static_assert(sizeof(size_t) * CHAR_BIT <= 64);
89
+
90
+/* SL_INDEX_COUNT must be <= number of bits in sl_bitmap's storage type. */
91
+tlsf_static_assert(sizeof(unsigned int) * CHAR_BIT >= SL_INDEX_COUNT);
92
+
93
+/* Ensure we've properly tuned our sizes. */
94
+tlsf_static_assert(ALIGN_SIZE == SMALL_BLOCK_SIZE / SL_INDEX_COUNT);
95
+
96
+/*
97
+** Data structures and associated constants.
98
+*/
99
+
100
+/*
101
+** Block header structure.
102
+**
103
+** There are several implementation subtleties involved:
104
+** - The prev_phys_block field is only valid if the previous block is free.
105
+** - The prev_phys_block field is actually stored at the end of the
106
+**   previous block. It appears at the beginning of this structure only to
107
+**   simplify the implementation.
108
+** - The next_free / prev_free fields are only valid if the block is free.
109
+*/
110
+typedef struct block_header_t
111
+{
112
+	/* Points to the previous physical block. */
113
+	struct block_header_t* prev_phys_block;
114
+
115
+	/* The size of this block, excluding the block header. */
116
+	size_t size;
117
+
118
+	/* Next and previous free blocks. */
119
+	struct block_header_t* next_free;
120
+	struct block_header_t* prev_free;
121
+} block_header_t;
122
+
123
+/*
124
+** Since block sizes are always at least a multiple of 4, the two least
125
+** significant bits of the size field are used to store the block status:
126
+** - bit 0: whether block is busy or free
127
+** - bit 1: whether previous block is busy or free
128
+*/
129
+static const size_t block_header_free_bit = 1 << 0;
130
+static const size_t block_header_prev_free_bit = 1 << 1;
131
+
132
+/*
133
+** The size of the block header exposed to used blocks is the size field.
134
+** The prev_phys_block field is stored *inside* the previous free block.
135
+*/
136
+static const size_t block_header_overhead = sizeof(size_t);
137
+
138
+/* User data starts directly after the size field in a used block. */
139
+static const size_t block_start_offset =
140
+	offsetof(block_header_t, size) + sizeof(size_t);
141
+
142
+/*
143
+** A free block must be large enough to store its header minus the size of
144
+** the prev_phys_block field, and no larger than the number of addressable
145
+** bits for FL_INDEX.
146
+*/
147
+static const size_t block_size_min = 
148
+	sizeof(block_header_t) - sizeof(block_header_t*);
149
+static const size_t block_size_max = tlsf_cast(size_t, 1) << FL_INDEX_MAX;
150
+
151
+
152
+/* The TLSF control structure. */
153
+typedef struct control_t
154
+{
155
+	/* Empty lists point at this block to indicate they are free. */
156
+	block_header_t block_null;
157
+
158
+	/* Bitmaps for free lists. */
159
+	unsigned int fl_bitmap;
160
+	unsigned int sl_bitmap[FL_INDEX_COUNT];
161
+
162
+	/* Head of free lists. */
163
+	block_header_t* blocks[FL_INDEX_COUNT][SL_INDEX_COUNT];
164
+} control_t;
165
+
166
+/* A type used for casting when doing pointer arithmetic. */
167
+typedef ptrdiff_t tlsfptr_t;
168
+
169
+/*
170
+** block_header_t member functions.
171
+*/
172
+
173
+static size_t block_size(const block_header_t* block)
174
+{
175
+	return block->size & ~(block_header_free_bit | block_header_prev_free_bit);
176
+}
177
+
178
+static void block_set_size(block_header_t* block, size_t size)
179
+{
180
+	const size_t oldsize = block->size;
181
+	block->size = size | (oldsize & (block_header_free_bit | block_header_prev_free_bit));
182
+}
183
+
184
+static int block_is_last(const block_header_t* block)
185
+{
186
+	return 0 == block_size(block);
187
+}
188
+
189
+static int block_is_free(const block_header_t* block)
190
+{
191
+	return tlsf_cast(int, block->size & block_header_free_bit);
192
+}
193
+
194
+static void block_set_free(block_header_t* block)
195
+{
196
+	block->size |= block_header_free_bit;
197
+}
198
+
199
+static void block_set_used(block_header_t* block)
200
+{
201
+	block->size &= ~block_header_free_bit;
202
+}
203
+
204
+static int block_is_prev_free(const block_header_t* block)
205
+{
206
+	return tlsf_cast(int, block->size & block_header_prev_free_bit);
207
+}
208
+
209
+static void block_set_prev_free(block_header_t* block)
210
+{
211
+	block->size |= block_header_prev_free_bit;
212
+}
213
+
214
+static void block_set_prev_used(block_header_t* block)
215
+{
216
+	block->size &= ~block_header_prev_free_bit;
217
+}
218
+
219
+static block_header_t* block_from_ptr(const void* ptr)
220
+{
221
+	return tlsf_cast(block_header_t*,
222
+		tlsf_cast(unsigned char*, ptr) - block_start_offset);
223
+}
224
+
225
+static void* block_to_ptr(const block_header_t* block)
226
+{
227
+	return tlsf_cast(void*,
228
+		tlsf_cast(unsigned char*, block) + block_start_offset);
229
+}
230
+
231
+/* Return location of next block after block of given size. */
232
+static block_header_t* offset_to_block(const void* ptr, size_t size)
233
+{
234
+	return tlsf_cast(block_header_t*, tlsf_cast(tlsfptr_t, ptr) + size);
235
+}
236
+
237
+/* Return location of previous block. */
238
+static block_header_t* block_prev(const block_header_t* block)
239
+{
240
+	return block->prev_phys_block;
241
+}
242
+
243
+/* Return location of next existing block. */
244
+static block_header_t* block_next(const block_header_t* block)
245
+{
246
+	block_header_t* next = offset_to_block(block_to_ptr(block),
247
+		block_size(block) - block_header_overhead);
248
+	tlsf_assert(!block_is_last(block));
249
+	return next;
250
+}
251
+
252
+/* Link a new block with its physical neighbor, return the neighbor. */
253
+static block_header_t* block_link_next(block_header_t* block)
254
+{
255
+	block_header_t* next = block_next(block);
256
+	next->prev_phys_block = block;
257
+	return next;
258
+}
259
+
260
+static void block_mark_as_free(block_header_t* block)
261
+{
262
+	/* Link the block to the next block, first. */
263
+	block_header_t* next = block_link_next(block);
264
+	block_set_prev_free(next);
265
+	block_set_free(block);
266
+}
267
+
268
+static void block_mark_as_used(block_header_t* block)
269
+{
270
+	block_header_t* next = block_next(block);
271
+	block_set_prev_used(next);
272
+	block_set_used(block);
273
+}
274
+
275
+static size_t align_up(size_t x, size_t align)
276
+{
277
+	tlsf_assert(0 == (align & (align - 1)) && "must align to a power of two");
278
+	return (x + (align - 1)) & ~(align - 1);
279
+}
280
+
281
+static size_t align_down(size_t x, size_t align)
282
+{
283
+	tlsf_assert(0 == (align & (align - 1)) && "must align to a power of two");
284
+	return x - (x & (align - 1));
285
+}
286
+
287
+static void* align_ptr(const void* ptr, size_t align)
288
+{
289
+	const tlsfptr_t aligned =
290
+		(tlsf_cast(tlsfptr_t, ptr) + (align - 1)) & ~(align - 1);
291
+	tlsf_assert(0 == (align & (align - 1)) && "must align to a power of two");
292
+	return tlsf_cast(void*, aligned);
293
+}
294
+
295
+/*
296
+** Adjust an allocation size to be aligned to word size, and no smaller
297
+** than internal minimum.
298
+*/
299
+static size_t adjust_request_size(size_t size, size_t align)
300
+{
301
+	size_t adjust = 0;
302
+	if (size && size < block_size_max)
303
+	{
304
+		const size_t aligned = align_up(size, align);
305
+		adjust = tlsf_max(aligned, block_size_min);
306
+	}
307
+	return adjust;
308
+}
309
+
310
+/*
311
+** TLSF utility functions. In most cases, these are direct translations of
312
+** the documentation found in the white paper.
313
+*/
314
+
315
+static void mapping_insert(size_t size, int* fli, int* sli)
316
+{
317
+	int fl, sl;
318
+	if (size < SMALL_BLOCK_SIZE)
319
+	{
320
+		/* Store small blocks in first list. */
321
+		fl = 0;
322
+		sl = tlsf_cast(int, size) / (SMALL_BLOCK_SIZE / SL_INDEX_COUNT);
323
+	}
324
+	else
325
+	{
326
+		fl = tlsf_fls_sizet(size);
327
+		sl = tlsf_cast(int, size >> (fl - SL_INDEX_COUNT_LOG2)) ^ (1 << SL_INDEX_COUNT_LOG2);
328
+		fl -= (FL_INDEX_SHIFT - 1);
329
+	}
330
+	*fli = fl;
331
+	*sli = sl;
332
+}
333
+
334
+/* This version rounds up to the next block size (for allocations) */
335
+static void mapping_search(size_t size, int* fli, int* sli)
336
+{
337
+	if (size >= (1 << SL_INDEX_COUNT_LOG2))
338
+	{
339
+		const size_t round = (1 << (tlsf_fls_sizet(size) - SL_INDEX_COUNT_LOG2)) - 1;
340
+		size += round;
341
+	}
342
+	mapping_insert(size, fli, sli);
343
+}
344
+
345
+static block_header_t* search_suitable_block(control_t* control, int* fli, int* sli)
346
+{
347
+	int fl = *fli;
348
+	int sl = *sli;
349
+
350
+	/*
351
+	** First, search for a block in the list associated with the given
352
+	** fl/sl index.
353
+	*/
354
+	unsigned int sl_map = control->sl_bitmap[fl] & (~0 << sl);
355
+	if (!sl_map)
356
+	{
357
+		/* No block exists. Search in the next largest first-level list. */
358
+		const unsigned int fl_map = control->fl_bitmap & (~0 << (fl + 1));
359
+		if (!fl_map)
360
+		{
361
+			/* No free blocks available, memory has been exhausted. */
362
+			return 0;
363
+		}
364
+
365
+		fl = tlsf_ffs(fl_map);
366
+		*fli = fl;
367
+		sl_map = control->sl_bitmap[fl];
368
+	}
369
+	tlsf_assert(sl_map && "internal error - second level bitmap is null");
370
+	sl = tlsf_ffs(sl_map);
371
+	*sli = sl;
372
+
373
+	/* Return the first block in the free list. */
374
+	return control->blocks[fl][sl];
375
+}
376
+
377
+/* Remove a free block from the free list.*/
378
+static void remove_free_block(control_t* control, block_header_t* block, int fl, int sl)
379
+{
380
+	block_header_t* prev = block->prev_free;
381
+	block_header_t* next = block->next_free;
382
+	tlsf_assert(prev && "prev_free field can not be null");
383
+	tlsf_assert(next && "next_free field can not be null");
384
+	next->prev_free = prev;
385
+	prev->next_free = next;
386
+
387
+	/* If this block is the head of the free list, set new head. */
388
+	if (control->blocks[fl][sl] == block)
389
+	{
390
+		control->blocks[fl][sl] = next;
391
+
392
+		/* If the new head is null, clear the bitmap. */
393
+		if (next == &control->block_null)
394
+		{
395
+			control->sl_bitmap[fl] &= ~(1 << sl);
396
+
397
+			/* If the second bitmap is now empty, clear the fl bitmap. */
398
+			if (!control->sl_bitmap[fl])
399
+			{
400
+				control->fl_bitmap &= ~(1 << fl);
401
+			}
402
+		}
403
+	}
404
+}
405
+
406
+/* Insert a free block into the free block list. */
407
+static void insert_free_block(control_t* control, block_header_t* block, int fl, int sl)
408
+{
409
+	block_header_t* current = control->blocks[fl][sl];
410
+	tlsf_assert(current && "free list cannot have a null entry");
411
+	tlsf_assert(block && "cannot insert a null entry into the free list");
412
+	block->next_free = current;
413
+	block->prev_free = &control->block_null;
414
+	current->prev_free = block;
415
+
416
+	tlsf_assert(block_to_ptr(block) == align_ptr(block_to_ptr(block), ALIGN_SIZE)
417
+		&& "block not aligned properly");
418
+	/*
419
+	** Insert the new block at the head of the list, and mark the first-
420
+	** and second-level bitmaps appropriately.
421
+	*/
422
+	control->blocks[fl][sl] = block;
423
+	control->fl_bitmap |= (1 << fl);
424
+	control->sl_bitmap[fl] |= (1 << sl);
425
+}
426
+
427
+/* Remove a given block from the free list. */
428
+static void block_remove(control_t* control, block_header_t* block)
429
+{
430
+	int fl, sl;
431
+	mapping_insert(block_size(block), &fl, &sl);
432
+	remove_free_block(control, block, fl, sl);
433
+}
434
+
435
+/* Insert a given block into the free list. */
436
+static void block_insert(control_t* control, block_header_t* block)
437
+{
438
+	int fl, sl;
439
+	mapping_insert(block_size(block), &fl, &sl);
440
+	insert_free_block(control, block, fl, sl);
441
+}
442
+
443
+static int block_can_split(block_header_t* block, size_t size)
444
+{
445
+	return block_size(block) >= sizeof(block_header_t) + size;
446
+}
447
+
448
+/* Split a block into two, the second of which is free. */
449
+static block_header_t* block_split(block_header_t* block, size_t size)
450
+{
451
+	/* Calculate the amount of space left in the remaining block. */
452
+	block_header_t* remaining =
453
+		offset_to_block(block_to_ptr(block), size - block_header_overhead);
454
+
455
+	const size_t remain_size = block_size(block) - (size + block_header_overhead);
456
+
457
+	tlsf_assert(block_to_ptr(remaining) == align_ptr(block_to_ptr(remaining), ALIGN_SIZE)
458
+		&& "remaining block not aligned properly");
459
+
460
+	tlsf_assert(block_size(block) == remain_size + size + block_header_overhead);
461
+	block_set_size(remaining, remain_size);
462
+	tlsf_assert(block_size(remaining) >= block_size_min && "block split with invalid size");
463
+
464
+	block_set_size(block, size);
465
+	block_mark_as_free(remaining);
466
+
467
+	return remaining;
468
+}
469
+
470
+/* Absorb a free block's storage into an adjacent previous free block. */
471
+static block_header_t* block_absorb(block_header_t* prev, block_header_t* block)
472
+{
473
+	tlsf_assert(!block_is_last(prev) && "previous block can't be last!");
474
+	/* Note: Leaves flags untouched. */
475
+	prev->size += block_size(block) + block_header_overhead;
476
+	block_link_next(prev);
477
+	return prev;
478
+}
479
+
480
+/* Merge a just-freed block with an adjacent previous free block. */
481
+static block_header_t* block_merge_prev(control_t* control, block_header_t* block)
482
+{
483
+	if (block_is_prev_free(block))
484
+	{
485
+		block_header_t* prev = block_prev(block);
486
+		tlsf_assert(prev && "prev physical block can't be null");
487
+		tlsf_assert(block_is_free(prev) && "prev block is not free though marked as such");
488
+		block_remove(control, prev);
489
+		block = block_absorb(prev, block);
490
+	}
491
+
492
+	return block;
493
+}
494
+
495
+/* Merge a just-freed block with an adjacent free block. */
496
+static block_header_t* block_merge_next(control_t* control, block_header_t* block)
497
+{
498
+	block_header_t* next = block_next(block);
499
+	tlsf_assert(next && "next physical block can't be null");
500
+
501
+	if (block_is_free(next))
502
+	{
503
+		tlsf_assert(!block_is_last(block) && "previous block can't be last!");
504
+		block_remove(control, next);
505
+		block = block_absorb(block, next);
506
+	}
507
+
508
+	return block;
509
+}
510
+
511
+/* Trim any trailing block space off the end of a block, return to pool. */
512
+static void block_trim_free(control_t* control, block_header_t* block, size_t size)
513
+{
514
+	tlsf_assert(block_is_free(block) && "block must be free");
515
+	if (block_can_split(block, size))
516
+	{
517
+		block_header_t* remaining_block = block_split(block, size);
518
+		block_link_next(block);
519
+		block_set_prev_free(remaining_block);
520
+		block_insert(control, remaining_block);
521
+	}
522
+}
523
+
524
+/* Trim any trailing block space off the end of a used block, return to pool. */
525
+static void block_trim_used(control_t* control, block_header_t* block, size_t size)
526
+{
527
+	tlsf_assert(!block_is_free(block) && "block must be used");
528
+	if (block_can_split(block, size))
529
+	{
530
+		/* If the next block is free, we must coalesce. */
531
+		block_header_t* remaining_block = block_split(block, size);
532
+		block_set_prev_used(remaining_block);
533
+
534
+		remaining_block = block_merge_next(control, remaining_block);
535
+		block_insert(control, remaining_block);
536
+	}
537
+}
538
+
539
+static block_header_t* block_trim_free_leading(control_t* control, block_header_t* block, size_t size)
540
+{
541
+	block_header_t* remaining_block = block;
542
+	if (block_can_split(block, size))
543
+	{
544
+		/* We want the 2nd block. */
545
+		remaining_block = block_split(block, size - block_header_overhead);
546
+		block_set_prev_free(remaining_block);
547
+
548
+		block_link_next(block);
549
+		block_insert(control, block);
550
+	}
551
+
552
+	return remaining_block;
553
+}
554
+
555
+static block_header_t* block_locate_free(control_t* control, size_t size)
556
+{
557
+	int fl = 0, sl = 0;
558
+	block_header_t* block = 0;
559
+
560
+	if (size)
561
+	{
562
+		mapping_search(size, &fl, &sl);
563
+		block = search_suitable_block(control, &fl, &sl);
564
+	}
565
+
566
+	if (block)
567
+	{
568
+		tlsf_assert(block_size(block) >= size);
569
+		remove_free_block(control, block, fl, sl);
570
+	}
571
+
572
+	return block;
573
+}
574
+
575
+static void* block_prepare_used(control_t* control, block_header_t* block, size_t size)
576
+{
577
+	void* p = 0;
578
+	if (block)
579
+	{
580
+		block_trim_free(control, block, size);
581
+		block_mark_as_used(block);
582
+		p = block_to_ptr(block);
583
+	}
584
+	return p;
585
+}
586
+
587
+/* Clear structure and point all empty lists at the null block. */
588
+static void control_construct(control_t* control)
589
+{
590
+	int i, j;
591
+
592
+	control->block_null.next_free = &control->block_null;
593
+	control->block_null.prev_free = &control->block_null;
594
+
595
+	control->fl_bitmap = 0;
596
+	for (i = 0; i < FL_INDEX_COUNT; ++i)
597
+	{
598
+		control->sl_bitmap[i] = 0;
599
+		for (j = 0; j < SL_INDEX_COUNT; ++j)
600
+		{
601
+			control->blocks[i][j] = &control->block_null;
602
+		}
603
+	}
604
+}
605
+
606
+/*
607
+** Debugging utilities.
608
+*/
609
+
610
+typedef struct integrity_t
611
+{
612
+	int prev_status;
613
+	int status;
614
+} integrity_t;
615
+
616
+#define tlsf_insist(x) { tlsf_assert(x); if (!(x)) { status--; } }
617
+
618
+static void integrity_walker(void* ptr, size_t size, int used, void* user)
619
+{
620
+	block_header_t* block = block_from_ptr(ptr);
621
+	integrity_t* integ = tlsf_cast(integrity_t*, user);
622
+	const int this_prev_status = block_is_prev_free(block) ? 1 : 0;
623
+	const int this_status = block_is_free(block) ? 1 : 0;
624
+	const size_t this_block_size = block_size(block);
625
+
626
+	int status = 0;
627
+	tlsf_insist(integ->prev_status == this_prev_status && "prev status incorrect");
628
+	tlsf_insist(size == this_block_size && "block size incorrect");
629
+
630
+	integ->prev_status = this_status;
631
+	integ->status += status;
632
+}
633
+
634
+int tlsf_check(tlsf_t tlsf)
635
+{
636
+	int i, j;
637
+
638
+	control_t* control = tlsf_cast(control_t*, tlsf);
639
+	int status = 0;
640
+
641
+	/* Check that the free lists and bitmaps are accurate. */
642
+	for (i = 0; i < FL_INDEX_COUNT; ++i)
643
+	{
644
+		for (j = 0; j < SL_INDEX_COUNT; ++j)
645
+		{
646
+			const int fl_map = control->fl_bitmap & (1 << i);
647
+			const int sl_list = control->sl_bitmap[i];
648
+			const int sl_map = sl_list & (1 << j);
649
+			const block_header_t* block = control->blocks[i][j];
650
+
651
+			/* Check that first- and second-level lists agree. */
652
+			if (!fl_map)
653
+			{
654
+				tlsf_insist(!sl_map && "second-level map must be null");
655
+			}
656
+
657
+			if (!sl_map)
658
+			{
659
+				tlsf_insist(block == &control->block_null && "block list must be null");
660
+				continue;
661
+			}
662
+
663
+			/* Check that there is at least one free block. */
664
+			tlsf_insist(sl_list && "no free blocks in second-level map");
665
+			tlsf_insist(block != &control->block_null && "block should not be null");
666
+
667
+			while (block != &control->block_null)
668
+			{
669
+				int fli, sli;
670
+				tlsf_insist(block_is_free(block) && "block should be free");
671
+				tlsf_insist(!block_is_prev_free(block) && "blocks should have coalesced");
672
+				tlsf_insist(!block_is_free(block_next(block)) && "blocks should have coalesced");
673
+				tlsf_insist(block_is_prev_free(block_next(block)) && "block should be free");
674
+				tlsf_insist(block_size(block) >= block_size_min && "block not minimum size");
675
+
676
+				mapping_insert(block_size(block), &fli, &sli);
677
+				tlsf_insist(fli == i && sli == j && "block size indexed in wrong list");
678
+				block = block->next_free;
679
+			}
680
+		}
681
+	}
682
+
683
+	return status;
684
+}
685
+
686
+#undef tlsf_insist
687
+
688
+static void default_walker(void* ptr, size_t size, int used, void* user)
689
+{
690
+	(void)user;
691
+	printf("\t%p %s size: %x (%p)\n", ptr, used ? "used" : "free", (unsigned int)size, block_from_ptr(ptr));
692
+}
693
+
694
+void tlsf_walk_pool(pool_t pool, tlsf_walker walker, void* user)
695
+{
696
+	tlsf_walker pool_walker = walker ? walker : default_walker;
697
+	block_header_t* block =
698
+		offset_to_block(pool, -(int)block_header_overhead);
699
+
700
+	while (block && !block_is_last(block))
701
+	{
702
+		pool_walker(
703
+			block_to_ptr(block),
704
+			block_size(block),
705
+			!block_is_free(block),
706
+			user);
707
+		block = block_next(block);
708
+	}
709
+}
710
+
711
+size_t tlsf_block_size(void* ptr)
712
+{
713
+	size_t size = 0;
714
+	if (ptr)
715
+	{
716
+		const block_header_t* block = block_from_ptr(ptr);
717
+		size = block_size(block);
718
+	}
719
+	return size;
720
+}
721
+
722
+int tlsf_check_pool(pool_t pool)
723
+{
724
+	/* Check that the blocks are physically correct. */
725
+	integrity_t integ = { 0, 0 };
726
+	tlsf_walk_pool(pool, integrity_walker, &integ);
727
+
728
+	return integ.status;
729
+}
730
+
731
+/*
732
+** Size of the TLSF structures in a given memory block passed to
733
+** tlsf_create, equal to the size of a control_t
734
+*/
735
+size_t tlsf_size()
736
+{
737
+	return sizeof(control_t);
738
+}
739
+
740
+size_t tlsf_align_size()
741
+{
742
+	return ALIGN_SIZE;
743
+}
744
+
745
+size_t tlsf_block_size_min()
746
+{
747
+	return block_size_min;
748
+}
749
+
750
+size_t tlsf_block_size_max()
751
+{
752
+	return block_size_max;
753
+}
754
+
755
+/*
756
+** Overhead of the TLSF structures in a given memory block passes to
757
+** tlsf_add_pool, equal to the overhead of a free block and the
758
+** sentinel block.
759
+*/
760
+size_t tlsf_pool_overhead()
761
+{
762
+	return 2 * block_header_overhead;
763
+}
764
+
765
+size_t tlsf_alloc_overhead()
766
+{
767
+	return block_header_overhead;
768
+}
769
+
770
+pool_t tlsf_add_pool(tlsf_t tlsf, void* mem, size_t bytes)
771
+{
772
+	block_header_t* block;
773
+	block_header_t* next;
774
+
775
+	const size_t pool_overhead = tlsf_pool_overhead();
776
+	const size_t pool_bytes = align_down(bytes - pool_overhead, ALIGN_SIZE);
777
+
778
+	if (((ptrdiff_t)mem % ALIGN_SIZE) != 0)
779
+	{
780
+		printf("tlsf_add_pool: Memory must be aligned by %u bytes.\n",
781
+			(unsigned int)ALIGN_SIZE);
782
+		return 0;
783
+	}
784
+
785
+	if (pool_bytes < block_size_min || pool_bytes > block_size_max)
786
+	{
787
+#if defined (TLSF_64BIT)
788
+		printf("tlsf_add_pool: Memory size must be between 0x%x and 0x%x00 bytes.\n", 
789
+			(unsigned int)(pool_overhead + block_size_min),
790
+			(unsigned int)((pool_overhead + block_size_max) / 256));
791
+#else
792
+		printf("tlsf_add_pool: Memory size must be between %u and %u bytes.\n", 
793
+			(unsigned int)(pool_overhead + block_size_min),
794
+			(unsigned int)(pool_overhead + block_size_max));
795
+#endif
796
+		return 0;
797
+	}
798
+
799
+	/*
800
+	** Create the main free block. Offset the start of the block slightly
801
+	** so that the prev_phys_block field falls outside of the pool -
802
+	** it will never be used.
803
+	*/
804
+	block = offset_to_block(mem, -(tlsfptr_t)block_header_overhead);
805
+	block_set_size(block, pool_bytes);
806
+	block_set_free(block);
807
+	block_set_prev_used(block);
808
+	block_insert(tlsf_cast(control_t*, tlsf), block);
809
+
810
+	/* Split the block to create a zero-size sentinel block. */
811
+	next = block_link_next(block);
812
+	block_set_size(next, 0);
813
+	block_set_used(next);
814
+	block_set_prev_free(next);
815
+
816
+	return mem;
817
+}
818
+
819
+void tlsf_remove_pool(tlsf_t tlsf, pool_t pool)
820
+{
821
+	control_t* control = tlsf_cast(control_t*, tlsf);
822
+	block_header_t* block = offset_to_block(pool, -(int)block_header_overhead);
823
+
824
+	int fl = 0, sl = 0;
825
+
826
+	tlsf_assert(block_is_free(block) && "block should be free");
827
+	tlsf_assert(!block_is_free(block_next(block)) && "next block should not be free");
828
+	tlsf_assert(block_size(block_next(block)) == 0 && "next block size should be zero");
829
+
830
+	mapping_insert(block_size(block), &fl, &sl);
831
+	remove_free_block(control, block, fl, sl);
832
+}
833
+
834
+/*
835
+** TLSF main interface.
836
+*/
837
+
838
+#if _DEBUG
839
+int test_ffs_fls()
840
+{
841
+	/* Verify ffs/fls work properly. */
842
+	int rv = 0;
843
+	rv += (tlsf_ffs(0) == -1) ? 0 : 0x1;
844
+	rv += (tlsf_fls(0) == -1) ? 0 : 0x2;
845
+	rv += (tlsf_ffs(1) == 0) ? 0 : 0x4;
846
+	rv += (tlsf_fls(1) == 0) ? 0 : 0x8;
847
+	rv += (tlsf_ffs(0x80000000) == 31) ? 0 : 0x10;
848
+	rv += (tlsf_ffs(0x80008000) == 15) ? 0 : 0x20;
849
+	rv += (tlsf_fls(0x80000008) == 31) ? 0 : 0x40;
850
+	rv += (tlsf_fls(0x7FFFFFFF) == 30) ? 0 : 0x80;
851
+
852
+#if defined (TLSF_64BIT)
853
+	rv += (tlsf_fls_sizet(0x80000000) == 31) ? 0 : 0x100;
854
+	rv += (tlsf_fls_sizet(0x100000000) == 32) ? 0 : 0x200;
855
+	rv += (tlsf_fls_sizet(0xffffffffffffffff) == 63) ? 0 : 0x400; 
856
+#endif
857
+
858
+	if (rv)
859
+	{
860
+		printf("tlsf_create: %x ffs/fls tests failed!\n", rv);
861
+	}
862
+	return rv;
863
+}
864
+#endif
865
+
866
+tlsf_t tlsf_create(void* mem)
867
+{
868
+#if _DEBUG
869
+	if (test_ffs_fls())
870
+	{
871
+		return 0;
872
+	}
873
+#endif
874
+
875
+	if (((tlsfptr_t)mem % ALIGN_SIZE) != 0)
876
+	{
877
+		printf("tlsf_create: Memory must be aligned to %u bytes.\n",
878
+			(unsigned int)ALIGN_SIZE);
879
+		return 0;
880
+	}
881
+
882
+	control_construct(tlsf_cast(control_t*, mem));
883
+
884
+	return tlsf_cast(tlsf_t, mem);
885
+}
886
+
887
+tlsf_t tlsf_create_with_pool(void* mem, size_t bytes)
888
+{
889
+	tlsf_t tlsf = tlsf_create(mem);
890
+	tlsf_add_pool(tlsf, (char*)mem + tlsf_size(), bytes - tlsf_size());
891
+	return tlsf;
892
+}
893
+
894
+void tlsf_destroy(tlsf_t tlsf)
895
+{
896
+	/* Nothing to do. */
897
+	(void)tlsf;
898
+}
899
+
900
+pool_t tlsf_get_pool(tlsf_t tlsf)
901
+{
902
+	return tlsf_cast(pool_t, (char*)tlsf + tlsf_size());
903
+}
904
+
905
+void* tlsf_malloc(tlsf_t tlsf, size_t size)
906
+{
907
+	control_t* control = tlsf_cast(control_t*, tlsf);
908
+	const size_t adjust = adjust_request_size(size, ALIGN_SIZE);
909
+	block_header_t* block = block_locate_free(control, adjust);
910
+	return block_prepare_used(control, block, adjust);
911
+}
912
+
913
+void* tlsf_memalign(tlsf_t tlsf, size_t align, size_t size)
914
+{
915
+	control_t* control = tlsf_cast(control_t*, tlsf);
916
+	const size_t adjust = adjust_request_size(size, ALIGN_SIZE);
917
+
918
+	/*
919
+	** We must allocate an additional minimum block size bytes so that if
920
+	** our free block will leave an alignment gap which is smaller, we can
921
+	** trim a leading free block and release it back to the pool. We must
922
+	** do this because the previous physical block is in use, therefore
923
+	** the prev_phys_block field is not valid, and we can't simply adjust
924
+	** the size of that block.
925
+	*/
926
+	const size_t gap_minimum = sizeof(block_header_t);
927
+	const size_t size_with_gap = adjust_request_size(adjust + align + gap_minimum, align);
928
+
929
+	/* If alignment is less than or equals base alignment, we're done. */
930
+	const size_t aligned_size = (align <= ALIGN_SIZE) ? adjust : size_with_gap;
931
+
932
+	block_header_t* block = block_locate_free(control, aligned_size);
933
+
934
+	/* This can't be a static assert. */
935
+	tlsf_assert(sizeof(block_header_t) == block_size_min + block_header_overhead);
936
+
937
+	if (block)
938
+	{
939
+		void* ptr = block_to_ptr(block);
940
+		void* aligned = align_ptr(ptr, align);
941
+		size_t gap = tlsf_cast(size_t,
942
+			tlsf_cast(tlsfptr_t, aligned) - tlsf_cast(tlsfptr_t, ptr));
943
+
944
+		/* If gap size is too small, offset to next aligned boundary. */
945
+		if (gap && gap < gap_minimum)
946
+		{
947
+			const size_t gap_remain = gap_minimum - gap;
948
+			const size_t offset = tlsf_max(gap_remain, align);
949
+			const void* next_aligned = tlsf_cast(void*,
950
+				tlsf_cast(tlsfptr_t, aligned) + offset);
951
+
952
+			aligned = align_ptr(next_aligned, align);
953
+			gap = tlsf_cast(size_t,
954
+				tlsf_cast(tlsfptr_t, aligned) - tlsf_cast(tlsfptr_t, ptr));
955
+		}
956
+
957
+		if (gap)
958
+		{
959
+			tlsf_assert(gap >= gap_minimum && "gap size too small");
960
+			block = block_trim_free_leading(control, block, gap);
961
+		}
962
+	}
963
+
964
+	return block_prepare_used(control, block, adjust);
965
+}
966
+
967
+void tlsf_free(tlsf_t tlsf, void* ptr)
968
+{
969
+	/* Don't attempt to free a NULL pointer. */
970
+	if (ptr)
971
+	{
972
+		control_t* control = tlsf_cast(control_t*, tlsf);
973
+		block_header_t* block = block_from_ptr(ptr);
974
+		tlsf_assert(!block_is_free(block) && "block already marked as free");
975
+		block_mark_as_free(block);
976
+		block = block_merge_prev(control, block);
977
+		block = block_merge_next(control, block);
978
+		block_insert(control, block);
979
+	}
980
+}
981
+
982
+/*
983
+** The TLSF block information provides us with enough information to
984
+** provide a reasonably intelligent implementation of realloc, growing or
985
+** shrinking the currently allocated block as required.
986
+**
987
+** This routine handles the somewhat esoteric edge cases of realloc:
988
+** - a non-zero size with a null pointer will behave like malloc
989
+** - a zero size with a non-null pointer will behave like free
990
+** - a request that cannot be satisfied will leave the original buffer
991
+**   untouched
992
+** - an extended buffer size will leave the newly-allocated area with
993
+**   contents undefined
994
+*/
995
+void* tlsf_realloc(tlsf_t tlsf, void* ptr, size_t size)
996
+{
997
+	control_t* control = tlsf_cast(control_t*, tlsf);
998
+	void* p = 0;
999
+
1000
+	/* Zero-size requests are treated as free. */
1001
+	if (ptr && size == 0)
1002
+	{
1003
+		tlsf_free(tlsf, ptr);
1004
+	}
1005
+	/* Requests with NULL pointers are treated as malloc. */
1006
+	else if (!ptr)
1007
+	{
1008
+		p = tlsf_malloc(tlsf, size);
1009
+	}
1010
+	else
1011
+	{
1012
+		block_header_t* block = block_from_ptr(ptr);
1013
+		block_header_t* next = block_next(block);
1014
+
1015
+		const size_t cursize = block_size(block);
1016
+		const size_t combined = cursize + block_size(next) + block_header_overhead;
1017
+		const size_t adjust = adjust_request_size(size, ALIGN_SIZE);
1018
+
1019
+		tlsf_assert(!block_is_free(block) && "block already marked as free");
1020
+
1021
+		/*
1022
+		** If the next block is used, or when combined with the current
1023
+		** block, does not offer enough space, we must reallocate and copy.
1024
+		*/
1025
+		if (adjust > cursize && (!block_is_free(next) || adjust > combined))
1026
+		{
1027
+			p = tlsf_malloc(tlsf, size);
1028
+			if (p)
1029
+			{
1030
+				const size_t minsize = tlsf_min(cursize, size);
1031
+				memcpy(p, ptr, minsize);
1032
+				tlsf_free(tlsf, ptr);
1033
+			}
1034
+		}
1035
+		else
1036
+		{
1037
+			/* Do we need to expand to the next block? */
1038
+			if (adjust > cursize)
1039
+			{
1040
+				block_merge_next(control, block);
1041
+				block_mark_as_used(block);
1042
+			}
1043
+
1044
+			/* Trim the resulting block and return the original pointer. */
1045
+			block_trim_used(control, block, adjust);
1046
+			p = ptr;
1047
+		}
1048
+	}
1049
+
1050
+	return p;
1051
+}
0 1052
new file mode 100644
... ...
@@ -0,0 +1,68 @@
1
+#ifndef INCLUDED_tlsf
2
+#define INCLUDED_tlsf
3
+
4
+/*
5
+** Two Level Segregated Fit memory allocator, version 3.0.
6
+** Written by Matthew Conte, and placed in the Public Domain.
7
+**	http://tlsf.baisoku.org
8
+**
9
+** Based on the original documentation by Miguel Masmano:
10
+**	http://rtportal.upv.es/rtmalloc/allocators/tlsf/index.shtml
11
+**
12
+** Please see the accompanying Readme.txt for implementation
13
+** notes and caveats.
14
+**
15
+** This implementation was written to the specification
16
+** of the document, therefore no GPL restrictions apply.
17
+*/
18
+
19
+#include <stddef.h>
20
+
21
+#if defined(__cplusplus)
22
+extern "C" {
23
+#endif
24
+
25
+/* tlsf_t: a TLSF structure. Can contain 1 to N pools. */
26
+/* pool_t: a block of memory that TLSF can manage. */
27
+typedef void* tlsf_t;
28
+typedef void* pool_t;
29
+
30
+/* Create/destroy a memory pool. */
31
+tlsf_t tlsf_create(void* mem);
32
+tlsf_t tlsf_create_with_pool(void* mem, size_t bytes);
33
+void tlsf_destroy(tlsf_t tlsf);
34
+pool_t tlsf_get_pool(tlsf_t tlsf);
35
+
36
+/* Add/remove memory pools. */
37
+pool_t tlsf_add_pool(tlsf_t tlsf, void* mem, size_t bytes);
38
+void tlsf_remove_pool(tlsf_t tlsf, pool_t pool);
39
+
40
+/* malloc/memalign/realloc/free replacements. */
41
+void* tlsf_malloc(tlsf_t tlsf, size_t bytes);
42
+void* tlsf_memalign(tlsf_t tlsf, size_t align, size_t bytes);
43
+void* tlsf_realloc(tlsf_t tlsf, void* ptr, size_t size);
44
+void tlsf_free(tlsf_t tlsf, void* ptr);
45
+
46
+/* Returns internal block size, not original request size */
47
+size_t tlsf_block_size(void* ptr);
48
+
49
+/* Overheads/limits of internal structures. */
50
+size_t tlsf_size();
51
+size_t tlsf_align_size();
52
+size_t tlsf_block_size_min();
53
+size_t tlsf_block_size_max();
54
+size_t tlsf_pool_overhead();
55
+size_t tlsf_alloc_overhead();
56
+
57
+/* Debugging. */
58
+typedef void (*tlsf_walker)(void* ptr, size_t size, int used, void* user);
59
+void tlsf_walk_pool(pool_t pool, tlsf_walker walker, void* user);
60
+/* Returns nonzero if any internal consistency check fails. */
61
+int tlsf_check(tlsf_t tlsf);
62
+int tlsf_check_pool(pool_t pool);
63
+
64
+#if defined(__cplusplus)
65
+};
66
+#endif
67
+
68
+#endif
0 69
new file mode 100644
... ...
@@ -0,0 +1,180 @@
1
+#ifndef INCLUDED_tlsfbits
2
+#define INCLUDED_tlsfbits
3
+
4
+#if defined(__cplusplus)
5
+#define tlsf_decl inline
6
+#else
7
+#define tlsf_decl static
8
+#endif
9
+
10
+/*
11
+** Architecture-specific bit manipulation routines.
12
+**
13
+** TLSF achieves O(1) cost for malloc and free operations by limiting
14
+** the search for a free block to a free list of guaranteed size
15
+** adequate to fulfill the request, combined with efficient free list
16
+** queries using bitmasks and architecture-specific bit-manipulation
17
+** routines.
18
+**
19
+** Most modern processors provide instructions to count leading zeroes
20
+** in a word, find the lowest and highest set bit, etc. These
21
+** specific implementations will be used when available, falling back
22
+** to a reasonably efficient generic implementation.
23
+**
24
+** NOTE: TLSF spec relies on ffs/fls returning value 0..31.
25
+** ffs/fls return 1-32 by default, returning 0 for error.
26
+*/
27
+
28
+/*
29
+** Detect whether or not we are building for a 32- or 64-bit (LP/LLP)
30
+** architecture. There is no reliable portable method at compile-time.
31
+*/
32
+#if defined (__alpha__) || defined (__ia64__) || defined (__x86_64__) \
33
+	|| defined (_WIN64) || defined (__LP64__) || defined (__LLP64__)
34
+#define TLSF_64BIT
35
+#endif
36
+
37
+/*
38
+** gcc 3.4 and above have builtin support, specialized for architecture.
39
+** Some compilers masquerade as gcc; patchlevel test filters them out.
40
+*/
41
+#if defined (__GNUC__) && (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) \
42
+	&& defined (__GNUC_PATCHLEVEL__)
43
+
44
+tlsf_decl int tlsf_ffs(unsigned int word)
45
+{
46
+	return __builtin_ffs(word) - 1;
47
+}
48
+
49
+tlsf_decl int tlsf_fls(unsigned int word)
50
+{
51
+	const int bit = word ? 32 - __builtin_clz(word) : 0;
52
+	return bit - 1;
53
+}
54
+
55
+#elif defined (_MSC_VER) && (_MSC_VER >= 1400) && (defined (_M_IX86) || defined (_M_X64))
56
+/* Microsoft Visual C++ support on x86/X64 architectures. */
57
+
58
+#include <intrin.h>
59
+
60
+#pragma intrinsic(_BitScanReverse)
61
+#pragma intrinsic(_BitScanForward)
62
+
63
+tlsf_decl int tlsf_fls(unsigned int word)
64
+{
65
+	unsigned long index;
66
+	return _BitScanReverse(&index, word) ? index : -1;
67
+}
68
+
69
+tlsf_decl int tlsf_ffs(unsigned int word)
70
+{
71
+	unsigned long index;
72
+	return _BitScanForward(&index, word) ? index : -1;
73
+}
74
+
75
+#elif defined (_MSC_VER) && defined (_M_PPC)
76
+/* Microsoft Visual C++ support on PowerPC architectures. */
77
+
78
+#include <ppcintrinsics.h>
79
+
80
+tlsf_decl int tlsf_fls(unsigned int word)
81
+{
82
+	const int bit = 32 - _CountLeadingZeros(word);
83
+	return bit - 1;
84
+}
85
+
86
+tlsf_decl int tlsf_ffs(unsigned int word)
87
+{
88
+	const unsigned int reverse = word & (~word + 1);
89
+	const int bit = 32 - _CountLeadingZeros(reverse);
90
+	return bit - 1;
91
+}
92
+
93
+#elif defined (__ARMCC_VERSION)
94
+/* RealView Compilation Tools for ARM */
95
+
96
+tlsf_decl int tlsf_ffs(unsigned int word)
97
+{
98
+	const unsigned int reverse = word & (~word + 1);
99
+	const int bit = 32 - __clz(reverse);
100
+	return bit - 1;
101
+}
102
+
103
+tlsf_decl int tlsf_fls(unsigned int word)
104
+{
105
+	const int bit = word ? 32 - __clz(word) : 0;
106
+	return bit - 1;
107
+}
108
+
109
+#elif defined (__ghs__)
110
+/* Green Hills support for PowerPC */
111
+
112
+#include <ppc_ghs.h>
113
+
114
+tlsf_decl int tlsf_ffs(unsigned int word)
115
+{
116
+	const unsigned int reverse = word & (~word + 1);
117
+	const int bit = 32 - __CLZ32(reverse);
118
+	return bit - 1;
119
+}
120
+
121
+tlsf_decl int tlsf_fls(unsigned int word)
122
+{
123
+	const int bit = word ? 32 - __CLZ32(word) : 0;
124
+	return bit - 1;
125
+}
126
+
127
+#else
128
+/* Fall back to generic implementation. */
129
+
130
+tlsf_decl int tlsf_fls_generic(unsigned int word)
131
+{
132
+	int bit = 32;
133
+
134
+	if (!word) bit -= 1;
135
+	if (!(word & 0xffff0000)) { word <<= 16; bit -= 16; }
136
+	if (!(word & 0xff000000)) { word <<= 8; bit -= 8; }
137
+	if (!(word & 0xf0000000)) { word <<= 4; bit -= 4; }
138
+	if (!(word & 0xc0000000)) { word <<= 2; bit -= 2; }
139
+	if (!(word & 0x80000000)) { word <<= 1; bit -= 1; }
140
+
141
+	return bit;
142
+}
143
+
144
+/* Implement ffs in terms of fls. */
145
+tlsf_decl int tlsf_ffs(unsigned int word)
146
+{
147
+	return tlsf_fls_generic(word & (~word + 1)) - 1;
148
+}
149
+
150
+tlsf_decl int tlsf_fls(unsigned int word)
151
+{
152
+	return tlsf_fls_generic(word) - 1;
153
+}
154
+
155
+#endif
156
+
157
+/* Possibly 64-bit version of tlsf_fls. */
158
+#if defined (TLSF_64BIT)
159
+tlsf_decl int tlsf_fls_sizet(size_t size)
160
+{
161
+	int high = (int)(size >> 32);
162
+	int bits = 0;
163
+	if (high)
164
+	{
165
+		bits = 32 + tlsf_fls(high);
166
+	}
167
+	else
168
+	{
169
+		bits = tlsf_fls((int)size & 0xffffffff);
170
+
171
+	}
172
+	return bits;
173
+}
174
+#else
175
+#define tlsf_fls_sizet tlsf_fls
176
+#endif
177
+
178
+#undef tlsf_decl
179
+
180
+#endif