You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

malloc.c 16KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562
  1. /*
  2. * Copyright (C) 2006 Michael Brown <mbrown@fensystems.co.uk>.
  3. *
  4. * This program is free software; you can redistribute it and/or
  5. * modify it under the terms of the GNU General Public License as
  6. * published by the Free Software Foundation; either version 2 of the
  7. * License, or any later version.
  8. *
  9. * This program is distributed in the hope that it will be useful, but
  10. * WITHOUT ANY WARRANTY; without even the implied warranty of
  11. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  12. * General Public License for more details.
  13. *
  14. * You should have received a copy of the GNU General Public License
  15. * along with this program; if not, write to the Free Software
  16. * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  17. * 02110-1301, USA.
  18. */
  19. FILE_LICENCE ( GPL2_OR_LATER );
  20. #include <stddef.h>
  21. #include <stdint.h>
  22. #include <string.h>
  23. #include <strings.h>
  24. #include <ipxe/io.h>
  25. #include <ipxe/list.h>
  26. #include <ipxe/init.h>
  27. #include <ipxe/refcnt.h>
  28. #include <ipxe/malloc.h>
  29. #include <valgrind/memcheck.h>
  30. /** @file
  31. *
  32. * Dynamic memory allocation
  33. *
  34. */
  35. /** A free block of memory */
  36. struct memory_block {
  37. /** Size of this block */
  38. size_t size;
  39. /** Padding
  40. *
  41. * This padding exists to cover the "count" field of a
  42. * reference counter, in the common case where a reference
  43. * counter is the first element of a dynamically-allocated
  44. * object. It avoids clobbering the "count" field as soon as
  45. * the memory is freed, and so allows for the possibility of
  46. * detecting reference counting errors.
  47. */
  48. char pad[ offsetof ( struct refcnt, count ) +
  49. sizeof ( ( ( struct refcnt * ) NULL )->count ) ];
  50. /** List of free blocks */
  51. struct list_head list;
  52. };
  53. #define MIN_MEMBLOCK_SIZE \
  54. ( ( size_t ) ( 1 << ( fls ( sizeof ( struct memory_block ) - 1 ) ) ) )
  55. /** A block of allocated memory complete with size information */
  56. struct autosized_block {
  57. /** Size of this block */
  58. size_t size;
  59. /** Remaining data */
  60. char data[0];
  61. };
  62. /**
  63. * Address for zero-length memory blocks
  64. *
  65. * @c malloc(0) or @c realloc(ptr,0) will return the special value @c
  66. * NOWHERE. Calling @c free(NOWHERE) will have no effect.
  67. *
  68. * This is consistent with the ANSI C standards, which state that
  69. * "either NULL or a pointer suitable to be passed to free()" must be
  70. * returned in these cases. Using a special non-NULL value means that
  71. * the caller can take a NULL return value to indicate failure,
  72. * without first having to check for a requested size of zero.
  73. *
  74. * Code outside of malloc.c do not ever need to refer to the actual
  75. * value of @c NOWHERE; this is an internal definition.
  76. */
  77. #define NOWHERE ( ( void * ) ~( ( intptr_t ) 0 ) )
  78. /** List of free memory blocks */
  79. static LIST_HEAD ( free_blocks );
  80. /** Total amount of free memory */
  81. size_t freemem;
  82. /**
  83. * Heap size
  84. *
  85. * Currently fixed at 512kB.
  86. */
  87. #define HEAP_SIZE ( 512 * 1024 )
  88. /** The heap itself */
  89. static char heap[HEAP_SIZE] __attribute__ (( aligned ( __alignof__(void *) )));
  90. /**
  91. * Mark all blocks in free list as defined
  92. *
  93. */
  94. static inline void valgrind_make_blocks_defined ( void ) {
  95. struct memory_block *block;
  96. if ( RUNNING_ON_VALGRIND <= 0 )
  97. return;
  98. /* Traverse free block list, marking each block structure as
  99. * defined. Some contortions are necessary to avoid errors
  100. * from list_check().
  101. */
  102. /* Mark block list itself as defined */
  103. VALGRIND_MAKE_MEM_DEFINED ( &free_blocks, sizeof ( free_blocks ) );
  104. /* Mark areas accessed by list_check() as defined */
  105. VALGRIND_MAKE_MEM_DEFINED ( &free_blocks.prev->next,
  106. sizeof ( free_blocks.prev->next ) );
  107. VALGRIND_MAKE_MEM_DEFINED ( free_blocks.next,
  108. sizeof ( *free_blocks.next ) );
  109. VALGRIND_MAKE_MEM_DEFINED ( &free_blocks.next->next->prev,
  110. sizeof ( free_blocks.next->next->prev ) );
  111. /* Mark each block in list as defined */
  112. list_for_each_entry ( block, &free_blocks, list ) {
  113. /* Mark block as defined */
  114. VALGRIND_MAKE_MEM_DEFINED ( block, sizeof ( *block ) );
  115. /* Mark areas accessed by list_check() as defined */
  116. VALGRIND_MAKE_MEM_DEFINED ( block->list.next,
  117. sizeof ( *block->list.next ) );
  118. VALGRIND_MAKE_MEM_DEFINED ( &block->list.next->next->prev,
  119. sizeof ( block->list.next->next->prev ) );
  120. }
  121. }
  122. /**
  123. * Mark all blocks in free list as inaccessible
  124. *
  125. */
  126. static inline void valgrind_make_blocks_noaccess ( void ) {
  127. struct memory_block *block;
  128. struct memory_block *prev = NULL;
  129. if ( RUNNING_ON_VALGRIND <= 0 )
  130. return;
  131. /* Traverse free block list, marking each block structure as
  132. * inaccessible. Some contortions are necessary to avoid
  133. * errors from list_check().
  134. */
  135. /* Mark each block in list as inaccessible */
  136. list_for_each_entry ( block, &free_blocks, list ) {
  137. /* Mark previous block (if any) as inaccessible. (Current
  138. * block will be accessed by list_check().)
  139. */
  140. if ( prev )
  141. VALGRIND_MAKE_MEM_NOACCESS ( prev, sizeof ( *prev ) );
  142. prev = block;
  143. /* At the end of the list, list_check() will end up
  144. * accessing the first list item. Temporarily mark
  145. * this area as defined.
  146. */
  147. VALGRIND_MAKE_MEM_DEFINED ( &free_blocks.next->prev,
  148. sizeof ( free_blocks.next->prev ) );
  149. }
  150. /* Mark last block (if any) as inaccessible */
  151. if ( prev )
  152. VALGRIND_MAKE_MEM_NOACCESS ( prev, sizeof ( *prev ) );
  153. /* Mark as inaccessible the area that was temporarily marked
  154. * as defined to avoid errors from list_check().
  155. */
  156. VALGRIND_MAKE_MEM_NOACCESS ( &free_blocks.next->prev,
  157. sizeof ( free_blocks.next->prev ) );
  158. /* Mark block list itself as inaccessible */
  159. VALGRIND_MAKE_MEM_NOACCESS ( &free_blocks, sizeof ( free_blocks ) );
  160. }
  161. /**
  162. * Discard some cached data
  163. *
  164. * @ret discarded Number of cached items discarded
  165. */
  166. static unsigned int discard_cache ( void ) {
  167. struct cache_discarder *discarder;
  168. unsigned int discarded;
  169. for_each_table_entry ( discarder, CACHE_DISCARDERS ) {
  170. discarded = discarder->discard();
  171. if ( discarded )
  172. return discarded;
  173. }
  174. return 0;
  175. }
  176. /**
  177. * Discard all cached data
  178. *
  179. */
  180. static void discard_all_cache ( void ) {
  181. unsigned int discarded;
  182. do {
  183. discarded = discard_cache();
  184. } while ( discarded );
  185. }
  186. /**
  187. * Allocate a memory block
  188. *
  189. * @v size Requested size
  190. * @v align Physical alignment
  191. * @v offset Offset from physical alignment
  192. * @ret ptr Memory block, or NULL
  193. *
  194. * Allocates a memory block @b physically aligned as requested. No
  195. * guarantees are provided for the alignment of the virtual address.
  196. *
  197. * @c align must be a power of two. @c size may not be zero.
  198. */
  199. void * alloc_memblock ( size_t size, size_t align, size_t offset ) {
  200. struct memory_block *block;
  201. size_t align_mask;
  202. size_t pre_size;
  203. ssize_t post_size;
  204. struct memory_block *pre;
  205. struct memory_block *post;
  206. struct memory_block *ptr;
  207. valgrind_make_blocks_defined();
  208. /* Round up size to multiple of MIN_MEMBLOCK_SIZE and
  209. * calculate alignment mask.
  210. */
  211. size = ( size + MIN_MEMBLOCK_SIZE - 1 ) & ~( MIN_MEMBLOCK_SIZE - 1 );
  212. align_mask = ( align - 1 ) | ( MIN_MEMBLOCK_SIZE - 1 );
  213. DBG ( "Allocating %#zx (aligned %#zx+%zx)\n", size, align, offset );
  214. while ( 1 ) {
  215. /* Search through blocks for the first one with enough space */
  216. list_for_each_entry ( block, &free_blocks, list ) {
  217. pre_size = ( ( offset - virt_to_phys ( block ) )
  218. & align_mask );
  219. post_size = ( block->size - pre_size - size );
  220. if ( post_size >= 0 ) {
  221. /* Split block into pre-block, block, and
  222. * post-block. After this split, the "pre"
  223. * block is the one currently linked into the
  224. * free list.
  225. */
  226. pre = block;
  227. block = ( ( ( void * ) pre ) + pre_size );
  228. post = ( ( ( void * ) block ) + size );
  229. DBG ( "[%p,%p) -> [%p,%p) + [%p,%p)\n", pre,
  230. ( ( ( void * ) pre ) + pre->size ),
  231. pre, block, post,
  232. ( ( ( void * ) pre ) + pre->size ) );
  233. /* If there is a "post" block, add it in to
  234. * the free list. Leak it if it is too small
  235. * (which can happen only at the very end of
  236. * the heap).
  237. */
  238. if ( (size_t) post_size >= MIN_MEMBLOCK_SIZE ) {
  239. VALGRIND_MAKE_MEM_DEFINED ( post,
  240. sizeof ( *post ) );
  241. post->size = post_size;
  242. list_add ( &post->list, &pre->list );
  243. }
  244. /* Shrink "pre" block, leaving the main block
  245. * isolated and no longer part of the free
  246. * list.
  247. */
  248. pre->size = pre_size;
  249. /* If there is no "pre" block, remove it from
  250. * the list. Also remove it (i.e. leak it) if
  251. * it is too small, which can happen only at
  252. * the very start of the heap.
  253. */
  254. if ( pre_size < MIN_MEMBLOCK_SIZE )
  255. list_del ( &pre->list );
  256. /* Update total free memory */
  257. freemem -= size;
  258. /* Return allocated block */
  259. DBG ( "Allocated [%p,%p)\n", block,
  260. ( ( ( void * ) block ) + size ) );
  261. ptr = block;
  262. goto done;
  263. }
  264. }
  265. /* Try discarding some cached data to free up memory */
  266. if ( ! discard_cache() ) {
  267. /* Nothing available to discard */
  268. DBG ( "Failed to allocate %#zx (aligned %#zx)\n",
  269. size, align );
  270. ptr = NULL;
  271. goto done;
  272. }
  273. }
  274. done:
  275. valgrind_make_blocks_noaccess();
  276. return ptr;
  277. }
  278. /**
  279. * Free a memory block
  280. *
  281. * @v ptr Memory allocated by alloc_memblock(), or NULL
  282. * @v size Size of the memory
  283. *
  284. * If @c ptr is NULL, no action is taken.
  285. */
  286. void free_memblock ( void *ptr, size_t size ) {
  287. struct memory_block *freeing;
  288. struct memory_block *block;
  289. struct memory_block *tmp;
  290. ssize_t gap_before;
  291. ssize_t gap_after = -1;
  292. /* Allow for ptr==NULL */
  293. if ( ! ptr )
  294. return;
  295. valgrind_make_blocks_defined();
  296. /* Round up size to match actual size that alloc_memblock()
  297. * would have used.
  298. */
  299. size = ( size + MIN_MEMBLOCK_SIZE - 1 ) & ~( MIN_MEMBLOCK_SIZE - 1 );
  300. freeing = ptr;
  301. VALGRIND_MAKE_MEM_DEFINED ( freeing, sizeof ( *freeing ) );
  302. freeing->size = size;
  303. DBG ( "Freeing [%p,%p)\n", freeing, ( ( ( void * ) freeing ) + size ));
  304. /* Insert/merge into free list */
  305. list_for_each_entry_safe ( block, tmp, &free_blocks, list ) {
  306. /* Calculate gaps before and after the "freeing" block */
  307. gap_before = ( ( ( void * ) freeing ) -
  308. ( ( ( void * ) block ) + block->size ) );
  309. gap_after = ( ( ( void * ) block ) -
  310. ( ( ( void * ) freeing ) + freeing->size ) );
  311. /* Merge with immediately preceding block, if possible */
  312. if ( gap_before == 0 ) {
  313. DBG ( "[%p,%p) + [%p,%p) -> [%p,%p)\n", block,
  314. ( ( ( void * ) block ) + block->size ), freeing,
  315. ( ( ( void * ) freeing ) + freeing->size ),block,
  316. ( ( ( void * ) freeing ) + freeing->size ) );
  317. block->size += size;
  318. list_del ( &block->list );
  319. freeing = block;
  320. }
  321. /* Stop processing as soon as we reach a following block */
  322. if ( gap_after >= 0 )
  323. break;
  324. }
  325. /* Insert before the immediately following block. If
  326. * possible, merge the following block into the "freeing"
  327. * block.
  328. */
  329. DBG ( "[%p,%p)\n", freeing, ( ( ( void * ) freeing ) + freeing->size));
  330. list_add_tail ( &freeing->list, &block->list );
  331. if ( gap_after == 0 ) {
  332. DBG ( "[%p,%p) + [%p,%p) -> [%p,%p)\n", freeing,
  333. ( ( ( void * ) freeing ) + freeing->size ), block,
  334. ( ( ( void * ) block ) + block->size ), freeing,
  335. ( ( ( void * ) block ) + block->size ) );
  336. freeing->size += block->size;
  337. list_del ( &block->list );
  338. }
  339. /* Update free memory counter */
  340. freemem += size;
  341. valgrind_make_blocks_noaccess();
  342. }
  343. /**
  344. * Reallocate memory
  345. *
  346. * @v old_ptr Memory previously allocated by malloc(), or NULL
  347. * @v new_size Requested size
  348. * @ret new_ptr Allocated memory, or NULL
  349. *
  350. * Allocates memory with no particular alignment requirement. @c
  351. * new_ptr will be aligned to at least a multiple of sizeof(void*).
  352. * If @c old_ptr is non-NULL, then the contents of the newly allocated
  353. * memory will be the same as the contents of the previously allocated
  354. * memory, up to the minimum of the old and new sizes. The old memory
  355. * will be freed.
  356. *
  357. * If allocation fails the previously allocated block is left
  358. * untouched and NULL is returned.
  359. *
  360. * Calling realloc() with a new size of zero is a valid way to free a
  361. * memory block.
  362. */
  363. void * realloc ( void *old_ptr, size_t new_size ) {
  364. struct autosized_block *old_block;
  365. struct autosized_block *new_block;
  366. size_t old_total_size;
  367. size_t new_total_size;
  368. size_t old_size;
  369. void *new_ptr = NOWHERE;
  370. /* Allocate new memory if necessary. If allocation fails,
  371. * return without touching the old block.
  372. */
  373. if ( new_size ) {
  374. new_total_size = ( new_size +
  375. offsetof ( struct autosized_block, data ) );
  376. new_block = alloc_memblock ( new_total_size, 1, 0 );
  377. if ( ! new_block )
  378. return NULL;
  379. VALGRIND_MAKE_MEM_UNDEFINED ( new_block, offsetof ( struct autosized_block, data ) );
  380. new_block->size = new_total_size;
  381. VALGRIND_MAKE_MEM_NOACCESS ( new_block, offsetof ( struct autosized_block, data ) );
  382. new_ptr = &new_block->data;
  383. VALGRIND_MALLOCLIKE_BLOCK ( new_ptr, new_size, 0, 0 );
  384. }
  385. /* Copy across relevant part of the old data region (if any),
  386. * then free it. Note that at this point either (a) new_ptr
  387. * is valid, or (b) new_size is 0; either way, the memcpy() is
  388. * valid.
  389. */
  390. if ( old_ptr && ( old_ptr != NOWHERE ) ) {
  391. old_block = container_of ( old_ptr, struct autosized_block,
  392. data );
  393. VALGRIND_MAKE_MEM_DEFINED ( old_block, offsetof ( struct autosized_block, data ) );
  394. old_total_size = old_block->size;
  395. old_size = ( old_total_size -
  396. offsetof ( struct autosized_block, data ) );
  397. memcpy ( new_ptr, old_ptr,
  398. ( ( old_size < new_size ) ? old_size : new_size ) );
  399. free_memblock ( old_block, old_total_size );
  400. VALGRIND_MAKE_MEM_NOACCESS ( old_block, offsetof ( struct autosized_block, data ) );
  401. VALGRIND_FREELIKE_BLOCK ( old_ptr, 0 );
  402. }
  403. return new_ptr;
  404. }
  405. /**
  406. * Allocate memory
  407. *
  408. * @v size Requested size
  409. * @ret ptr Memory, or NULL
  410. *
  411. * Allocates memory with no particular alignment requirement. @c ptr
  412. * will be aligned to at least a multiple of sizeof(void*).
  413. */
  414. void * malloc ( size_t size ) {
  415. return realloc ( NULL, size );
  416. }
  417. /**
  418. * Free memory
  419. *
  420. * @v ptr Memory allocated by malloc(), or NULL
  421. *
  422. * Memory allocated with malloc_dma() cannot be freed with free(); it
  423. * must be freed with free_dma() instead.
  424. *
  425. * If @c ptr is NULL, no action is taken.
  426. */
  427. void free ( void *ptr ) {
  428. realloc ( ptr, 0 );
  429. }
  430. /**
  431. * Allocate cleared memory
  432. *
  433. * @v size Requested size
  434. * @ret ptr Allocated memory
  435. *
  436. * Allocate memory as per malloc(), and zero it.
  437. *
  438. * This function name is non-standard, but pretty intuitive.
  439. * zalloc(size) is always equivalent to calloc(1,size)
  440. */
  441. void * zalloc ( size_t size ) {
  442. void *data;
  443. data = malloc ( size );
  444. if ( data )
  445. memset ( data, 0, size );
  446. return data;
  447. }
  448. /**
  449. * Add memory to allocation pool
  450. *
  451. * @v start Start address
  452. * @v end End address
  453. *
  454. * Adds a block of memory [start,end) to the allocation pool. This is
  455. * a one-way operation; there is no way to reclaim this memory.
  456. *
  457. * @c start must be aligned to at least a multiple of sizeof(void*).
  458. */
  459. void mpopulate ( void *start, size_t len ) {
  460. /* Prevent free_memblock() from rounding up len beyond the end
  461. * of what we were actually given...
  462. */
  463. free_memblock ( start, ( len & ~( MIN_MEMBLOCK_SIZE - 1 ) ) );
  464. }
  465. /**
  466. * Initialise the heap
  467. *
  468. */
  469. static void init_heap ( void ) {
  470. VALGRIND_MAKE_MEM_NOACCESS ( heap, sizeof ( heap ) );
  471. mpopulate ( heap, sizeof ( heap ) );
  472. }
  473. /** Memory allocator initialisation function */
  474. struct init_fn heap_init_fn __init_fn ( INIT_EARLY ) = {
  475. .initialise = init_heap,
  476. };
  477. /**
  478. * Discard all cached data on shutdown
  479. *
  480. */
  481. static void shutdown_cache ( int booting __unused ) {
  482. discard_all_cache();
  483. }
  484. /** Memory allocator shutdown function */
  485. struct startup_fn heap_startup_fn __startup_fn ( STARTUP_EARLY ) = {
  486. .shutdown = shutdown_cache,
  487. };
  488. #if 0
  489. #include <stdio.h>
  490. /**
  491. * Dump free block list
  492. *
  493. */
  494. void mdumpfree ( void ) {
  495. struct memory_block *block;
  496. printf ( "Free block list:\n" );
  497. list_for_each_entry ( block, &free_blocks, list ) {
  498. printf ( "[%p,%p] (size %#zx)\n", block,
  499. ( ( ( void * ) block ) + block->size ), block->size );
  500. }
  501. }
  502. #endif