arena: allocate the per-bin abandoned-page bitmaps on first abandon

Every (heap, arena) pair carried MI_ARENA_BIN_COUNT abandoned-page
bitmaps next to its `pages` bitmap, laid out and initialized up front:
once in the arena's info slices for the main heap, and once more in a
zeroed allocation for every other heap that touched the arena
(`mi_heap_ensure_arena_pages`). Each bitmap is sized by the arena
(2 KiB per GiB of arena), so with the default 1 GiB reservation that is
~110 KiB per heap, and ~410 KiB for a heap in a 4 GiB arena. Writing the
header of each bitmap touched one OS page per bitmap, so a heap paid
about 50 page faults (or the memset, for external memory that is not
known to be zero) before its first allocation, and most heaps never
abandon a page at all: short-lived heaps are destroyed, not abandoned.

Bun creates one heap per transpile and one for JSC's 4 GiB structure
heap, so this was about 250 of the ~1000 page faults of `bun file.js`.

Now `pages_abandoned[bin]` starts NULL and is allocated the first time a
page of that bin is abandoned, from the subproc meta-data heap (safe on
the abandon paths, where a regular heap allocation is not), published
with a CAS. Readers treat NULL as all-clear; they were already guarded
by `heap->abandoned_count[bin]`. `_mi_arena_pages_free` frees them with
the heap's arena pages. If the allocation fails the page is abandoned
unmapped, like a full page, and mapped once a free brings it back.

The new test-abandoned-lazy exercises the abandon, reclaim,
re-abandon, visit and delete paths from several threads at once.

Author: robobun

CMakeLists.txt

@@ -877,7 +877,7 @@ if (MI_BUILD_TESTS)
   enable_testing()
 
   # static link tests
-  set(mi_static_tests api api-fill stress-heaps stress-subprocs stress heap-mt heap-teardown heap-delete-race heap-churn heap-aba heap-burst-destroy fork-user-heap snapshot prof prof-adversarial purge-zero park-handoff free-before-init)
+  set(mi_static_tests api api-fill stress-heaps stress-subprocs stress heap-mt heap-teardown heap-delete-race heap-churn heap-aba heap-burst-destroy abandoned-lazy fork-user-heap snapshot prof prof-adversarial purge-zero park-handoff free-before-init)
   if(NOT (MI_DEBUG_TSAN OR MI_TRACK_ASAN OR MI_DEBUG_UBSAN))
     list(APPEND mi_static_tests thp-optout)  # counts madvise calls by interposing it, which a sanitizer runtime does first
   endif()

include/mimalloc/internal.h

@@ -277,6 +277,7 @@ void          _mi_arenas_abandoned_page_free(mi_page_t* page, mi_theap_t* curren
 void          _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theap);
 void          _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx /* can be NULL */);
 bool          _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page);
+void          _mi_arena_pages_free(mi_arena_pages_t* arena_pages);
 size_t        mi_arenas_get_count(mi_subproc_t* subproc);
 uint8_t*      mi_arena_slice_start(mi_arena_t* arena, size_t slice_index);
 

include/mimalloc/types.h

@@ -756,8 +756,13 @@ typedef struct mi_bbitmap_s mi_bbitmap_t;   // atomic binned bitmap (defined in
 
 struct mi_arena_pages_s {
   mi_bitmap_t* pages;                // all registered pages (abandoned and owned)
-  mi_bitmap_t* pages_abandoned[MI_ARENA_BIN_COUNT];  // abandoned pages per size bin (a set bit means the start of the page)
-  // followed by the bitmaps (whose siz`es depend on the arena size)
+  // Abandoned pages per size bin (a set bit means the start of the page). Each bitmap is
+  // allocated the first time a page of that bin is abandoned (`mi_arena_pages_abandoned_ensure`);
+  // NULL means no page of that bin was ever abandoned. Eagerly laying out all MI_ARENA_BIN_COUNT
+  // of them cost a page fault per bitmap (one header write each, on its own OS page) for every
+  // heap that touched an arena, and most heaps never abandon a page.
+  _Atomic(mi_bitmap_t*) pages_abandoned[MI_ARENA_BIN_COUNT];
+  // followed by the `pages` bitmap (whose size depends on the arena size)
 };
 
 

src/arena.c

@@ -29,6 +29,9 @@ The arena allocation needs to be thread safe and we use an atomic bitmap to allo
 #error "The page_t.page_ma_offset field is not large enough to cover a full arena"
 #endif
 
+static mi_bitmap_t* mi_arena_pages_abandoned(mi_arena_pages_t* arena_pages, size_t bin);
+static mi_bitmap_t* mi_arena_pages_abandoned_ensure(mi_arena_t* arena, mi_arena_pages_t* arena_pages, size_t bin);
+
 /* -----------------------------------------------------------
   Arena id's
 ----------------------------------------------------------- */
@@ -726,9 +729,9 @@ static mi_page_t* mi_arenas_page_try_find_abandoned(mi_theap_t* theap, size_t sl
   mi_forall_suitable_arenas(heap, req_arena, tseq, match_numa, any_numa, allow_large, arena)
   {
     mi_arena_pages_t* const arena_pages = mi_heap_arena_pages(heap, arena);
-    if (arena_pages != NULL) {
+    mi_bitmap_t* const bitmap = (arena_pages != NULL ? mi_arena_pages_abandoned(arena_pages, bin) : NULL);
+    if (bitmap != NULL) {
       size_t slice_index;
-      mi_bitmap_t* const bitmap = arena_pages->pages_abandoned[bin];
 
       if (mi_bitmap_try_find_and_claim(bitmap, tseq, &slice_index, &mi_arena_try_claim_abandoned, arena)) {
         // found an abandoned page of the right size
@@ -1119,7 +1122,7 @@ static void mi_arenas_page_free_prim(mi_page_t* page, mi_subproc_t* subproc, mi_
       const size_t bin = _mi_bin(mi_page_block_size(page));
       mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));
       mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
-      mi_assert_internal(bin >= MI_ARENA_BIN_COUNT || mi_bitmap_is_clearN(arena_pages->pages_abandoned[bin], slice_index, 1));
+      mi_assert_internal(bin >= MI_ARENA_BIN_COUNT || mi_arena_pages_abandoned(arena_pages, bin) == NULL || mi_bitmap_is_clearN(mi_arena_pages_abandoned(arena_pages, bin), slice_index, 1));
       // note: we cannot check for `!mi_page_is_abandoned_and_mapped` since that may
       // be (temporarily) not true if the free happens while trying to reclaim
       // see `mi_arena_try_claim_abandoned`
@@ -1224,13 +1227,18 @@ void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theapx) {
       mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
       mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count));
 
-      mi_page_set_abandoned_mapped(page);
-      const bool was_clear = mi_bitmap_set(arena_pages->pages_abandoned[bin], slice_index);
-      MI_UNUSED(was_clear); mi_assert_internal(was_clear);
-      mi_atomic_increment_relaxed(&heap->abandoned_count[bin]);
-      mi_theapx_stat_increase(heap, current_theapx, pages_abandoned, 1);
-      mi_abandoned_page_unown(page, current_theapx);
-      return;
+      // If the bin's bitmap cannot be allocated the page is abandoned unmapped, like a full
+      // page: it is still reachable through `pages` and is reclaimed once a block in it is freed.
+      mi_bitmap_t* const bitmap = mi_arena_pages_abandoned_ensure(arena, arena_pages, bin);
+      if mi_likely(bitmap != NULL) {
+        mi_page_set_abandoned_mapped(page);
+        const bool was_clear = mi_bitmap_set(bitmap, slice_index);
+        MI_UNUSED(was_clear); mi_assert_internal(was_clear);
+        mi_atomic_increment_relaxed(&heap->abandoned_count[bin]);
+        mi_theapx_stat_increase(heap, current_theapx, pages_abandoned, 1);
+        mi_abandoned_page_unown(page, current_theapx);
+        return;
+      }
     }
   }
   // otherwise,
@@ -1298,7 +1306,9 @@ void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx) {
     mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
 
     // this busy waits until a concurrent reader (from alloc_abandoned) is done
-    mi_bitmap_clear_once_set(arena->subproc, arena_pages->pages_abandoned[bin], slice_index);
+    mi_bitmap_t* const bitmap = mi_arena_pages_abandoned(arena_pages, bin);
+    mi_assert_internal(bitmap != NULL);  // a mapped page was set in it
+    mi_bitmap_clear_once_set(arena->subproc, bitmap, slice_index);
     mi_page_clear_abandoned_mapped(page);
     mi_atomic_decrement_relaxed(&heap->abandoned_count[bin]);
   }
@@ -1384,7 +1394,8 @@ void _mi_arenas_purge_abandoned_holes(mi_heap_t* heap, mi_tld_t* tld) {
       // singleton bins have no abandoned bitmap (upstream ad1bcdbf, to shrink arena meta).
       for (size_t bin = 0; bin < MI_ARENA_BIN_COUNT; bin++) {
         if (mi_atomic_load_relaxed(&heap->abandoned_count[bin]) == 0) continue;
-        mi_bitmap_t* const bitmap = arena_pages->pages_abandoned[bin];
+        mi_bitmap_t* const bitmap = mi_arena_pages_abandoned(arena_pages, bin);
+        if (bitmap == NULL) continue;
         mi_purge_holes_arg_t parg = { bitmap, tld };
         (void)_mi_bitmap_forall_set(bitmap, &mi_arena_page_purge_holes_at, arena, &parg);
       }
@@ -1427,7 +1438,8 @@ void _mi_arenas_holes_report(mi_heap_t* heap, mi_holes_report_t* rep) {
     if (arena_pages != NULL) {
       for (size_t bin = 0; bin < MI_ARENA_BIN_COUNT; bin++) {   // see above: not MI_BIN_COUNT
         if (mi_atomic_load_relaxed(&heap->abandoned_count[bin]) == 0) continue;
-        mi_arena_holes_report_arg_t ra = { arena_pages->pages_abandoned[bin], rep };
+        mi_arena_holes_report_arg_t ra = { mi_arena_pages_abandoned(arena_pages, bin), rep };
+        if (ra.bitmap == NULL) continue;
         (void)_mi_bitmap_forall_set(ra.bitmap, &mi_arena_page_holes_report_at, arena, &ra);
       }
     }
@@ -1651,8 +1663,7 @@ static size_t mi_arena_pages_size(size_t slice_count, size_t* bitmap_base) {
   if (slice_count == 0) slice_count = MI_BCHUNK_BITS;
   mi_assert_internal((slice_count % MI_BCHUNK_BITS) == 0);
   const size_t base_size = _mi_align_up(sizeof(mi_arena_pages_t), MI_BCHUNK_SIZE);
-  const size_t bitmaps_count = 1 + MI_ARENA_BIN_COUNT; // pages, and abandoned
-  const size_t bitmaps_size = bitmaps_count * mi_bitmap_size(slice_count, NULL);
+  const size_t bitmaps_size = mi_bitmap_size(slice_count, NULL); // pages (the abandoned bitmaps are allocated on demand)
   const size_t size = base_size + bitmaps_size;
   if (bitmap_base != NULL) *bitmap_base = base_size;
   return size;
@@ -1662,7 +1673,7 @@ static size_t mi_arena_info_slices_needed(size_t slice_count, size_t* bitmap_bas
   if (slice_count == 0) slice_count = MI_BCHUNK_BITS;
   mi_assert_internal((slice_count % MI_BCHUNK_BITS) == 0);
   const size_t base_size = _mi_align_up(sizeof(mi_arena_t), MI_BCHUNK_SIZE);
-  const size_t bitmaps_count = 4 + MI_ARENA_BIN_COUNT; // commit, dirty, purge, pages, and abandoned
+  const size_t bitmaps_count = 4; // commit, dirty, purge, and pages (the abandoned bitmaps are allocated on demand)
   const size_t bitmaps_size = bitmaps_count * mi_bitmap_size(slice_count, NULL) + mi_bbitmap_size(slice_count, NULL); // + free
   #if MI_PAGE_META_IS_SEPARATED
   const size_t pages_size = slice_count * sizeof(mi_page_t);
@@ -1701,12 +1712,55 @@ static mi_arena_pages_t* mi_arena_pages_alloc(mi_arena_t* arena) {
   uint8_t* base = (uint8_t*)arena_pages + bitmap_base;
   mi_assert_internal(_mi_is_aligned(base, MI_BCHUNK_SIZE));
   arena_pages->pages = mi_arena_bitmap_init(slice_count, &base);
-  for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) {
-    arena_pages->pages_abandoned[i] = mi_arena_bitmap_init(slice_count, &base);
-  }
+  // `pages_abandoned[]` stays NULL (the allocation is zeroed) until a page of that bin is abandoned.
   return arena_pages;
 }
 
+// The abandoned-pages bitmap of `bin`, or NULL if no page of that bin was ever abandoned in
+// this (heap, arena) pair. A NULL bitmap reads as all-clear.
+static mi_bitmap_t* mi_arena_pages_abandoned(mi_arena_pages_t* arena_pages, size_t bin) {
+  mi_assert_internal(bin < MI_ARENA_BIN_COUNT);
+  return mi_atomic_load_ptr_acquire(mi_bitmap_t, &arena_pages->pages_abandoned[bin]);
+}
+
+// The abandoned-pages bitmap of `bin`, allocated on first use. Allocated from the subproc
+// meta-data heap so this is safe on the abandon paths (a thread tearing down its theaps, a
+// foreign free re-abandoning a page), where allocating from a regular heap is not. Publishing
+// is a CAS so no lock is needed: a loser frees its copy and uses the winner's. Returns NULL only
+// if the allocation failed.
+static mi_bitmap_t* mi_arena_pages_abandoned_ensure(mi_arena_t* arena, mi_arena_pages_t* arena_pages, size_t bin) {
+  mi_bitmap_t* bitmap = mi_arena_pages_abandoned(arena_pages, bin);
+  if mi_likely(bitmap != NULL) return bitmap;
+  const size_t slice_count = arena->slice_count;
+  const size_t size = mi_bitmap_size(slice_count, NULL);
+  mi_bitmap_t* fresh = (mi_bitmap_t*)_mi_meta_zalloc_aligned(arena->subproc, size, MI_BCHUNK_SIZE, NULL);
+  if (fresh == NULL) return NULL;
+  mi_bitmap_init(fresh, slice_count, true /* already zero */);
+  mi_bitmap_t* expected = NULL;
+  if (mi_atomic_cas_ptr_strong_acq_rel(mi_bitmap_t, &arena_pages->pages_abandoned[bin], &expected, fresh)) {
+    return fresh;
+  }
+  // another thread published one first
+  _mi_free_subproc_safe(fresh);
+  mi_assert_internal(expected != NULL);
+  return expected;
+}
+
+// Release the on-demand abandoned bitmaps of a heap's arena pages (the `pages` bitmap is part of
+// the `arena_pages` allocation itself).
+static void mi_arena_pages_free_abandoned(mi_arena_pages_t* arena_pages) {
+  for (size_t bin = 0; bin < MI_ARENA_BIN_COUNT; bin++) {
+    mi_bitmap_t* bitmap = mi_atomic_exchange_ptr_acq_rel(mi_bitmap_t, &arena_pages->pages_abandoned[bin], NULL);
+    if (bitmap != NULL) { _mi_free_subproc_safe(bitmap); }
+  }
+}
+
+void _mi_arena_pages_free(mi_arena_pages_t* arena_pages) {
+  if (arena_pages == NULL) return;
+  mi_arena_pages_free_abandoned(arena_pages);
+  _mi_free_subproc_safe(arena_pages);
+}
+
 static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start,
                                         size_t slice_count, mi_arena_t* parent, size_t total_size,
                                         int numa_node, bool exclusive,
@@ -1799,7 +1853,7 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start,
   arena->slices_purge = mi_arena_bitmap_init(slice_count, &base);
   arena->pages_main.pages = mi_arena_bitmap_init(slice_count, &base);
   for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) {
-    arena->pages_main.pages_abandoned[i] = mi_arena_bitmap_init(slice_count, &base);
+    mi_atomic_store_ptr_relaxed(mi_bitmap_t, &arena->pages_main.pages_abandoned[i], NULL);  // allocated on first abandon
   }
   #if MI_PAGE_META_IS_SEPARATED
   arena->pages_meta = (mi_page_t*)base;
@@ -2690,7 +2744,9 @@ bool _mi_heap_visit_blocks(mi_heap_t* heap, bool abandoned_only, bool visit_bloc
         for (size_t bin = 0; ok && bin < MI_ARENA_BIN_COUNT; bin++) {
           // todo: if we had a single abandoned page map as well, this can be faster.
           if (mi_atomic_load_relaxed(&heap->abandoned_count[bin]) > 0) {
-            ok = _mi_bitmap_forall_set(arena_pages->pages_abandoned[bin], &mi_heap_visit_page_at, arena, &visit_info);
+            mi_bitmap_t* const bitmap = mi_arena_pages_abandoned(arena_pages, bin);
+            if (bitmap == NULL) continue;
+            ok = _mi_bitmap_forall_set(bitmap, &mi_heap_visit_page_at, arena, &visit_info);
           }
         }
       }

src/heap.c

@@ -229,7 +229,7 @@ static void mi_heap_free(mi_heap_t* heap, bool acquire_heaps_lock) {
         mi_arena_pages_t* arena_pages = mi_atomic_load_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i]);
         if (arena_pages!=NULL) {
           mi_atomic_store_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i], NULL);
-          _mi_free_subproc_safe(arena_pages);
+          _mi_arena_pages_free(arena_pages);
         }
       }
     }