Merge branch 'dev' into dev2

Author: daanx

include/mimalloc/internal.h

@@ -690,6 +690,13 @@ static inline bool mi_page_is_in_full(const mi_page_t* page) {
 }
 
 static inline void mi_page_set_in_full(mi_page_t* page, bool in_full) {
+  if (page->flags.x.in_full != in_full) {
+    // optimize: maintain pages_full_size to avoid visiting the full queue (issue #1220)
+    mi_heap_t* const heap = mi_page_heap(page);
+    const size_t size = page->capacity * mi_page_block_size(page);
+    if (in_full) { heap->pages_full_size += size; }
+            else { mi_assert_internal(size <= heap->pages_full_size); heap->pages_full_size -= size; }
+  }
   page->flags.x.in_full = in_full;
 }
 

include/mimalloc/types.h

@@ -562,6 +562,7 @@ struct mi_heap_s {
   size_t                page_count;                          // total number of pages in the `pages` queues.
   size_t                page_retired_min;                    // smallest retired index (retired pages are fully free, but still in the page queues)
   size_t                page_retired_max;                    // largest retired index into the `pages` array.
+  size_t                pages_full_size;                     // optimization: total size of blocks in the pages of the full queue (issue #1220)
   long                  generic_count;                       // how often is `_mi_malloc_generic` called?
   long                  generic_collect_count;               // how often is `_mi_malloc_generic` called without collecting?
   mi_heap_t*            next;                                // list of heaps per thread

src/init.c

@@ -119,6 +119,7 @@ mi_decl_cache_align const mi_heap_t _mi_heap_empty = {
   { {0}, {0}, 0, true }, // random
   0,                // page count
   MI_BIN_FULL, 0,   // page retired min/max
+  0,                // pages_full_size
   0, 0,             // generic count
   NULL,             // next
   false,            // can reclaim
@@ -168,6 +169,7 @@ mi_decl_cache_align mi_heap_t _mi_heap_main = {
   { {0x846ca68b}, {0}, 0, true },  // random
   0,                // page count
   MI_BIN_FULL, 0,   // page retired min/max
+  0,                // pages_full_size
   0, 0,             // generic count
   NULL,             // next heap
   false,            // can reclaim