maintain the total size of the pages in the full queue (issue #1220)

Author: daanx

include/mimalloc/internal.h

@@ -822,17 +822,29 @@ static inline mi_page_flags_t mi_page_flags(const mi_page_t* page) {
   return (mi_page_xthread_id(page) & MI_PAGE_FLAG_MASK);
 }
 
-static inline void mi_page_flags_set(mi_page_t* page, bool set, mi_page_flags_t newflag) {
-  if (set) { mi_atomic_or_relaxed(&page->xthread_id, newflag); }
-      else { mi_atomic_and_relaxed(&page->xthread_id, ~newflag); }
+static inline bool mi_page_flags_set(mi_page_t* page, bool set, mi_page_flags_t newflag) {
+  mi_page_flags_t old;
+  if (set) { old = mi_atomic_or_relaxed(&page->xthread_id, newflag); }
+      else { old = mi_atomic_and_relaxed(&page->xthread_id, ~newflag); }
+  return ((old & newflag) == newflag);
 }
 
 static inline bool mi_page_is_in_full(const mi_page_t* page) {
   return ((mi_page_flags(page) & MI_PAGE_IN_FULL_QUEUE) != 0);
 }
 
 static inline void mi_page_set_in_full(mi_page_t* page, bool in_full) {
-  mi_page_flags_set(page, in_full, MI_PAGE_IN_FULL_QUEUE);
+  const bool was_in_full = mi_page_flags_set(page, in_full, MI_PAGE_IN_FULL_QUEUE);
+  if (was_in_full != in_full) {
+    // optimize: maintain pages_full_size to avoid visiting the full queue (issue #1220)
+    mi_theap_t* const theap = page->theap;
+    mi_assert_internal(theap!=NULL);
+    if (theap != NULL) {
+      const size_t size = page->capacity * mi_page_block_size(page);
+      if (in_full) { theap->pages_full_size += size; }
+              else { mi_assert_internal(size <= theap->pages_full_size); theap->pages_full_size -= size; }
+    }
+  }
 }
 
 static inline bool mi_page_has_interior_pointers(const mi_page_t* page) {

include/mimalloc/types.h

@@ -517,6 +517,7 @@ struct mi_theap_s {
   size_t                page_count;                          // total number of pages in the `pages` queues.
   size_t                page_retired_min;                    // smallest retired index (retired pages are fully free, but still in the page queues)
   size_t                page_retired_max;                    // largest retired index into the `pages` array.
+  size_t                pages_full_size;                     // optimization: total size of blocks in the pages of the full queue (issue #1220)
   long                  generic_count;                       // how often is `_mi_malloc_generic` called?
   long                  generic_collect_count;               // how often is `_mi_malloc_generic` called without collecting?
 

src/init.c

@@ -126,6 +126,7 @@ mi_decl_cache_align const mi_theap_t _mi_theap_empty = {
   { {0}, {0}, 0, true },  // random
   0,                      // page count
   MI_BIN_FULL, 0,         // page retired min/max
+  0,                      // pages_full_size
   0, 0,                   // generic count
   NULL, NULL,             // tnext, tprev
   NULL, NULL,             // hnext, hprev
@@ -150,6 +151,7 @@ mi_decl_cache_align const mi_theap_t _mi_theap_empty_wrong = {
   { {0}, {0}, 0, true },  // random
   0,                      // page count
   MI_BIN_FULL, 0,         // page retired min/max
+  0,                      // pages_full_size
   0, 0,                   // generic count
   NULL, NULL,             // tnext, tprev
   NULL, NULL,             // hnext, hprev
@@ -191,6 +193,7 @@ mi_decl_cache_align mi_theap_t theap_main = {
   { {0x846ca68b}, {0}, 0, true },  // random
   0,                      // page count
   MI_BIN_FULL, 0,         // page retired min/max
+  0,                      // pages_full_size
   0, 0,                   // generic count
   NULL, NULL,             // tnext, tprev
   NULL, NULL,             // hnext, hprev