Merge pull request #111 from struct/6_2022_perf_fixes

6 2022 perf fixes

Author: struct

include/conf.h

@@ -45,7 +45,7 @@
 
 /* Zones can be retired after a certain number of
  * allocations. This is computed as the total count
- * of chunks the zone can handle multiplied by this
+ * of chunks the zone can hold multiplied by this
  * value. The zone is replaced at that point if all
  * of its current chunks are free */
 #define ZONE_ALLOC_RETIRE 32

include/iso_alloc_internal.h

@@ -450,6 +450,7 @@ typedef struct {
     uint16_t zones_used;
     void *guard_below;
     void *guard_above;
+    uint32_t zone_retirement_shf;
     uintptr_t *chunk_quarantine;
     size_t chunk_quarantine_count;
     /* Zones are linked by their next_sz_index member which

src/iso_alloc.c

@@ -177,6 +177,7 @@ INTERNAL_HIDDEN void iso_alloc_initialize_global_root(void) {
      * result in a soft page fault */
     MLOCK(&_root, sizeof(iso_alloc_root));
 
+    _root->zone_retirement_shf = _log2(ZONE_ALLOC_RETIRE);
     _root->zones_size = (MAX_ZONES * sizeof(iso_alloc_zone_t));
     _root->zones_size += (g_page_size * 2);
     _root->zones_size = ROUND_UP_PAGE(_root->zones_size);
@@ -529,7 +530,6 @@ INTERNAL_HIDDEN iso_alloc_zone_t *_iso_new_zone(size_t size, bool internal, int3
     if(new_zone->tagged == true) {
         create_guard_page(p + g_page_size + tag_mapping_size);
         new_zone->user_pages_start = (p + g_page_size + tag_mapping_size + g_page_size);
-
         uint64_t *_mtp = p + g_page_size;
 
         /* Generate random tags */
@@ -779,8 +779,7 @@ INTERNAL_HIDDEN bit_slot_t iso_scan_zone_free_slot_slow(iso_alloc_zone_t *zone)
         bit_slot_t bts = bm[i];
 
         for(int64_t j = 0; j < BITS_PER_QWORD; j += BITS_PER_CHUNK) {
-            /* We can easily check if every bitslot represented by
-             * this qword is allocated with or without canaries */
+            /* Check each bit to see if its available */
             if((GET_BIT(bts, j)) == 0) {
                 return ((i << BITS_PER_QWORD_SHIFT) + j);
             }
@@ -963,7 +962,7 @@ INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_alloc_bitslot_from_zone(bit_slot_t bit
 
     /* This chunk was either previously allocated and free'd
      * or it's a canary chunk. In either case this means it
-     * has a canary written in its first dword. Here we check
+     * has a canary written in its first qword. Here we check
      * that canary and abort if its been corrupted */
 #if !ENABLE_ASAN && !DISABLE_CANARY
     if((GET_BIT(b, (which_bit + 1))) == 1) {
@@ -972,6 +971,9 @@ INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_alloc_bitslot_from_zone(bit_slot_t bit
     }
 #endif
 
+    zone->af_count++;
+    zone->alloc_count++;
+
     /* Set the in-use bit */
     SET_BIT(b, which_bit);
 
@@ -981,19 +983,17 @@ INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_alloc_bitslot_from_zone(bit_slot_t bit
      * as a canary chunk. This bit is set again upon free */
     UNSET_BIT(b, (which_bit + 1));
     bm[dwords_to_bit_slot] = b;
-    zone->af_count++;
-    zone->alloc_count++;
     return p;
 }
 
 /* Does not require the root is locked */
 INTERNAL_HIDDEN INLINE void populate_zone_cache(iso_alloc_zone_t *zone) {
-    if(UNLIKELY(zone->internal == false)) {
+    /* Don't cache this zone if it was recently cached */
+    if(zone_cache_count != 0 && zone_cache[zone_cache_count - 1].zone == zone) {
         return;
     }
 
-    /* Don't cache this zone if it was recently cached */
-    if(zone_cache_count != 0 && zone_cache[zone_cache_count - 1].zone == zone) {
+    if(UNLIKELY(zone->internal == false)) {
         return;
     }
 
@@ -1408,7 +1408,9 @@ INTERNAL_HIDDEN void iso_free_chunk_from_zone(iso_alloc_zone_t *zone, void *rest
      * which could result in a page fault */
     bitmap_index_t b = bm[dwords_to_bit_slot];
 
-    /* Ensure the pointer is a multiple of chunk size */
+    /* Ensure the pointer is a multiple of chunk size. Chunk size
+     * should always be a power of 2 so this bitwise AND works and
+     * is generally faster than modulo */
     if(UNLIKELY((chunk_offset & (zone->chunk_size - 1)) != 0)) {
         LOG_AND_ABORT("Chunk at 0x%p is not a multiple of zone[%d] chunk size %d. Off by %lu bits",
                       p, zone->index, zone->chunk_size, (chunk_offset & (zone->chunk_size - 1)));
@@ -1619,7 +1621,7 @@ INTERNAL_HIDDEN bool _is_zone_retired(iso_alloc_zone_t *zone) {
      * and has allocated and freed more than ZONE_ALLOC_RETIRE
      * chunks in its lifetime then we destroy and replace it with
      * a new zone */
-    if(UNLIKELY(zone->af_count == 0 && zone->alloc_count > (zone->chunk_count * ZONE_ALLOC_RETIRE))) {
+    if(UNLIKELY(zone->af_count == 0 && zone->alloc_count > (zone->chunk_count << _root->zone_retirement_shf))) {
         if(zone->internal == true && zone->chunk_size < (MAX_DEFAULT_ZONE_SZ * 2)) {
             return true;
         }

src/iso_alloc_mem_tags.c

@@ -51,7 +51,10 @@ INTERNAL_HIDDEN void *_untag_ptr(void *p, iso_alloc_zone_t *zone) {
 
 INTERNAL_HIDDEN bool _refresh_zone_mem_tags(iso_alloc_zone_t *zone) {
 #if MEMORY_TAGGING
-    if(UNLIKELY(zone->af_count == 0 && zone->alloc_count > (zone->chunk_count * ZONE_ALLOC_RETIRE)) >> 2) {
+    /* This implements a similar policy to zone retirement.
+     * The only difference is that we refresh all tags at
+     * %25 of the configured zone retirement age */
+    if(UNLIKELY(zone->af_count == 0 && zone->alloc_count > (zone->chunk_count << _root->zone_retirement_shf)) >> 2) {
         size_t s = ROUND_UP_PAGE(zone->chunk_count * MEM_TAG_SIZE);
         uint64_t *_mtp = (zone->user_pages_start - g_page_size - s);