Merge pull request #88 from struct/5_19_2022_perf_formatting

5 19 2022 perf formatting

Author: struct

.cirrus.yml

@@ -9,7 +9,7 @@ freebsd_instance:
 task:
   name: testsuite-freebsd-amd64
   install_script:
-    - pkg install bash gmake
+    - pkg install -y bash gmake
   script:
     - gmake tests
     - gmake cpp_tests

include/conf.h

@@ -49,9 +49,6 @@
  * of its current chunks are free */
 #define ZONE_ALLOC_RETIRE 32
 
-/* The size of our bit slot freelist */
-#define BIT_SLOT_CACHE_SZ 255
-
 /* This byte value will overwrite the contents
  * of all free'd user chunks if -DSANITIZE_CHUNKS
  * is enabled in the Makefile */
@@ -60,7 +57,7 @@
 /* See PERFORMANCE.md for notes on huge page sizes.
  * If your system uses a non-default value for huge
  * page sizes you will need to adjust that here */
-#if (__linux__ && MAP_HUGETLB) || (__APPLE__ && VM_FLAGS_SUPERPAGE_SIZE_2MB) || (__FreeBSD__ && MAP_HUGETLB) && HUGE_PAGES
+#if(__linux__ && MAP_HUGETLB) || (__APPLE__ && VM_FLAGS_SUPERPAGE_SIZE_2MB) || (__FreeBSD__ && MAP_HUGETLB) && HUGE_PAGES
 #define HUGE_PAGE_SZ 2097152
 #endif
 

include/iso_alloc_internal.h

@@ -236,20 +236,17 @@ using namespace std;
     n &= ~(1UL << k);
 
 #define ALIGN_SZ_UP(n) \
-    ((((n) + (ALIGNMENT) - 1) >> 3 ) * ALIGNMENT)
+    ((((n + ALIGNMENT) - 1) >> 3) * ALIGNMENT)
 
 #define ALIGN_SZ_DOWN(n) \
-    ((((n) + (ALIGNMENT) -1) >> 3) * ALIGNMENT) - ALIGNMENT
+    ((((n + ALIGNMENT) - 1) >> 3) * ALIGNMENT) - ALIGNMENT
 
 #define ROUND_UP_PAGE(n) \
-  ((((n) + (g_page_size) - 1) >> g_page_size_shift) * (g_page_size))
+    ((((n + g_page_size) - 1) >> g_page_size_shift) * (g_page_size))
 
 #define ROUND_DOWN_PAGE(n) \
     (ROUND_UP_PAGE(n) - g_page_size)
 
-#define GET_MAX_BITMASK_INDEX(zone) \
-    (zone->bitmap_size >> 3)
-
 #define MASK_ZONE_PTRS(zone) \
     MASK_BITMAP_PTRS(zone);  \
     MASK_USER_PTRS(zone);
@@ -336,8 +333,7 @@ extern uint32_t g_page_size_shift;
 /* iso_alloc makes a number of default zones for common
  * allocation sizes. Allocations are 'first fit' up until
  * ZONE_1024 at which point a new zone is created for that
- * specific size request. You can create additional startup
- * profile by adjusting the next few lines below. */
+ * specific size request. */
 #define DEFAULT_ZONE_COUNT sizeof(default_zones) >> 3
 
 #define MEM_TAG_SIZE 1
@@ -351,6 +347,8 @@ typedef int64_t bitmap_index_t;
 typedef uint16_t zone_lookup_table_t;
 typedef uint16_t chunk_lookup_table_t;
 
+#define BIT_SLOT_CACHE_SZ 255
+
 typedef struct {
     void *user_pages_start;     /* Start of the pages backing this zone */
     void *bitmap_start;         /* Start of the bitmap */
@@ -363,14 +361,15 @@ typedef struct {
     uint64_t pointer_mask;                             /* Each zone has its own pointer protection secret */
     uint32_t chunk_size;                               /* Size of chunks managed by this zone */
     uint32_t bitmap_size;                              /* Size of the bitmap in bytes */
+    bitmap_index_t max_bitmap_idx;                     /* Max bitmap index for this bitmap */
     bool internal;                                     /* Zones can be managed by iso_alloc or private */
     bool is_full;                                      /* Flags whether this zone is full to avoid bit slot searches */
     uint16_t index;                                    /* Zone index */
     uint16_t next_sz_index;                            /* What is the index of the next zone of this size */
     uint32_t alloc_count;                              /* Total number of lifetime allocations */
     uint32_t af_count;                                 /* Increment/Decrement with each alloc/free operation */
     uint32_t chunk_count;                              /* Total number of chunks in this zone */
-    uint8_t chunk_size_pow2;                           /* Computed by _log2(chunk_size) */
+    uint8_t chunk_size_pow2;                           /* Computed by _log2(chunk_size) at zone creation */
 #if MEMORY_TAGGING
     bool tagged; /* Zone supports memory tagging */
 #endif

src/iso_alloc.c

@@ -72,7 +72,7 @@ INTERNAL_HIDDEN void create_canary_chunks(iso_alloc_zone_t *zone) {
     bitmap_index_t *bm = (bitmap_index_t *) zone->bitmap_start;
     bit_slot_t bit_slot;
 
-    const bitmap_index_t max_bitmap_idx = GET_MAX_BITMASK_INDEX(zone) - 1;
+    const bitmap_index_t max_bitmap_idx = (zone->max_bitmap_idx - 1);
 
     /* Roughly %1 of the chunks in this zone will become a canary */
     const uint64_t canary_count = (zone->chunk_count >> CANARY_COUNT_DIV);
@@ -174,7 +174,6 @@ INTERNAL_HIDDEN void _verify_all_zones(void) {
 INTERNAL_HIDDEN void _verify_zone(iso_alloc_zone_t *zone) {
     UNMASK_ZONE_PTRS(zone);
     const bitmap_index_t *bm = (bitmap_index_t *) zone->bitmap_start;
-    const bitmap_index_t max_bm_idx = GET_MAX_BITMASK_INDEX(zone);
     bit_slot_t bit_slot;
 
     if(zone->next_sz_index > _root->zones_used) {
@@ -188,7 +187,7 @@ INTERNAL_HIDDEN void _verify_zone(iso_alloc_zone_t *zone) {
         }
     }
 
-    for(bitmap_index_t i = 0; i < max_bm_idx; i++) {
+    for(bitmap_index_t i = 0; i < zone->max_bitmap_idx; i++) {
         for(int64_t j = 1; j < BITS_PER_QWORD; j += BITS_PER_CHUNK) {
             /* If this bit is set it is either a free chunk or
              * a canary chunk. Either way it should have a set
@@ -213,18 +212,17 @@ INTERNAL_HIDDEN void _verify_zone(iso_alloc_zone_t *zone) {
  * find any free slots. */
 INTERNAL_HIDDEN INLINE void fill_free_bit_slot_cache(iso_alloc_zone_t *zone) {
     const bitmap_index_t *bm = (bitmap_index_t *) zone->bitmap_start;
-    const bitmap_index_t max_bitmap_idx = GET_MAX_BITMASK_INDEX(zone);
 
     /* This gives us an arbitrary spot in the bitmap to
      * start searching but may mean we end up with a smaller
      * cache. This may negatively affect performance but
      * leads to a less predictable free list */
     bitmap_index_t bm_idx;
 
-    /* The largest max_bitmap_idx we will ever
+    /* The largest zone->max_bitmap_idx we will ever
      * have is 8192 for SMALLEST_CHUNK_SZ (16) */
-    if(max_bitmap_idx > ALIGNMENT) {
-        bm_idx = ((uint32_t) rand_uint64() * (max_bitmap_idx - 1) >> 32);
+    if(zone->max_bitmap_idx > ALIGNMENT) {
+        bm_idx = ((uint32_t) rand_uint64() * (zone->max_bitmap_idx - 1) >> 32);
     } else {
         bm_idx = 0;
     }
@@ -236,7 +234,7 @@ INTERNAL_HIDDEN INLINE void fill_free_bit_slot_cache(iso_alloc_zone_t *zone) {
     for(free_bit_slot_cache_index = 0; free_bit_slot_cache_index < BIT_SLOT_CACHE_SZ; bm_idx++) {
         /* Don't index outside of the bitmap or
          * we will return inaccurate bit slots */
-        if(UNLIKELY(bm_idx >= max_bitmap_idx)) {
+        if(UNLIKELY(bm_idx >= zone->max_bitmap_idx)) {
             zone->free_bit_slot_cache_index = free_bit_slot_cache_index;
             return;
         }
@@ -452,18 +450,11 @@ INTERNAL_HIDDEN void _unmap_zone(iso_alloc_zone_t *zone) {
     chunk_lookup_table[ADDR_TO_CHUNK_TABLE(zone->user_pages_start)] = 0;
 
     munmap(zone->bitmap_start, zone->bitmap_size);
-    madvise(zone->bitmap_start, zone->bitmap_size, MADV_DONTNEED);
     munmap(zone->bitmap_start - _root->system_page_size, _root->system_page_size);
-    madvise(zone->bitmap_start - _root->system_page_size, _root->system_page_size, MADV_DONTNEED);
     munmap(zone->bitmap_start + zone->bitmap_size, _root->system_page_size);
-    madvise(zone->bitmap_start + zone->bitmap_size, _root->system_page_size, MADV_DONTNEED);
-
     munmap(zone->user_pages_start, ZONE_USER_SIZE);
-    madvise(zone->user_pages_start, ZONE_USER_SIZE, MADV_DONTNEED);
     munmap(zone->user_pages_start - _root->system_page_size, _root->system_page_size);
-    madvise(zone->user_pages_start - _root->system_page_size, _root->system_page_size, MADV_DONTNEED);
     munmap(zone->user_pages_start + ZONE_USER_SIZE, _root->system_page_size);
-    madvise(zone->user_pages_start + ZONE_USER_SIZE, _root->system_page_size, MADV_DONTNEED);
 }
 
 INTERNAL_HIDDEN void _iso_alloc_destroy_zone(iso_alloc_zone_t *zone) {
@@ -690,6 +681,7 @@ INTERNAL_HIDDEN iso_alloc_zone_t *_iso_new_zone(size_t size, bool internal) {
      * then we need to allocate a minimum size bitmap */
     uint32_t bitmap_size = (new_zone->chunk_count << BITS_PER_CHUNK_SHIFT) >> BITS_PER_BYTE_SHIFT;
     new_zone->bitmap_size = (bitmap_size > sizeof(bitmap_index_t)) ? bitmap_size : sizeof(bitmap_index_t);
+    new_zone->max_bitmap_idx = new_zone->bitmap_size >> 3;
 
     /* All of the following fields are immutable
      * and should not change once they are set */
@@ -840,10 +832,9 @@ INTERNAL_HIDDEN iso_alloc_zone_t *_iso_new_zone(size_t size, bool internal) {
  * looking for empty holes (i.e. slot == 0) */
 INTERNAL_HIDDEN bit_slot_t iso_scan_zone_free_slot(iso_alloc_zone_t *zone) {
     const bitmap_index_t *bm = (bitmap_index_t *) zone->bitmap_start;
-    const bitmap_index_t max_bm_idx = GET_MAX_BITMASK_INDEX(zone);
 
     /* Iterate the entire bitmap a qword at a time */
-    for(bitmap_index_t i = 0; i < max_bm_idx; i++) {
+    for(bitmap_index_t i = 0; i < zone->max_bitmap_idx; i++) {
         /* If the byte is 0 then there are some free
          * slots we can use at this location */
         if(bm[i] == 0x0) {
@@ -861,9 +852,8 @@ INTERNAL_HIDDEN bit_slot_t iso_scan_zone_free_slot(iso_alloc_zone_t *zone) {
  * that indicates there is at least 1 free bit slot  */
 INTERNAL_HIDDEN bit_slot_t iso_scan_zone_free_slot_slow(iso_alloc_zone_t *zone) {
     const bitmap_index_t *bm = (bitmap_index_t *) zone->bitmap_start;
-    const bitmap_index_t max_bm_idx = GET_MAX_BITMASK_INDEX(zone);
 
-    for(bitmap_index_t i = 0; i < max_bm_idx; i++) {
+    for(bitmap_index_t i = 0; i < zone->max_bitmap_idx; i++) {
         for(int64_t j = 0; j < BITS_PER_QWORD; j += BITS_PER_CHUNK) {
             /* We can easily check if every bitslot represented by
              * this qword is allocated with or without canaries */
@@ -1175,10 +1165,6 @@ INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_alloc_bitslot_from_zone(bit_slot_t bit
     void *p = POINTER_FROM_BITSLOT(zone, bitslot);
     UNPOISON_ZONE_CHUNK(zone, p);
 
-#if !ENABLE_ASAN && !DISABLE_CANARY
-    __builtin_prefetch(p, 1);
-#endif
-
     bitmap_index_t *bm = (bitmap_index_t *) zone->bitmap_start;
 
     /* Read out 64 bits from the bitmap. We will write
@@ -1187,9 +1173,9 @@ INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_alloc_bitslot_from_zone(bit_slot_t bit
      * which could result in a page fault */
     bitmap_index_t b = bm[dwords_to_bit_slot];
 
-    if(UNLIKELY(p > zone->user_pages_start + ZONE_USER_SIZE)) {
+    if(UNLIKELY(p >= zone->user_pages_start + ZONE_USER_SIZE)) {
         LOG_AND_ABORT("Allocating an address 0x%p from zone[%d], bit slot %lu %ld bytes %ld pages outside zones user pages 0x%p 0x%p",
-                      p, zone->index, bitslot, p - (zone->user_pages_start + ZONE_USER_SIZE), (p - (zone->user_pages_start + ZONE_USER_SIZE)) / _root->system_page_size,
+                      p, zone->index, bitslot, p - zone->user_pages_start + ZONE_USER_SIZE, (p - zone->user_pages_start + ZONE_USER_SIZE) / _root->system_page_size,
                       zone->user_pages_start, zone->user_pages_start + ZONE_USER_SIZE);
     }
 
@@ -1257,7 +1243,7 @@ INTERNAL_HIDDEN uint8_t _iso_alloc_get_mem_tag(void *p, iso_alloc_zone_t *zone)
         LOG_AND_ABORT("Chunk offset %d not an alignment of %d", chunk_offset, zone->chunk_size);
     }
 
-    _mtp += (chunk_offset / zone->chunk_size);
+    _mtp += (chunk_offset >> zone->chunk_size_pow2);
     return *_mtp;
 #else
     return 0;
@@ -1351,7 +1337,7 @@ INTERNAL_HIDDEN ASSUME_ALIGNED void *_iso_alloc(iso_alloc_zone_t *zone, size_t s
              * thread recently used for an alloc/free operation.
              * It's likely we are allocating a similar size chunk
              * and this will speed up that operation */
-            for(int64_t i = 0; i < zone_cache_count; i++) {
+            for(size_t i = 0; i < zone_cache_count; i++) {
                 if(zone_cache[i].chunk_size >= size) {
                     bool fit = iso_does_zone_fit(zone_cache[i].zone, size);
 
@@ -1504,7 +1490,7 @@ INTERNAL_HIDDEN iso_alloc_zone_t *iso_find_zone_bitmap_range(const void *restric
     iso_alloc_zone_t *tmp_zone = NULL;
 
     /* Now we check the MRU thread zone cache */
-    for(int64_t i = 0; i < zone_cache_count; i++) {
+    for(size_t i = 0; i < zone_cache_count; i++) {
         tmp_zone = zone_cache[i].zone;
         bitmap_start = UNMASK_BITMAP_PTR(tmp_zone);
 
@@ -1548,7 +1534,7 @@ INTERNAL_HIDDEN iso_alloc_zone_t *iso_find_zone_range(const void *restrict p) {
     iso_alloc_zone_t *tmp_zone = NULL;
 
     /* Now we check the MRU thread zone cache */
-    for(int64_t i = 0; i < zone_cache_count; i++) {
+    for(size_t i = 0; i < zone_cache_count; i++) {
         tmp_zone = zone_cache[i].zone;
         user_pages_start = UNMASK_USER_PTR(tmp_zone);
 
@@ -1757,10 +1743,6 @@ INTERNAL_HIDDEN void iso_free_chunk_from_zone(iso_alloc_zone_t *zone, void *rest
     /* Set the next bit so we know this chunk was used */
     SET_BIT(b, (which_bit + 1));
 
-#if !ENABLE_ASAN && (!DISABLE_CANARY || SANITIZE_CHUNKS)
-    __builtin_prefetch(p, 1);
-#endif
-
     /* Unset the bit and write the value into the bitmap
      * if this is not a permanent free. A permanent free
      * means this chunk will be marked as if it is a canary */
@@ -1850,7 +1832,7 @@ INTERNAL_HIDDEN void _iso_free(void *p, bool permanent) {
     }
 
 #if NO_ZERO_ALLOCATIONS
-    if(p == _zero_alloc_page) {
+    if(UNLIKELY(p == _zero_alloc_page)) {
         return;
     }
 #endif
@@ -1894,7 +1876,7 @@ INTERNAL_HIDDEN void _iso_free_size(void *p, size_t size) {
     }
 
 #if NO_ZERO_ALLOCATIONS
-    if(p == _zero_alloc_page && size != 0) {
+    if(UNLIKELY(p == _zero_alloc_page && size != 0)) {
         LOG_AND_ABORT("Zero sized chunk (0x%p) with non-zero (%d) size passed to free", p, size);
     }
 

src/iso_alloc_profiler.c

@@ -104,7 +104,7 @@ INTERNAL_HIDDEN uint64_t _iso_alloc_zone_leak_detector(iso_alloc_zone_t *zone, b
                  * canary value. If it doesn't validate then we assume
                  * its a true leak and increment the in_use counter */
                 bit_slot_t bit_slot = (i * BITS_PER_QWORD) + j;
-                const void *leak = (zone->user_pages_start + ((bit_slot / BITS_PER_CHUNK) * zone->chunk_size));
+                const void *leak = (zone->user_pages_start + ((bit_slot >> 1) * zone->chunk_size));
 
                 if(bit_two == 1 && (check_canary_no_abort(zone, leak) != ERR)) {
                     continue;

src/iso_alloc_util.c

@@ -48,15 +48,15 @@ INTERNAL_HIDDEN void *mmap_pages(size_t size, bool populate, const char *name, i
 #if MAP_HUGETLB && HUGE_PAGES
     /* If we are allocating pages for a user zone
      * then take advantage of the huge TLB */
-    if(size == ZONE_USER_SIZE || size == (ZONE_USER_SIZE / 2)) {
+    if(size == ZONE_USER_SIZE || size == (ZONE_USER_SIZE >> 1)) {
         flags |= MAP_HUGETLB;
     }
 #endif
 #elif __APPLE__
 #if VM_FLAGS_SUPERPAGE_SIZE_2MB && HUGE_PAGES
     /* If we are allocating pages for a user zone
      * we are going to use the 2 MB superpage flag */
-    if(size == ZONE_USER_SIZE || size == (ZONE_USER_SIZE / 2)) {
+    if(size == ZONE_USER_SIZE || size == (ZONE_USER_SIZE >> 1)) {
         fd = VM_FLAGS_SUPERPAGE_SIZE_2MB;
     }
 #endif
@@ -70,7 +70,7 @@ INTERNAL_HIDDEN void *mmap_pages(size_t size, bool populate, const char *name, i
     }
 
 #if __linux__ && MAP_HUGETLB && HUGE_PAGES && MADV_HUGEPAGE
-    if(size == ZONE_USER_SIZE || size == (ZONE_USER_SIZE / 2)) {
+    if(size == ZONE_USER_SIZE || size == (ZONE_USER_SIZE >> 1)) {
         madvise(p, size, MADV_HUGEPAGE);
     }
 #endif
@@ -121,10 +121,10 @@ INTERNAL_HIDDEN INLINE CONST size_t next_pow2(size_t sz) {
 }
 
 const uint32_t _log_table[32] = {
-    0,  9,  1, 10, 13, 21,  2, 29,
-    11, 14, 16, 18, 22, 25,  3, 30,
-    8, 12, 20, 28, 15, 17, 24,  7,
-    19, 27, 23,  6, 26,  5,  4, 31};
+    0, 9, 1, 10, 13, 21, 2, 29,
+    11, 14, 16, 18, 22, 25, 3, 30,
+    8, 12, 20, 28, 15, 17, 24, 7,
+    19, 27, 23, 6, 26, 5, 4, 31};
 
 /* Fast log2() implementation for 32 bit integers */
 INTERNAL_HIDDEN uint32_t _log2(uint32_t v) {
@@ -133,5 +133,5 @@ INTERNAL_HIDDEN uint32_t _log2(uint32_t v) {
     v |= v >> 4;
     v |= v >> 8;
     v |= v >> 16;
-    return _log_table[(uint32_t)(v*0x07C4ACDD) >> 27];
+    return _log_table[(uint32_t) (v * 0x07C4ACDD) >> 27];
 }

tests/alloc_fuzz.c

@@ -13,7 +13,7 @@ uint32_t allocation_sizes[] = {ZONE_16, ZONE_32, ZONE_64, ZONE_128,
                                ZONE_256, ZONE_512, ZONE_1024,
                                ZONE_2048, ZONE_4096, ZONE_8192,
                                SMALL_SZ_MAX / 4, SMALL_SZ_MAX / 2,
-                               SMALL_SZ_MAX - 1, SMALL_SZ_MAX };
+                               SMALL_SZ_MAX - 1, SMALL_SZ_MAX};
 
 uint32_t array_sizes[] = {16, 32, 64, 128, 256, 512, 1024, 2048};