- Temp allocator now supports more than 2 in-flight stacks.

- Printing stacktrace uses its own temp allocator.
- `@pool` no longer takes an argument.
- `Allocator` interface removes `mark` and `reset`.
- DynamicArenaAllocator has changed init function.
- Added `BackedArenaAllocator` which is allocated to a fixed size, then allocates on the backing allocator and supports mark/reset.
This commit is contained in:
Christoffer Lerno
2025-03-18 15:16:22 +01:00
parent 82cc49b388
commit 72608ce01d
27 changed files with 519 additions and 334 deletions

View File

@@ -50,8 +50,8 @@ fn void ArenaAllocator.release(&self, void* ptr, bool) @dynamic
}
}
fn usz ArenaAllocator.mark(&self) @dynamic => self.used;
fn void ArenaAllocator.reset(&self, usz mark) @dynamic => self.used = mark;
fn usz ArenaAllocator.mark(&self) => self.used;
fn void ArenaAllocator.reset(&self, usz mark) => self.used = mark;
<*
@require !alignment || math::is_power_of_2(alignment)

View File

@@ -0,0 +1,210 @@
module std::core::mem::allocator;
import std::io, std::math;
struct AllocChunk @local
{
usz size;
char[*] data;
}
struct BackedArenaAllocator (Allocator)
{
Allocator backing_allocator;
ExtraPage* last_page;
usz used;
usz capacity;
char[*] data;
}
const usz PAGE_IS_ALIGNED @local = (usz)isz.max + 1u;
struct ExtraPage @local
{
ExtraPage* prev_page;
void* start;
usz mark;
usz size;
usz ident;
char[*] data;
}
macro usz ExtraPage.pagesize(&self) => self.size & ~PAGE_IS_ALIGNED;
macro bool ExtraPage.is_aligned(&self) => self.size & PAGE_IS_ALIGNED == PAGE_IS_ALIGNED;
<*
@require size >= 16
*>
fn BackedArenaAllocator*? new_backed_allocator(usz size, Allocator allocator)
{
BackedArenaAllocator* temp = allocator::alloc_with_padding(allocator, BackedArenaAllocator, size)!;
temp.last_page = null;
temp.backing_allocator = allocator;
temp.used = 0;
temp.capacity = size;
return temp;
}
fn void BackedArenaAllocator.destroy(&self)
{
self.reset(0);
if (self.last_page) (void)self._free_page(self.last_page);
allocator::free(self.backing_allocator, self);
}
fn usz BackedArenaAllocator.mark(&self) => self.used;
fn void BackedArenaAllocator.release(&self, void* old_pointer, bool) @dynamic
{
usz old_size = *(usz*)(old_pointer - DEFAULT_SIZE_PREFIX);
if (old_pointer + old_size == &self.data[self.used])
{
self.used -= old_size;
asan::poison_memory_region(&self.data[self.used], old_size);
}
}
fn void BackedArenaAllocator.reset(&self, usz mark)
{
ExtraPage *last_page = self.last_page;
while (last_page && last_page.mark > mark)
{
self.used = last_page.mark;
ExtraPage *to_free = last_page;
last_page = last_page.prev_page;
self._free_page(to_free)!!;
}
self.last_page = last_page;
$if env::COMPILER_SAFE_MODE || env::ADDRESS_SANITIZER:
if (!last_page)
{
usz cleaned = self.used - mark;
if (cleaned > 0)
{
$if env::COMPILER_SAFE_MODE && !env::ADDRESS_SANITIZER:
self.data[mark : cleaned] = 0xAA;
$endif
asan::poison_memory_region(&self.data[mark], cleaned);
}
}
$endif
self.used = mark;
}
fn void? BackedArenaAllocator._free_page(&self, ExtraPage* page) @inline @local
{
void* mem = page.start;
return self.backing_allocator.release(mem, page.is_aligned());
}
fn void*? BackedArenaAllocator._realloc_page(&self, ExtraPage* page, usz size, usz alignment) @inline @local
{
// Then the actual start pointer:
void* real_pointer = page.start;
// Walk backwards to find the pointer to this page.
ExtraPage **pointer_to_prev = &self.last_page;
// Remove the page from the list
while (*pointer_to_prev != page)
{
pointer_to_prev = &((*pointer_to_prev).prev_page);
}
*pointer_to_prev = page.prev_page;
usz page_size = page.pagesize();
// Clear on size > original size.
void* data = self.acquire(size, NO_ZERO, alignment)!;
mem::copy(data, &page.data[0], page_size, mem::DEFAULT_MEM_ALIGNMENT, mem::DEFAULT_MEM_ALIGNMENT);
self.backing_allocator.release(real_pointer, page.is_aligned());
return data;
}
fn void*? BackedArenaAllocator.resize(&self, void* pointer, usz size, usz alignment) @dynamic
{
AllocChunk *chunk = pointer - AllocChunk.sizeof;
if (chunk.size == (usz)-1)
{
assert(self.last_page, "Realloc of unrelated pointer");
// First grab the page
ExtraPage *page = pointer - ExtraPage.sizeof;
return self._realloc_page(page, size, alignment);
}
AllocChunk* data = self.acquire(size, NO_ZERO, alignment)!;
mem::copy(data, pointer, chunk.size, mem::DEFAULT_MEM_ALIGNMENT, mem::DEFAULT_MEM_ALIGNMENT);
return data;
}
<*
@require size > 0
@require !alignment || math::is_power_of_2(alignment)
@require alignment <= mem::MAX_MEMORY_ALIGNMENT : `alignment too big`
*>
fn void*? BackedArenaAllocator.acquire(&self, usz size, AllocInitType init_type, usz alignment) @dynamic
{
alignment = alignment_for_allocation(alignment);
void* start_mem = &self.data;
void* starting_ptr = start_mem + self.used;
void* aligned_header_start = mem::aligned_pointer(starting_ptr, AllocChunk.alignof);
void* mem = aligned_header_start + AllocChunk.sizeof;
if (alignment > AllocChunk.alignof)
{
mem = mem::aligned_pointer(mem, alignment);
}
usz new_usage = (usz)(mem - start_mem) + size;
// Arena allocation, simple!
if (new_usage <= self.capacity)
{
asan::unpoison_memory_region(starting_ptr, new_usage - self.used);
AllocChunk* chunk_start = mem - AllocChunk.sizeof;
chunk_start.size = size;
self.used = new_usage;
if (init_type == ZERO) mem::clear(mem, size, mem::DEFAULT_MEM_ALIGNMENT);
return mem;
}
// Fallback to backing allocator
ExtraPage* page;
// We have something we need to align.
if (alignment > mem::DEFAULT_MEM_ALIGNMENT)
{
// This is actually simpler, since it will create the offset for us.
usz total_alloc_size = mem::aligned_offset(ExtraPage.sizeof + size, alignment);
if (init_type == ZERO)
{
mem = allocator::calloc_aligned(self.backing_allocator, total_alloc_size, alignment)!;
}
else
{
mem = allocator::malloc_aligned(self.backing_allocator, total_alloc_size, alignment)!;
}
void* start = mem;
mem += mem::aligned_offset(ExtraPage.sizeof, alignment);
page = (ExtraPage*)mem - 1;
page.start = start;
page.size = size | PAGE_IS_ALIGNED;
}
else
{
// Here we might need to pad
usz padded_header_size = mem::aligned_offset(ExtraPage.sizeof, mem::DEFAULT_MEM_ALIGNMENT);
usz total_alloc_size = padded_header_size + size;
void* alloc = self.backing_allocator.acquire(total_alloc_size, init_type, 0)!;
// Find the page.
page = alloc + padded_header_size - ExtraPage.sizeof;
assert(mem::ptr_is_aligned(page, BackedArenaAllocator.alignof));
assert(mem::ptr_is_aligned(&page.data[0], mem::DEFAULT_MEM_ALIGNMENT));
page.start = alloc;
page.size = size;
}
// Mark it as a page
page.ident = ~(usz)0;
// Store when it was created
page.mark = ++self.used;
// Hook up the page.
page.prev_page = self.last_page;
self.last_page = page;
return &page.data[0];
}

View File

@@ -16,7 +16,7 @@ struct DynamicArenaAllocator (Allocator)
@param [&inout] allocator
@require page_size >= 128
*>
fn void DynamicArenaAllocator.init(&self, usz page_size, Allocator allocator)
fn void DynamicArenaAllocator.init(&self, Allocator allocator, usz page_size)
{
self.page = null;
self.unused_page = null;
@@ -110,9 +110,8 @@ fn void*? DynamicArenaAllocator.resize(&self, void* old_pointer, usz size, usz a
return new_mem;
}
fn void DynamicArenaAllocator.reset(&self, usz mark = 0) @dynamic
fn void DynamicArenaAllocator.reset(&self)
{
assert(mark == 0, "Unexpectedly reset dynamic arena allocator with mark %d", mark);
DynamicArenaPage* page = self.page;
DynamicArenaPage** unused_page_ptr = &self.unused_page;
while (page)

View File

@@ -11,8 +11,11 @@ struct TempAllocator (Allocator)
{
Allocator backing_allocator;
TempAllocatorPage* last_page;
TempAllocator* derived;
bool allocated;
usz used;
usz capacity;
usz original_capacity;
char[*] data;
}
@@ -23,7 +26,6 @@ struct TempAllocatorPage
{
TempAllocatorPage* prev_page;
void* start;
usz mark;
usz size;
usz ident;
char[*] data;
@@ -34,25 +36,92 @@ macro bool TempAllocatorPage.is_aligned(&self) => self.size & PAGE_IS_ALIGNED ==
<*
@require size >= 16
@require allocator.type != TempAllocator.typeid : "You may not create a temp allocator with a TempAllocator as the backing allocator."
*>
fn TempAllocator*? new_temp_allocator(usz size, Allocator allocator)
fn TempAllocator*? new_temp_allocator(Allocator allocator, usz size)
{
TempAllocator* temp = allocator::alloc_with_padding(allocator, TempAllocator, size)!;
temp.last_page = null;
temp.backing_allocator = allocator;
temp.used = 0;
temp.capacity = size;
temp.allocated = true;
temp.derived = null;
temp.original_capacity = temp.capacity = size;
return temp;
}
fn void TempAllocator.destroy(&self)
<*
@require !self.derived
@require min_size > TempAllocator.sizeof + 64 : "Min size must meaningfully hold the data + some bytes"
@require mult > 0 : "The multiple can never be zero"
*>
fn TempAllocator*? TempAllocator.derive_allocator(&self, usz min_size, usz buffer, usz mult)
{
self.reset(0);
if (self.last_page) (void)self._free_page(self.last_page);
allocator::free(self.backing_allocator, self);
usz remaining = self.capacity - self.used;
void* mem @noinit;
usz size @noinit;
if (min_size + buffer > remaining)
{
return self.derived = new_temp_allocator(self.backing_allocator, min_size * mult)!;
}
usz start = mem::aligned_offset(self.used + buffer, mem::DEFAULT_MEM_ALIGNMENT);
void* ptr = &self.data[start];
TempAllocator* temp = (TempAllocator*)ptr;
temp.last_page = null;
temp.backing_allocator = self.backing_allocator;
temp.used = 0;
temp.allocated = false;
temp.derived = null;
temp.original_capacity = temp.capacity = self.capacity - start - TempAllocator.sizeof;
self.capacity = start;
self.derived = temp;
return temp;
}
fn usz TempAllocator.mark(&self) @dynamic => self.used;
fn void TempAllocator.reset(&self)
{
TempAllocator* child = self.derived;
if (!child) return;
while (child)
{
TempAllocator* old = child;
old.destroy();
child = old.derived;
}
self.capacity = self.original_capacity;
self.derived = null;
}
<*
@require self.allocated : "Only a top level allocator should be freed."
*>
fn void TempAllocator.free(&self)
{
self.reset();
self.destroy();
}
fn void TempAllocator.destroy(&self) @local
{
TempAllocatorPage *last_page = self.last_page;
while (last_page)
{
TempAllocatorPage *to_free = last_page;
last_page = last_page.prev_page;
self._free_page(to_free)!!;
}
if (self.allocated)
{
allocator::free(self.backing_allocator, self);
return;
}
$if env::COMPILER_SAFE_MODE || env::ADDRESS_SANITIZER:
$if env::COMPILER_SAFE_MODE && !env::ADDRESS_SANITIZER:
self.data[0 : self.used] = 0xAA;
$else
asan::poison_memory_region(&self.data[0], self.used);
$endif
$endif
}
fn void TempAllocator.release(&self, void* old_pointer, bool) @dynamic
{
@@ -63,32 +132,7 @@ fn void TempAllocator.release(&self, void* old_pointer, bool) @dynamic
asan::poison_memory_region(&self.data[self.used], old_size);
}
}
fn void TempAllocator.reset(&self, usz mark) @dynamic
{
TempAllocatorPage *last_page = self.last_page;
while (last_page && last_page.mark > mark)
{
self.used = last_page.mark;
TempAllocatorPage *to_free = last_page;
last_page = last_page.prev_page;
self._free_page(to_free)!!;
}
self.last_page = last_page;
$if env::COMPILER_SAFE_MODE || env::ADDRESS_SANITIZER:
if (!last_page)
{
usz cleaned = self.used - mark;
if (cleaned > 0)
{
$if env::COMPILER_SAFE_MODE && !env::ADDRESS_SANITIZER:
self.data[mark : cleaned] = 0xAA;
$endif
asan::poison_memory_region(&self.data[mark], cleaned);
}
}
$endif
self.used = mark;
}
fn void? TempAllocator._free_page(&self, TempAllocatorPage* page) @inline @local
{
@@ -202,29 +246,9 @@ fn void*? TempAllocator.acquire(&self, usz size, AllocInitType init_type, usz al
// Mark it as a page
page.ident = ~(usz)0;
// Store when it was created
page.mark = ++self.used;
// Hook up the page.
page.prev_page = self.last_page;
self.last_page = page;
return &page.data[0];
}
fn void? TempAllocator.print_pages(&self, File* f)
{
TempAllocatorPage *last_page = self.last_page;
if (!last_page)
{
io::fprintf(f, "No pages.\n")!;
return;
}
io::fprintf(f, "---Pages----\n")!;
uint index = 0;
while (last_page)
{
bool is_not_aligned = !(last_page.size & (1u64 << 63));
io::fprintf(f, "%d. Alloc: %d %d at %p%s\n", ++index,
last_page.size & ~(1u64 << 63), last_page.mark, &last_page.data[0], is_not_aligned ? "" : " [aligned]")!;
last_page = last_page.prev_page;
}
}

View File

@@ -4,6 +4,7 @@
module std::core::builtin;
import libc, std::hash, std::io, std::os::backtrace;
<*
EMPTY_MACRO_SLOT is a value used for implementing optional arguments for macros in an efficient
way. It relies on the fact that distinct types are not implicitly convertable.
@@ -87,8 +88,16 @@ macro anycast(any v, $Type) @builtin
return ($Type*)v.ptr;
}
fn bool print_backtrace(String message, int backtraces_to_ignore) @if(env::NATIVE_STACKTRACE) => @pool()
fn bool print_backtrace(String message, int backtraces_to_ignore) @if(env::NATIVE_STACKTRACE) => @stack_mem(0x1100; Allocator smem)
{
TempAllocator* t = allocator::current_temp;
TempAllocator* new_t = allocator::new_temp_allocator(smem, 0x1000)!!;
allocator::current_temp = new_t;
defer
{
allocator::current_temp = t;
new_t.free();
}
void*[256] buffer;
void*[] backtraces = backtrace::capture_current(&buffer);
backtraces_to_ignore++;

View File

@@ -69,7 +69,8 @@ fn void DString.replace(&self, String needle, String replacement)
self.replace_char(needle[0], replacement[0]);
return;
}
@pool(data.allocator) {
@pool()
{
String str = self.tcopy_str();
self.clear();
usz len = str.len;
@@ -538,7 +539,7 @@ macro void DString.insert_at(&self, usz index, value)
fn usz? DString.appendf(&self, String format, args...) @maydiscard
{
if (!self.data()) self.tinit(format.len + 20);
@pool(self.data().allocator)
@pool()
{
Formatter formatter;
formatter.init(&out_string_append_fn, self);
@@ -549,7 +550,7 @@ fn usz? DString.appendf(&self, String format, args...) @maydiscard
fn usz? DString.appendfn(&self, String format, args...) @maydiscard
{
if (!self.data()) self.tinit(format.len + 20);
@pool(self.data().allocator)
@pool()
{
Formatter formatter;
formatter.init(&out_string_append_fn, self);

View File

@@ -555,61 +555,32 @@ macro void @stack_pool(usz $size; @body) @builtin
};
}
struct TempState
{
TempAllocator* old;
TempAllocator* current;
usz mark;
}
<*
Push the current temp allocator. A push must always be balanced with a pop using the current state.
*>
fn TempState temp_push(TempAllocator* other = null)
fn PoolState temp_push()
{
TempAllocator* current = allocator::temp();
TempAllocator* old = current;
if (other == current)
{
current = allocator::temp_allocator_next();
}
return { old, current, current.used };
return allocator::push_pool() @inline;
}
<*
Pop the current temp allocator. A pop must always be balanced with a push.
*>
fn void temp_pop(TempState old_state)
fn void temp_pop(PoolState old_state)
{
assert(allocator::thread_temp_allocator == old_state.current, "Tried to pop temp allocators out of order.");
assert(old_state.current.used >= old_state.mark, "Tried to pop temp allocators out of order.");
old_state.current.reset(old_state.mark);
allocator::thread_temp_allocator = old_state.old;
allocator::pop_pool(old_state) @inline;
}
<*
@require @is_empty_macro_slot(#other_temp) ||| $assignable(#other_temp, Allocator) : "Must be an allocator"
*>
macro void @pool(#other_temp = EMPTY_MACRO_SLOT; @body) @builtin
macro void @pool(;@body) @builtin
{
TempAllocator* current = allocator::temp();
$if @is_valid_macro_slot(#other_temp):
TempAllocator* original = current;
if (current == #other_temp.ptr) current = allocator::temp_allocator_next();
$endif
usz mark = current.used;
PoolState state = allocator::push_pool() @inline;
defer
{
current.reset(mark);
$if @is_valid_macro_slot(#other_temp):
allocator::thread_temp_allocator = original;
$endif;
allocator::pop_pool(state) @inline;
}
@body();
}
import libc;
module std::core::mem @if(WASM_NOLIBC);
import std::core::mem::allocator @public;
SimpleHeapAllocator wasm_allocator @private;
@@ -624,7 +595,6 @@ fn void initialize_wasm_mem() @init(1024) @private
wasm_allocator.init(fn (x) => allocator::wasm_memory.allocate_block(x));
allocator::thread_allocator = &wasm_allocator;
allocator::temp_base_allocator = &wasm_allocator;
allocator::init_default_temp_allocators();
}
module std::core::mem;

View File

@@ -1,5 +1,6 @@
module std::core::mem::allocator;
const DEFAULT_SIZE_PREFIX = usz.sizeof;
const DEFAULT_SIZE_PREFIX_ALIGNMENT = usz.alignof;
@@ -18,8 +19,6 @@ enum AllocInitType
interface Allocator
{
fn void reset(usz mark) @optional;
fn usz mark() @optional;
<*
@require !alignment || math::is_power_of_2(alignment)
@require alignment <= mem::MAX_MEMORY_ALIGNMENT : `alignment too big`
@@ -356,14 +355,32 @@ macro void*? @aligned_realloc(#calloc_fn, #free_fn, void* old_pointer, usz bytes
// All allocators
alias mem @builtin = thread_allocator ;
tlocal Allocator thread_allocator @private = base_allocator();
Allocator temp_base_allocator @private = base_allocator();
tlocal TempAllocator* thread_temp_allocator @private = null;
tlocal TempAllocator*[2] temp_allocator_pair @private;
typedef PoolState = void*;
tlocal TempAllocator* current_temp;
tlocal TempAllocator* top_temp;
usz temp_allocator_min_size = temp_allocator_default_min_size();
usz temp_allocator_buffer_size = temp_allocator_default_buffer_size();
usz temp_allocator_new_mult = 4;
fn PoolState push_pool()
{
TempAllocator* old = current_temp ?: create_temp_allocator();
current_temp = current_temp.derive_allocator(temp_allocator_min_size, temp_allocator_buffer_size, temp_allocator_new_mult)!!;
return (PoolState)old;
}
fn void pop_pool(PoolState old)
{
current_temp = (TempAllocator*)old;
current_temp.reset();
}
macro Allocator base_allocator() @private
{
@@ -374,46 +391,51 @@ macro Allocator base_allocator() @private
$endif
}
macro TempAllocator* create_default_sized_temp_allocator(Allocator allocator) @local
macro usz temp_allocator_size() @local
{
$switch env::MEMORY_ENV:
$case NORMAL:
return new_temp_allocator(1024 * 256, allocator)!!;
$case SMALL:
return new_temp_allocator(1024 * 16, allocator)!!;
$case TINY:
return new_temp_allocator(1024 * 2, allocator)!!;
$case NONE:
unreachable("Temp allocator must explicitly created when memory-env is set to 'none'.");
$case NORMAL: return 256 * 1024;
$case SMALL: return 1024 * 32;
$case TINY: return 1024 * 4;
$case NONE: return 0;
$endswitch
}
macro usz temp_allocator_default_min_size() @local
{
$switch env::MEMORY_ENV:
$case NORMAL: return 16 * 1024;
$case SMALL: return 1024 * 2;
$case TINY: return 256;
$case NONE: return 256;
$endswitch
}
macro usz temp_allocator_default_buffer_size() @local
{
$switch env::MEMORY_ENV:
$case NORMAL: return 1024;
$case SMALL: return 128;
$case TINY: return 64;
$case NONE: return 64;
$endswitch
}
macro Allocator heap() => thread_allocator;
macro TempAllocator* temp()
<*
@require !current_temp : "This should never be called when temp already exists"
*>
fn TempAllocator* create_temp_allocator() @private
{
if (!thread_temp_allocator)
{
init_default_temp_allocators();
}
return thread_temp_allocator;
return top_temp = current_temp = allocator::new_temp_allocator(base_allocator(), temp_allocator_size())!!;
}
macro Allocator temp()
{
return current_temp ?: create_temp_allocator();
}
macro TempAllocator* tmem() @builtin
{
if (!thread_temp_allocator)
{
init_default_temp_allocators();
}
return thread_temp_allocator;
}
fn void init_default_temp_allocators() @private
{
temp_allocator_pair[0] = create_default_sized_temp_allocator(temp_base_allocator);
temp_allocator_pair[1] = create_default_sized_temp_allocator(temp_base_allocator);
thread_temp_allocator = temp_allocator_pair[0];
}
alias tmem @builtin = temp;
fn void destroy_temp_allocators_after_exit() @finalizer(65535) @local @if(env::LIBC)
{
@@ -425,22 +447,8 @@ fn void destroy_temp_allocators_after_exit() @finalizer(65535) @local @if(env::L
*>
fn void destroy_temp_allocators()
{
if (!thread_temp_allocator) return;
temp_allocator_pair[0].destroy();
temp_allocator_pair[1].destroy();
temp_allocator_pair[..] = null;
thread_temp_allocator = null;
}
fn TempAllocator* temp_allocator_next() @private
{
if (!thread_temp_allocator)
{
init_default_temp_allocators();
return thread_temp_allocator;
}
usz index = thread_temp_allocator == temp_allocator_pair[0] ? 1 : 0;
return thread_temp_allocator = temp_allocator_pair[index];
if (!top_temp) return;
top_temp.free();
}
const NullAllocator NULL_ALLOCATOR = {};

View File

@@ -244,7 +244,7 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private
name.append_repeat('-', len - len / 2);
if (!context.is_quiet_mode) io::printn(name);
name.clear();
TempState temp_state = mem::temp_push();
PoolState temp_state = mem::temp_push();
defer mem::temp_pop(temp_state);
foreach(unit : tests)
{

View File

@@ -59,7 +59,7 @@ fn ZString tformat_zstr(String fmt, args...)
@param [inout] allocator : `The allocator to use`
@param [in] fmt : `The formatting string`
*>
fn String format(Allocator allocator, String fmt, args...) => @pool(allocator)
fn String format(Allocator allocator, String fmt, args...) => @pool()
{
DString str = dstring::temp_with_capacity(fmt.len + args.len * 8);
str.appendf(fmt, ...args);
@@ -104,7 +104,7 @@ fn String join(Allocator allocator, String[] s, String joiner)
{
total_size += str.len;
}
@pool(allocator)
@pool()
{
DString res = dstring::temp_with_capacity(total_size);
res.append(s[0]);