Add location tracking for memory allocations.

This commit is contained in:
Christoffer Lerno
2023-11-09 11:08:36 +01:00
committed by Christoffer Lerno
parent 1e38ccdd2b
commit e31f2a03ba
16 changed files with 578 additions and 296 deletions

View File

@@ -29,7 +29,7 @@ struct ArenaAllocatorHeader @local
char[*] data;
}
fn void ArenaAllocator.release(&self, void* ptr, bool) @dynamic
fn void ArenaAllocator.release(&self, void* ptr, bool, TrackingEnv* env = null) @dynamic
{
if (!ptr) return;
assert((uptr)ptr >= (uptr)self.data.ptr, "Pointer originates from a different allocator.");
@@ -50,7 +50,7 @@ fn void ArenaAllocator.reset(&self, usz mark) @dynamic => self.used = mark;
* @require offset <= size && offset >= 0
* @require mem::aligned_offset(offset, ArenaAllocatorHeader.alignof) == offset
**/
fn void*! ArenaAllocator.acquire(&self, usz size, bool clear, usz alignment, usz offset) @dynamic
fn void*! ArenaAllocator.acquire(&self, usz size, bool clear, usz alignment, usz offset, TrackingEnv* env = null) @dynamic
{
if (!size) return null;
alignment = alignment_for_allocation(alignment);
@@ -76,7 +76,7 @@ fn void*! ArenaAllocator.acquire(&self, usz size, bool clear, usz alignment, usz
* @require offset <= size && offset >= 0
* @require mem::aligned_offset(offset, ArenaAllocatorHeader.alignof) == offset
**/
fn void*! ArenaAllocator.resize(&self, void *old_pointer, usz size, usz alignment, usz offset) @dynamic
fn void*! ArenaAllocator.resize(&self, void *old_pointer, usz size, usz alignment, usz offset, TrackingEnv* env) @dynamic
{
if (!size)
{
@@ -110,7 +110,7 @@ fn void*! ArenaAllocator.resize(&self, void *old_pointer, usz size, usz alignmen
return old_pointer;
}
// Otherwise just allocate new memory.
void* mem = self.acquire(size, false, alignment, offset)!;
void* mem = self.acquire(size, false, alignment, offset, env)!;
mem::copy(mem, old_pointer, old_size, mem::DEFAULT_MEM_ALIGNMENT, mem::DEFAULT_MEM_ALIGNMENT);
return mem;
}

View File

@@ -60,7 +60,7 @@ struct DynamicArenaChunk @local
/**
* @require self.page `tried to free pointer on invalid allocator`
*/
fn void DynamicArenaAllocator.release(&self, void* ptr, bool) @dynamic
fn void DynamicArenaAllocator.release(&self, void* ptr, bool, TrackingEnv* env) @dynamic
{
if (!ptr) return;
DynamicArenaPage* current_page = self.page;
@@ -74,16 +74,16 @@ fn void DynamicArenaAllocator.release(&self, void* ptr, bool) @dynamic
/**
* @require self.page `tried to realloc pointer on invalid allocator`
*/
fn void*! DynamicArenaAllocator.resize(&self, void* old_pointer, usz size, usz alignment, usz offset) @dynamic
fn void*! DynamicArenaAllocator.resize(&self, void* old_pointer, usz size, usz alignment, usz offset, TrackingEnv* env) @dynamic
{
if (!size)
{
self.release(old_pointer, alignment > 0);
self.release(old_pointer, alignment > 0, env);
return null;
}
if (!old_pointer)
{
return self.acquire(size, true, alignment, offset);
return self.acquire(size, true, alignment, offset, env);
}
DynamicArenaPage* current_page = self.page;
alignment = alignment_for_allocation(alignment);
@@ -108,7 +108,7 @@ fn void*! DynamicArenaAllocator.resize(&self, void* old_pointer, usz size, usz a
current_page.used += add_size;
return old_pointer;
}
void* new_mem = self.acquire(size, false, alignment, offset)!;
void* new_mem = self.acquire(size, false, alignment, offset, env)!;
mem::copy(new_mem, old_pointer, old_size, mem::DEFAULT_MEM_ALIGNMENT);
return new_mem;
}
@@ -163,7 +163,7 @@ fn void*! DynamicArenaAllocator._alloc_new(&self, usz size, usz alignment, usz o
/**
* @require !alignment || math::is_power_of_2(alignment)
*/
fn void*! DynamicArenaAllocator.acquire(&self, usz size, bool clear, usz alignment, usz offset) @dynamic
fn void*! DynamicArenaAllocator.acquire(&self, usz size, bool clear, usz alignment, usz offset, TrackingEnv* env) @dynamic
{
if (!size) return null;
alignment = alignment_for_allocation(alignment);

View File

@@ -21,7 +21,7 @@ fn void SimpleHeapAllocator.init(&self, MemoryAllocFn allocator)
self.free_list = null;
}
fn void*! SimpleHeapAllocator.acquire(&self, usz size, bool clear, usz alignment, usz offset) @dynamic
fn void*! SimpleHeapAllocator.acquire(&self, usz size, bool clear, usz alignment, usz offset, TrackingEnv* env) @dynamic
{
if (!size) return null;
if (clear)
@@ -31,23 +31,23 @@ fn void*! SimpleHeapAllocator.acquire(&self, usz size, bool clear, usz alignment
return alignment > 0 ? @aligned_alloc(self._alloc, size, alignment, offset) : self._alloc(size);
}
fn void*! SimpleHeapAllocator.resize(&self, void* old_pointer, usz size, usz alignment, usz offset) @dynamic
fn void*! SimpleHeapAllocator.resize(&self, void* old_pointer, usz size, usz alignment, usz offset, TrackingEnv* env) @dynamic
{
if (!size)
{
self.release(old_pointer, alignment > 0);
self.release(old_pointer, alignment > 0, env);
return null;
}
if (!old_pointer)
{
return self.acquire(size, true, alignment, offset);
return self.acquire(size, true, alignment, offset, env);
}
return alignment > 0
? @aligned_realloc(self._calloc, self._free, old_pointer, size, alignment, offset)
: self._realloc(old_pointer, size);
}
fn void SimpleHeapAllocator.release(&self, void* old_pointer, bool aligned) @dynamic
fn void SimpleHeapAllocator.release(&self, void* old_pointer, bool aligned, TrackingEnv* env) @dynamic
{
if (aligned)
{

View File

@@ -0,0 +1,63 @@
// Copyright (c) 2021 Christoffer Lerno. All rights reserved.
// Use of this source code is governed by the MIT license
// a copy of which can be found in the LICENSE_STDLIB file.
module std::core::mem::allocator;
import libc;
const LibcAllocator LIBC_ALLOCATOR = {};
distinct LibcAllocator (Allocator) = uptr;
fn void*! LibcAllocator.acquire(&self, usz bytes, bool clear, usz alignment, usz offset, TrackingEnv* env) @dynamic
{
assert(alignment != 0 || offset == 0);
if (clear)
{
void* data = alignment ? @aligned_calloc(fn void*(usz bytes) => libc::calloc(bytes, 1), bytes, alignment, offset)!! : libc::calloc(bytes, 1);
return data ?: AllocationFailure.OUT_OF_MEMORY?;
}
else
{
void* data = alignment ? @aligned_alloc(libc::malloc, bytes, alignment, offset)!! : libc::malloc(bytes);
if (!data) return AllocationFailure.OUT_OF_MEMORY?;
$if env::TESTING:
for (usz i = 0; i < bytes; i++) ((char*)data)[i] = 0xAA;
$endif
return data;
}
}
fn void*! LibcAllocator.resize(&self, void* old_ptr, usz new_bytes, usz alignment, usz offset, TrackingEnv* env) @dynamic
{
assert(alignment != 0 || offset == 0);
if (!new_bytes)
{
self.release(old_ptr, alignment > 0, env);
return null;
}
if (!old_ptr)
{
return self.acquire(new_bytes, true, alignment, offset, env);
}
if (alignment)
{
void* data = @aligned_realloc(fn void*(usz bytes) => libc::calloc(bytes, 1), libc::free, old_ptr, new_bytes, alignment, offset)!!;
return data ?: AllocationFailure.OUT_OF_MEMORY?;
}
return libc::realloc(old_ptr, new_bytes) ?: AllocationFailure.OUT_OF_MEMORY?;
}
fn void LibcAllocator.release(&self, void* old_ptr, bool aligned, TrackingEnv* env) @dynamic
{
if (aligned)
{
@aligned_free(libc::free, old_ptr)!!;
}
else
{
libc::free(old_ptr);
}
}

View File

@@ -1,133 +0,0 @@
// Copyright (c) 2021 Christoffer Lerno. All rights reserved.
// Use of this source code is governed by the MIT license
// a copy of which can be found in the LICENSE_STDLIB file.
module std::core::mem::allocator;
import libc;
struct AlignedBlock
{
usz len;
void* start;
}
/**
* @require bytes > 0
* @require alignment > 0
**/
macro void*! @aligned_alloc(#alloc_fn, usz bytes, usz alignment, usz offset)
{
usz header = mem::aligned_offset(AlignedBlock.sizeof + offset, alignment) - offset;
$if @typekind(#alloc_fn(bytes)) == OPTIONAL:
void* data = #alloc_fn(header + bytes)!;
$else
void* data = #alloc_fn(header + bytes);
$endif
void* mem = mem::aligned_pointer(data + header + offset, alignment) - offset;
assert(mem > data);
AlignedBlock* desc = (AlignedBlock*)mem - 1;
*desc = { bytes, data };
return mem;
}
/**
* @require bytes > 0
* @require alignment > 0
**/
macro void*! @aligned_calloc(#calloc_fn, usz bytes, usz alignment, usz offset)
{
usz header = mem::aligned_offset(AlignedBlock.sizeof + offset, alignment) - offset;
$if @typekind(#calloc_fn(bytes)) == OPTIONAL:
void* data = #calloc_fn(header + bytes)!;
$else
void* data = #calloc_fn(header + bytes);
$endif
void* mem = mem::aligned_pointer(data + header + offset, alignment) - offset;
AlignedBlock* desc = (AlignedBlock*)mem - 1;
assert(mem > data);
*desc = { bytes, data };
return mem;
}
/**
* @require bytes > 0
* @require alignment > 0
**/
macro void*! @aligned_realloc(#calloc_fn, #free_fn, void* old_pointer, usz bytes, usz alignment, usz offset)
{
AlignedBlock* desc = (AlignedBlock*)old_pointer - 1;
void* data_start = desc.start;
void* new_data = @aligned_calloc(#calloc_fn, bytes, alignment, offset)!;
mem::copy(new_data, old_pointer, desc.len < bytes ? desc.len : bytes, mem::DEFAULT_MEM_ALIGNMENT, mem::DEFAULT_MEM_ALIGNMENT);
$if @typekind(#free_fn(data_start)) == OPTIONAL:
#free_fn(data_start)!;
$else
#free_fn(data_start);
$endif
return new_data;
}
macro void! @aligned_free(#free_fn, void* old_pointer)
{
AlignedBlock* desc = (AlignedBlock*)old_pointer - 1;
$if @typekind(#free_fn(desc.start)) == OPTIONAL:
#free_fn(desc.start)!;
$else
#free_fn(desc.start);
$endif
}
distinct LibcAllocator (Allocator) = uptr;
fn void*! LibcAllocator.acquire(&self, usz bytes, bool clear, usz alignment, usz offset) @dynamic
{
assert(alignment != 0 || offset == 0);
if (clear)
{
void* data = alignment ? @aligned_calloc(fn void*(usz bytes) => libc::calloc(bytes, 1), bytes, alignment, offset)!! : libc::calloc(bytes, 1);
return data ?: AllocationFailure.OUT_OF_MEMORY?;
}
else
{
void* data = alignment ? @aligned_alloc(libc::malloc, bytes, alignment, offset)!! : libc::malloc(bytes);
if (!data) return AllocationFailure.OUT_OF_MEMORY?;
$if env::TESTING:
for (usz i = 0; i < bytes; i++) ((char*)data)[i] = 0xAA;
$endif
return data;
}
}
fn void*! LibcAllocator.resize(&self, void* old_ptr, usz new_bytes, usz alignment, usz offset) @dynamic
{
assert(alignment != 0 || offset == 0);
if (!new_bytes)
{
self.release(old_ptr, alignment > 0);
return null;
}
if (!old_ptr)
{
return self.acquire(new_bytes, true, alignment, offset);
}
if (alignment)
{
void* data = @aligned_realloc(fn void*(usz bytes) => libc::calloc(bytes, 1), libc::free, old_ptr, new_bytes, alignment, offset)!!;
return data ?: AllocationFailure.OUT_OF_MEMORY?;
}
return libc::realloc(old_ptr, new_bytes) ?: AllocationFailure.OUT_OF_MEMORY?;
}
fn void LibcAllocator.release(&self, void* old_ptr, bool aligned) @dynamic
{
if (aligned)
{
@aligned_free(libc::free, old_ptr)!!;
}
else
{
libc::free(old_ptr);
}
}

View File

@@ -54,7 +54,7 @@ struct OnStackAllocatorHeader
char[*] data;
}
fn void OnStackAllocator.release(&self, void* old_pointer, bool aligned) @dynamic
fn void OnStackAllocator.release(&self, void* old_pointer, bool aligned, TrackingEnv* env = null) @dynamic
{
if (!old_pointer) return;
if (allocation_in_stack_mem(self, old_pointer)) return;
@@ -103,18 +103,18 @@ fn OnStackAllocatorExtraChunk* on_stack_allocator_find_chunk(OnStackAllocator* a
* @require offset <= size && offset >= 0
* @require mem::aligned_offset(offset, OnStackAllocatorExtraChunk.alignof) == offset
**/
fn void*! OnStackAllocator.resize(&self, void* old_pointer, usz size, usz alignment, usz offset) @dynamic
fn void*! OnStackAllocator.resize(&self, void* old_pointer, usz size, usz alignment, usz offset, TrackingEnv* env) @dynamic
{
if (!allocation_in_stack_mem(self, old_pointer))
{
OnStackAllocatorExtraChunk* chunk = on_stack_allocator_find_chunk(self, old_pointer);
assert(chunk, "Tried to realloc pointer not belonging to the allocator");
return chunk.data = self.backing_allocator.resize(old_pointer, size, alignment, offset)!;
return chunk.data = self.backing_allocator.resize(old_pointer, size, alignment, offset, env)!;
}
OnStackAllocatorHeader* header = old_pointer - OnStackAllocatorHeader.sizeof;
usz old_size = header.size;
void* mem = self.acquire(size, true, alignment, offset)!;
void* mem = self.acquire(size, true, alignment, offset, env)!;
mem::copy(mem, old_pointer, old_size, mem::DEFAULT_MEM_ALIGNMENT, mem::DEFAULT_MEM_ALIGNMENT);
return mem;
}
@@ -126,7 +126,7 @@ fn void*! OnStackAllocator.resize(&self, void* old_pointer, usz size, usz alignm
* @require offset == 0 || alignment > 0
* @require mem::aligned_offset(offset, OnStackAllocatorHeader.alignof) == offset
**/
fn void*! OnStackAllocator.acquire(&self, usz size, bool clear, usz alignment, usz offset) @dynamic
fn void*! OnStackAllocator.acquire(&self, usz size, bool clear, usz alignment, usz offset, TrackingEnv* env) @dynamic
{
if (size == 0) return null;
bool aligned = alignment > 0;
@@ -144,7 +144,7 @@ fn void*! OnStackAllocator.acquire(&self, usz size, bool clear, usz alignment, u
defer catch backing_allocator.free(chunk);
defer try self.chunk = chunk;
*chunk = { .prev = self.chunk, .is_aligned = aligned };
return chunk.data = backing_allocator.acquire(size, clear, aligned ? alignment : 0, offset)!;
return chunk.data = backing_allocator.acquire(size, clear, aligned ? alignment : 0, offset, env)!;
}
self.used = end;
void *mem = aligned_pointer_to_offset - offset;

View File

@@ -47,7 +47,7 @@ fn TempAllocator*! new_temp(usz size, Allocator* allocator)
fn usz TempAllocator.mark(&self) @dynamic => self.used;
fn void TempAllocator.release(&self, void* old_pointer, bool) @dynamic
fn void TempAllocator.release(&self, void* old_pointer, bool, TrackingEnv* env) @dynamic
{
usz old_size = *(usz*)(old_pointer - DEFAULT_SIZE_PREFIX);
if (old_pointer + old_size == &self.data[self.used])
@@ -75,7 +75,7 @@ fn void! TempAllocator._free_page(&self, TempAllocatorPage* page) @inline @local
return self.backing_allocator.free(mem);
}
fn void*! TempAllocator._realloc_page(&self, TempAllocatorPage* page, usz size, usz alignment, usz offset) @inline @local
fn void*! TempAllocator._realloc_page(&self, TempAllocatorPage* page, usz size, usz alignment, usz offset, TrackingEnv* env) @inline @local
{
// Then the actual start pointer:
void* real_pointer = page.start;
@@ -90,7 +90,7 @@ fn void*! TempAllocator._realloc_page(&self, TempAllocatorPage* page, usz size,
*pointer_to_prev = page.prev_page;
usz page_size = page.pagesize();
// Clear on size > original size.
void* data = self.acquire(size, size > page_size, alignment, offset)!;
void* data = self.acquire(size, size > page_size, alignment, offset, env)!;
mem::copy(data, &page.data[0], page_size, mem::DEFAULT_MEM_ALIGNMENT, mem::DEFAULT_MEM_ALIGNMENT);
if (page.is_aligned())
{
@@ -103,16 +103,16 @@ fn void*! TempAllocator._realloc_page(&self, TempAllocatorPage* page, usz size,
return data;
}
fn void*! TempAllocator.resize(&self, void* pointer, usz size, usz alignment, usz offset) @dynamic
fn void*! TempAllocator.resize(&self, void* pointer, usz size, usz alignment, usz offset, TrackingEnv* env) @dynamic
{
if (!size)
{
self.release(pointer, alignment > 0);
self.release(pointer, alignment > 0, env);
return null;
}
if (!pointer)
{
return self.acquire(size, true, alignment, offset);
return self.acquire(size, true, alignment, offset, env);
}
TempAllocatorChunk *chunk = pointer - TempAllocatorChunk.sizeof;
if (chunk.size == (usz)-1)
@@ -120,11 +120,11 @@ fn void*! TempAllocator.resize(&self, void* pointer, usz size, usz alignment, us
assert(self.last_page, "Realloc of non temp pointer");
// First grab the page
TempAllocatorPage *page = pointer - TempAllocatorPage.sizeof;
return self._realloc_page(page, size, alignment, offset);
return self._realloc_page(page, size, alignment, offset, env);
}
// TODO optimize last allocation
TempAllocatorChunk* data = self.acquire(size, size > chunk.size, alignment, offset)!;
TempAllocatorChunk* data = self.acquire(size, size > chunk.size, alignment, offset, env)!;
mem::copy(data, pointer, chunk.size, mem::DEFAULT_MEM_ALIGNMENT, mem::DEFAULT_MEM_ALIGNMENT);
return data;
@@ -134,7 +134,7 @@ fn void*! TempAllocator.resize(&self, void* pointer, usz size, usz alignment, us
* @require !alignment || math::is_power_of_2(alignment)
* @require alignment <= mem::MAX_MEMORY_ALIGNMENT `alignment too big`
**/
fn void*! TempAllocator.acquire(&self, usz size, bool clear, usz alignment, usz offset) @dynamic
fn void*! TempAllocator.acquire(&self, usz size, bool clear, usz alignment, usz offset, TrackingEnv* env) @dynamic
{
if (!size) return null;
alignment = alignment_for_allocation(alignment);
@@ -182,7 +182,7 @@ fn void*! TempAllocator.acquire(&self, usz size, bool clear, usz alignment, usz
// Here we might need to pad
usz padded_header_size = mem::aligned_offset(TempAllocatorPage.sizeof, mem::DEFAULT_MEM_ALIGNMENT);
usz total_alloc_size = padded_header_size + size;
void* alloc = self.backing_allocator.acquire(total_alloc_size, clear)!;
void* alloc = self.backing_allocator.acquire(total_alloc_size, clear, 0, 0, null)!;
// Find the page.
page = alloc + padded_header_size - TempAllocatorPage.sizeof;

View File

@@ -4,8 +4,16 @@
module std::core::mem::allocator;
import std::collections::map;
import std::collections::list;
def PtrMap = HashMap(<uptr, usz>);
struct Allocation
{
usz size;
TrackingEnv tracking_env;
void* ptr;
}
def AllocMap = HashMap(<uptr, Allocation>);
// A simple tracking allocator.
// It tracks allocations using a hash map but
@@ -13,7 +21,7 @@ def PtrMap = HashMap(<uptr, usz>);
struct TrackingAllocator (Allocator)
{
Allocator* inner_allocator;
PtrMap map;
AllocMap map;
usz mem_total;
usz allocs_total;
}
@@ -46,7 +54,7 @@ fn usz TrackingAllocator.allocated(&self)
usz allocated = 0;
@pool()
{
foreach (usz allocation : self.map.value_tlist()) allocated += allocation;
foreach (&allocation : self.map.value_tlist()) allocated += allocation.size;
};
return allocated;
}
@@ -61,43 +69,48 @@ fn usz TrackingAllocator.total_allocated(&self) => self.mem_total;
**/
fn usz TrackingAllocator.total_allocation_count(&self) => self.allocs_total;
fn Allocation[] TrackingAllocator.allocations_tlist(&self, Allocator* allocator)
{
return self.map.value_tlist();
}
/**
* @return "the number of non-freed allocations."
**/
fn usz TrackingAllocator.allocation_count(&self) => self.map.count;
fn void*! TrackingAllocator.acquire(&self, usz size, bool clear, usz alignment, usz offset) @dynamic
fn void*! TrackingAllocator.acquire(&self, usz size, bool clear, usz alignment, usz offset, TrackingEnv* env) @dynamic
{
void* data = self.inner_allocator.acquire(size, clear, alignment, offset)!;
void* data = self.inner_allocator.acquire(size, clear, alignment, offset, env)!;
self.allocs_total++;
if (data)
{
self.map.set((uptr)data, size);
self.map.set((uptr)data, { .size = size, .ptr = data, .tracking_env = env ? *env : TrackingEnv{} });
self.mem_total += size;
self.allocs_total++;
}
return data;
}
fn void*! TrackingAllocator.resize(&self, void* old_pointer, usz size, usz alignment, usz offset) @dynamic
fn void*! TrackingAllocator.resize(&self, void* old_pointer, usz size, usz alignment, usz offset, TrackingEnv* env) @dynamic
{
void* data = self.inner_allocator.resize(old_pointer, size, alignment, offset)!;
void* data = self.inner_allocator.resize(old_pointer, size, alignment, offset, env)!;
if (old_pointer)
{
self.map.remove((uptr)old_pointer);
}
if (data)
{
self.map.set((uptr)data, size);
self.map.set((uptr)data, { .size = size, .ptr = data, .tracking_env = env ? *env : TrackingEnv{} });
self.mem_total += size;
self.allocs_total++;
}
return data;
}
fn void TrackingAllocator.release(&self, void* old_pointer, bool is_aligned) @dynamic
fn void TrackingAllocator.release(&self, void* old_pointer, bool is_aligned, TrackingEnv* env) @dynamic
{
self.inner_allocator.release(old_pointer, is_aligned);
self.inner_allocator.release(old_pointer, is_aligned, env);
if (old_pointer) self.map.remove((uptr)old_pointer);
}
@@ -105,3 +118,49 @@ fn void TrackingAllocator.clear(&self)
{
self.map.clear();
}
fn void TrackingAllocator.print_report(&self) => self.fprint_report(io::stdout())!!;
fn void! TrackingAllocator.fprint_report(&self, OutStream* out)
{
usz total = 0;
usz entries = 0;
@pool()
{
Allocation[] allocs = self.map.value_tlist();
if (allocs.len)
{
$if (!env::TRACK_MEMORY):
io::fprintn(out, "======== Memory Report ========")!;
io::fprintn(out, "Size in bytes Address")!;
foreach (i, &allocation : allocs)
{
entries++;
total += allocation.size;
io::fprintfn(out, "%13s %p", allocation.size, allocation.ptr)!;
}
io::fprintn(out, "===============================")!;
$else
io::fprintn(out, "================================== Memory Report ==================================")!;
io::fprintn(out, "Size in bytes Address Function File")!;
foreach (i, &allocation : allocs)
{
entries++;
total += allocation.size;
TrackingEnv *env = &allocation.tracking_env;
io::fprintfn(out, "%13s %p %-30s %s:%d", allocation.size, allocation.ptr, env.function, env.file, env.line)!;
}
io::fprintn(out, "===================================================================================")!;
$endif
}
else
{
io::fprintn(out, "* NO ALLOCATIONS FOUND *")!;
}
};
io::fprintfn(out, "- Total currently allocated memory: %d", total)!;
io::fprintfn(out, "- Total current allocations: %d", entries)!;
io::fprintfn(out, "- Total allocations (freed and retained): %d", self.allocs_total)!;
io::fprintfn(out, "- Total allocated memory (freed and retained): %d", self.mem_total)!;
}

View File

@@ -129,7 +129,7 @@ const usz LLVM_VERSION = $$LLVM_VERSION;
const bool BENCHMARKING = $$BENCHMARKING;
const bool TESTING = $$TESTING;
const MemoryEnvironment MEMORY_ENV = (MemoryEnvironment)$$MEMORY_ENVIRONMENT;
const bool TRACK_MEMORY = DEBUG_SYMBOLS && (COMPILER_SAFE_MODE || TESTING);
const bool X86_64 = ARCH_TYPE == X86_64;
const bool AARCH64 = ARCH_TYPE == AARCH64;

View File

@@ -368,92 +368,12 @@ macro bool equals(a, b, isz len = -1, usz $align = 0)
return true;
}
macro @clone(value) @builtin
{
return mem::heap().clone(value);
}
macro @tclone(value) @builtin
{
return mem::temp().clone(value);
}
macro type_alloc_must_be_aligned($Type)
{
return $Type.alignof > DEFAULT_MEM_ALIGNMENT;
}
fn void* malloc(usz size) @builtin @inline
{
return mem::heap().alloc(size);
}
fn void* tmalloc(usz size, usz alignment = 0, usz offset = 0) @builtin @inline
{
return temp().acquire(size, false, alignment, offset)!!;
}
macro new($Type)
{
return heap().new($Type);
}
macro new_clear($Type)
{
return heap().new_clear($Type);
}
macro new_temp($Type)
{
return tmalloc($Type.sizeof);
}
macro new_temp_clear($Type)
{
return tcalloc($Type.sizeof);
}
macro new_array($Type, usz elements)
{
return heap().new_array($Type, elements);
}
macro temp_array($Type, usz elements)
{
return (($Type*)tmalloc($Type.sizeof * elements, $Type.alignof))[:elements];
}
macro new_zero_array($Type, usz elements)
{
return heap().new_zero_array($Type, elements);
}
macro temp_zero_array($Type, usz elements)
{
return (($Type*)tcalloc($Type.sizeof * elements, $Type.alignof))[:elements];
}
fn void* calloc(usz size) @builtin @inline
{
return heap().calloc(size);
}
fn void* tcalloc(usz size, usz alignment = 0, usz offset = 0) @builtin @inline
{
return temp().acquire(size, false, alignment, offset)!!;
}
fn void* realloc(void *ptr, usz new_size) @builtin @inline
{
return heap().realloc(ptr, new_size);
}
fn void free(void* ptr) @builtin @inline
{
heap().free(ptr);
}
/**
* Run with a specific allocator inside of the macro body.
**/
@@ -465,10 +385,6 @@ macro void @scoped(Allocator* allocator; @body())
@body();
}
fn void* trealloc(void* ptr, usz size, usz alignment = mem::DEFAULT_MEM_ALIGNMENT) @builtin @inline
{
return temp().resize(ptr, size, alignment, 0)!!;
}
macro void @stack_mem(usz $size; @body(Allocator* mem)) @builtin
{
@@ -576,4 +492,92 @@ fn void initialize_wasm_mem() @init(1) @private
wasm_allocator.init(fn (x) => allocator::wasm_memory.allocate_block(x));
temp_base_allocator = &wasm_allocator;
thread_allocator = &wasm_allocator;
}
}
module std::core::mem @if(!env::TRACK_MEMORY);
macro @clone(value) @builtin
{
return mem::heap().clone(value);
}
macro @tclone(value) @builtin
{
return mem::temp().clone(value);
}
fn void* malloc(usz size) @builtin @inline
{
return mem::heap().alloc(size);
}
fn void* tmalloc(usz size, usz alignment = 0, usz offset = 0) @builtin @inline
{
return temp().acquire(size, false, alignment, offset, null)!!;
}
macro new($Type)
{
return heap().new($Type);
}
macro new_clear($Type)
{
return heap().new_clear($Type);
}
macro new_temp($Type)
{
return tmalloc($Type.sizeof);
}
macro new_temp_clear($Type)
{
return tcalloc($Type.sizeof);
}
macro new_array($Type, usz elements)
{
return heap().new_array($Type, elements);
}
macro temp_array($Type, usz elements)
{
return (($Type*)tmalloc($Type.sizeof * elements, $Type.alignof))[:elements];
}
macro new_zero_array($Type, usz elements)
{
return heap().new_zero_array($Type, elements);
}
macro temp_zero_array($Type, usz elements)
{
return (($Type*)tcalloc($Type.sizeof * elements, $Type.alignof))[:elements];
}
fn void* calloc(usz size) @builtin @inline
{
return heap().calloc(size);
}
fn void* tcalloc(usz size, usz alignment = 0, usz offset = 0) @builtin @inline
{
return temp().acquire(size, false, alignment, offset, null)!!;
}
fn void* realloc(void *ptr, usz new_size) @builtin @inline
{
return heap().realloc(ptr, new_size);
}
fn void free(void* ptr) @builtin @inline
{
heap().free(ptr);
}
fn void* trealloc(void* ptr, usz size, usz alignment = mem::DEFAULT_MEM_ALIGNMENT) @builtin @inline
{
return temp().resize(ptr, size, alignment, 0, null)!!;
}

View File

@@ -3,34 +3,123 @@ module std::core::mem::allocator;
const DEFAULT_SIZE_PREFIX = usz.sizeof;
const DEFAULT_SIZE_PREFIX_ALIGNMENT = usz.alignof;
struct TrackingEnv
{
String file;
String function;
uint line;
}
interface Allocator
{
fn void reset(usz mark) @optional;
fn usz mark() @optional;
fn void*! acquire(usz size, bool clear = false, usz alignment = 0, usz offset = 0);
fn void*! resize(void* ptr, usz new_size, usz alignment = 0, usz offset = 0);
fn void release(void* ptr, bool aligned = false);
fn void*! acquire(usz size, bool clear, usz alignment, usz offset, TrackingEnv* env);
fn void*! resize(void* ptr, usz new_size, usz alignment, usz offset, TrackingEnv* env);
fn void release(void* ptr, bool aligned, TrackingEnv* env);
}
const LibcAllocator LIBC_ALLOCATOR = {};
struct AlignedBlock
{
usz len;
void* start;
}
/**
* @require bytes > 0
* @require alignment > 0
**/
macro void*! @aligned_alloc(#alloc_fn, usz bytes, usz alignment, usz offset)
{
usz header = mem::aligned_offset(AlignedBlock.sizeof + offset, alignment) - offset;
$if @typekind(#alloc_fn(bytes)) == OPTIONAL:
void* data = #alloc_fn(header + bytes)!;
$else
void* data = #alloc_fn(header + bytes);
$endif
void* mem = mem::aligned_pointer(data + header + offset, alignment) - offset;
assert(mem > data);
AlignedBlock* desc = (AlignedBlock*)mem - 1;
*desc = { bytes, data };
return mem;
}
/**
* @require bytes > 0
* @require alignment > 0
**/
macro void*! @aligned_calloc(#calloc_fn, usz bytes, usz alignment, usz offset)
{
usz header = mem::aligned_offset(AlignedBlock.sizeof + offset, alignment) - offset;
$if @typekind(#calloc_fn(bytes)) == OPTIONAL:
void* data = #calloc_fn(header + bytes)!;
$else
void* data = #calloc_fn(header + bytes);
$endif
void* mem = mem::aligned_pointer(data + header + offset, alignment) - offset;
AlignedBlock* desc = (AlignedBlock*)mem - 1;
assert(mem > data);
*desc = { bytes, data };
return mem;
}
/**
* @require bytes > 0
* @require alignment > 0
**/
macro void*! @aligned_realloc(#calloc_fn, #free_fn, void* old_pointer, usz bytes, usz alignment, usz offset)
{
AlignedBlock* desc = (AlignedBlock*)old_pointer - 1;
void* data_start = desc.start;
void* new_data = @aligned_calloc(#calloc_fn, bytes, alignment, offset)!;
mem::copy(new_data, old_pointer, desc.len < bytes ? desc.len : bytes, mem::DEFAULT_MEM_ALIGNMENT, mem::DEFAULT_MEM_ALIGNMENT);
$if @typekind(#free_fn(data_start)) == OPTIONAL:
#free_fn(data_start)!;
$else
#free_fn(data_start);
$endif
return new_data;
}
macro void! @aligned_free(#free_fn, void* old_pointer)
{
AlignedBlock* desc = (AlignedBlock*)old_pointer - 1;
$if @typekind(#free_fn(desc.start)) == OPTIONAL:
#free_fn(desc.start)!;
$else
#free_fn(desc.start);
$endif
}
def MemoryAllocFn = fn char[]!(usz);
fault AllocationFailure
{
OUT_OF_MEMORY,
CHUNK_TOO_LARGE,
}
fn usz alignment_for_allocation(usz alignment) @inline @private
{
return alignment < mem::DEFAULT_MEM_ALIGNMENT ? alignment = mem::DEFAULT_MEM_ALIGNMENT : alignment;
}
module std::core::mem::allocator @if(!env::TRACK_MEMORY);
// Allocator "functions"
macro void*! Allocator.alloc_checked(&self, usz size)
{
$if env::TESTING:
char* data = self.acquire(size, false, 0, 0)!;
char* data = self.acquire(size, false, 0, 0, null)!;
mem::set(data, 0xAA, size, mem::DEFAULT_MEM_ALIGNMENT);
return data;
$else
return self.acquire(size, false, 0, 0);
return self.acquire(size, false, 0, 0, null);
$endif
}
macro void*! Allocator.calloc_checked(&self, usz size) => self.acquire(size, true, 0, 0);
macro void*! Allocator.realloc_checked(&self, void* ptr, usz new_size) => self.resize(ptr, new_size, 0, 0);
macro void*! Allocator.calloc_checked(&self, usz size) => self.acquire(size, true, 0, 0, null);
macro void*! Allocator.realloc_checked(&self, void* ptr, usz new_size) => self.resize(ptr, new_size, 0, 0, null);
macro Allocator.new_array(&self, $Type, usz size, usz end_padding = 0)
{
@@ -79,46 +168,51 @@ macro Allocator.clone(&self, value)
return x;
}
macro void* Allocator.alloc(&self, usz size) @nodiscard => self.alloc_checked(size)!!;
macro void* Allocator.calloc(&self, usz size) @nodiscard => self.acquire(size, true, 0, 0)!!;
macro void* Allocator.realloc(&self, void* ptr, usz new_size) @nodiscard => self.resize(ptr, new_size, 0, 0)!!;
macro void* Allocator.alloc(&self, usz size) @nodiscard
{
return self.alloc_checked(size)!!;
}
macro void* Allocator.calloc(&self, usz size) @nodiscard
{
return self.acquire(size, true, 0, 0, null)!!;
}
macro void* Allocator.realloc(&self, void* ptr, usz new_size) @nodiscard
{
return self.resize(ptr, new_size, 0, 0, null)!!;
}
macro void*! Allocator.alloc_aligned(&self, usz size, usz alignment, usz offset = 0)
{
$if env::TESTING:
char* data = self.acquire(size, false, alignment, offset)!;
char* data = self.acquire(size, false, alignment, offset, null)!;
mem::set(data, 0xAA, size, mem::DEFAULT_MEM_ALIGNMENT);
return data;
$else
return self.acquire(size, false, alignment, offset);
return self.acquire(size, false, alignment, offset, null);
$endif
}
macro void*! Allocator.calloc_aligned(&self, usz size, usz alignment, usz offset = 0) => self.acquire(size, true, alignment, offset);
macro void*! Allocator.realloc_aligned(&self, void* ptr, usz new_size, usz alignment = 0, usz offset = 0) => self.resize(ptr, new_size, alignment, offset);
macro void*! Allocator.calloc_aligned(&self, usz size, usz alignment, usz offset = 0)
{
return self.acquire(size, true, alignment, offset, null);
}
macro void*! Allocator.realloc_aligned(&self, void* ptr, usz new_size, usz alignment = 0, usz offset = 0)
{
return self.resize(ptr, new_size, alignment, offset, null);
}
macro void Allocator.free(&self, void* ptr)
{
$if env::TESTING:
if (ptr) ((char*)ptr)[0] = 0xBA;
$endif
self.release(ptr, false);
self.release(ptr, false, null);
}
macro void Allocator.free_aligned(&self, void* ptr)
{
$if env::TESTING:
if (ptr) ((char*)ptr)[0] = 0xBA;
$endif
self.release(ptr, true);
}
fault AllocationFailure
{
OUT_OF_MEMORY,
CHUNK_TOO_LARGE,
}
fn usz alignment_for_allocation(usz alignment) @inline @private
{
return alignment < mem::DEFAULT_MEM_ALIGNMENT ? alignment = mem::DEFAULT_MEM_ALIGNMENT : alignment;
self.release(ptr, true, null);
}

194
lib/std/core/mem_tracked.c3 Normal file
View File

@@ -0,0 +1,194 @@
module std::core::mem @if(env::TRACK_MEMORY);
macro @clone(value, String file = $$FILE, String func = $$FUNC, uint line = $$LINE) @builtin
{
return mem::heap().clone(value, .file = file, .func = func, .line = line);
}
macro @tclone(value, String file = $$FILE, String func = $$FUNC, uint line = $$LINE) @builtin
{
return mem::temp().clone(value, .file = file, .func = func, .line = line);
}
fn void* malloc(usz size, String file = $$FILE, String func = $$FUNC, uint line = $$LINE) @builtin @inline
{
return mem::heap().alloc(size, .file = file, .func = func, .line = line);
}
fn void* tmalloc(usz size, usz alignment = 0, usz offset = 0, String file = $$FILE, String func = $$FUNC, uint line = $$LINE) @builtin @inline
{
return temp().acquire(size, false, alignment, offset, .env = &&TrackingEnv{ file, func, line})!!;
}
macro new($Type, String file = $$FILE, String func = $$FUNC, uint line = $$LINE)
{
return heap().new($Type, .file = file, .func = func, .line = line);
}
macro new_clear($Type, String file = $$FILE, String func = $$FUNC, uint line = $$LINE)
{
return heap().new_clear($Type, .file = file, .func = func, .line = line);
}
macro new_temp($Type, String file = $$FILE, String func = $$FUNC, uint line = $$LINE)
{
return tmalloc($Type.sizeof, .file = file, .func = func, .line = line);
}
macro new_temp_clear($Type, String file = $$FILE, String func = $$FUNC, uint line = $$LINE)
{
return tcalloc($Type.sizeof, .file = file, .func = func, .line = line);
}
macro new_array($Type, usz elements, String file = $$FILE, String func = $$FUNC, uint line = $$LINE)
{
return heap().new_array($Type, elements, .file = file, .func = func, .line = line);
}
macro temp_array($Type, usz elements, String file = $$FILE, String func = $$FUNC, uint line = $$LINE)
{
return (($Type*)tmalloc($Type.sizeof * elements, $Type.alignof, .file = file, .func = func, .line = line))[:elements];
}
macro new_zero_array($Type, usz elements, String file = $$FILE, String func = $$FUNC, uint line = $$LINE)
{
return heap().new_zero_array($Type, elements, .file = file, .func = func, .line = line);
}
macro temp_zero_array($Type, usz elements, String file = $$FILE, String func = $$FUNC, uint line = $$LINE)
{
return (($Type*)tcalloc($Type.sizeof * elements, $Type.alignof, .file = file, .func = func, .line = line))[:elements];
}
fn void* calloc(usz size, String file = $$FILE, String func = $$FUNC, uint line = $$LINE) @builtin @inline
{
return heap().calloc(size, .file = file, .func = func, .line = line);
}
fn void* tcalloc(usz size, usz alignment = 0, usz offset = 0, String file = $$FILE, String func = $$FUNC, uint line = $$LINE) @builtin @inline
{
return temp().acquire(size, false, alignment, offset, .env = &&TrackingEnv{ file, func, line})!!;
}
fn void* realloc(void *ptr, usz new_size, String file = $$FILE, String func = $$FUNC, uint line = $$LINE) @builtin @inline
{
return heap().realloc(ptr, new_size, .file = file, .func = func, .line = line);
}
fn void free(void* ptr, String file = $$FILE, String func = $$FUNC, uint line = $$LINE) @builtin @inline
{
heap().free(ptr, .file = file, .func = func, .line = line);
}
fn void* trealloc(void* ptr, usz size, usz alignment = mem::DEFAULT_MEM_ALIGNMENT, String file = $$FILE, String func = $$FUNC, uint line = $$LINE) @builtin @inline
{
return temp().resize(ptr, size, alignment, 0, .env = &&TrackingEnv{ file, func, line})!!;
}
module std::core::mem::allocator @if(env::TRACK_MEMORY);
macro void*! Allocator.alloc_checked(&self, usz size, String file = $$FILE, String func = $$FUNC, uint line = $$LINE)
{
char* data = self.acquire(size, false, 0, 0, .env = &&TrackingEnv{ file, func, line})!;
mem::set(data, 0xAA, size, mem::DEFAULT_MEM_ALIGNMENT);
return data;
}
macro void*! Allocator.calloc_checked(&self, usz size, String file = $$FILE, String func = $$FUNC, uint line = $$LINE)
{
return self.acquire(size, true, 0, 0, .env = &&TrackingEnv{ file, func, line});
}
macro void*! Allocator.realloc_checked(&self, void* ptr, usz new_size, String file = $$FILE, String func = $$FUNC, uint line = $$LINE)
{
return self.resize(ptr, new_size, 0, 0, .env = &&TrackingEnv{ file, func, line});
}
macro Allocator.new_array(&self, $Type, usz size, usz end_padding = 0, String file = $$FILE, String func = $$FUNC, uint line = $$LINE)
{
return (($Type*)self.alloc_checked($Type.sizeof * size + end_padding, .file = file, .func = func, .line = line))[:size]!!;
}
macro Allocator.new_array_checked(&self, $Type, usz size, usz end_padding = 0, String file = $$FILE, String func = $$FUNC, uint line = $$LINE)
{
return (($Type*)self.alloc_checked($Type.sizeof * size + end_padding, .file = file, .func = func, .line = line))[:size];
}
macro Allocator.new_zero_array(&self, $Type, usz size, usz end_padding = 0, String file = $$FILE, String func = $$FUNC, uint line = $$LINE)
{
return (($Type*)self.calloc_checked($Type.sizeof * size + end_padding, .file = file, .func = func, .line = line))[:size]!!;
}
macro Allocator.new_zero_array_checked(&self, $Type, usz size, usz end_padding = 0, String file = $$FILE, String func = $$FUNC, uint line = $$LINE)
{
return (($Type*)self.calloc_checked($Type.sizeof * size + end_padding, .file = file, .func = func, .line = line))[:size];
}
macro Allocator.new(&self, $Type, usz end_padding = 0, String file = $$FILE, String func = $$FUNC, uint line = $$LINE) @nodiscard
{
return ($Type*)self.alloc_checked($Type.sizeof + end_padding, .file = file, .func = func, .line = line)!!;
}
macro Allocator.new_checked(&self, $Type, usz end_padding = 0, String file = $$FILE, String func = $$FUNC, uint line = $$LINE) @nodiscard
{
return ($Type*)self.alloc_checked($Type.sizeof + end_padding, .file = file, .func = func, .line = line);
}
macro Allocator.new_clear(&self, $Type, usz end_padding = 0, String file = $$FILE, String func = $$FUNC, uint line = $$LINE) @nodiscard
{
return ($Type*)self.calloc_checked($Type.sizeof + end_padding, .file = file, .func = func, .line = line)!!;
}
macro Allocator.new_clear_checked(&self, $Type, usz end_padding = 0, String file = $$FILE, String func = $$FUNC, uint line = $$LINE) @nodiscard
{
return ($Type*)self.calloc_checked($Type.sizeof + end_padding, .file = file, .func = func, .line = line);
}
macro Allocator.clone(&self, value, String file = $$FILE, String func = $$FUNC, uint line = $$LINE)
{
var x = self.alloc($typeof(value), .file = file, .func = func, .line = line);
*x = value;
return x;
}
macro void* Allocator.alloc(&self, usz size, String file = $$FILE, String func = $$FUNC, uint line = $$LINE) @nodiscard
{
return self.alloc_checked(size, .file = file, .func = func, .line = line)!!;
}
macro void* Allocator.calloc(&self, usz size, String file = $$FILE, String func = $$FUNC, uint line = $$LINE) @nodiscard
{
return self.acquire(size, true, 0, 0, .env = &&TrackingEnv{ file, func, line})!!;
}
macro void* Allocator.realloc(&self, void* ptr, usz new_size, String file = $$FILE, String func = $$FUNC, uint line = $$LINE) @nodiscard
{
return self.resize(ptr, new_size, 0, 0, .env = &&TrackingEnv{ file, func, line})!!;
}
macro void*! Allocator.alloc_aligned(&self, usz size, usz alignment, usz offset = 0, String file = $$FILE, String func = $$FUNC, uint line = $$LINE)
{
$if env::TESTING:
char* data = self.acquire(size, false, alignment, offset, .env = &&TrackingEnv{ file, func, line})!;
mem::set(data, 0xAA, size, mem::DEFAULT_MEM_ALIGNMENT);
return data;
$else
return self.acquire(size, false, alignment, offset, .env = &&TrackingEnv{ file, func, line});
$endif
}
macro void*! Allocator.calloc_aligned(&self, usz size, usz alignment, usz offset = 0, String file = $$FILE, String func = $$FUNC, uint line = $$LINE)
{
return self.acquire(size, true, alignment, offset, .env = &&TrackingEnv{ file, func, line});
}
macro void*! Allocator.realloc_aligned(&self, void* ptr, usz new_size, usz alignment = 0, usz offset = 0, String file = $$FILE, String func = $$FUNC, uint line = $$LINE)
{
return self.resize(ptr, new_size, alignment, offset, .env = &&TrackingEnv{ file, func, line});
}
macro void Allocator.free(&self, void* ptr, String file = $$FILE, String func = $$FUNC, uint line = $$LINE)
{
if (ptr) ((char*)ptr)[0] = 0xBA;
self.release(ptr, false, .env = &&TrackingEnv{ file, func, line});
}
macro void Allocator.free_aligned(&self, void* ptr, String file = $$FILE, String func = $$FUNC, uint line = $$LINE)
{
if (ptr) ((char*)ptr)[0] = 0xBA;
self.release(ptr, true, .env = &&TrackingEnv{ file, func, line});
}

View File

@@ -131,6 +131,7 @@
### Stdlib changes
- Tracking allocator with location.
- `init_new`/`init_temp` for allocating init methods.
- `DString.printf` is now `DString.appendf`.
- Tuple and Maybe types.

View File

@@ -1 +1 @@
#define COMPILER_VERSION "0.4.697"
#define COMPILER_VERSION "0.4.698"

View File

@@ -101,12 +101,12 @@ cache_hit: ; preds = %entry
missing_function: ; preds = %8
%10 = load ptr, ptr @std.core.builtin.panic, align 8
call void %10(ptr @.panic_msg, i64 44, ptr @.file, i64 16, ptr @.func, i64 4, i32 28)
call void %10(ptr @.panic_msg, i64 44, ptr @.file, i64 16
unreachable
match: ; preds = %8
%11 = load ptr, ptr %2, align 8
%12 = call i64 %fn_phi(ptr %retparam, ptr %11, i64 8, i8 zeroext 0, i64 0, i64 0)
%12 = call i64 %fn_phi(ptr %retparam, ptr %11, i64 8, i8 zeroext 0, i64 0, i64 0, ptr null)
%not_err = icmp eq i64 %12, 0
%13 = call i1 @llvm.expect.i1(i1 %not_err, i1 true)
br i1 %13, label %after_check, label %assign_optional
@@ -265,4 +265,4 @@ dtable_next4: ; preds = %dtable_check1
dtable_found6: ; preds = %dtable_check1
store ptr @"$ct.dyn.inherit.Test.hello", ptr %dtable_ref2, align 8
ret void
}
}

View File

@@ -88,12 +88,12 @@ cache_hit: ; preds = %entry
missing_function: ; preds = %8
%10 = load ptr, ptr @std.core.builtin.panic, align 8
call void %10(ptr @.panic_msg, i64 44, ptr @.file, i64 16, ptr @.func, i64 4, i32 28)
call void %10(ptr @.panic_msg, i64 44, ptr @.file, i64 16, ptr @.func, i64 4
unreachable
match: ; preds = %8
%11 = load ptr, ptr %2, align 8
%12 = call i64 %fn_phi(ptr %retparam, ptr %11, i64 8, i8 zeroext 0, i64 0, i64 0)
%12 = call i64 %fn_phi(ptr %retparam, ptr %11, i64 8, i8 zeroext 0, i64 0, i64 0, ptr null)
%not_err = icmp eq i64 %12, 0
%13 = call i1 @llvm.expect.i1(i1 %not_err, i1 true)
br i1 %13, label %after_check, label %assign_optional