Added Vmem allocator

lib/std/core/allocators/temp_allocator.c3

@@ -1,6 +1,5 @@
-module std::core::mem::allocator;
+module std::core::mem::allocator @if(!env::POSIX || !$feature(VMEM_TEMP));
 import std::io, std::math;
-import std::core::sanitizer::asan;
 
 // This implements the temp allocator.
 // The temp allocator is a specialized allocator only intended for use where
@@ -327,3 +326,81 @@ fn void*? TempAllocator.acquire(&self, usz size, AllocInitType init_type, usz al
 	return &page.data[0];
 }
 
+module std::core::mem::allocator @if(env::POSIX && $feature(VMEM_TEMP));
+import std::math;
+
+tlocal VmemOptions temp_allocator_default_options = {
+	.shrink_on_reset = env::MEMORY_ENV != NORMAL,
+	.protect_unused_pages = env::COMPILER_OPT_LEVEL <= O1 || env::COMPILER_SAFE_MODE,
+	.scratch_released_data = env::COMPILER_SAFE_MODE
+};
+
+
+fn TempAllocator*? new_temp_allocator(Allocator allocator, usz size, usz reserve = temp_allocator_reserve_size, usz min_size = temp_allocator_min_size, usz realloc_size = temp_allocator_realloc_size)
+{
+	Vmem mem;
+	TempAllocator* t = allocator::new(allocator, TempAllocator);
+	defer catch allocator::free(allocator, t);
+	t.vmem.init(preferred_size: isz.sizeof > 4 ? 4 * mem::GB : 512 * mem::MB,
+	            reserve_page_size: isz.sizeof > 4 ? 256 * mem::KB : 0,
+	            options: temp_allocator_default_options)!;
+	t.allocator = allocator;
+	return t;
+}
+
+struct TempAllocator (Allocator)
+{
+	Vmem vmem;
+	TempAllocator* derived;
+	Allocator allocator;
+}
+
+<*
+ @require size > 0
+ @require !alignment || math::is_power_of_2(alignment)
+ @require alignment <= mem::MAX_MEMORY_ALIGNMENT : `alignment too big`
+*>
+fn void*? TempAllocator.acquire(&self, usz size, AllocInitType init_type, usz alignment) @dynamic
+{
+	return self.vmem.acquire(size, init_type, alignment) @inline;
+}
+
+fn TempAllocator*? TempAllocator.derive_allocator(&self, usz reserve = 0)
+{
+	if (self.derived) return self.derived;
+	return self.derived = new_temp_allocator(self.allocator, 0)!;
+}
+
+<*
+ Reset the entire temp allocator, destroying all children
+*>
+fn void TempAllocator.reset(&self)
+{
+	TempAllocator* child = self.derived;
+	if (!child) return;
+	child.reset();
+	child.vmem.reset(0);
+}
+fn void TempAllocator.free(&self)
+{
+	self.destroy();
+}
+
+fn void TempAllocator.destroy(&self) @local
+{
+	TempAllocator* child = self.derived;
+	if (!child) return;
+	child.destroy();
+	self.vmem.free() @inline;
+	allocator::free(self.allocator, self) @inline;
+}
+
+fn void*? TempAllocator.resize(&self, void* pointer, usz size, usz alignment) @dynamic
+{
+	return self.vmem.resize(pointer, size, alignment) @inline;
+}
+
+fn void TempAllocator.release(&self, void* old_pointer, bool b) @dynamic
+{
+	self.vmem.release(old_pointer, b) @inline;
+}

lib/std/core/allocators/vmem.c3

@@ -0,0 +1,239 @@
+module std::core::mem::allocator @if(env::POSIX);
+import std::math, std::os::posix, libc, std::bits;
+
+// Virtual Memory allocator
+
+faultdef VMEM_RESERVE_FAILED, VMEM_PROTECT_FAILED;
+
+struct Vmem (Allocator)
+{
+	void* ptr;
+	usz allocated;
+	usz capacity;
+	usz pagesize;
+	usz page_pot;
+	usz last_page;
+	usz high_water;
+	VmemOptions options;
+}
+
+bitstruct VmemOptions : int
+{
+	bool shrink_on_reset;         // Release memory on reset
+	bool protect_unused_pages;    // Protect unused pages on reset
+	bool scratch_released_data;   // Overwrite released data with 0xAA
+}
+
+<*
+ Implements the Allocator interface method.
+
+ @require !reserve_page_size || math::is_power_of_2(reserve_page_size)
+ @require reserve_page_size <= preferred_size : "The min reserve_page_size size must be less or equal to the preferred size"
+ @require preferred_size >= 1 * mem::KB : "The preferred size must exceed 1 KB"
+ @return? mem::INVALID_ALLOC_SIZE, mem::OUT_OF_MEMORY, VMEM_RESERVE_FAILED
+*>
+fn void? Vmem.init(&self, usz preferred_size, usz reserve_page_size = 0, VmemOptions options = { true, true, env::COMPILER_SAFE_MODE }, usz min_size = 0)
+{
+	void* ptr;
+    static usz page_size = 0;
+    if (!page_size) page_size = posix::getpagesize();
+    if (page_size < reserve_page_size) page_size = reserve_page_size;
+    preferred_size = mem::aligned_offset(preferred_size, page_size);
+    if (!min_size) min_size = max(preferred_size / 1024, 1);
+	while (preferred_size >= min_size)
+	{
+		ptr = posix::mmap(null, preferred_size, posix::PROT_NONE, posix::MAP_PRIVATE | posix::MAP_ANONYMOUS, -1, 0);
+		// It worked?
+		if (ptr != posix::MAP_FAILED && ptr) break;
+		// Did it fail in a non-retriable way?
+		switch (libc::errno())
+		{
+			case errno::ENOMEM:
+			case errno::EOVERFLOW:
+			case errno::EAGAIN:
+				// Try a smaller size.
+				preferred_size /= 2;
+				continue;
+		}
+		break;
+	}
+	// Check if we ended on a failure.
+	if ((ptr == posix::MAP_FAILED) || !ptr) return VMEM_RESERVE_FAILED?;
+	if (page_size > preferred_size) page_size = preferred_size;
+	$if env::ADDRESS_SANITIZER:
+		asan::poison_memory_region(self.ptr, self.capacity);
+	$endif
+	*self = { .ptr = ptr, .high_water = 0,
+			  .capacity = preferred_size,
+			  .pagesize = page_size,
+			  .page_pot = page_size.ctz(),
+			  .options  = options,
+	        };
+}
+
+<*
+ Implements the Allocator interface method.
+
+ @require !alignment || math::is_power_of_2(alignment)
+ @require alignment <= mem::MAX_MEMORY_ALIGNMENT : `alignment too big`
+ @require size > 0
+ @return? mem::INVALID_ALLOC_SIZE, mem::OUT_OF_MEMORY
+*>
+fn void*? Vmem.acquire(&self, usz size, AllocInitType init_type, usz alignment) @dynamic
+{
+	alignment = alignment_for_allocation(alignment);
+	usz total_len = self.capacity;
+	if (size > total_len) return mem::INVALID_ALLOC_SIZE?;
+	void* start_mem = self.ptr;
+	void* unaligned_pointer_to_offset = start_mem + self.allocated + VmemHeader.sizeof;
+	void* mem = mem::aligned_pointer(unaligned_pointer_to_offset, alignment);
+	usz after = (usz)(mem - self.ptr) + size;
+	if (after > total_len) return mem::OUT_OF_MEMORY?;
+	if (init_type == ZERO && self.high_water <= self.allocated)
+	{
+		init_type = NO_ZERO;
+	}
+	protect(self, after)!;
+	VmemHeader* header = mem - VmemHeader.sizeof;
+	header.size = size;
+	if (init_type == ZERO) mem::clear(mem, size, mem::DEFAULT_MEM_ALIGNMENT);
+	return mem;
+}
+
+fn bool Vmem.owns_pointer(&self, void* ptr) @inline
+{
+	return (uptr)ptr >= (uptr)self.ptr && (uptr)ptr < (uptr)self.ptr + self.capacity;
+}
+<*
+ Implements the Allocator interface method.
+
+ @require !alignment || math::is_power_of_2(alignment)
+ @require alignment <= mem::MAX_MEMORY_ALIGNMENT : `alignment too big`
+ @require old_pointer != null
+ @require size > 0
+ @return? mem::INVALID_ALLOC_SIZE, mem::OUT_OF_MEMORY
+*>
+fn void*? Vmem.resize(&self, void *old_pointer, usz size, usz alignment) @dynamic
+{
+	if (size > self.capacity) return mem::INVALID_ALLOC_SIZE?;
+	alignment = alignment_for_allocation(alignment);
+	assert(self.owns_pointer(old_pointer), "Pointer originates from a different allocator: %p, not in %p - %p", old_pointer, self.ptr, self.ptr + self.allocated);
+	VmemHeader* header = old_pointer - VmemHeader.sizeof;
+    usz old_size = header.size;
+    // Do last allocation and alignment match?
+    if (self.ptr + self.allocated == old_pointer + old_size && mem::ptr_is_aligned(old_pointer, alignment))
+    {
+        if (old_size == size) return old_pointer;
+    	if (old_size >= size)
+    	{
+    		unprotect(self, self.allocated + size - old_size);
+    	}
+    	else
+    	{
+    		usz allocated = self.allocated + size - old_size;
+    		if (allocated > self.capacity) return mem::OUT_OF_MEMORY?;
+    		protect(self, allocated)!;
+    	}
+    	header.size = size;
+    	return old_pointer;
+    }
+    // Otherwise just allocate new memory.
+    void* mem = self.acquire(size, NO_ZERO, alignment)!;
+    mem::copy(mem, old_pointer, old_size, mem::DEFAULT_MEM_ALIGNMENT, mem::DEFAULT_MEM_ALIGNMENT);
+    return mem;
+}
+
+<*
+ Implements the Allocator interface method.
+
+ @require ptr != null
+*>
+fn void Vmem.release(&self, void* ptr, bool) @dynamic
+{
+	assert(self.owns_pointer(ptr), "Pointer originates from a different allocator %p.", ptr);
+	VmemHeader* header = ptr - VmemHeader.sizeof;
+	// Reclaim memory if it's the last element.
+	if (ptr + header.size == self.ptr + self.allocated)
+	{
+		unprotect(self, self.allocated - header.size - VmemHeader.sizeof);
+	}
+}
+
+fn usz Vmem.mark(&self)
+{
+	return self.allocated;
+}
+
+<*
+ @require mark <= self.allocated : "Invalid mark"
+*>
+fn void Vmem.reset(&self, usz mark)
+{
+	if (mark == self.allocated) return;
+	unprotect(self, mark);
+}
+
+fn void Vmem.free(&self)
+{
+	if (!self.ptr) return;
+	$switch:
+		$case env::ADDRESS_SANITIZER:
+			asan::poison_memory_region(self.ptr, self.capacity);
+		$case env::COMPILER_SAFE_MODE:
+    	    ((char*)self.ptr)[0:self.allocated] = 0xAA;
+	$endswitch
+	posix::munmap(self.ptr, self.capacity);
+	*self = {};
+}
+
+// Internal data
+
+struct VmemHeader @local
+{
+	usz size;
+	char[*] data;
+}
+
+macro void? protect(Vmem* mem, usz after) @local
+{
+	usz shift = mem.page_pot;
+	usz page_after = (after + mem.pagesize - 1) >> shift;
+	usz last_page = mem.last_page;
+	bool over_high_water = mem.high_water < after;
+	if (page_after > last_page)
+	{
+		if (mem.options.protect_unused_pages || over_high_water)
+		{
+			if (posix::mprotect(mem.ptr + last_page << shift, (page_after - last_page) << shift, posix::PROT_WRITE | posix::PROT_READ)) return VMEM_PROTECT_FAILED?;
+		}
+		mem.last_page = page_after;
+	}
+	$if env::ADDRESS_SANITIZER:
+		asan::unpoison_memory_region(mem.ptr + mem.allocated, after - mem.allocated);
+	$endif
+	mem.allocated = after;
+	if (over_high_water) mem.high_water = after;
+}
+
+macro void unprotect(Vmem* mem, usz after) @local
+{
+	usz shift = mem.page_pot;
+	usz last_page = mem.last_page;
+	usz page_after = mem.last_page = (after + mem.pagesize - 1) >> shift;
+	$if env::ADDRESS_SANITIZER:
+		asan::poison_memory_region(mem.ptr + after, mem.allocated - after);
+	$else
+		if (mem.options.scratch_released_data)
+		{
+			mem::set(mem.ptr + after, 0xAA, mem.allocated - after);
+		}
+	$endif
+	if ((mem.options.shrink_on_reset || mem.options.protect_unused_pages) && page_after < last_page)
+	{
+		void* start = mem.ptr + page_after << shift;
+		usz len = (last_page - page_after) << shift;
+		if (mem.options.shrink_on_reset) posix::madvise(start, len, posix::MADV_DONTNEED);
+		if (mem.options.protect_unused_pages) posix::mprotect(start, len, posix::PROT_NONE);
+	}
+	mem.allocated = after;
+}

lib/std/core/mem.c3

@@ -286,7 +286,7 @@ macro compare_exchange_volatile(ptr, compare, value, AtomicOrdering $success = S
 *>
 fn usz aligned_offset(usz offset, usz alignment)
 {
-	return alignment * ((offset + alignment - 1) / alignment);
+	return (offset + alignment - 1) & ~(alignment - 1);
 }
 
 macro void* aligned_pointer(void* ptr, usz alignment)

lib/std/core/mem_allocator.c3

@@ -1,7 +1,7 @@
 module std::core::mem::allocator;
 import std::math;
 
-// C3 has multiple different allocators available:
+// C3 has several different allocators available:
 //
 // Name                     Arena  Uses buffer  OOM Fallback?     Mark?   Reset?
 // ArenaAllocator             Yes          Yes             No       Yes      Yes
@@ -12,6 +12,7 @@ import std::math;
 // OnStackAllocator           Yes          Yes            Yes        No       No   *Note: Used by @stack_mem
 // TempAllocator              Yes           No            Yes        No*      No*  *Note: Mark/reset using @pool
 // TrackingAllocator           No           No            N/A        No       No   *Note: Wraps other heap allocator
+// Vmem                       Yes           No             No       Yes      Yes   *Note: Can be set to huge sizes
 
 const DEFAULT_SIZE_PREFIX = usz.sizeof;
 const DEFAULT_SIZE_PREFIX_ALIGNMENT = usz.alignof;