diff --git a/lib/std/core/allocators/temp_allocator.c3 b/lib/std/core/allocators/temp_allocator.c3 index a51d85cdb..d63150584 100644 --- a/lib/std/core/allocators/temp_allocator.c3 +++ b/lib/std/core/allocators/temp_allocator.c3 @@ -1,4 +1,4 @@ -module std::core::mem::allocator @if(!env::POSIX || !$feature(VMEM_TEMP)); +module std::core::mem::allocator @if(!(env::POSIX || env::WIN32) || !$feature(VMEM_TEMP)); import std::io, std::math; // This implements the temp allocator. @@ -326,7 +326,7 @@ fn void*? TempAllocator.acquire(&self, usz size, AllocInitType init_type, usz al return &page.data[0]; } -module std::core::mem::allocator @if(env::POSIX && $feature(VMEM_TEMP)); +module std::core::mem::allocator @if((env::POSIX || env::WIN32) && $feature(VMEM_TEMP)); import std::math; tlocal VmemOptions temp_allocator_default_options = { diff --git a/lib/std/core/allocators/vmem.c3 b/lib/std/core/allocators/vmem.c3 index 7728b5bca..b56e1e3b5 100644 --- a/lib/std/core/allocators/vmem.c3 +++ b/lib/std/core/allocators/vmem.c3 @@ -1,5 +1,7 @@ -module std::core::mem::allocator @if(env::POSIX); +module std::core::mem::allocator @if(env::POSIX || env::WIN32); import std::math, std::os::posix, libc, std::bits; +import std::core::mem; + // Virtual Memory allocator @@ -7,9 +9,8 @@ faultdef VMEM_RESERVE_FAILED, VMEM_PROTECT_FAILED; struct Vmem (Allocator) { - void* ptr; + VirtualMemory memory; usz allocated; - usz capacity; usz pagesize; usz page_pot; usz last_page; @@ -34,37 +35,35 @@ bitstruct VmemOptions : int *> fn void? Vmem.init(&self, usz preferred_size, usz reserve_page_size = 0, VmemOptions options = { true, true, env::COMPILER_SAFE_MODE }, usz min_size = 0) { - void* ptr; - static usz page_size = 0; - if (!page_size) page_size = posix::getpagesize(); + static usz page_size = 0; + if (!page_size) page_size = mem::os_pagesize(); if (page_size < reserve_page_size) page_size = reserve_page_size; preferred_size = mem::aligned_offset(preferred_size, page_size); if (!min_size) min_size = max(preferred_size / 1024, 1); + VirtualMemory? memory = mem::OUT_OF_MEMORY?; while (preferred_size >= min_size) { - ptr = posix::mmap(null, preferred_size, posix::PROT_NONE, posix::MAP_PRIVATE | posix::MAP_ANONYMOUS, -1, 0); + memory = mem::virtual_alloc(preferred_size, PROTECTED); // It worked? - if (ptr != posix::MAP_FAILED && ptr) break; - // Did it fail in a non-retriable way? - switch (libc::errno()) + if (try memory) break; + switch (@catch(memory)) { - case errno::ENOMEM: - case errno::EOVERFLOW: - case errno::EAGAIN: + case mem::OUT_OF_MEMORY: + case mem::VMEM_OVERFLOW: // Try a smaller size. preferred_size /= 2; continue; + default: + break; } - break; } - // Check if we ended on a failure. - if ((ptr == posix::MAP_FAILED) || !ptr) return VMEM_RESERVE_FAILED?; + if (catch memory) return VMEM_RESERVE_FAILED?; if (page_size > preferred_size) page_size = preferred_size; $if env::ADDRESS_SANITIZER: - asan::poison_memory_region(self.ptr, self.capacity); + asan::poison_memory_region(memory.ptr, memory.size); $endif - *self = { .ptr = ptr, .high_water = 0, - .capacity = preferred_size, + *self = { .memory = memory, + .high_water = 0, .pagesize = page_size, .page_pot = page_size.ctz(), .options = options, @@ -82,12 +81,12 @@ fn void? Vmem.init(&self, usz preferred_size, usz reserve_page_size = 0, VmemOpt fn void*? Vmem.acquire(&self, usz size, AllocInitType init_type, usz alignment) @dynamic { alignment = alignment_for_allocation(alignment); - usz total_len = self.capacity; + usz total_len = self.memory.size; if (size > total_len) return mem::INVALID_ALLOC_SIZE?; - void* start_mem = self.ptr; + void* start_mem = self.memory.ptr; void* unaligned_pointer_to_offset = start_mem + self.allocated + VmemHeader.sizeof; void* mem = mem::aligned_pointer(unaligned_pointer_to_offset, alignment); - usz after = (usz)(mem - self.ptr) + size; + usz after = (usz)(mem - start_mem) + size; if (after > total_len) return mem::OUT_OF_MEMORY?; if (init_type == ZERO && self.high_water <= self.allocated) { @@ -102,7 +101,7 @@ fn void*? Vmem.acquire(&self, usz size, AllocInitType init_type, usz alignment) fn bool Vmem.owns_pointer(&self, void* ptr) @inline { - return (uptr)ptr >= (uptr)self.ptr && (uptr)ptr < (uptr)self.ptr + self.capacity; + return (uptr)ptr >= (uptr)self.memory.ptr && (uptr)ptr < (uptr)self.memory.ptr + self.memory.size; } <* Implements the Allocator interface method. @@ -115,13 +114,13 @@ fn bool Vmem.owns_pointer(&self, void* ptr) @inline *> fn void*? Vmem.resize(&self, void *old_pointer, usz size, usz alignment) @dynamic { - if (size > self.capacity) return mem::INVALID_ALLOC_SIZE?; + if (size > self.memory.size) return mem::INVALID_ALLOC_SIZE?; alignment = alignment_for_allocation(alignment); - assert(self.owns_pointer(old_pointer), "Pointer originates from a different allocator: %p, not in %p - %p", old_pointer, self.ptr, self.ptr + self.allocated); + assert(self.owns_pointer(old_pointer), "Pointer originates from a different allocator: %p, not in %p - %p", old_pointer, self.memory.ptr, self.memory.ptr + self.allocated); VmemHeader* header = old_pointer - VmemHeader.sizeof; usz old_size = header.size; // Do last allocation and alignment match? - if (self.ptr + self.allocated == old_pointer + old_size && mem::ptr_is_aligned(old_pointer, alignment)) + if (self.memory.ptr + self.allocated == old_pointer + old_size && mem::ptr_is_aligned(old_pointer, alignment)) { if (old_size == size) return old_pointer; if (old_size >= size) @@ -131,7 +130,7 @@ fn void*? Vmem.resize(&self, void *old_pointer, usz size, usz alignment) @dynami else { usz allocated = self.allocated + size - old_size; - if (allocated > self.capacity) return mem::OUT_OF_MEMORY?; + if (allocated > self.memory.size) return mem::OUT_OF_MEMORY?; protect(self, allocated)!; } header.size = size; @@ -153,7 +152,7 @@ fn void Vmem.release(&self, void* ptr, bool) @dynamic assert(self.owns_pointer(ptr), "Pointer originates from a different allocator %p.", ptr); VmemHeader* header = ptr - VmemHeader.sizeof; // Reclaim memory if it's the last element. - if (ptr + header.size == self.ptr + self.allocated) + if (ptr + header.size == self.memory.ptr + self.allocated) { unprotect(self, self.allocated - header.size - VmemHeader.sizeof); } @@ -175,14 +174,14 @@ fn void Vmem.reset(&self, usz mark) fn void Vmem.free(&self) { - if (!self.ptr) return; + if (!self.memory.ptr) return; $switch: $case env::ADDRESS_SANITIZER: - asan::poison_memory_region(self.ptr, self.capacity); + asan::poison_memory_region(self.memory.ptr, self.memory.size); $case env::COMPILER_SAFE_MODE: - ((char*)self.ptr)[0:self.allocated] = 0xAA; + ((char*)self.memory.ptr)[0:self.allocated] = 0xAA; $endswitch - posix::munmap(self.ptr, self.capacity); + self.memory.destroy(); *self = {}; } @@ -202,14 +201,17 @@ macro void? protect(Vmem* mem, usz after) @local bool over_high_water = mem.high_water < after; if (page_after > last_page) { + usz page_start = last_page << shift; + usz page_len = (page_after - last_page) << shift; + mem.memory.commit(page_start, page_len)!; if (mem.options.protect_unused_pages || over_high_water) { - if (posix::mprotect(mem.ptr + last_page << shift, (page_after - last_page) << shift, posix::PROT_WRITE | posix::PROT_READ)) return VMEM_PROTECT_FAILED?; + mem.memory.set_access(page_start, page_len, READWRITE)!; } mem.last_page = page_after; } $if env::ADDRESS_SANITIZER: - asan::unpoison_memory_region(mem.ptr + mem.allocated, after - mem.allocated); + asan::unpoison_memory_region(mem.memory.ptr + mem.allocated, after - mem.allocated); $endif mem.allocated = after; if (over_high_water) mem.high_water = after; @@ -221,19 +223,19 @@ macro void unprotect(Vmem* mem, usz after) @local usz last_page = mem.last_page; usz page_after = mem.last_page = (after + mem.pagesize - 1) >> shift; $if env::ADDRESS_SANITIZER: - asan::poison_memory_region(mem.ptr + after, mem.allocated - after); + asan::poison_memory_region(mem.memory.ptr + after, mem.allocated - after); $else if (mem.options.scratch_released_data) { - mem::set(mem.ptr + after, 0xAA, mem.allocated - after); + mem::set(mem.memory.ptr + after, 0xAA, mem.allocated - after); } $endif if ((mem.options.shrink_on_reset || mem.options.protect_unused_pages) && page_after < last_page) { - void* start = mem.ptr + page_after << shift; + usz start = page_after << shift; usz len = (last_page - page_after) << shift; - if (mem.options.shrink_on_reset) posix::madvise(start, len, posix::MADV_DONTNEED); - if (mem.options.protect_unused_pages) posix::mprotect(start, len, posix::PROT_NONE); + if (mem.options.shrink_on_reset) (void)mem.memory.release(start, len); + if (mem.options.protect_unused_pages) (void)mem.memory.set_access(start, len, PROTECTED); } mem.allocated = after; } diff --git a/lib/std/core/mem.c3 b/lib/std/core/mem.c3 index e8f87505a..40025f456 100644 --- a/lib/std/core/mem.c3 +++ b/lib/std/core/mem.c3 @@ -3,6 +3,7 @@ // a copy of which can be found in the LICENSE_STDLIB file. module std::core::mem; import std::core::mem::allocator @public; +import std::os::posix; import std::math; const MAX_MEMORY_ALIGNMENT = 0x1000_0000; @@ -19,6 +20,19 @@ macro bool @constant_is_power_of_2($x) @const @private return $x != 0 && ($x & ($x - 1)) == 0; } +fn usz os_pagesize() +{ + $switch: + $case env::POSIX: + return posix::getpagesize(); + $case env::WIN32: + // Possibly improve this + return 4096; + $default: + return 4096; + $endswitch +} + <* Load a vector from memory according to a mask assuming default alignment. diff --git a/lib/std/core/os/virtual_mem.c3 b/lib/std/core/os/virtual_mem.c3 new file mode 100644 index 000000000..f6c1257e7 --- /dev/null +++ b/lib/std/core/os/virtual_mem.c3 @@ -0,0 +1,176 @@ +module std::core::mem; +import std::core::env; + +struct VirtualMemory +{ + void* ptr; + usz size; +} + +faultdef VMEM_OVERFLOW, VMEM_UNKNOWN_ERROR, VMEM_ACCESS_NOT_ALLOWED, VMEM_PAGE_NOT_ALIGNED, VMEM_UPDATE_FAILED; + + +module std::core::mem @if(env::POSIX); +import std::os::posix, libc::errno, std::math; + +enum VirtualMemoryAccess : int(int posix_val) +{ + PROTECTED = posix::PROT_NONE, + READABLE = posix::PROT_READ, + WRITABLE = posix::PROT_WRITE, + READWRITE = posix::PROT_READ | posix::PROT_WRITE +} + +<* + @param size : "The size of the memory to allocate." + @param access : "The initial access." + @require size > 0 : "The size must be non-zero" + @return? OUT_OF_MEMORY, VMEM_OVERFLOW, VMEM_UNKNOWN_ERROR +*> +fn VirtualMemory? virtual_alloc(usz size, VirtualMemoryAccess access) +{ + void *ptr = posix::mmap(null, size, access.posix_val, posix::MAP_PRIVATE | posix::MAP_ANONYMOUS, -1, 0); + if (ptr != posix::MAP_FAILED && ptr) return { ptr, size }; + switch (libc::errno()) + { + case errno::ENOMEM: return OUT_OF_MEMORY?; + case errno::EOVERFLOW: return VMEM_OVERFLOW?; + default: return VMEM_UNKNOWN_ERROR?; + } +} + +macro void? VirtualMemory.commit(self, usz offset, usz len) {} + +<* + @param offset : "Starting from what offset to update" + @param len : "To what len to update" + @param access : "The new access" + @require offset < self.size : "Offset out of range" + @require offset + len < self.size : "Length out of range" + @require offset == 0 || math::is_power_of_2(offset) : "Offset should be a power of 2" + @require math::is_power_of_2(len) : "Length should be a multiple of the page size" + @return? VMEM_ACCESS_NOT_ALLOWED, VMEM_PAGE_NOT_ALIGNED, VMEM_OVERFLOW, VMEM_UNKNOWN_ERROR +*> +fn void? VirtualMemory.set_access(self, usz offset, usz len, VirtualMemoryAccess access) +{ + if (posix::mprotect(self.ptr + offset, len, access.posix_val)) + { + switch (libc::errno()) + { + case errno::EACCES: return VMEM_ACCESS_NOT_ALLOWED?; + case errno::EINVAL: return VMEM_PAGE_NOT_ALIGNED?; + case errno::EOVERFLOW: return VMEM_OVERFLOW?; + case errno::ENOMEM: abort("Vmem access out of range"); + default: return VMEM_UNKNOWN_ERROR?; + } + } +} + +<* + @param offset : "Starting from what offset to release" + @param len : "To what len to release" + @require offset < self.size : "Offset out of range" + @require offset + len < self.size : "Length out of range" + @require offset == 0 || math::is_power_of_2(offset) : "Offset should be a power of 2" + @require math::is_power_of_2(len) : "Length should be a multiple of the page size" + @return? VMEM_UPDATE_FAILED +*> +fn void? VirtualMemory.release(self, usz offset, usz len) +{ + if (posix::madvise(self.ptr + offset, len, posix::MADV_DONTNEED)) return VMEM_UPDATE_FAILED?; +} + +<* + @require self.ptr != null : "Virtual memory must be initialized to call destroy" +*> +fn void VirtualMemory.destroy(&self) +{ + posix::munmap(self.ptr, self.size); + self.ptr = null; +} + +module std::core::mem @if(env::WIN32); +import std::os::win32, std::math; + +enum VirtualMemoryAccess : int(Win32_Protect win32_val) +{ + PROTECTED = PAGE_NOACCESS, + READABLE = PAGE_READONLY, + WRITABLE = PAGE_READWRITE, + READWRITE = PAGE_READWRITE +} + +<* + @param size : "The size of the memory to allocate." + @param access : "The initial access." + @require size > 0 : "The size must be non-zero" + @return? OUT_OF_MEMORY, VMEM_UNKNOWN_ERROR +*> +fn VirtualMemory? virtual_alloc(usz size, VirtualMemoryAccess access) +{ + void *ptr = win32::virtualAlloc(null, size, MEM_RESERVE, access.win32_val); + if (ptr) return { ptr, size }; + switch (win32::getLastError()) + { + case win32::ERROR_NOT_ENOUGH_MEMORY: + case win32::ERROR_COMMITMENT_LIMIT: return OUT_OF_MEMORY?; + default: return VMEM_UNKNOWN_ERROR?; + } +} + +<* + @param offset : "Starting from what offset to commit" + @param len : "To what len to commit" + @require offset < self.size : "Offset out of range" + @require offset + len < self.size : "Length out of range" + @require offset == 0 || math::is_power_of_2(offset) : "Offset should be a power of 2" + @require math::is_power_of_2(len) : "Length should be a multiple of the page size" + @return? VMEM_UPDATE_FAILED +*> +macro void? VirtualMemory.commit(self, usz offset, usz len) +{ + void *res = win32::virtualAlloc(self.ptr + offset, len, MEM_COMMIT, PAGE_READWRITE); + if (!res) return VMEM_UPDATE_FAILED?; +} + +<* + @param offset : "Starting from what offset to commit" + @param len : "To what len to commit" + @require offset < self.size : "Offset out of range" + @require offset + len < self.size : "Length out of range" + @require offset == 0 || math::is_power_of_2(offset) : "Offset should be a power of 2" + @require math::is_power_of_2(len) : "Length should be a multiple of the page size" + @return? VMEM_UPDATE_FAILED +*> +macro void? VirtualMemory.set_access(self, usz offset, usz len, VirtualMemoryAccess access) +{ + Win32_Protect old; + if (!win32::virtualProtect(self.ptr + offset, len, access.win32_val, &old)) + { + return VMEM_UPDATE_FAILED?; + } +} + +<* + @param offset : "Starting from what offset to release" + @param len : "To what len to release" + @require offset < self.size : "Offset out of range" + @require offset + len < self.size : "Length out of range" + @require offset == 0 || math::is_power_of_2(offset) : "Offset should be a power of 2" + @require math::is_power_of_2(len) : "Length should be a multiple of the page size" + @return? VMEM_UPDATE_FAILED +*> +fn void? VirtualMemory.release(self, usz offset, usz len) +{ + if (!win32::virtualFree(self.ptr + offset, len, MEM_DECOMMIT)) return VMEM_UPDATE_FAILED?; +} + +<* + @require self.ptr != null : "Virtual memory must be initialized to call destroy" +*> +fn void VirtualMemory.destroy(&self) +{ + win32::virtualFree(self.ptr, 0, MEM_RELEASE); + self.ptr = null; +} + diff --git a/lib/std/os/win32/general.c3 b/lib/std/os/win32/general.c3 index 355910703..da6db3f64 100644 --- a/lib/std/os/win32/general.c3 +++ b/lib/std/os/win32/general.c3 @@ -223,4 +223,5 @@ const Win32_DWORD ERROR_SCOPE_NOT_FOUND = 0x13E; const Win32_DWORD ERROR_UNDEFINED_SCOPE = 0x13F; const Win32_DWORD ERROR_IO_INCOMPLETE = 0x3E4; const Win32_DWORD ERROR_IO_PENDING = 0x3E5; -const Win32_DWORD ERROR_TIMEOUT = 0x5B4; \ No newline at end of file +const Win32_DWORD ERROR_TIMEOUT = 0x5B4; +const Win32_DWORD ERROR_COMMITMENT_LIMIT = 0x5AF; \ No newline at end of file diff --git a/lib/std/os/win32/memoryapi.c3 b/lib/std/os/win32/memoryapi.c3 index 1919a0eda..a3e2e47ed 100644 --- a/lib/std/os/win32/memoryapi.c3 +++ b/lib/std/os/win32/memoryapi.c3 @@ -36,6 +36,7 @@ enum Win32_FreeType : const Win32_DWORD MEM_COALESCE_PLACEHOLDERS = 0x00000001, MEM_PRESERVE_PLACEHOLDER = 0x00000002, } -extern fn Win32_LPVOID virtualAlloc(Win32_LPVOID lpAddres, Win32_SIZE_T dwSize, Win32_AllocationType flAllocationType, Win32_DWORD flProtect) @extern("VirtualAlloc"); +extern fn Win32_LPVOID virtualAlloc(Win32_LPVOID lpAddres, Win32_SIZE_T dwSize, Win32_AllocationType flAllocationType, Win32_Protect flProtect) @extern("VirtualAlloc"); extern fn Win32_PVOID virtualAlloc2(Win32_HANDLE process, Win32_PVOID baseAddress, Win32_SIZE_T size, Win32_AllocationType allocationType, Win32_ULONG pageProtection, Win32_MEM_EXTENDED_PARAMETER* extendedParameters, Win32_ULONG parameterCount) @extern("VirtualAlloc2"); extern fn Win32_BOOL virtualFree(Win32_LPVOID lpAddress, Win32_SIZE_T dwSize, Win32_FreeType dwFreeType) @extern("VirtualFree"); +extern fn Win32_BOOL virtualProtect(Win32_LPVOID lpAddress, Win32_SIZE_T dwSize, Win32_Protect flNewProtect, Win32_Protect* lpflOldProtect) @extern("VirtualProtect"); \ No newline at end of file diff --git a/releasenotes.md b/releasenotes.md index 1587b560d..78564f12f 100644 --- a/releasenotes.md +++ b/releasenotes.md @@ -55,6 +55,7 @@ - Improve contract for readline. #2280 - Added Whirlpool hash. - Added string::bformat. +- VirtualMemory type and functions. ## 0.7.3 Change list diff --git a/test/test_suite/debug_symbols/defer_macro.c3t b/test/test_suite/debug_symbols/defer_macro.c3t index 013eac22e..2bbb2c46c 100644 --- a/test/test_suite/debug_symbols/defer_macro.c3t +++ b/test/test_suite/debug_symbols/defer_macro.c3t @@ -445,7 +445,7 @@ cache_hit: ; preds = %if.exit missing_function: ; preds = %17 %19 = load ptr, ptr @std.core.builtin.panic, align 8, !dbg !167 - call void %19(ptr @.panic_msg, i64 44, ptr @.file, i64 16, ptr @.func, i64 6, i32 85) #5, !dbg !167 + call void %19(ptr @.panic_msg, i64 44, ptr @.file, unreachable, !dbg !167 match: ; preds = %17 @@ -681,7 +681,7 @@ no_match: ; preds = %compare !72 = !DILocation(line: 35, column: 49, scope: !66) !73 = !DILocalVariable(name: "name", arg: 3, scope: !66, file: !7, line: 35, type: !39) !74 = !DILocation(line: 35, column: 63, scope: !66) -!76 = distinct !DISubprogram(name: "new", linkageName: "new", scope: !77, file: !77, line: 710, scopeLine: 710, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !6) +!76 = distinct !DISubprogram(name: "new", linkageName: "new", scope: !77, file: !77 !77 = !DIFile(filename: "mem.c3", directory: !78 = !DILocation(line: 37, column: 9, scope: !66) !79 = distinct !DISubprogram(name: "test", linkageName: "test.test", scope: !7, file: !7, line: 45, type: !80, scopeLine: 45, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !21) diff --git a/test/test_suite/defer/defer_catch_mix.c3t b/test/test_suite/defer/defer_catch_mix.c3t index 7e5503b86..f4230bc19 100644 --- a/test/test_suite/defer/defer_catch_mix.c3t +++ b/test/test_suite/defer/defer_catch_mix.c3t @@ -343,7 +343,7 @@ missing_function: ; preds = %6 store %"char[]" { ptr @.func, i64 4 }, ptr %taddr5, align 8 %10 = load [2 x i64], ptr %taddr5, align 8 %11 = load ptr, ptr @std.core.builtin.panic, align 8 - call void %11([2 x i64] %8, [2 x i64] %9, [2 x i64] %10, i32 97) #4 + call void %11([2 x i64] %8, [2 x i64] %9, [2 x i64] %10, unreachable match: ; preds = %6 @@ -379,7 +379,7 @@ panic_block: ; preds = %assign_optional %"$$temp" = insertvalue %"any[]" %24, i64 1, 1 store %"any[]" %"$$temp", ptr %taddr10, align 8 %25 = load [2 x i64], ptr %taddr10, align 8 - call void @std.core.builtin.panicf([2 x i64] %21, [2 x i64] %22, [2 x i64] %23, i32 261, [2 x i64] %25) #4 + call void @std.core.builtin.panicf([2 x i64] %21, [2 x i64] %22, [2 x i64] %23, unreachable noerr_block: ; preds = %after_check