From 440df8415e1638240b3ce88ba9a0a2ad1459ce97 Mon Sep 17 00:00:00 2001 From: hamkoroke <147899529+hamkoroke@users.noreply.github.com> Date: Mon, 4 Aug 2025 06:26:52 +0900 Subject: [PATCH] Support memory mapped files and add File.map (#2321) * Support memory mapped files and add File.map --------- Co-authored-by: Christoffer Lerno --- lib/std/core/os/mem_vm.c3 | 32 ++++++++- lib/std/io/file_mmap.c3 | 92 +++++++++++++++++++++++++ releasenotes.md | 2 + test/unit/stdlib/io/file_mmap.c3 | 114 +++++++++++++++++++++++++++++++ 4 files changed, 239 insertions(+), 1 deletion(-) create mode 100644 lib/std/io/file_mmap.c3 create mode 100644 test/unit/stdlib/io/file_mmap.c3 diff --git a/lib/std/core/os/mem_vm.c3 b/lib/std/core/os/mem_vm.c3 index a74f963c9..900119246 100644 --- a/lib/std/core/os/mem_vm.c3 +++ b/lib/std/core/os/mem_vm.c3 @@ -2,7 +2,7 @@ The VM module holds code for working with virtual memory on supported platforms (currently Win32 and Posix) *> module std::core::mem::vm; -import std::os::win32, std::os::posix, libc; +import std::io, std::os::win32, std::os::posix, libc; <* VirtualMemory is an abstraction for working with an allocated virtual memory area. It will invoke vm:: functions @@ -219,6 +219,36 @@ fn void? decommit(void* ptr, usz len, bool block = true) $endswitch } +<* + Map a portion of an already-opened file into memory. + + @param fd : "File descriptor" + @param size : "Number of bytes to map, will be rounded up to page size" + @param offset : "Byte offset in file, must be page size aligned" + @param access : "The initial access permissions" + @param shared : "if True then MAP_SHARED else MAP_PRIVATE" + @return? mem::OUT_OF_MEMORY, RANGE_OVERFLOW, UNKNOWN_ERROR, ACCESS_DENIED, INVALID_ARGS, io::NO_PERMISSION, io::FILE_NOT_VALID, io::WOULD_BLOCK, io::FILE_NOT_FOUND + @return "Pointer to the mapped region" +*> +fn void*? mmap_file(Fd fd, usz size, usz offset = 0, VirtualMemoryAccess access = READ, bool shared = false) @if (env::POSIX) +{ + CInt flags = shared ? posix::MAP_SHARED : posix::MAP_PRIVATE; + void* ptr = posix::mmap(null, aligned_alloc_size(size), access.to_posix(), flags, fd, offset); + if (ptr != posix::MAP_FAILED) return ptr; + switch (libc::errno()) + { + case errno::ENOMEM: return mem::OUT_OF_MEMORY?; + case errno::EOVERFLOW: return RANGE_OVERFLOW?; + case errno::EPERM: return ACCESS_DENIED?; + case errno::EINVAL: return INVALID_ARGS?; + case errno::EACCES: return io::NO_PERMISSION?; + case errno::EBADF: return io::FILE_NOT_VALID?; + case errno::EAGAIN: return io::WOULD_BLOCK?; + case errno::ENXIO: return io::FILE_NOT_FOUND?; + default: return UNKNOWN_ERROR?; + } +} + <* Create a VirtualMemory using diff --git a/lib/std/io/file_mmap.c3 b/lib/std/io/file_mmap.c3 new file mode 100644 index 000000000..a50f346b0 --- /dev/null +++ b/lib/std/io/file_mmap.c3 @@ -0,0 +1,92 @@ +module std::io::file::mmap @if(env::LIBC &&& env::POSIX); +import std::core::mem::vm, std::io::file; + +struct FileMmap +{ + File file; + VirtualMemory vm; + usz offset; + usz len; +} + +<* + Provides a slice of bytes to the expected mapped range discarding the extra bytes due to misaligment of offset and/or size. + + @return "Slice of the mapped range where the first byte matches the file's byte at the offset specified to File::file_mmap()" +*> +fn char[] FileMmap.bytes(&self) +{ + return self.vm.ptr[self.offset:self.len]; +} + +<* + Destroys the underlyng VirtualMemory object ie. calls munmap()" +*> +fn void? FileMmap.destroy(&self) @maydiscard +{ + fault err1 = @catch(self.file.close()); + fault err2 = @catch(self.vm.destroy()); + if (err1) return err1?; + if (err2) return err2?; +} + +module std::io::file @if(env::LIBC &&& env::POSIX); + +<* + Maps a region of an already-opened file into memory + + @param file : "Already opened file created on the caller scope" + @param offset : "Byte offset in file, will be rounded down to page size" + @param len : "Size in bytes to map starting from offset, will be rounded up to page size" + @return? mem::OUT_OF_MEMORY, vm::ACCESS_DENIED, vm::RANGE_OVERFLOW, vm::INVALID_ARGS, vm::UNKNOWN_ERROR, io::NO_PERMISSION, io::FILE_NOT_VALID, io::WOULD_BLOCK, io::FILE_NOT_FOUND + @return "Memory mapped region. Must be released with FileMmap.destroy(). Provided File will not be closed" +*> +fn mmap::FileMmap? mmap_file(File file, usz offset = 0, usz len = 0, vm::VirtualMemoryAccess access = READ, bool shared = false) +{ + if (len == 0) + { + usz cur = file.seek(0, CURSOR)!; + defer file.seek(cur, SET)!!; + usz file_size = file.seek(0, END)!; + len = file_size - offset; + } + + // get the page size + usz page_size = vm::aligned_alloc_size(0); + + // align the offset specified by the user (might be not aligned) + usz page_offset = offset & (page_size - 1); + usz map_offset = offset - page_offset; + + // adjust map length (both the region start and the region end might be not aligned) + usz map_len = len + page_offset; // when region start not aligned + map_len = vm::aligned_alloc_size(map_len); // when region end not aligned + + void* ptr = vm::mmap_file(file.fd(), map_len, map_offset, access, shared)!; + + // FileMmap does not own the supplied file + return {{}, {ptr, map_len, access}, page_offset, len}; +} + +<* + Maps a region of the given file into memory + + @param filename : "File path" + @param mode : "File opening mode" + @param offset : "Byte offset in file, will be rounded down to page size" + @param len : "Size in bytes to map starting from offset, will be rounded up to page size" + @return? mem::OUT_OF_MEMORY, vm::ACCESS_DENIED, vm::RANGE_OVERFLOW, vm::INVALID_ARGS, vm::UNKNOWN_ERROR, io::NO_PERMISSION, io::FILE_NOT_VALID, io::WOULD_BLOCK, io::FILE_NOT_FOUND + @return "Memory mapped region. Must be released with FileMmap.destroy()" +*> +fn mmap::FileMmap? mmap_open(String filename, String mode, usz offset = 0, usz len = 0, vm::VirtualMemoryAccess access = READ, bool shared = false) +{ + File file = open(filename, mode)!; + defer catch (void)file.close(); + FileMmap mm = mmap_file(file, offset, len, access, shared)!; + + // FileMmap owns the file and it will close it on destroy() + mm.file = file; + + return mm; +} + diff --git a/releasenotes.md b/releasenotes.md index 1e41d2092..8d6e35260 100644 --- a/releasenotes.md +++ b/releasenotes.md @@ -16,6 +16,8 @@ ### Stdlib changes - Add `==` to `Pair`, `Triple` and TzDateTime. Add print to `Pair` and `Triple`. - Add OpenBSD to `env::INET_DEVICES` and add required socket constants. +- Added `FileMmap` to manage memory mapped files. +- Add `vm::mmap_file` to memory map a file. ## 0.7.4 Change list diff --git a/test/unit/stdlib/io/file_mmap.c3 b/test/unit/stdlib/io/file_mmap.c3 new file mode 100644 index 000000000..53496d53f --- /dev/null +++ b/test/unit/stdlib/io/file_mmap.c3 @@ -0,0 +1,114 @@ +module std::io::file @if(env::LIBC &&& env::POSIX); + +import std::io; + +fn void write_file(String path, char[] data) +{ + File f = file::open(path, "wb")!!; + defer f.close()!!; + f.write(data)!!; +} + +fn void map_read_only() @test +{ + const String FNAME = "tmp/map_test_ro"; + char[] msg = "Hello, world"; + write_file(FNAME, msg); + + File f = file::open(FNAME, "rb")!!; + defer f.close()!!; + + FileMmap mapped_file = file::mmap_file(f, 0, 0, vm::VirtualMemoryAccess.READ)!!; + defer mapped_file.destroy()!!; + + char[] view = mapped_file.bytes(); + assert(view.len == msg.len); + for (usz i = 0; i < view.len; i += 1) + { + assert(view[i] == msg[i]); + } +} + +fn void map_read_write() @test +{ + const String FNAME = "tmp/map_test_rw"; + char[4] data = "ABCD"; + write_file(FNAME, data[..]); + + bool shared = true; + File f = file::open(FNAME, "r+")!!; + mmap::FileMmap region = file::mmap_file(f, 0, 0, vm::VirtualMemoryAccess.READWRITE, shared)!!; + + region.bytes()[1] = 'Z'; + + region.destroy()!!; + f.close()!!; + + File fr = file::open(FNAME, "r")!!; + defer fr.close()!!; + char[4] check; + fr.read(check[..])!!; + assert(check[0] == 'A' && check[1] == 'Z' && check[2] == 'C'); +} + +fn void map_with_offset(String fname, usz offset, usz size, char[] reference_msg) +{ + File f = file::open(fname, "r")!!; + defer f.close()!!; + + mmap::FileMmap mapped_file = file::mmap_file(f, offset, size, vm::VirtualMemoryAccess.READ)!!; + defer mapped_file.destroy()!!; + + usz expected_size = size; + if (size == 0) + { + expected_size = reference_msg.len - offset; + } + char[] view = mapped_file.bytes(); + assert(view.len == expected_size); + assert(view[0] == reference_msg[offset]); +} + +fn void map_read_only_with_offset_and_size_zero() @test +{ + const String FNAME = "tmp/map_test_ro_with_offset"; + char[] msg = "Hello, world"; + write_file(FNAME, msg); + + usz map_size = 0; + for (usz i = 0; i < msg.len; i += 1) + { + map_with_offset(FNAME, i, map_size, msg); + } +} + +fn void map_read_only_with_offset() @test +{ + const String FNAME = "tmp/map_test_ro_with_offset"; + char[] msg = "Hello, world"; + write_file(FNAME, msg); + + usz map_size = 3; + for (usz i = 0; i < msg.len - map_size; i++) + { + map_with_offset(FNAME, i, map_size, msg); + } +} + +fn void map_from_filename() @test +{ + const String FNAME = "tmp/map_test_ro"; + char[] msg = "Hello, world"; + write_file(FNAME, msg); + + mmap::FileMmap mapped_file = file::mmap_open(FNAME, "rb", 0, 0, vm::VirtualMemoryAccess.READ)!!; + defer mapped_file.destroy()!!; + + char[] view = mapped_file.bytes(); + test::eq(view.len, msg.len); + for (usz i = 0; i < view.len; i++) + { + test::eq(view[i], msg[i]); + } +} +