Support memory mapped files and add File.map (#2321)

* Support memory mapped files and add File.map

---------

Co-authored-by: Christoffer Lerno <christoffer@aegik.com>

lib/std/core/os/mem_vm.c3

@@ -2,7 +2,7 @@
  The VM module holds code for working with virtual memory on supported platforms (currently Win32 and Posix)
 *>
 module std::core::mem::vm;
-import std::os::win32, std::os::posix, libc;
+import std::io, std::os::win32, std::os::posix, libc;
 
 <*
  VirtualMemory is an abstraction for working with an allocated virtual memory area. It will invoke vm:: functions
@@ -219,6 +219,36 @@ fn void? decommit(void* ptr, usz len, bool block = true)
 	$endswitch
 }
 
+<*
+ Map a portion of an already-opened file into memory.
+
+ @param fd : "File descriptor"
+ @param size : "Number of bytes to map, will be rounded up to page size"
+ @param offset : "Byte offset in file, must be page size aligned"
+ @param access : "The initial access permissions"
+ @param shared : "if True then MAP_SHARED else MAP_PRIVATE"
+ @return? mem::OUT_OF_MEMORY, RANGE_OVERFLOW, UNKNOWN_ERROR, ACCESS_DENIED, INVALID_ARGS, io::NO_PERMISSION, io::FILE_NOT_VALID, io::WOULD_BLOCK, io::FILE_NOT_FOUND
+ @return "Pointer to the mapped region"
+*>
+fn void*? mmap_file(Fd fd, usz size, usz offset = 0, VirtualMemoryAccess access = READ, bool shared = false) @if (env::POSIX)
+{
+	CInt flags = shared ? posix::MAP_SHARED : posix::MAP_PRIVATE;
+	void* ptr = posix::mmap(null, aligned_alloc_size(size), access.to_posix(), flags, fd, offset);
+	if (ptr != posix::MAP_FAILED) return ptr;
+	switch (libc::errno())
+	{
+		case errno::ENOMEM:    return mem::OUT_OF_MEMORY?;
+		case errno::EOVERFLOW: return RANGE_OVERFLOW?;
+		case errno::EPERM:     return ACCESS_DENIED?;
+		case errno::EINVAL:    return INVALID_ARGS?;
+		case errno::EACCES:    return io::NO_PERMISSION?;
+		case errno::EBADF:     return io::FILE_NOT_VALID?;
+		case errno::EAGAIN:    return io::WOULD_BLOCK?;
+		case errno::ENXIO:     return io::FILE_NOT_FOUND?;
+		default:               return UNKNOWN_ERROR?;
+	}
+}
+
 <*
  Create a VirtualMemory using
 

lib/std/io/file_mmap.c3

@@ -0,0 +1,92 @@
+module std::io::file::mmap @if(env::LIBC &&& env::POSIX);
+import std::core::mem::vm, std::io::file;
+
+struct FileMmap
+{
+	File file;
+	VirtualMemory vm;
+	usz offset;
+	usz len;
+}
+
+<*
+ Provides a slice of bytes to the expected mapped range discarding the extra bytes due to misaligment of offset and/or size.
+
+ @return "Slice of the mapped range where the first byte matches the file's byte at the offset specified to File::file_mmap()"
+*>
+fn char[] FileMmap.bytes(&self)
+{
+	return self.vm.ptr[self.offset:self.len];
+}
+
+<*
+  Destroys the underlyng VirtualMemory object ie. calls munmap()"
+*>
+fn void? FileMmap.destroy(&self) @maydiscard
+{
+	fault err1 = @catch(self.file.close());
+	fault err2 = @catch(self.vm.destroy());
+	if (err1) return err1?;
+	if (err2) return err2?;
+}
+
+module std::io::file @if(env::LIBC &&& env::POSIX);
+
+<*
+ Maps a region of an already-opened file into memory
+
+ @param file : "Already opened file created on the caller scope"
+ @param offset : "Byte offset in file, will be rounded down to page size"
+ @param len : "Size in bytes to map starting from offset, will be rounded up to page size"
+ @return? mem::OUT_OF_MEMORY, vm::ACCESS_DENIED, vm::RANGE_OVERFLOW, vm::INVALID_ARGS, vm::UNKNOWN_ERROR, io::NO_PERMISSION, io::FILE_NOT_VALID, io::WOULD_BLOCK, io::FILE_NOT_FOUND
+ @return "Memory mapped region. Must be released with FileMmap.destroy(). Provided File will not be closed"
+*>
+fn mmap::FileMmap? mmap_file(File file, usz offset = 0, usz len = 0, vm::VirtualMemoryAccess access = READ, bool shared = false)
+{
+	if (len == 0)
+	{
+		usz cur = file.seek(0, CURSOR)!;
+		defer file.seek(cur, SET)!!;
+		usz file_size = file.seek(0, END)!;
+		len = file_size - offset;
+	}
+
+	// get the page size
+	usz page_size = vm::aligned_alloc_size(0);
+
+	// align the offset specified by the user (might be not aligned)
+	usz page_offset = offset & (page_size - 1);
+	usz map_offset = offset - page_offset;
+	
+	// adjust map length (both the region start and the region end might be not aligned)
+	usz map_len = len + page_offset; // when region start not aligned
+	map_len = vm::aligned_alloc_size(map_len); // when region end not aligned
+
+	void* ptr = vm::mmap_file(file.fd(), map_len, map_offset, access, shared)!;
+
+	// FileMmap does not own the supplied file
+	return {{}, {ptr, map_len, access}, page_offset, len};
+}
+
+<*
+ Maps a region of the given file into memory
+
+ @param filename : "File path"
+ @param mode : "File opening mode"
+ @param offset : "Byte offset in file, will be rounded down to page size"
+ @param len : "Size in bytes to map starting from offset, will be rounded up to page size"
+ @return? mem::OUT_OF_MEMORY, vm::ACCESS_DENIED, vm::RANGE_OVERFLOW, vm::INVALID_ARGS, vm::UNKNOWN_ERROR, io::NO_PERMISSION, io::FILE_NOT_VALID, io::WOULD_BLOCK, io::FILE_NOT_FOUND
+ @return "Memory mapped region. Must be released with FileMmap.destroy()"
+*>
+fn mmap::FileMmap? mmap_open(String filename, String mode, usz offset = 0, usz len = 0, vm::VirtualMemoryAccess access = READ, bool shared = false)
+{
+	File file = open(filename, mode)!;
+	defer catch (void)file.close();
+	FileMmap mm = mmap_file(file, offset, len, access, shared)!;
+
+	// FileMmap owns the file and it will close it on destroy()
+	mm.file = file;
+
+	return mm;
+}
+

releasenotes.md

@@ -16,6 +16,8 @@
 ### Stdlib changes
 - Add `==` to `Pair`, `Triple` and TzDateTime. Add print to `Pair` and `Triple`.
 - Add OpenBSD to `env::INET_DEVICES` and add required socket constants.
+- Added `FileMmap` to manage memory mapped files.
+- Add `vm::mmap_file` to memory map a file.
 
 ## 0.7.4 Change list
 

test/unit/stdlib/io/file_mmap.c3

@@ -0,0 +1,114 @@
+module std::io::file @if(env::LIBC &&& env::POSIX);
+
+import std::io;
+
+fn void write_file(String path, char[] data)
+{
+	File f = file::open(path, "wb")!!;
+	defer f.close()!!;
+	f.write(data)!!;
+}
+
+fn void map_read_only() @test
+{
+	const String FNAME = "tmp/map_test_ro";
+	char[] msg = "Hello, world";
+	write_file(FNAME, msg);
+
+	File f = file::open(FNAME, "rb")!!;
+	defer f.close()!!;
+
+	FileMmap mapped_file = file::mmap_file(f, 0, 0, vm::VirtualMemoryAccess.READ)!!;
+	defer mapped_file.destroy()!!;
+	
+	char[] view = mapped_file.bytes();
+	assert(view.len == msg.len);
+	for (usz i = 0; i < view.len; i += 1)
+	{
+		assert(view[i] == msg[i]);
+	}
+}
+
+fn void map_read_write() @test
+{
+	const String FNAME = "tmp/map_test_rw";
+	char[4] data = "ABCD";
+	write_file(FNAME, data[..]);
+
+	bool shared = true;
+	File f = file::open(FNAME, "r+")!!;
+	mmap::FileMmap region = file::mmap_file(f, 0, 0, vm::VirtualMemoryAccess.READWRITE, shared)!!;
+
+	region.bytes()[1] = 'Z';
+
+	region.destroy()!!;
+	f.close()!!;
+
+	File fr = file::open(FNAME, "r")!!;
+	defer fr.close()!!;
+	char[4] check;
+	fr.read(check[..])!!;
+	assert(check[0] == 'A' && check[1] == 'Z' && check[2] == 'C');
+}
+
+fn void map_with_offset(String fname, usz offset, usz size, char[] reference_msg)
+{
+	File f = file::open(fname, "r")!!;
+	defer f.close()!!;
+
+	mmap::FileMmap mapped_file = file::mmap_file(f, offset, size, vm::VirtualMemoryAccess.READ)!!;
+	defer mapped_file.destroy()!!;
+
+	usz expected_size = size;
+	if (size == 0)
+	{
+		expected_size = reference_msg.len - offset;
+	}
+	char[] view = mapped_file.bytes();
+	assert(view.len == expected_size);
+	assert(view[0] == reference_msg[offset]);
+}
+
+fn void map_read_only_with_offset_and_size_zero() @test
+{
+	const String FNAME = "tmp/map_test_ro_with_offset";
+	char[] msg = "Hello, world";
+	write_file(FNAME, msg);
+
+	usz map_size = 0;
+	for (usz i = 0; i < msg.len; i += 1)
+	{
+		map_with_offset(FNAME, i, map_size, msg);
+	}
+}
+
+fn void map_read_only_with_offset() @test
+{
+	const String FNAME = "tmp/map_test_ro_with_offset";
+	char[] msg = "Hello, world";
+	write_file(FNAME, msg);
+
+	usz map_size = 3;
+	for (usz i = 0; i < msg.len - map_size; i++)
+	{
+		map_with_offset(FNAME, i, map_size, msg);
+	}
+}
+
+fn void map_from_filename() @test
+{
+	const String FNAME = "tmp/map_test_ro";
+	char[] msg = "Hello, world";
+	write_file(FNAME, msg);
+
+	mmap::FileMmap mapped_file = file::mmap_open(FNAME, "rb", 0, 0, vm::VirtualMemoryAccess.READ)!!;
+	defer mapped_file.destroy()!!;
+
+	char[] view = mapped_file.bytes();
+	test::eq(view.len, msg.len);
+	for (usz i = 0; i < view.len; i++)
+	{
+		test::eq(view[i], msg[i]);
+	}
+}
+