mirror of
https://github.com/c3lang/c3c.git
synced 2026-02-27 03:51:18 +00:00
stdlib: std::compression::zip and std::compression::deflate (#2930)
* stdlib: implement `std::compression::zip` and `std::compression::deflate` - C3 implementation of DEFLATE (RFC 1951) and ZIP archive handling. - Support for reading and writing archives using STORE and DEFLATE methods. - Decompression supports both fixed and dynamic Huffman blocks. - Compression using greedy LZ77 matching. - Zero dependencies on libc. - Stream-based entry reading and writing. - Full unit test coverage. NOTE: This is an initial implementation. Future improvements could be: - Optimization of the LZ77 matching (lazy matching). - Support for dynamic Huffman blocks in compression. - ZIP64 support for large files/archives. - Support for encryption and additional compression methods. * optimizations+refactoring deflate: - replace linear search with hash-based match finding. - implement support for dynamic Huffman blocks using the Package-Merge algorithm. - add streaming decompression. - add buffered StreamBitReader. zip: - add ZIP64 support. - add CP437 and UTF-8 filename encoding detection. - add DOS date/time conversion and timestamp preservation. - add ZipEntryReader for streaming entry reads. - implement ZipArchive.extract and ZipArchive.recover helpers. other: - Add `set_modified_time` to std::io; - Add benchmarks and a few more unit tests. * zip: add archive comment support add tests * forgot to rename the benchmark :( * detect utf8 names on weird zips fix method not passed to open_writer * another edge case where directory doesn't end with / * testing utilities - detect encrypted zip - `ZipArchive.open_writer` default to DEFLATE * fix zip64 creation, add tests * fix ZIP header endianness for big-endian compatibility Update ZipLFH, ZipCDH, ZipEOCD, Zip64EOCD, and Zip64Locator structs to use little-endian bitstruct types from std::core::bitorder * fix ZipEntryReader position tracking and seek logic ZIP_METHOD_STORE added a test to track this * add package-merge algorithm attribution Thanks @konimarti * standalone deflate_benchmark.c3 against `miniz` * fix integer overflows, leaks and improve safety * a few safety for 32-bit systems and tests * deflate compress optimization * improve match finding, hash updates, and buffer usage * use ulong for zip offsets * style changes (#18) * style changes * update tests * style changes in `deflate.c3` * fix typo * Allocator first. Some changes to deflate to use `copy_to` * Fix missing conversion on 32 bits. * Fix deflate stream. Formatting. Prefer switch over if-elseif * - Stream functions now use long/ulong rather than isz/usz for seek/available. - `instream.seek` is replaced by `set_cursor` and `cursor`. - `instream.available`, `cursor` etc are long/ulong rather than isz/usz to be correct on 32-bit. * Update to constdef * Fix test --------- Co-authored-by: Book-reader <thevoid@outlook.co.nz> Co-authored-by: Christoffer Lerno <christoffer@aegik.com>
This commit is contained in:
51
benchmarks/stdlib/compression/deflate.c3
Normal file
51
benchmarks/stdlib/compression/deflate.c3
Normal file
@@ -0,0 +1,51 @@
|
||||
module deflate_benchmarks;
|
||||
import std::compression::deflate;
|
||||
|
||||
const uint SMALL_ITERATIONS = 50000;
|
||||
const uint LARGE_ITERATIONS = 100;
|
||||
|
||||
// Data to compress
|
||||
const char[] SMALL_DATA = { [0..1023] = 'A' };
|
||||
const char[] LARGE_DATA = { [0..1048575] = 'B' };
|
||||
|
||||
char[] small_compressed;
|
||||
char[] large_compressed;
|
||||
|
||||
fn void initialize_bench() @init
|
||||
{
|
||||
small_compressed = deflate::compress(mem, SMALL_DATA)!!;
|
||||
large_compressed = deflate::compress(mem, LARGE_DATA)!!;
|
||||
set_benchmark_warmup_iterations(2);
|
||||
set_benchmark_max_iterations(10);
|
||||
|
||||
set_benchmark_func_iterations($qnameof(deflate_compress_small), SMALL_ITERATIONS);
|
||||
set_benchmark_func_iterations($qnameof(deflate_decompress_small), SMALL_ITERATIONS);
|
||||
set_benchmark_func_iterations($qnameof(deflate_compress_large), LARGE_ITERATIONS);
|
||||
set_benchmark_func_iterations($qnameof(deflate_decompress_large), LARGE_ITERATIONS);
|
||||
}
|
||||
|
||||
// =======================================================================================
|
||||
module deflate_benchmarks @benchmark;
|
||||
|
||||
import std::compression::deflate;
|
||||
import std::core::mem;
|
||||
|
||||
fn void deflate_compress_small() => @pool()
|
||||
{
|
||||
char[]? compressed = deflate::compress(tmem, SMALL_DATA);
|
||||
}
|
||||
|
||||
fn void deflate_decompress_small() => @pool()
|
||||
{
|
||||
char[]? decompressed = deflate::decompress(tmem, small_compressed);
|
||||
}
|
||||
|
||||
fn void deflate_compress_large() => @pool()
|
||||
{
|
||||
char[]? compressed = deflate::compress(tmem, LARGE_DATA);
|
||||
}
|
||||
|
||||
fn void deflate_decompress_large() => @pool()
|
||||
{
|
||||
char[]? decompressed = deflate::decompress(tmem, large_compressed);
|
||||
}
|
||||
@@ -32,8 +32,8 @@ fn void initialize_bench() @init
|
||||
$qnameof(sha1_16)[..^4],
|
||||
$qnameof(sha2_256_16)[..^4],
|
||||
$qnameof(sha2_512_16)[..^4],
|
||||
$qnameof(blake2s_256_16)[..^4],
|
||||
$qnameof(blake2b_256_16)[..^4],
|
||||
//$qnameof(blake2s_256_16)[..^4],
|
||||
//$qnameof(blake2b_256_16)[..^4],
|
||||
$qnameof(blake3_16)[..^4],
|
||||
$qnameof(ripemd_160_16)[..^4],
|
||||
$qnameof(whirlpool_16)[..^4],
|
||||
@@ -68,8 +68,8 @@ fn void md5_16() => md5::hash(common_16);
|
||||
fn void sha1_16() => sha1::hash(common_16);
|
||||
fn void sha2_256_16() => sha256::hash(common_16);
|
||||
fn void sha2_512_16() => sha512::hash(common_16);
|
||||
fn void blake2s_256_16() => blake2::s(256, common_16);
|
||||
fn void blake2b_256_16() => blake2::b(256, common_16);
|
||||
//fn void blake2s_256_16() => blake2::s(256, common_16);
|
||||
//fn void blake2b_256_16() => blake2::b(256, common_16);
|
||||
fn void blake3_16() => blake3::hash(common_16);
|
||||
fn void ripemd_160_16() => ripemd::hash{160}(common_16);
|
||||
fn void whirlpool_16() => whirlpool::hash(common_16);
|
||||
@@ -80,8 +80,8 @@ fn void md5_256() => md5::hash(common_256);
|
||||
fn void sha1_256() => sha1::hash(common_256);
|
||||
fn void sha2_256_256() => sha256::hash(common_256);
|
||||
fn void sha2_512_256() => sha512::hash(common_256);
|
||||
fn void blake2s_256_256() => blake2::s(256, common_256);
|
||||
fn void blake2b_256_256() => blake2::b(256, common_256);
|
||||
//fn void blake2s_256_256() => blake2::s(256, common_256);
|
||||
//fn void blake2b_256_256() => blake2::b(256, common_256);
|
||||
fn void blake3_256() => blake3::hash(common_256);
|
||||
fn void ripemd_160_256() => ripemd::hash{160}(common_256);
|
||||
fn void whirlpool_256() => whirlpool::hash(common_256);
|
||||
@@ -92,8 +92,8 @@ fn void md5_4kib() => md5::hash(common_4kib);
|
||||
fn void sha1_4kib() => sha1::hash(common_4kib);
|
||||
fn void sha2_256_4kib() => sha256::hash(common_4kib);
|
||||
fn void sha2_512_4kib() => sha512::hash(common_4kib);
|
||||
fn void blake2s_256_4kib() => blake2::s(256, common_4kib);
|
||||
fn void blake2b_256_4kib() => blake2::b(256, common_4kib);
|
||||
//fn void blake2s_256_4kib() => blake2::s(256, common_4kib);
|
||||
//fn void blake2b_256_4kib() => blake2::b(256, common_4kib);
|
||||
fn void blake3_4kib() => blake3::hash(common_4kib);
|
||||
fn void ripemd_160_4kib() => ripemd::hash{160}(common_4kib);
|
||||
fn void whirlpool_4kib() => whirlpool::hash(common_4kib);
|
||||
@@ -104,8 +104,8 @@ fn void md5_1mib() => md5::hash(common_1mib);
|
||||
fn void sha1_1mib() => sha1::hash(common_1mib);
|
||||
fn void sha2_256_1mib() => sha256::hash(common_1mib);
|
||||
fn void sha2_512_1mib() => sha512::hash(common_1mib);
|
||||
fn void blake2s_256_1mib() => blake2::s(256, common_1mib);
|
||||
fn void blake2b_256_1mib() => blake2::b(256, common_1mib);
|
||||
//fn void blake2s_256_1mib() => blake2::s(256, common_1mib);
|
||||
//fn void blake2b_256_1mib() => blake2::b(256, common_1mib);
|
||||
fn void blake3_1mib() => blake3::hash(common_1mib);
|
||||
fn void ripemd_160_1mib() => ripemd::hash{160}(common_1mib);
|
||||
fn void whirlpool_1mib() => whirlpool::hash(common_1mib);
|
||||
|
||||
1108
lib/std/compression/deflate.c3
Normal file
1108
lib/std/compression/deflate.c3
Normal file
File diff suppressed because it is too large
Load Diff
1215
lib/std/compression/zip.c3
Normal file
1215
lib/std/compression/zip.c3
Normal file
File diff suppressed because it is too large
Load Diff
@@ -658,9 +658,10 @@ fn usz? DString.read_from_stream(&self, InStream reader)
|
||||
if (&reader.available)
|
||||
{
|
||||
usz total_read = 0;
|
||||
while (usz available = reader.available()!)
|
||||
while (ulong available = reader.available()!)
|
||||
{
|
||||
self.reserve(available);
|
||||
if (available > isz.max) available = (ulong)isz.max;
|
||||
self.reserve((usz)available);
|
||||
StringData* data = self.data();
|
||||
usz len = reader.read(data.chars[data.len..(data.capacity - 1)])!;
|
||||
total_read += len;
|
||||
|
||||
@@ -126,6 +126,7 @@ const bool ARCH_64_BIT = $$REGISTER_SIZE == 64;
|
||||
const bool LIBC = $$COMPILER_LIBC_AVAILABLE;
|
||||
const bool NO_LIBC = !LIBC && !CUSTOM_LIBC;
|
||||
const bool CUSTOM_LIBC = $$CUSTOM_LIBC;
|
||||
const bool OLD_IO = $feature(OLD_IO);
|
||||
const CompilerOptLevel COMPILER_OPT_LEVEL = CompilerOptLevel.from_ordinal($$COMPILER_OPT_LEVEL);
|
||||
const bool BIG_ENDIAN = $$PLATFORM_BIG_ENDIAN;
|
||||
const bool I128_NATIVE_SUPPORT = $$PLATFORM_I128_SUPPORTED;
|
||||
|
||||
@@ -125,10 +125,11 @@ fn bool run_benchmarks(BenchmarkUnit[] benchmarks)
|
||||
char[] perc_str = { [0..19] = ' ', [20] = 0 };
|
||||
int perc = 0;
|
||||
uint print_step = current_benchmark_iterations / 100;
|
||||
if (print_step == 0) print_step = 1;
|
||||
|
||||
for (this_iteration = 0; this_iteration < current_benchmark_iterations; ++this_iteration, benchmark_nano_seconds = {})
|
||||
{
|
||||
if (0 == this_iteration % print_step) // only print right about when the % will update
|
||||
if (this_iteration % print_step == 0) // only print right about when the % will update
|
||||
{
|
||||
perc_str[0..(uint)math::floor((this_iteration / (float)current_benchmark_iterations) * 20)] = '#';
|
||||
perc = (uint)math::ceil(100 * (this_iteration / (float)current_benchmark_iterations));
|
||||
|
||||
@@ -142,7 +142,7 @@ fn void mute_output() @local
|
||||
File* stderr = io::stderr();
|
||||
*stderr = test_context.fake_stdout;
|
||||
*stdout = test_context.fake_stdout;
|
||||
(void)test_context.fake_stdout.seek(0, Seek.SET)!!;
|
||||
(void)test_context.fake_stdout.set_cursor(0)!!;
|
||||
}
|
||||
|
||||
fn void unmute_output(bool has_error) @local
|
||||
@@ -155,7 +155,7 @@ fn void unmute_output(bool has_error) @local
|
||||
*stderr = test_context.stored.stderr;
|
||||
*stdout = test_context.stored.stdout;
|
||||
|
||||
usz log_size = test_context.fake_stdout.seek(0, Seek.CURSOR)!!;
|
||||
ulong log_size = test_context.fake_stdout.cursor()!!;
|
||||
if (has_error)
|
||||
{
|
||||
io::printn(test_context.has_ansi_codes ? "[\e[0;31mFAIL\e[0m]" : "[FAIL]");
|
||||
@@ -165,7 +165,7 @@ fn void unmute_output(bool has_error) @local
|
||||
{
|
||||
test_context.fake_stdout.write_byte('\n')!!;
|
||||
test_context.fake_stdout.write_byte('\0')!!;
|
||||
(void)test_context.fake_stdout.seek(0, Seek.SET)!!;
|
||||
test_context.fake_stdout.set_cursor(0)!!;
|
||||
|
||||
io::printfn("\n========== TEST LOG ============");
|
||||
io::printfn("%s\n", test_context.current_test_name);
|
||||
|
||||
@@ -39,11 +39,16 @@ fn bool is_dir(String path)
|
||||
return os::native_is_dir(path);
|
||||
}
|
||||
|
||||
fn usz? get_size(String path)
|
||||
fn ulong? get_size(String path)
|
||||
{
|
||||
return os::native_file_size(path);
|
||||
}
|
||||
|
||||
fn void? set_modified_time(String path, Time_t time)
|
||||
{
|
||||
return os::native_set_modified_time(path, time);
|
||||
}
|
||||
|
||||
fn void? delete(String filename)
|
||||
{
|
||||
return os::native_remove(filename) @inline;
|
||||
@@ -63,10 +68,25 @@ fn void? File.reopen(&self, String filename, String mode)
|
||||
*>
|
||||
fn usz? File.seek(&self, isz offset, Seek seek_mode = Seek.SET) @dynamic
|
||||
{
|
||||
os::native_fseek(self.file, offset, seek_mode)!;
|
||||
return os::native_ftell(self.file);
|
||||
os::native_fseek(self.file, offset, (SeekOrigin)seek_mode.ordinal)!;
|
||||
return (usz)os::native_ftell(self.file);
|
||||
}
|
||||
|
||||
<*
|
||||
@require self.file != null
|
||||
*>
|
||||
fn void? File.set_cursor(&self, long offset, SeekOrigin whence = FROM_START) @dynamic
|
||||
{
|
||||
return os::native_fseek(self.file, offset, whence);
|
||||
}
|
||||
|
||||
<*
|
||||
@require self.file != null
|
||||
*>
|
||||
fn long? File.cursor(&self) @dynamic
|
||||
{
|
||||
return os::native_ftell(self.file);
|
||||
}
|
||||
|
||||
/*
|
||||
Implement later
|
||||
@@ -118,6 +138,14 @@ fn void? File.close(&self) @inline @dynamic
|
||||
self.file = null;
|
||||
}
|
||||
|
||||
fn ulong? File.size(&self) @dynamic
|
||||
{
|
||||
long curr = self.cursor()!;
|
||||
defer (void)self.set_cursor(curr);
|
||||
self.set_cursor(0, FROM_END)!;
|
||||
return self.cursor()!;
|
||||
}
|
||||
|
||||
<*
|
||||
@require self.file != null
|
||||
*>
|
||||
@@ -171,9 +199,8 @@ fn char[]? load_buffer(String filename, char[] buffer)
|
||||
{
|
||||
File file = open(filename, "rb")!;
|
||||
defer (void)file.close();
|
||||
usz len = file.seek(0, END)!;
|
||||
long len = file.size()!;
|
||||
if (len > buffer.len) return io::OVERFLOW~;
|
||||
file.seek(0, SET)!;
|
||||
usz read = 0;
|
||||
while (read < len)
|
||||
{
|
||||
@@ -187,16 +214,16 @@ fn char[]? load(Allocator allocator, String filename)
|
||||
{
|
||||
File file = open(filename, "rb")!;
|
||||
defer (void)file.close();
|
||||
usz len = file.seek(0, END)!;
|
||||
file.seek(0, SET)!;
|
||||
char* data = allocator::malloc_try(allocator, len)!;
|
||||
ulong len = file.size()!;
|
||||
if (len > usz.max) return io::OUT_OF_SPACE~;
|
||||
char* data = allocator::malloc_try(allocator, (usz)len)!;
|
||||
defer catch allocator::free(allocator, data);
|
||||
usz read = 0;
|
||||
while (read < len)
|
||||
while (read < (usz)len)
|
||||
{
|
||||
read += file.read(data[read:len - read])!;
|
||||
read += file.read(data[read:(usz)len - read])!;
|
||||
}
|
||||
return data[:len];
|
||||
return data[:(usz)len];
|
||||
}
|
||||
|
||||
fn char[]? load_path(Allocator allocator, Path path) => load(allocator, path.str_view());
|
||||
|
||||
@@ -45,10 +45,9 @@ fn FileMmap? mmap_file(File file, usz offset = 0, usz len = 0, VirtualMemoryAcce
|
||||
{
|
||||
if (len == 0)
|
||||
{
|
||||
usz cur = file.seek(0, CURSOR)!;
|
||||
defer file.seek(cur, SET)!!;
|
||||
usz file_size = file.seek(0, END)!;
|
||||
len = file_size - offset;
|
||||
ulong new_len = file.size()! - offset;
|
||||
if (new_len > (ulong)isz.max) return mem::OUT_OF_MEMORY~;
|
||||
len = (usz)new_len;
|
||||
}
|
||||
|
||||
// get the page size
|
||||
|
||||
@@ -11,6 +11,14 @@ enum Seek
|
||||
END
|
||||
}
|
||||
|
||||
enum SeekOrigin
|
||||
{
|
||||
FROM_START,
|
||||
FROM_CURSOR,
|
||||
FROM_END
|
||||
}
|
||||
|
||||
|
||||
faultdef
|
||||
ALREADY_EXISTS,
|
||||
BUSY,
|
||||
|
||||
@@ -49,16 +49,16 @@ fn void*? native_freopen(void* file, String filename, String mode) @inline => @
|
||||
return file ?: file_open_errno()~;
|
||||
}
|
||||
|
||||
fn void? native_fseek(void* file, isz offset, Seek seek_mode) @inline
|
||||
fn void? native_fseek(void* file, long offset, SeekOrigin seek_mode) @inline
|
||||
{
|
||||
if (libc::fseek(file, (SeekIndex)offset, seek_mode.ordinal)) return file_seek_errno()~;
|
||||
}
|
||||
|
||||
|
||||
fn usz? native_ftell(CFile file) @inline
|
||||
fn long? native_ftell(CFile file) @inline
|
||||
{
|
||||
long index = libc::ftell(file);
|
||||
return index >= 0 ? (usz)index : file_seek_errno()~;
|
||||
return index >= 0 ? index : file_seek_errno()~;
|
||||
}
|
||||
|
||||
fn usz? native_fwrite(CFile file, char[] buffer) @inline
|
||||
@@ -123,3 +123,22 @@ macro fault file_seek_errno() @local
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
struct Utimbuf
|
||||
{
|
||||
Time_t actime;
|
||||
Time_t modtime;
|
||||
}
|
||||
|
||||
extern fn int utime(char* filename, void* times) @if(!env::WIN32);
|
||||
extern fn int _wutime(WChar* filename, void* times) @if(env::WIN32);
|
||||
|
||||
fn void? native_set_modified_time(String filename, libc::Time_t time) => @stack_mem(256; Allocator smem)
|
||||
{
|
||||
Utimbuf times = { time, time };
|
||||
$if env::WIN32:
|
||||
if (_wutime(filename.to_wstring(smem)!, ×)) return io::GENERAL_ERROR~;
|
||||
$else
|
||||
if (utime(filename.zstr_copy(smem), ×)) return io::GENERAL_ERROR~;
|
||||
$endif
|
||||
}
|
||||
|
||||
@@ -4,12 +4,13 @@ import libc;
|
||||
alias FopenFn = fn void*?(String, String);
|
||||
alias FreopenFn = fn void*?(void*, String, String);
|
||||
alias FcloseFn = fn void?(void*);
|
||||
alias FseekFn = fn void?(void*, isz, Seek);
|
||||
alias FtellFn = fn usz?(void*);
|
||||
alias FseekFn = fn void?(void*, long, SeekOrigin);
|
||||
alias FtellFn = fn long?(void*);
|
||||
alias FwriteFn = fn usz?(void*, char[] buffer);
|
||||
alias FreadFn = fn usz?(void*, char[] buffer);
|
||||
alias RemoveFn = fn void?(String);
|
||||
alias FputcFn = fn void?(int, void*);
|
||||
alias SetModifiedTimeFn = fn void?(String, libc::Time_t);
|
||||
|
||||
FopenFn native_fopen_fn @weak @if(!$defined(native_fopen_fn));
|
||||
FcloseFn native_fclose_fn @weak @if(!$defined(native_fclose_fn));
|
||||
@@ -20,6 +21,7 @@ FwriteFn native_fwrite_fn @weak @if(!$defined(native_fwrite_fn));
|
||||
FreadFn native_fread_fn @weak @if(!$defined(native_fread_fn));
|
||||
RemoveFn native_remove_fn @weak @if(!$defined(native_remove_fn));
|
||||
FputcFn native_fputc_fn @weak @if(!$defined(native_fputc_fn));
|
||||
SetModifiedTimeFn native_set_modified_time_fn @weak @if(!$defined(native_set_modified_time_fn));
|
||||
|
||||
<*
|
||||
@require mode.len > 0
|
||||
@@ -52,13 +54,13 @@ fn void*? native_freopen(void* file, String filename, String mode) @inline
|
||||
return io::UNSUPPORTED_OPERATION~;
|
||||
}
|
||||
|
||||
fn void? native_fseek(void* file, isz offset, Seek seek_mode) @inline
|
||||
fn void? native_fseek(void* file, long offset, SeekOrigin whence) @inline
|
||||
{
|
||||
if (native_fseek_fn) return native_fseek_fn(file, offset, seek_mode);
|
||||
if (native_fseek_fn) return native_fseek_fn(file, offset, whence);
|
||||
return io::UNSUPPORTED_OPERATION~;
|
||||
}
|
||||
|
||||
fn usz? native_ftell(CFile file) @inline
|
||||
fn ulong? native_ftell(CFile file) @inline
|
||||
{
|
||||
if (native_ftell_fn) return native_ftell_fn(file);
|
||||
return io::UNSUPPORTED_OPERATION~;
|
||||
@@ -81,3 +83,9 @@ fn void? native_fputc(CInt c, CFile stream) @inline
|
||||
if (native_fputc_fn) return native_fputc_fn(c, stream);
|
||||
return io::UNSUPPORTED_OPERATION~;
|
||||
}
|
||||
|
||||
fn void? native_set_modified_time(String filename, libc::Time_t time) @inline
|
||||
{
|
||||
if (native_set_modified_time_fn) return native_set_modified_time_fn(filename, time);
|
||||
return io::UNSUPPORTED_OPERATION~;
|
||||
}
|
||||
|
||||
@@ -47,14 +47,15 @@ fn usz? native_file_size(String path) @if(env::WIN32) => @pool()
|
||||
return (usz)size.quadPart;
|
||||
}
|
||||
|
||||
fn usz? native_file_size(String path) @if(!env::WIN32 && !env::DARWIN && !env::LINUX && !env::ANDROID && !env::BSD_FAMILY)
|
||||
fn ulong? native_file_size(String path) @if(!env::WIN32 && !env::DARWIN && !env::LINUX && !env::ANDROID && !env::BSD_FAMILY)
|
||||
{
|
||||
File f = file::open(path, "r")!;
|
||||
defer (void)f.close();
|
||||
return f.seek(0, Seek.END)!;
|
||||
f.set_cursor(0, FROM_END)!;
|
||||
return f.cursor();
|
||||
}
|
||||
|
||||
fn usz? native_file_size(String path) @if(env::DARWIN || env::LINUX || env::ANDROID || env::BSD_FAMILY)
|
||||
fn ulong? native_file_size(String path) @if(env::DARWIN || env::LINUX || env::ANDROID || env::BSD_FAMILY)
|
||||
{
|
||||
Stat stat;
|
||||
native_stat(&stat, path)!;
|
||||
|
||||
@@ -36,7 +36,7 @@ fn Path? cwd(Allocator allocator)
|
||||
|
||||
fn bool is_dir(Path path) => os::native_is_dir(path.str_view());
|
||||
fn bool is_file(Path path) => os::native_is_file(path.str_view());
|
||||
fn usz? file_size(Path path) => os::native_file_size(path.str_view());
|
||||
fn ulong? file_size(Path path) => os::native_file_size(path.str_view());
|
||||
fn bool exists(Path path) => os::native_file_or_dir_exists(path.str_view());
|
||||
fn Path? tcwd() => cwd(tmem) @inline;
|
||||
|
||||
|
||||
@@ -1,12 +1,20 @@
|
||||
module std::io;
|
||||
import std::math;
|
||||
|
||||
|
||||
|
||||
alias SetCursorFn = fn void?(void*, long offset, SeekOrigin whence = START);
|
||||
|
||||
|
||||
interface InStream
|
||||
{
|
||||
fn void? close() @optional;
|
||||
fn long? cursor() @optional;
|
||||
fn void? set_cursor(long offset, SeekOrigin whence = FROM_START) @optional;
|
||||
fn usz? seek(isz offset, Seek seek) @optional;
|
||||
fn usz len() @optional;
|
||||
fn usz? available() @optional;
|
||||
fn ulong? size() @optional;
|
||||
fn ulong? available() @optional;
|
||||
fn usz? read(char[] buffer);
|
||||
fn char? read_byte();
|
||||
fn usz? write_to(OutStream out) @optional;
|
||||
@@ -24,15 +32,23 @@ interface OutStream
|
||||
fn usz? read_to(InStream in) @optional;
|
||||
}
|
||||
|
||||
fn usz? available(InStream s)
|
||||
fn ulong? available(InStream s)
|
||||
{
|
||||
if (&s.available) return s.available();
|
||||
if (&s.set_cursor && &s.cursor)
|
||||
{
|
||||
long curr = s.cursor()!;
|
||||
s.set_cursor(0, FROM_END)!;
|
||||
ulong len = s.cursor()!;
|
||||
s.set_cursor(curr)!;
|
||||
return len - curr;
|
||||
}
|
||||
if (&s.seek)
|
||||
{
|
||||
usz curr = s.seek(0, Seek.CURSOR)!;
|
||||
usz len = s.seek(0, Seek.END)!;
|
||||
s.seek(curr, Seek.SET)!;
|
||||
return len - curr;
|
||||
return (ulong)len - (ulong)curr;
|
||||
}
|
||||
return io::UNSUPPORTED_OPERATION~;
|
||||
}
|
||||
@@ -177,6 +193,11 @@ macro usz? write_using_write_byte(s, char[] bytes)
|
||||
|
||||
macro void? pushback_using_seek(s)
|
||||
{
|
||||
if (&s.set_cursor)
|
||||
{
|
||||
s.set_cursor(-1, FROM_CURSOR)!;
|
||||
return;
|
||||
}
|
||||
s.seek(-1, CURSOR)!;
|
||||
}
|
||||
|
||||
@@ -621,14 +642,8 @@ macro void? skip(stream, usz bytes)
|
||||
{
|
||||
if (!bytes) return;
|
||||
$switch:
|
||||
$case !$defined(stream.seek):
|
||||
for (usz i = 0; i < bytes; i++)
|
||||
{
|
||||
stream.read()!;
|
||||
}
|
||||
return;
|
||||
$case $typeof(stream) == InStream:
|
||||
if (!&stream.seek)
|
||||
if (!&stream.seek && !&stream.set_cursor)
|
||||
{
|
||||
for (usz i = 0; i < bytes; i++)
|
||||
{
|
||||
@@ -636,9 +651,21 @@ macro void? skip(stream, usz bytes)
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (!&stream.set_cursor)
|
||||
{
|
||||
stream.seek(bytes, CURSOR)!;
|
||||
return;
|
||||
}
|
||||
stream.set_cursor(bytes, FROM_CURSOR)!;
|
||||
$case $defined(stream.set_cursor):
|
||||
stream.set_cursor(bytes, FROM_CURSOR)!;
|
||||
$case $defined(stream.seek):
|
||||
stream.seek(bytes, CURSOR)!;
|
||||
$default:
|
||||
stream.seek(bytes, CURSOR)!;
|
||||
for (usz i = 0; i < bytes; i++)
|
||||
{
|
||||
stream.read()!;
|
||||
}
|
||||
$endswitch
|
||||
}
|
||||
|
||||
|
||||
@@ -104,28 +104,37 @@ fn void? ByteBuffer.pushback_byte(&self) @dynamic
|
||||
self.has_last = false;
|
||||
}
|
||||
|
||||
fn usz? ByteBuffer.seek(&self, isz offset, Seek seek) @dynamic
|
||||
fn long? ByteBuffer.cursor(&self) @dynamic
|
||||
{
|
||||
switch (seek)
|
||||
{
|
||||
case SET:
|
||||
if (offset < 0 || offset > self.write_idx) return INVALID_POSITION~;
|
||||
self.read_idx = offset;
|
||||
return offset;
|
||||
case CURSOR:
|
||||
if ((offset < 0 && self.read_idx < -offset) ||
|
||||
(offset > 0 && self.read_idx + offset > self.write_idx)) return INVALID_POSITION~;
|
||||
self.read_idx += offset;
|
||||
case END:
|
||||
if (offset < 0 || offset > self.write_idx) return INVALID_POSITION~;
|
||||
self.read_idx = self.write_idx - offset;
|
||||
}
|
||||
return self.read_idx;
|
||||
}
|
||||
|
||||
fn usz? ByteBuffer.available(&self) @inline @dynamic
|
||||
fn void? ByteBuffer.set_cursor(&self, long offset, SeekOrigin whence = FROM_START) @dynamic
|
||||
{
|
||||
return self.write_idx - self.read_idx;
|
||||
switch (whence)
|
||||
{
|
||||
case FROM_START:
|
||||
if (offset < 0 || offset > self.write_idx) return INVALID_POSITION~;
|
||||
self.read_idx = (usz)offset;
|
||||
case FROM_CURSOR:
|
||||
if ((offset < 0 && self.read_idx < -offset) ||
|
||||
(offset > 0 && self.read_idx + offset > self.write_idx)) return INVALID_POSITION~;
|
||||
self.read_idx += (usz)offset;
|
||||
case FROM_END:
|
||||
if (offset < 0 || offset > self.write_idx) return INVALID_POSITION~;
|
||||
self.read_idx = self.write_idx - (usz)offset;
|
||||
}
|
||||
}
|
||||
|
||||
fn usz? ByteBuffer.seek(&self, isz offset, Seek seek) @dynamic
|
||||
{
|
||||
self.set_cursor(offset, (SeekOrigin)seek.ordinal)!;
|
||||
return (usz)self.cursor();
|
||||
}
|
||||
|
||||
fn ulong? ByteBuffer.available(&self) @inline @dynamic
|
||||
{
|
||||
return (ulong)self.write_idx - self.read_idx;
|
||||
}
|
||||
|
||||
fn void ByteBuffer.grow(&self, usz n)
|
||||
|
||||
@@ -41,16 +41,26 @@ fn void? ByteReader.pushback_byte(&self) @dynamic
|
||||
|
||||
fn usz? ByteReader.seek(&self, isz offset, Seek seek) @dynamic
|
||||
{
|
||||
isz new_index;
|
||||
switch (seek)
|
||||
{
|
||||
case SET: new_index = offset;
|
||||
case CURSOR: new_index = self.index + offset;
|
||||
case END: new_index = self.bytes.len + offset;
|
||||
self.set_cursor((long)offset, (SeekOrigin)seek.ordinal)!;
|
||||
return (usz)self.cursor();
|
||||
}
|
||||
if (new_index < 0) return INVALID_POSITION~;
|
||||
self.index = new_index;
|
||||
return new_index;
|
||||
|
||||
fn long? ByteReader.cursor(&self) @dynamic
|
||||
{
|
||||
return self.index;
|
||||
}
|
||||
|
||||
fn void? ByteReader.set_cursor(&self, long offset, SeekOrigin whence = FROM_START) @dynamic
|
||||
{
|
||||
long new_index;
|
||||
switch (whence)
|
||||
{
|
||||
case FROM_START: new_index = offset;
|
||||
case FROM_CURSOR: new_index = self.index + offset;
|
||||
case FROM_END: new_index = self.bytes.len + offset;
|
||||
}
|
||||
if (new_index < 0 || new_index > self.bytes.len) return INVALID_POSITION~;
|
||||
self.index = (usz)new_index;
|
||||
}
|
||||
|
||||
fn usz? ByteReader.write_to(&self, OutStream writer) @dynamic
|
||||
@@ -62,7 +72,7 @@ fn usz? ByteReader.write_to(&self, OutStream writer) @dynamic
|
||||
return written;
|
||||
}
|
||||
|
||||
fn usz? ByteReader.available(&self) @inline @dynamic
|
||||
fn ulong? ByteReader.available(&self) @inline @dynamic
|
||||
{
|
||||
return max(0, self.bytes.len - self.index);
|
||||
}
|
||||
@@ -86,9 +86,10 @@ fn usz? ByteWriter.read_from(&self, InStream reader) @dynamic
|
||||
usz start_index = self.index;
|
||||
if (&reader.available)
|
||||
{
|
||||
while (usz available = reader.available()!)
|
||||
while (ulong available = reader.available()!)
|
||||
{
|
||||
self.ensure_capacity(self.index + available)!;
|
||||
if (available > usz.max) return OUT_OF_SPACE~;
|
||||
self.ensure_capacity(self.index + (usz)available)!;
|
||||
usz read = reader.read(self.bytes[self.index..])!;
|
||||
self.index += read;
|
||||
}
|
||||
|
||||
@@ -38,7 +38,7 @@ fn char? LimitReader.read_byte(&self) @dynamic
|
||||
return self.wrapped_stream.read_byte();
|
||||
}
|
||||
|
||||
fn usz? LimitReader.available(&self) @inline @dynamic
|
||||
fn ulong? LimitReader.available(&self) @inline @dynamic
|
||||
{
|
||||
return self.limit;
|
||||
}
|
||||
@@ -186,7 +186,7 @@ fn ulong? elf_module_image_base(String path) @local
|
||||
bool is_little_endian = file.read_byte()! == 1;
|
||||
// Actually, not supported.
|
||||
if (!is_little_endian) return backtrace::IMAGE_NOT_FOUND~;
|
||||
file.seek(0)!;
|
||||
file.set_cursor(0)!;
|
||||
if (is_64)
|
||||
{
|
||||
Elf64_Ehdr file_header;
|
||||
@@ -195,7 +195,7 @@ fn ulong? elf_module_image_base(String path) @local
|
||||
for (isz i = 0; i < file_header.e_phnum; i++)
|
||||
{
|
||||
Elf64_Phdr header;
|
||||
file.seek((usz)file_header.e_phoff + (usz)file_header.e_phentsize * i)!;
|
||||
file.set_cursor(file_header.e_phoff + (long)file_header.e_phentsize * i)!;
|
||||
io::read_any(&file, &header)!;
|
||||
if (header.p_type == PT_PHDR) return header.p_vaddr - header.p_offset;
|
||||
}
|
||||
@@ -207,7 +207,7 @@ fn ulong? elf_module_image_base(String path) @local
|
||||
for (isz i = 0; i < file_header.e_phnum; i++)
|
||||
{
|
||||
Elf32_Phdr header;
|
||||
file.seek(file_header.e_phoff + (usz)file_header.e_phentsize * i)!;
|
||||
file.set_cursor(file_header.e_phoff + (long)file_header.e_phentsize * i)!;
|
||||
io::read_any(&file, &header)!;
|
||||
if (header.p_type == PT_PHDR) return (ulong)header.p_vaddr - header.p_offset;
|
||||
}
|
||||
|
||||
@@ -33,6 +33,9 @@
|
||||
- Add `array::even`, `array::odd`, and `array::unlace` macros. #2892
|
||||
- Add discrete and continuous distributions in `std::math::distributions`.
|
||||
- Add bitorder functions `store_le`, `load_le`, `store_be`, `store_le`.
|
||||
- Stream functions now use long/ulong rather than isz/usz for seek/available.
|
||||
- `instream.seek` is replaced by `set_cursor` and `cursor`.
|
||||
- `instream.available`, `cursor` etc are long/ulong rather than isz/usz to be correct on 32-bit.
|
||||
|
||||
### Fixes
|
||||
- Add error message if directory with output file name already exists
|
||||
|
||||
207
test/compression/deflate_benchmark.c3
Normal file
207
test/compression/deflate_benchmark.c3
Normal file
@@ -0,0 +1,207 @@
|
||||
// 1. `gcc -O3 -c dependencies/miniz/miniz.c -o build/miniz.o`
|
||||
// 2. `build/c3c -O3 compile-run test/compression/deflate_benchmark.c3 build/miniz.o`
|
||||
|
||||
module deflate_benchmark;
|
||||
import std, std::time::clock;
|
||||
|
||||
const int AMOUNT_OF_WORK = 10; // Increase this to scale test data sizes
|
||||
|
||||
fn int main(String[] args)
|
||||
{
|
||||
io::printf("\n%s%s DEFLATE BENCHMARK %s", Ansi.BOLD, Ansi.BG_CYAN, Ansi.RESET);
|
||||
io::printfn(" Comparing C3 std::compression::deflate with miniz (in-process)\n");
|
||||
io::printfn(" Work Scale: %dx\n", AMOUNT_OF_WORK);
|
||||
|
||||
io::printfn("%-26s | %7s | %7s | %7s | %7s | %-10s", "Test Case", "C3 Rat.", "Miz Rat.", "C3 MB/s", "Miz MB/s", "Winner");
|
||||
io::printfn("---------------------------+---------+---------+---------+---------+-----------");
|
||||
|
||||
// Test 1: Redundant data
|
||||
usz redundant_size = 10_000_000 * (usz)AMOUNT_OF_WORK;
|
||||
char[] redundant = allocator::alloc_array(tmem, char, redundant_size);
|
||||
mem::set(redundant.ptr, 'A', redundant_size);
|
||||
run_bench(string::tformat("Redundant (%dMB 'A')", (int)(redundant_size / 1_000_000)), redundant);
|
||||
|
||||
// Test 2: Large Source Project (All .c files in src/compiler)
|
||||
DString project_src;
|
||||
Path src_dir = path::new(tmem, "src/compiler")!!;
|
||||
PathList? compiler_files = path::ls(tmem, src_dir);
|
||||
if (try files = compiler_files) {
|
||||
for (int i = 0; i < AMOUNT_OF_WORK; i++) {
|
||||
foreach (p : files) {
|
||||
if (p.basename().ends_with(".c")) {
|
||||
Path full_p = src_dir.tappend(p.str_view())!!;
|
||||
if (try data = file::load_path(tmem, full_p)) {
|
||||
project_src.append(data);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
run_bench("Compiler Source (Bulk)", project_src.str_view());
|
||||
|
||||
// Test 3: Standard Library (All .c3 files in lib/std)
|
||||
DString std_src;
|
||||
for (int i = 0; i < AMOUNT_OF_WORK; i++) {
|
||||
collect_files(path::new(tmem, "lib/std")!!, ".c3", &std_src);
|
||||
}
|
||||
run_bench("Stdlib Source (Bulk)", std_src.str_view());
|
||||
|
||||
// Test 4: Log Files (Simulated)
|
||||
DString log_data;
|
||||
for (int i = 0; i < 50_000 * AMOUNT_OF_WORK; i++) {
|
||||
log_data.appendf("2024-02-13 21:30:%02d.%03d [INFO] Connection established from 192.168.1.%d\n", i % 60, i % 1000, i % 255);
|
||||
log_data.appendf("2024-02-13 21:30:%02d.%03d [DEBUG] Buffer size: %d bytes\n", i % 60, i % 1000, (i * 123) % 4096);
|
||||
}
|
||||
run_bench("Log Files (Simulated)", log_data.str_view());
|
||||
|
||||
// Test 5: Web Content (Simulated HTML/CSS)
|
||||
DString web_data;
|
||||
web_data.append("<!DOCTYPE html><html><head><style>.item { color: red; margin: 10px; }</style></head><body>");
|
||||
for (int i = 0; i < 1000 * AMOUNT_OF_WORK; i++) {
|
||||
web_data.appendf("<div class='item' id='obj%d'>", i);
|
||||
web_data.append("<h1>Title of the item</h1><p>This is some repetitive descriptive text that might appear on a web page.</p>");
|
||||
web_data.append("<ul><li>Feature 1</li><li>Feature 2</li><li>Feature 3</li></ul></div>");
|
||||
}
|
||||
web_data.append("</body></html>");
|
||||
run_bench("Web Content (Simulated)", web_data.str_view());
|
||||
|
||||
// Test 6: CSV Data (Simulated)
|
||||
DString csv_data;
|
||||
csv_data.append("id,name,value1,value2,status,category\n");
|
||||
for (int i = 0; i < 20_000 * AMOUNT_OF_WORK; i++) {
|
||||
csv_data.appendf("%d,Product_%d,%d.5,%d,\"%s\",\"%s\"\n",
|
||||
i, i % 100, i * 10, i % 500,
|
||||
i % 3 == 0 ? "Active" : "Inactive",
|
||||
i % 5 == 0 ? "Electronics" : "Home");
|
||||
}
|
||||
run_bench("CSV Data (Simulated)", csv_data.str_view());
|
||||
|
||||
// Test 7: Binary Data (Structured)
|
||||
usz binary_size = 2_000_000 * (usz)AMOUNT_OF_WORK;
|
||||
char[] binary = allocator::alloc_array(tmem, char, binary_size);
|
||||
for (usz i = 0; i < binary.len; i += 4) {
|
||||
uint val = (uint)i ^ 0xDEADBEEF;
|
||||
mem::copy(binary.ptr + i, &val, 4);
|
||||
}
|
||||
run_bench("Binary Data (Structured)", binary);
|
||||
|
||||
// Test 8: Random Noise (1MB)
|
||||
usz noise_size = 1_000_000 * (usz)AMOUNT_OF_WORK;
|
||||
DString noise;
|
||||
for (usz i = 0; i < noise_size; i++) {
|
||||
noise.append((char)rand('z' - 'a' + 1) + 'a');
|
||||
}
|
||||
run_bench("Random Noise (Scaled)", noise.str_view());
|
||||
|
||||
// Test 9: Tiny File (Check overhead)
|
||||
run_bench("Tiny File (asd.c3)", "module asd; fn void main() {}\n");
|
||||
|
||||
// Test 10: Natural Language (Repetitive)
|
||||
String text = "The quick brown fox jumps over the lazy dog. ";
|
||||
DString long_text;
|
||||
for (int i = 0; i < 50_000 * AMOUNT_OF_WORK; i++) long_text.append(text);
|
||||
run_bench("Natural Text (Scaled)", long_text.str_view());
|
||||
|
||||
if (args.len > 1) {
|
||||
Path custom_p = path::new(tmem, args[1])!!;
|
||||
if (try custom_data = file::load_path(tmem, custom_p)) {
|
||||
run_bench(string::tformat("Custom: %s", custom_p.basename()), custom_data);
|
||||
}
|
||||
}
|
||||
// Final Summary
|
||||
double avg_c3 = totals.c3_speed_sum / totals.count;
|
||||
double avg_miniz = totals.miniz_speed_sum / totals.count;
|
||||
double total_factor = avg_c3 / avg_miniz;
|
||||
|
||||
io::printfn("\n%sOVERALL SUMMARY%s", Ansi.BOLD, Ansi.RESET);
|
||||
io::printfn(" Average Throughput C3: %8.1f MB/s", avg_c3);
|
||||
io::printfn(" Average Throughput Miniz: %8.1f MB/s", avg_miniz);
|
||||
io::printfn(" %sC3 is %.1fx faster on average!%s\n", Ansi.BOLD, total_factor, Ansi.RESET);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
struct BenchResult {
|
||||
long time_ns;
|
||||
usz size;
|
||||
double ratio;
|
||||
double throughput_mbs;
|
||||
}
|
||||
|
||||
struct BenchTotal {
|
||||
double c3_speed_sum;
|
||||
double miniz_speed_sum;
|
||||
int count;
|
||||
}
|
||||
BenchTotal totals;
|
||||
|
||||
fn void run_bench(String title, char[] data)
|
||||
{
|
||||
// C3 Bench
|
||||
Clock start = clock::now();
|
||||
char[] c3_compressed = deflate::compress(data, tmem)!!;
|
||||
Clock end = clock::now();
|
||||
BenchResult c3 = calculate_metrics(data.len, (long)(end - start), c3_compressed.len);
|
||||
|
||||
// Miniz Bench
|
||||
usz miniz_size;
|
||||
start = clock::now();
|
||||
void* miniz_ptr = tdefl_compress_mem_to_heap(data.ptr, data.len, &miniz_size, MINIZ_FLAGS);
|
||||
end = clock::now();
|
||||
BenchResult miniz = calculate_metrics(data.len, (long)(end - start), miniz_size);
|
||||
if (miniz_ptr) mz_free(miniz_ptr);
|
||||
|
||||
// Performance Delta
|
||||
double speed_factor = c3.throughput_mbs / miniz.throughput_mbs;
|
||||
|
||||
io::printf("%-26s | %6.2f%% | %6.2f%% | %7.1f | %7.1f | %s%s (%.1fx)%s\n",
|
||||
title[:(min(title.len, 26))],
|
||||
c3.ratio, miniz.ratio,
|
||||
c3.throughput_mbs, miniz.throughput_mbs,
|
||||
speed_factor > 1.0 ? Ansi.CYAN : Ansi.WHITE,
|
||||
speed_factor > 1.0 ? "C3" : "Miniz",
|
||||
speed_factor > 1.0 ? speed_factor : 1.0 / speed_factor,
|
||||
Ansi.RESET);
|
||||
|
||||
totals.c3_speed_sum += c3.throughput_mbs;
|
||||
totals.miniz_speed_sum += miniz.throughput_mbs;
|
||||
totals.count++;
|
||||
}
|
||||
|
||||
fn void collect_files(Path dir, String suffix, DString* out)
|
||||
{
|
||||
PathList? items = path::ls(tmem, dir);
|
||||
if (catch items) return;
|
||||
foreach (p : items) {
|
||||
Path full = dir.tappend(p.str_view())!!;
|
||||
if (path::is_dir(full)) {
|
||||
if (p.basename() != "." && p.basename() != "..") {
|
||||
collect_files(full, suffix, out);
|
||||
}
|
||||
} else if (p.basename().ends_with(suffix)) {
|
||||
if (try data = file::load_path(tmem, full)) {
|
||||
out.append(data);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn BenchResult calculate_metrics(usz original_len, long time_ns, usz compressed_len)
|
||||
{
|
||||
BenchResult res;
|
||||
res.time_ns = time_ns;
|
||||
res.size = compressed_len;
|
||||
res.ratio = (double)compressed_len / (double)original_len * 100.0;
|
||||
res.throughput_mbs = (double)original_len / (1024.0 * 1024.0) / ((double)time_ns / 1_000_000_000.0);
|
||||
return res;
|
||||
}
|
||||
|
||||
// External Miniz bindings
|
||||
extern fn void* tdefl_compress_mem_to_heap(void* pSrc_buf, usz src_buf_len, usz* pOut_len, int flags);
|
||||
extern fn void mz_free(void* p);
|
||||
|
||||
const int TDEFL_GREEDY_PARSING_FLAG = 0x04000;
|
||||
const int TDEFL_NONDETERMINISTIC_PARSING_FLAG = 0x08000; // Fastest init for miniz for a fair comparisson
|
||||
const int C3_EQUIVALENT_PROBES = 16; // C3 uses MAX_CHAIN = 16 as default (this should be exposed)
|
||||
|
||||
const int MINIZ_FLAGS = C3_EQUIVALENT_PROBES | TDEFL_GREEDY_PARSING_FLAG | TDEFL_NONDETERMINISTIC_PARSING_FLAG;
|
||||
194
test/compression/zip_compare_7z.c3
Normal file
194
test/compression/zip_compare_7z.c3
Normal file
@@ -0,0 +1,194 @@
|
||||
<*
|
||||
Compare `C3 zip` vs `7z` extraction
|
||||
External dependencies: 7z, diff
|
||||
*>
|
||||
module verify_zip;
|
||||
import std;
|
||||
import libc;
|
||||
|
||||
fn int main(String[] args)
|
||||
{
|
||||
if (args.len < 2)
|
||||
{
|
||||
io::printfn("Usage: %s [-r|--recursive] [-o|--output <dir>] <zip_dir>", args[0]);
|
||||
return 1;
|
||||
}
|
||||
|
||||
bool recursive = false;
|
||||
String zip_dir;
|
||||
String output_dir;
|
||||
|
||||
for (int i = 1; i < args.len; i++)
|
||||
{
|
||||
String arg = args[i];
|
||||
switch (arg)
|
||||
{
|
||||
case "-r":
|
||||
case "--recursive":
|
||||
recursive = true;
|
||||
case "-o":
|
||||
case "--output":
|
||||
if (++i >= args.len)
|
||||
{
|
||||
io::printfn("Error: %s requires a directory path", arg);
|
||||
return 1;
|
||||
}
|
||||
output_dir = args[i];
|
||||
default:
|
||||
if (arg.starts_with("-"))
|
||||
{
|
||||
io::printfn("Error: unknown option %s", arg);
|
||||
return 1;
|
||||
}
|
||||
if (zip_dir)
|
||||
{
|
||||
io::printfn("Error: multiple zip directories specified ('%s' and '%s')", zip_dir, arg);
|
||||
return 1;
|
||||
}
|
||||
zip_dir = arg;
|
||||
}
|
||||
}
|
||||
|
||||
if (!zip_dir)
|
||||
{
|
||||
io::printfn("Error: no zip directory specified.");
|
||||
return 1;
|
||||
}
|
||||
|
||||
return process_dir(zip_dir, recursive, output_dir);
|
||||
}
|
||||
|
||||
fn int process_dir(String dir, bool recursive, String output_dir)
|
||||
{
|
||||
PathList? files = path::ls(tmem, path::temp(dir)!!);
|
||||
if (catch excuse = files)
|
||||
{
|
||||
io::printfn("Could not open directory: %s (Excuse: %s)", dir, excuse);
|
||||
return 1;
|
||||
}
|
||||
|
||||
foreach (p : files)
|
||||
{
|
||||
String name = p.basename();
|
||||
if (name == "." || name == "..") continue;
|
||||
|
||||
String zip_path = path::temp(dir)!!.tappend(name)!!.str_view();
|
||||
|
||||
if (file::is_dir(zip_path))
|
||||
{
|
||||
if (recursive)
|
||||
{
|
||||
if (process_dir(zip_path, recursive, output_dir) != 0) return 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!name.ends_with(".zip")) continue;
|
||||
|
||||
ulong size = 0;
|
||||
File? f = file::open(zip_path, "rb");
|
||||
if (try fh = f)
|
||||
{
|
||||
(void)fh.seek(0, Seek.END);
|
||||
size = fh.seek(0, Seek.CURSOR) ?? 0;
|
||||
fh.close()!!;
|
||||
}
|
||||
io::printf("Verifying %-40s [%7d KB] ", name[:(min(name.len, 40))], size / 1024);
|
||||
|
||||
switch (verify_one(zip_path, output_dir))
|
||||
{
|
||||
case 0:
|
||||
io::printfn("%sFAILED%s ❌", Ansi.RED, Ansi.RESET);
|
||||
return 1;
|
||||
case 1:
|
||||
io::printfn("%sPASSED%s ✅", Ansi.GREEN, Ansi.RESET);
|
||||
default:
|
||||
io::printn();
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
fn int verify_one(String zip_path, String output_dir)
|
||||
{
|
||||
Path extract_root;
|
||||
if (output_dir)
|
||||
{
|
||||
extract_root = path::temp(output_dir)!!;
|
||||
}
|
||||
else
|
||||
{
|
||||
extract_root = path::temp_directory(tmem)!!;
|
||||
}
|
||||
|
||||
String name = (String)path::temp(zip_path)!!.basename();
|
||||
|
||||
Path temp_c3 = extract_root.tappend(name.tconcat("_c3"))!!;
|
||||
Path temp_7z = extract_root.tappend(name.tconcat("_7z"))!!;
|
||||
|
||||
(void)path::mkdir(temp_c3, true);
|
||||
(void)path::mkdir(temp_7z, true);
|
||||
|
||||
ZipArchive? archive = zip::open(zip_path, "r");
|
||||
if (catch excuse = archive)
|
||||
{
|
||||
io::printfn("%sFAIL%s (open: %s)", Ansi.RED, Ansi.RESET, excuse);
|
||||
return 0;
|
||||
}
|
||||
defer (void)archive.close();
|
||||
|
||||
Time start = time::now();
|
||||
if (catch excuse = archive.extract(temp_c3.str_view()))
|
||||
{
|
||||
if (excuse == zip::ENCRYPTED_FILE)
|
||||
{
|
||||
io::printf("%sSKIPPED%s (Encrypted)", Ansi.YELLOW, Ansi.RESET);
|
||||
return 2;
|
||||
}
|
||||
io::printfn("%sFAIL%s (extract: %s)", Ansi.RED, Ansi.RESET, excuse);
|
||||
return 0;
|
||||
}
|
||||
Duration c3_time = time::now() - start;
|
||||
|
||||
start = time::now();
|
||||
if (!extract_7z(zip_path, temp_7z.str_view()))
|
||||
{
|
||||
io::printfn("%sFAIL%s (7z extract)", Ansi.RED, Ansi.RESET);
|
||||
return 0;
|
||||
}
|
||||
Duration p7_time = time::now() - start;
|
||||
|
||||
io::printf(" [C3: %5d ms, 7z: %5d ms]", (long)c3_time / 1000, (long)p7_time / 1000);
|
||||
|
||||
io::print(" Comparing... ");
|
||||
if (!compare_dirs(temp_c3.str_view(), temp_7z.str_view()))
|
||||
{
|
||||
io::printfn("%sFAIL%s (Differences found)", Ansi.RED, Ansi.RESET);
|
||||
return 0;
|
||||
}
|
||||
|
||||
// keep files on error for manual verification
|
||||
(void)path::rmtree(temp_c3);
|
||||
(void)path::rmtree(temp_7z);
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
fn bool extract_7z(String zip_path, String output_dir)
|
||||
{
|
||||
String out_opt = "-o".tconcat(output_dir);
|
||||
String[] cmd = { "7z", "x", zip_path, out_opt, "-y", "-bb0" };
|
||||
SubProcess? proc = process::create(cmd, { .search_user_path = true });
|
||||
if (catch excuse = proc) return false;
|
||||
return (int)proc.join()!! == 0;
|
||||
}
|
||||
|
||||
fn bool compare_dirs(String dir1, String dir2)
|
||||
{
|
||||
String[] cmd = { "diff", "-r", dir1, dir2 };
|
||||
SubProcess? proc = process::create(cmd, { .search_user_path = true, .inherit_stdio = true });
|
||||
if (catch excuse = proc) return false;
|
||||
int res = (int)proc.join()!!;
|
||||
return res == 0;
|
||||
}
|
||||
210
test/unit/stdlib/compression/deflate.c3
Normal file
210
test/unit/stdlib/compression/deflate.c3
Normal file
@@ -0,0 +1,210 @@
|
||||
module deflate_test @test;
|
||||
|
||||
import std::compression::deflate, std::io, std::math;
|
||||
|
||||
fn void test_deflate_basic()
|
||||
{
|
||||
String original = "Hello, world! This is a test of the DEFLATE compression algorithm.";
|
||||
char[] compressed = deflate::compress(mem, original)!!;
|
||||
defer free(compressed.ptr);
|
||||
|
||||
char[] decompressed = deflate::decompress(mem, compressed)!!;
|
||||
defer free(decompressed.ptr);
|
||||
|
||||
assert((String)decompressed == original, "Decompressed data does not match original");
|
||||
}
|
||||
|
||||
fn void test_deflate_repetitive()
|
||||
{
|
||||
// 5000 bytes of repetitive data should compress very well
|
||||
usz len = 5000;
|
||||
char[] original = mem::malloc(len)[:len];
|
||||
defer free(original.ptr);
|
||||
|
||||
for (usz i = 0; i < len; i++)
|
||||
{
|
||||
original[i] = (char)((i % 10) + '0');
|
||||
}
|
||||
|
||||
char[] compressed = deflate::compress(mem, original)!!;
|
||||
defer free(compressed.ptr);
|
||||
|
||||
// Check that we actually achieved some compression
|
||||
assert(compressed.len < len / 10, "Repetitive data should compress well");
|
||||
|
||||
char[] decompressed = deflate::decompress(mem, compressed)!!;
|
||||
defer free(decompressed.ptr);
|
||||
|
||||
assert(decompressed.len == original.len, "Length mismatch");
|
||||
assert((String)decompressed == (String)original, "Data mismatch");
|
||||
}
|
||||
|
||||
fn void test_deflate_empty()
|
||||
{
|
||||
char[] original = {};
|
||||
char[] compressed = deflate::compress(mem, original)!!;
|
||||
defer free(compressed.ptr);
|
||||
|
||||
char[] decompressed = deflate::decompress(mem, compressed)!!;
|
||||
defer free(decompressed.ptr);
|
||||
|
||||
assert(decompressed.len == 0, "Expected empty decompression");
|
||||
}
|
||||
|
||||
fn void test_deflate_large_repetitive() @if($feature(SLOW_TESTS))
|
||||
{
|
||||
// Test larger buffer to trigger reallocs in inflater
|
||||
usz len = 100000;
|
||||
char[] original = mem::malloc(len)[:len];
|
||||
defer free(original.ptr);
|
||||
|
||||
mem::set(original.ptr, (char)'A', len);
|
||||
|
||||
char[] compressed = deflate::compress(mem, original)!!;
|
||||
defer free(compressed.ptr);
|
||||
|
||||
char[] decompressed = deflate::decompress(mem, compressed)!!;
|
||||
defer free(decompressed.ptr);
|
||||
|
||||
assert(decompressed.len == len, "Length mismatch");
|
||||
assert(decompressed[0] == 'A' && decompressed[len-1] == 'A', "Data mismatch");
|
||||
}
|
||||
|
||||
fn void test_deflate_random_ish()
|
||||
{
|
||||
// Data that doesn't compress well
|
||||
usz len = 1024;
|
||||
char[] original = mem::malloc(len)[:len];
|
||||
defer free(original.ptr);
|
||||
|
||||
for (usz i = 0; i < len; i++)
|
||||
{
|
||||
original[i] = (char)(i & 0xFF);
|
||||
}
|
||||
|
||||
char[] compressed = deflate::compress(mem, original)!!;
|
||||
defer free(compressed.ptr);
|
||||
|
||||
char[] decompressed = deflate::decompress(mem, compressed)!!;
|
||||
defer free(decompressed.ptr);
|
||||
|
||||
assert((String)decompressed == (String)original, "Data mismatch");
|
||||
}
|
||||
|
||||
fn void test_deflate_corrupted()
|
||||
{
|
||||
char[] compressed = deflate::compress(mem, "Some data")!!;
|
||||
defer free(compressed.ptr);
|
||||
|
||||
// Corrupt the block type (bits 1-2 of first byte) to 3 (reserved/invalid)
|
||||
compressed[0] |= 0x06;
|
||||
|
||||
char[]? decompressed = deflate::decompress(mem, compressed);
|
||||
assert(!@ok(decompressed), "Expected decompression to fail for corrupted data");
|
||||
}
|
||||
|
||||
fn void test_deflate_stream()
|
||||
{
|
||||
@pool()
|
||||
{
|
||||
String base = "This is a streaming test for DEFLATE. ";
|
||||
usz base_len = base.len;
|
||||
usz count = 50;
|
||||
char[] original_arr = mem::malloc(base_len * count)[:base_len * count];
|
||||
defer free(original_arr.ptr);
|
||||
for (usz i = 0; i < count; i++) {
|
||||
mem::copy(original_arr.ptr + i * base_len, base.ptr, base_len);
|
||||
}
|
||||
String original = (String)original_arr;
|
||||
|
||||
char[] compressed = deflate::compress(mem, original_arr)!!;
|
||||
defer free(compressed.ptr);
|
||||
|
||||
// Use a temporary file on disk to test the streaming interface
|
||||
File f = file::open("unittest_stream_deflate.bin", "wb+")!!;
|
||||
defer { (void)f.close(); (void)file::delete("unittest_stream_deflate.bin"); }
|
||||
|
||||
f.write(compressed)!!;
|
||||
f.seek(0, Seek.SET)!!;
|
||||
|
||||
// Decompress using stream
|
||||
File out_f = file::open("unittest_stream_out.bin", "wb+")!!;
|
||||
defer { (void)out_f.close(); (void)file::delete("unittest_stream_out.bin"); }
|
||||
|
||||
deflate::decompress_stream(&f, &out_f)!!;
|
||||
|
||||
usz out_size = out_f.seek(0, Seek.CURSOR)!!;
|
||||
assert(out_size == original.len, "Length mismatch in streaming decompression");
|
||||
|
||||
out_f.seek(0, Seek.SET)!!;
|
||||
char[] result = mem::malloc(out_size)[:out_size];
|
||||
defer free(result.ptr);
|
||||
out_f.read(result)!!;
|
||||
|
||||
assert((String)result == original, "Data mismatch in streaming decompression");
|
||||
};
|
||||
}
|
||||
|
||||
fn void test_deflate_embedded_stream()
|
||||
{
|
||||
String base = "This is a streaming test for DEFLATE. ";
|
||||
|
||||
char[] compressed = deflate::compress(mem, base[..])!!;
|
||||
defer free(compressed.ptr);
|
||||
|
||||
usz append_len = compressed.len + 1;
|
||||
char[] append = mem::malloc(append_len)[:append_len];
|
||||
defer free(append.ptr);
|
||||
|
||||
append[:compressed.len] = compressed[..];
|
||||
append[compressed.len..] = 'c';
|
||||
|
||||
ByteReader reader;
|
||||
reader.init(append);
|
||||
|
||||
ByteWriter writer;
|
||||
writer.tinit();
|
||||
|
||||
deflate::decompress_stream(&reader, &writer)!!;
|
||||
|
||||
assert(writer.str_view() == base);
|
||||
|
||||
assert(reader.read_byte()!! == 'c');
|
||||
}
|
||||
|
||||
fn void test_deflate_incremental()
|
||||
{
|
||||
@pool()
|
||||
{
|
||||
String original = "This is a test of incremental decompression. We will read it byte by byte.";
|
||||
char[] compressed = deflate::compress(mem, original)!!;
|
||||
defer free(compressed.ptr);
|
||||
|
||||
// Use a ByteReader for the compressed data
|
||||
io::ByteReader in_stream;
|
||||
in_stream.init(compressed);
|
||||
|
||||
Inflater* inflater = mem::new(Inflater);
|
||||
char[] bit_buf = mem::malloc(8192)[:8192];
|
||||
inflater.init(&in_stream, bit_buf);
|
||||
defer free(bit_buf.ptr);
|
||||
defer free(inflater);
|
||||
|
||||
char[] decompressed = mem::malloc(original.len)[:original.len];
|
||||
defer free(decompressed.ptr);
|
||||
|
||||
for (usz i = 0; i < original.len; i++)
|
||||
{
|
||||
char[1] one_byte;
|
||||
usz n = inflater.read(one_byte[..])!!;
|
||||
assert(n == 1, "Expected 1 byte");
|
||||
decompressed[i] = one_byte[0];
|
||||
}
|
||||
|
||||
// One more read should return 0 (or EOF)
|
||||
char[1] extra;
|
||||
assert(inflater.read(extra[..])!! == 0, "Expected EOF");
|
||||
|
||||
assert((String)original == (String)decompressed, "Incremental decompression failed");
|
||||
};
|
||||
}
|
||||
549
test/unit/stdlib/compression/zip.c3
Normal file
549
test/unit/stdlib/compression/zip.c3
Normal file
@@ -0,0 +1,549 @@
|
||||
module zip_test @test;
|
||||
|
||||
import std::io;
|
||||
import std::compression::zip;
|
||||
|
||||
fn void test_zip_store()
|
||||
{
|
||||
@pool()
|
||||
{
|
||||
// Create archive with uncompressed file
|
||||
ZipArchive zip = zip::open(mem, "unittest_store.zip", "w")!!;
|
||||
zip.write_file("test.txt", "Hello, World!", STORE)!!;
|
||||
(void)zip.close();
|
||||
defer (void)file::delete("unittest_store.zip");
|
||||
|
||||
// Read and verify
|
||||
ZipArchive read_zip = zip::open(mem, "unittest_store.zip", "r")!!;
|
||||
defer (void)read_zip.close();
|
||||
|
||||
assert(read_zip.count() == 1, "Expected 1 entry");
|
||||
|
||||
ZipEntry entry = read_zip.stat("test.txt")!!;
|
||||
assert(entry.method == STORE, "Expected STORE method");
|
||||
assert(entry.uncompressed_size == 13, "Expected 13 bytes");
|
||||
|
||||
char[] data = read_zip.read_file_all(mem, "test.txt")!!;
|
||||
defer free(data);
|
||||
assert((String)data == "Hello, World!", "Data mismatch");
|
||||
|
||||
};
|
||||
}
|
||||
|
||||
fn void test_zip_deflate()
|
||||
{
|
||||
@pool()
|
||||
{
|
||||
// Create archive with compressed file
|
||||
ZipArchive zip = zip::open(mem, "unittest_deflate.zip", "w")!!;
|
||||
|
||||
ZipEntryWriter writer = zip.open_writer("compressed.txt", DEFLATE)!!;
|
||||
String data = "This is a test. ";
|
||||
for (int i = 0; i < 100; i++)
|
||||
{
|
||||
writer.write((char[])data)!!;
|
||||
}
|
||||
writer.close()!!;
|
||||
(void)zip.close();
|
||||
defer (void)file::delete("unittest_deflate.zip");
|
||||
|
||||
// Read and verify
|
||||
ZipArchive read_zip = zip::open(mem, "unittest_deflate.zip", "r")!!;
|
||||
defer (void)read_zip.close();
|
||||
|
||||
assert(read_zip.count() == 1, "Expected 1 entry");
|
||||
|
||||
ZipEntry entry = read_zip.stat("compressed.txt")!!;
|
||||
assert(entry.method == DEFLATE, "Expected DEFLATE method");
|
||||
assert(entry.uncompressed_size == 1600, "Expected 1600 bytes");
|
||||
|
||||
char[] decompressed = read_zip.read_file_all(mem, "compressed.txt")!!;
|
||||
defer free(decompressed);
|
||||
assert(decompressed.len == 1600, "Decompressed size mismatch");
|
||||
|
||||
};
|
||||
}
|
||||
|
||||
fn void test_zip_directory()
|
||||
{
|
||||
@pool()
|
||||
{
|
||||
// Create archive with directory
|
||||
ZipArchive zip = zip::open(mem, "unittest_dir.zip", "w")!!;
|
||||
zip.add_directory("docs")!!;
|
||||
zip.write_file("docs/readme.txt", "README")!!;
|
||||
(void)zip.close();
|
||||
defer (void)file::delete("unittest_dir.zip");
|
||||
|
||||
// Read and verify
|
||||
ZipArchive read_zip = zip::open(mem, "unittest_dir.zip", "r")!!;
|
||||
defer (void)read_zip.close();
|
||||
|
||||
assert(read_zip.count() == 2, "Expected 2 entries");
|
||||
|
||||
ZipEntry dir_entry = read_zip.stat("docs/")!!;
|
||||
assert(dir_entry.is_directory, "Expected directory");
|
||||
assert(dir_entry.uncompressed_size == 0, "Directory should have 0 size");
|
||||
|
||||
ZipEntry file_entry = read_zip.stat("docs/readme.txt")!!;
|
||||
assert(!file_entry.is_directory, "Expected file");
|
||||
|
||||
char[] data = read_zip.read_file_all(mem, "docs/readme.txt")!!;
|
||||
defer free(data);
|
||||
assert((String)data == "README", "Data mismatch");
|
||||
|
||||
};
|
||||
}
|
||||
|
||||
fn void test_zip_crc32_verification()
|
||||
{
|
||||
@pool()
|
||||
{
|
||||
// Create archive
|
||||
ZipArchive zip = zip::open(mem, "unittest_crc.zip", "w")!!;
|
||||
zip.write_file("data.txt", "Test data for CRC32")!!;
|
||||
(void)zip.close();
|
||||
defer (void)file::delete("unittest_crc.zip");
|
||||
|
||||
ZipArchive read_zip = zip::open(mem, "unittest_crc.zip", "r")!!;
|
||||
defer (void)read_zip.close();
|
||||
|
||||
char[] data = read_zip.read_file_all(mem, "data.txt")!!;
|
||||
defer free(data);
|
||||
assert((String)data == "Test data for CRC32", "Data mismatch");
|
||||
|
||||
};
|
||||
}
|
||||
|
||||
fn void test_zip_multiple_files()
|
||||
{
|
||||
@pool()
|
||||
{
|
||||
// Create archive with multiple files
|
||||
ZipArchive zip = zip::open(mem, "unittest_multi.zip", "w")!!;
|
||||
zip.write_file("file1.txt", "First file")!!;
|
||||
zip.write_file("file2.txt", "Second file")!!;
|
||||
zip.write_file("file3.txt", "Third file")!!;
|
||||
(void)zip.close();
|
||||
defer (void)file::delete("unittest_multi.zip");
|
||||
|
||||
// Read and verify
|
||||
ZipArchive read_zip = zip::open(mem, "unittest_multi.zip", "r")!!;
|
||||
defer (void)read_zip.close();
|
||||
|
||||
assert(read_zip.count() == 3, "Expected 3 entries");
|
||||
|
||||
for (usz i = 0; i < read_zip.count(); i++)
|
||||
{
|
||||
ZipEntry entry = read_zip.stat_at(i)!!;
|
||||
assert(!entry.is_directory, "Expected files only");
|
||||
}
|
||||
|
||||
char[] data1 = read_zip.read_file_all(mem, "file1.txt")!!;
|
||||
defer free(data1);
|
||||
assert((String)data1 == "First file", "File1 mismatch");
|
||||
|
||||
char[] data2 = read_zip.read_file_all(mem, "file2.txt")!!;
|
||||
defer free(data2);
|
||||
assert((String)data2 == "Second file", "File2 mismatch");
|
||||
|
||||
char[] data3 = read_zip.read_file_all(mem, "file3.txt")!!;
|
||||
defer free(data3);
|
||||
assert((String)data3 == "Third file", "File3 mismatch");
|
||||
|
||||
};
|
||||
}
|
||||
|
||||
fn void test_zip_streaming()
|
||||
{
|
||||
@pool()
|
||||
{
|
||||
// Test streaming write
|
||||
ZipArchive zip = zip::open(mem, "unittest_stream.zip", "w")!!;
|
||||
|
||||
ZipEntryWriter writer = zip.open_writer("stream.txt", DEFLATE)!!;
|
||||
writer.write("Part 1. ")!!;
|
||||
writer.write("Part 2. ")!!;
|
||||
writer.write("Part 3.")!!;
|
||||
writer.close()!!;
|
||||
|
||||
(void)zip.close();
|
||||
defer (void)file::delete("unittest_stream.zip");
|
||||
|
||||
// Read and verify
|
||||
ZipArchive read_zip = zip::open(mem, "unittest_stream.zip", "r")!!;
|
||||
defer (void)read_zip.close();
|
||||
|
||||
char[] data = read_zip.read_file_all(mem, "stream.txt")!!;
|
||||
defer free(data);
|
||||
assert((String)data == "Part 1. Part 2. Part 3.", "Streaming write failed");
|
||||
|
||||
};
|
||||
}
|
||||
fn void test_zip_invalid_access()
|
||||
{
|
||||
@pool()
|
||||
{
|
||||
// Test non-existent archive
|
||||
ZipArchive? opt = zip::open(mem, "non_existent.zip", "r");
|
||||
assert(!@ok(opt), "Expected error when opening non-existent file");
|
||||
|
||||
// Test non-existent entry
|
||||
ZipArchive zip = zip::open(mem, "unittest_edge.zip", "w")!!;
|
||||
zip.write_file("exists.txt", "data")!!;
|
||||
(void)zip.close();
|
||||
defer (void)file::delete("unittest_edge.zip");
|
||||
|
||||
ZipArchive read_zip = zip::open(mem, "unittest_edge.zip", "r")!!;
|
||||
defer (void)read_zip.close();
|
||||
|
||||
ZipEntry? entry_opt = read_zip.stat("does_not_exist.txt");
|
||||
assert(!@ok(entry_opt), "Expected ENTRY_NOT_FOUND");
|
||||
|
||||
char[]? data_opt = read_zip.read_file_all(mem, "does_not_exist.txt");
|
||||
assert(!@ok(data_opt), "Expected error when reading non-existent file");
|
||||
assert(!@ok(data_opt), "Expected error when reading non-existent file");
|
||||
|
||||
};
|
||||
}
|
||||
|
||||
fn void test_zip_empty_archive()
|
||||
{
|
||||
@pool()
|
||||
{
|
||||
// Create empty archive
|
||||
ZipArchive zip = zip::open(mem, "unittest_empty.zip", "w")!!;
|
||||
(void)zip.close();
|
||||
defer (void)file::delete("unittest_empty.zip");
|
||||
|
||||
// Read empty archive
|
||||
ZipArchive read_zip = zip::open(mem, "unittest_empty.zip", "r")!!;
|
||||
defer (void)read_zip.close();
|
||||
|
||||
assert(read_zip.count() == 0, "Expected 0 entries");
|
||||
};
|
||||
}
|
||||
|
||||
fn void test_zip_recovery()
|
||||
{
|
||||
@pool()
|
||||
{
|
||||
String path = "unittest_embedded_broken.zip";
|
||||
// Create a "broken" ZIP (LFH + Data, but no Central Directory)
|
||||
// Filename: "a", Data: "bc"
|
||||
char[] broken_zip = {0x50,0x4B,0x03,0x04,0x14,0x00,0x00,0x08,0x00,0x00,0x00,0x00,0x00,0x00,0x38,0x2B,0xA9,0xC2,0x02,0x00,0x00,0x00,0x02,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x61,0x62,0x63};
|
||||
file::save(path, broken_zip)!!;
|
||||
defer (void)file::delete(path);
|
||||
|
||||
ZipArchive? normal = zip::open(mem, path, "r");
|
||||
assert(!@ok(normal), "Normal open should fail on broken ZIP");
|
||||
|
||||
ZipArchive recovered = zip::recover(mem, path)!!;
|
||||
defer (void)recovered.close();
|
||||
|
||||
assert(recovered.count() == 1, "Should have recovered 1 file");
|
||||
char[] data = recovered.read_file_all(mem, "a")!!;
|
||||
defer free(data);
|
||||
assert((String)data == "bc", "Recovered data mismatch");
|
||||
};
|
||||
}
|
||||
|
||||
fn void test_zip_cp437()
|
||||
{
|
||||
@pool()
|
||||
{
|
||||
String path = "unittest_embedded_cp437.zip";
|
||||
// Create a ZIP with CP437 encoding (Bit 11 NOT set)
|
||||
// Filename: 0x80 (Ç in CP437), Data: "x"
|
||||
char[] cp437_zip = {0x50,0x4B,0x03,0x04,0x14,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x83,0x16,0xDC,0x8C,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x80,0x78};
|
||||
|
||||
file::save(path, cp437_zip)!!;
|
||||
defer (void)file::delete(path);
|
||||
|
||||
ZipArchive recovered = zip::recover(mem, path)!!;
|
||||
defer (void)recovered.close();
|
||||
|
||||
ZipEntry entry = recovered.stat_at(0)!!;
|
||||
assert(entry.name == "Ç", "CP437 decoding failed");
|
||||
|
||||
char[] data = recovered.read_file_all(mem, "Ç")!!;
|
||||
defer free(data);
|
||||
assert((String)data == "x", "Data mismatch in CP437 test");
|
||||
};
|
||||
}
|
||||
|
||||
fn void test_zip_with_comment()
|
||||
{
|
||||
@pool()
|
||||
{
|
||||
// Create a ZIP file with a comment
|
||||
ZipArchive zip = zip::open(mem, "unittest_comment.zip", "w")!!;
|
||||
zip.write_file("test.txt", "Hello, World!")!!;
|
||||
(void)zip.close();
|
||||
defer (void)file::delete("unittest_comment.zip");
|
||||
|
||||
char[] zip_data = file::load(mem, "unittest_comment.zip")!!;
|
||||
defer free(zip_data);
|
||||
|
||||
isz eocd_pos = -1;
|
||||
for (isz i = (isz)zip_data.len - 22; i >= 0; i--)
|
||||
{
|
||||
uint sig = mem::load((uint*)&zip_data[i], 1);
|
||||
if (sig == 0x06054b50)
|
||||
{
|
||||
eocd_pos = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert(eocd_pos >= 0, "EOCD not found");
|
||||
|
||||
String comment = "This is a test comment!";
|
||||
mem::store((ushort*)&zip_data[eocd_pos + 20], (ushort)comment.len, 1);
|
||||
|
||||
char[] new_zip = mem::new_array(char, zip_data.len + comment.len);
|
||||
defer free(new_zip);
|
||||
mem::copy(new_zip.ptr, zip_data.ptr, zip_data.len);
|
||||
mem::copy(new_zip.ptr + zip_data.len, comment.ptr, comment.len);
|
||||
|
||||
file::save("unittest_comment.zip", new_zip[:zip_data.len + comment.len])!!;
|
||||
|
||||
// Try to open it
|
||||
ZipArchive read_zip = zip::open(mem, "unittest_comment.zip", "r")!!;
|
||||
defer (void)read_zip.close();
|
||||
|
||||
assert(read_zip.count() == 1, "Expected 1 entry");
|
||||
assert(read_zip.comment == comment, "Comment mismatch");
|
||||
|
||||
char[] data = read_zip.read_file_all(mem, "test.txt")!!;
|
||||
defer free(data);
|
||||
assert((String)data == "Hello, World!", "Data mismatch with comment");
|
||||
};
|
||||
}
|
||||
|
||||
fn void test_zip_write_comment()
|
||||
{
|
||||
@pool()
|
||||
{
|
||||
ZipArchive zip = zip::open(mem, "unittest_write_comment.zip", "w")!!;
|
||||
zip.comment = String.copy("Created by C3 ZIP library", zip.allocator);
|
||||
zip.write_file("test.txt", "Hello!")!!;
|
||||
(void)zip.close();
|
||||
defer (void)file::delete("unittest_write_comment.zip");
|
||||
|
||||
ZipArchive read_zip = zip::open(mem, "unittest_write_comment.zip", "r")!!;
|
||||
defer (void)read_zip.close();
|
||||
|
||||
assert(read_zip.comment == "Created by C3 ZIP library", "Comment not preserved");
|
||||
assert(read_zip.count() == 1, "Expected 1 entry");
|
||||
};
|
||||
}
|
||||
|
||||
fn void test_zip64_headers()
|
||||
{
|
||||
@pool()
|
||||
{
|
||||
String filename = "unittest_zip64.zip";
|
||||
ZipArchive zip = zip::open(mem, filename, "w")!!;
|
||||
|
||||
ZipEntryWriter writer = zip.open_writer("large.txt", STORE)!!;
|
||||
writer.write("data")!!;
|
||||
|
||||
// Manually set the size to > 4GB to trigger ZIP64 headers in the Central Directory.
|
||||
// This tests the fix for ZIP64 extra field serialization (ensuring no byte truncation).
|
||||
writer.entry.uncompressed_size = 0x100000001;
|
||||
writer.entry.compressed_size = 0x100000001;
|
||||
|
||||
writer.close()!!;
|
||||
(void)zip.close();
|
||||
defer (void)file::delete(filename);
|
||||
|
||||
ZipArchive read_zip = zip::open(mem, filename, "r")!!;
|
||||
defer (void)read_zip.close();
|
||||
|
||||
ZipEntry entry = read_zip.stat("large.txt")!!;
|
||||
assert(entry.uncompressed_size == 0x100000001, "Failed to read ZIP64 uncompressed size");
|
||||
assert(entry.compressed_size == 0x100000001, "Failed to read ZIP64 compressed size");
|
||||
};
|
||||
}
|
||||
|
||||
fn void test_zip_utf8()
|
||||
{
|
||||
@pool()
|
||||
{
|
||||
String filename = "unittest_utf8.zip";
|
||||
String utf8_name = "测试_🚀.txt";
|
||||
ZipArchive zip = zip::open(mem, filename, "w")!!;
|
||||
zip.write_file(utf8_name, "content")!!;
|
||||
(void)zip.close();
|
||||
defer (void)file::delete(filename);
|
||||
|
||||
ZipArchive read_zip = zip::open(mem, filename, "r")!!;
|
||||
defer (void)read_zip.close();
|
||||
|
||||
ZipEntry entry = read_zip.stat(utf8_name)!!;
|
||||
assert(entry.name == utf8_name, "UTF-8 filename mismatch");
|
||||
};
|
||||
}
|
||||
|
||||
fn void test_zip_zero_length()
|
||||
{
|
||||
@pool()
|
||||
{
|
||||
String filename = "unittest_zero.zip";
|
||||
ZipArchive zip = zip::open(mem, filename, "w")!!;
|
||||
zip.write_file("empty.txt", "")!!;
|
||||
(void)zip.close();
|
||||
defer (void)file::delete(filename);
|
||||
|
||||
ZipArchive read_zip = zip::open(mem, filename, "r")!!;
|
||||
defer (void)read_zip.close();
|
||||
|
||||
ZipEntry entry = read_zip.stat("empty.txt")!!;
|
||||
assert(entry.uncompressed_size == 0, "Size should be 0");
|
||||
|
||||
char[] data = read_zip.read_file_all(mem, "empty.txt")!!;
|
||||
defer free(data);
|
||||
assert(data.len == 0, "Read data should be empty");
|
||||
};
|
||||
}
|
||||
|
||||
fn void test_zip64_offset()
|
||||
{
|
||||
@pool()
|
||||
{
|
||||
String filename = "unittest_zip64_offset.zip";
|
||||
ZipArchive zip = zip::open(mem, filename, "w")!!;
|
||||
|
||||
ZipEntryWriter writer = zip.open_writer("offset_test.txt", STORE)!!;
|
||||
writer.write("data")!!;
|
||||
|
||||
// Manually set offset to > 4GB to trigger ZIP64 headers in the Central Directory
|
||||
writer.entry.offset = 0x100000005;
|
||||
|
||||
writer.close()!!;
|
||||
(void)zip.close();
|
||||
defer (void)file::delete(filename);
|
||||
|
||||
ZipArchive read_zip = zip::open(mem, filename, "r")!!;
|
||||
defer (void)read_zip.close();
|
||||
|
||||
ZipEntry entry = read_zip.stat("offset_test.txt")!!;
|
||||
assert(entry.offset == 0x100000005, "Failed to read ZIP64 offset");
|
||||
};
|
||||
}
|
||||
|
||||
fn void test_zip_reader_pos_and_seek()
|
||||
{
|
||||
@pool()
|
||||
{
|
||||
String path = "unittest_reader.zip";
|
||||
ZipArchive zip = zip::open(mem, path, "w")!!;
|
||||
zip.write_file("test.txt", "0123456789", STORE)!!;
|
||||
(void)zip.close();
|
||||
defer (void)file::delete(path);
|
||||
|
||||
ZipArchive read_zip = zip::open(mem, path, "r")!!;
|
||||
defer (void)read_zip.close();
|
||||
|
||||
ZipEntryReader reader = read_zip.open_reader("test.txt")!!;
|
||||
defer (void)reader.close();
|
||||
|
||||
assert(reader.len() == 10, "Expected length 10");
|
||||
assert(reader.available()!! == 10, "Expected 10 bytes available");
|
||||
assert(reader.pos == 0, "Expected pos 0");
|
||||
|
||||
char[3] buf;
|
||||
assert(reader.read(buf[..])!! == 3);
|
||||
assert((String)buf[..] == "012", "Expected '012'");
|
||||
assert(reader.pos == 3, "Expected pos 3");
|
||||
assert(reader.available()!! == 7, "Expected 7 bytes available");
|
||||
|
||||
assert(reader.seek(2, Seek.CURSOR)!! == 5, "Expected seek to 5");
|
||||
assert(reader.pos == 5, "Expected pos 5 after seek");
|
||||
assert(reader.available()!! == 5, "Expected 5 bytes available after seek");
|
||||
|
||||
assert(reader.read(buf[..])!! == 3);
|
||||
assert((String)buf[..] == "567", "Expected '567'");
|
||||
|
||||
assert(reader.seek(1, Seek.SET)!! == 1, "Expected seek to 1");
|
||||
assert(reader.read(buf[..])!! == 3);
|
||||
assert((String)buf[..] == "123", "Expected '123'");
|
||||
|
||||
assert(reader.seek(-2, Seek.END)!! == 8, "Expected seek to 8");
|
||||
assert(reader.read(buf[..])!! == 2);
|
||||
assert((String)buf[:2] == "89", "Expected '89'");
|
||||
assert(reader.available()!! == 0, "Expected 0 bytes available at end");
|
||||
|
||||
// Edge case: Negative seek SET
|
||||
assert(!@ok(reader.seek(-1, Seek.SET)), "Negative seek SET should fail");
|
||||
|
||||
// Edge case: Seek past end
|
||||
assert(reader.seek(100, Seek.SET)!! == 10, "Seek past end should cap at size");
|
||||
assert(reader.pos == 10, "Pos should be 10");
|
||||
};
|
||||
}
|
||||
|
||||
fn void test_zip_comment_boundary()
|
||||
{
|
||||
@pool()
|
||||
{
|
||||
String filename = "unittest_comment_limit.zip";
|
||||
|
||||
// 1. Test exactly 65535 bytes (Should pass)
|
||||
{
|
||||
ZipArchive zip = zip::open(mem, filename, "w")!!;
|
||||
char[] huge_comment = allocator::malloc(tmem, 65535)[:65535];
|
||||
mem::set(huge_comment.ptr, (char)'C', 65535);
|
||||
zip.comment = String.copy((String)huge_comment, zip.allocator);
|
||||
zip.write_file("t.txt", "d")!!;
|
||||
(void)zip.close();
|
||||
|
||||
ZipArchive read_zip = zip::open(mem, filename, "r")!!;
|
||||
assert(read_zip.comment.len == 65535, "Comment length mismatch at 65535");
|
||||
(void)read_zip.close();
|
||||
(void)file::delete(filename);
|
||||
}
|
||||
|
||||
// 2. Test 65536 bytes (Should fail with INVALID_ARGUMENT)
|
||||
{
|
||||
ZipArchive zip = zip::open(mem, filename, "w")!!;
|
||||
char[] too_huge = allocator::malloc(tmem, 65536)[:65536];
|
||||
mem::set(too_huge.ptr, (char)'X', 65536);
|
||||
zip.comment = String.copy((String)too_huge, zip.allocator);
|
||||
zip.write_file("t.txt", "d")!!;
|
||||
|
||||
fault res = @catch(zip.close());
|
||||
assert(res == zip::INVALID_ARGUMENT, "Expected INVALID_ARGUMENT for 64k+1 comment");
|
||||
(void)file::delete(filename);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
fn void test_zip_reader_available_capping()
|
||||
{
|
||||
@pool()
|
||||
{
|
||||
// We manually construct a reader to test the capping logic for huge entry sizes
|
||||
// that might exist on 32-bit systems (where usz < 64-bit).
|
||||
ZipEntryReader reader;
|
||||
mem::set(&reader, 0, ZipEntryReader.sizeof);
|
||||
reader.size = 0xFFFFFFFFFFFFFFFF;
|
||||
reader.pos = 0;
|
||||
|
||||
usz avail = reader.available()!!;
|
||||
assert(avail == usz.max, "Expected available to be capped at usz.max");
|
||||
|
||||
reader.pos = 100;
|
||||
avail = reader.available()!!;
|
||||
if (usz.max < 0xFFFFFFFFFFFFFFFF)
|
||||
{
|
||||
// triggers on 32-bit
|
||||
assert(avail == usz.max, "Expected available to still be capped at usz.max");
|
||||
}
|
||||
else
|
||||
{
|
||||
// on 64-bit
|
||||
assert(avail == usz.max - (usz)100, "Expected available size to be correct on 64-bit");
|
||||
}
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user