C3L zip support. Version bump.

CMakeLists.txt

@@ -168,6 +168,7 @@ message(STATUS "linking to llvm libs ${lld_libs}")
 message(STATUS "Found lld libs ${lld_libs}")
 
 add_library(c3c_wrappers STATIC wrapper/src/wrapper.cpp)
+add_library(miniz STATIC dependencies/miniz/miniz.c)
 
 add_executable(c3c
         src/build/builder.c
@@ -250,6 +251,7 @@ add_executable(c3c
         src/utils/vmem.h
         src/utils/whereami.c
         src/utils/cpus.c
+        src/utils/unzipper.c
         src/compiler/decltable.c
         src/compiler/mac_support.c
         src/compiler/tilde_codegen_storeload.c
@@ -273,10 +275,11 @@ target_include_directories(c3c PRIVATE
 target_include_directories(c3c_wrappers PRIVATE
         "${CMAKE_SOURCE_DIR}/wrapper/src/")
 
-
+target_include_directories(miniz PUBLIC
+        "${CMAKE_SOURCE_DIR}/dependencies/miniz/")
 
 target_link_libraries(c3c_wrappers ${llvm_libs} ${lld_libs})
-target_link_libraries(c3c ${llvm_libs} c3c_wrappers ${lld_libs})
+target_link_libraries(c3c ${llvm_libs} miniz c3c_wrappers ${lld_libs})
 if(C3_USE_TB)
     target_link_libraries(c3c c3c_wrappers ${TB_LIB})
     target_compile_definitions(c3c PUBLIC TB_BACKEND=1)
@@ -296,9 +299,11 @@ if(MSVC)
     if(CMAKE_BUILD_TYPE STREQUAL "Debug")
         target_compile_options(c3c PUBLIC /MTd)
         target_compile_options(c3c_wrappers PUBLIC /MTd)
+        target_compile_options(miniz PUBLIC /MTd)
     else()
         target_compile_options(c3c PUBLIC /MT)
         target_compile_options(c3c_wrappers PUBLIC /MT)
+        target_compile_options(miniz PUBLIC /MT)
     endif()
 else()
     message(STATUS "using gcc/clang warning switches")

resources/examples/raylib/raylib_snake.c3

@@ -1,4 +1,5 @@
 module snake;
+import raylib;
 /**
  *
  *   raylib - classic game: snake

resources/examples/spectralnorm.c3

@@ -1,10 +1,7 @@
 module spectralnorm;
-import std::mem;
 import std::array;
-
+import std::io;
-extern fn int atoi(char *s);
+import std::math;
-extern fn int printf(char *s, ...);
-extern fn double sqrt(double);
 
 double[] temparr;
 
@@ -13,47 +10,57 @@ fn double eval_A(int i, int j)
 	return 1.0 / ((i + j) * (i + j + 1) / 2 + i + 1);
 }
 
-fn void eval_A_times_u(double[] u, double[] au)
+fn void eval_A_times_u(double[] u, double[] au, double[] x)
 {
+	int len = au.len;
 	foreach (i, &val : au)
 	{
 		*val = 0;
 		foreach (j, uval : u)
 		{
-			*val += eval_A((int)(i), (int)(j)) * uval;
+			*val += x[i * len + j] * uval;
 		}
 	}
 }
 
-fn void eval_At_times_u(double[] u, double[] au)
+fn void eval_At_times_u(double[] u, double[] au, double[] x)
 {
+	int len = au.len;
 	foreach (i, &val : au)
 	{
 		*val = 0;
 		foreach (j, uval : u)
 		{
-			*val += eval_A((int)(j), (int)(i)) * uval;
+			*val += x[i * len + j] * uval;
 		}
 	}
 }
 
-fn void eval_AtA_times_u(double[] u, double[] atau) @noinline
+fn void eval_AtA_times_u(double[] u, double[] atau, double[] x)
 {
-	eval_A_times_u(u, temparr);
+	eval_A_times_u(u, temparr, x);
-	eval_At_times_u(temparr, atau);
+	eval_At_times_u(temparr, atau, x);
 }
 
-fn int main(int argc, char **argv)
+fn void main(char[][] args)
 {
-	int n = (argc == 2) ? atoi(argv[1]) : 2000;
+	int n = args.len == 2 ? str::to_int(args[1])!! : 2000;
 	temparr = array::alloc(double, n);
 	double[] u = array::alloc(double, n);
 	double[] v = array::alloc(double, n);
-	foreach(&uval : u) *uval = 1;
+	double[] x = array::alloc(double, n * n);
+	for (int i = 0; i < n; i++)
+	{
+		for (int j = 0; j < n; j++)
+		{
+			x[i * n + j] = eval_A(i, j);
+		}
+	}
+	u[..] = 1;
 	for (int i = 0; i < 10; i++)
 	{
-		eval_AtA_times_u(u, v);
+		eval_AtA_times_u(u, v, x);
-		eval_AtA_times_u(v, u);
+		eval_AtA_times_u(v, u, x);
 	}
 	double vBv;
 	double vv;
@@ -62,6 +69,5 @@ fn int main(int argc, char **argv)
 		vBv += u[i] * vval;
 		vv += vval * vval;
 	}
-	printf("%0.9f\n", sqrt(vBv / vv));
+	io::printf("%0.9f\n", math::sqrt(vBv / vv));
-	return 0;
 }

src/compiler/libraries.c

@@ -142,6 +142,71 @@ static void add_library_dependency(Library *library, Library **library_list, siz
 	}
 }
 
+INLINE void zip_check_err(const char *lib, const char *error)
+{
+	if (error) error_exit("Malformed compressed '%s' library: %s.", lib, error);
+}
+
+INLINE JSONObject* read_manifest(const char *lib, const char *manifest_data)
+{
+	JsonParser parser;
+	json_init_string(&parser, manifest_data, &malloc_arena);
+	JSONObject *json = json_parse(&parser);
+	if (parser.error_message)
+	{
+		error_exit("Error on line %d reading '%s':'%s'", parser.line, lib, parser.error_message);
+	}
+	return json;
+}
+
+static inline JSONObject *resolve_zip_library(const char *lib, const char **resulting_library)
+{
+	FILE *f = fopen(lib, "rb");
+	if (!f) error_exit("Failed to open library '%s' for reading.", lib);
+	ZipDirIterator iterator;
+	const char *error;
+
+	// Find the manifest.
+	ZipFile file;
+	zip_check_err(lib, zip_dir_iterator(f, &iterator));
+	do
+	{
+		if (iterator.current_file >= iterator.files) error_exit("Missing manifest in '%s'.", lib);
+		zip_check_err(lib, zip_dir_iterator_next(&iterator, &file));
+		if (strcmp(file.name, MANIFEST_FILE) == 0) break;
+	} while (1);
+
+	// Read the manifest.
+	char *manifest_data;
+	zip_check_err(lib, zip_file_read(f, &file, (void**)&manifest_data));
+
+	// Parse the JSON
+	JSONObject *json = read_manifest(lib, manifest_data);
+
+	// Create the directory for the temporary files.
+	const char *lib_name = filename(lib);
+	scratch_buffer_clear();
+	scratch_buffer_append(active_target.build_dir ? active_target.build_dir : "_temp_build");
+	scratch_buffer_printf("/_c3l/%s_%x/", lib_name, file.file_crc32);
+	const char *lib_dir = scratch_buffer_copy();
+	dir_make_recursive(scratch_buffer_to_string());
+	scratch_buffer_append_char('/');
+	char *dir = scratch_buffer_to_string();
+
+	// Iterate through all files.
+	zip_check_err(lib, zip_dir_iterator(f, &iterator));
+	while (iterator.current_file < iterator.files)
+	{
+		zip_check_err(lib, zip_dir_iterator_next(&iterator, &file));
+		if (file.uncompressed_size == 0 || file.name[0] == '.') continue;
+		// Copy file.
+		zip_file_write(f, &file, dir, false);
+	}
+	fclose(f);
+	*resulting_library = lib_dir;
+	return json;
+
+}
 void resolve_libraries(void)
 {
 	static const char *c3lib_suffix = ".c3l";
@@ -155,19 +220,17 @@ void resolve_libraries(void)
 	size_t lib_count = 0;
 	VECEACH(c3_libs, i)
 	{
-		size_t size;
 		const char *lib = c3_libs[i];
+		JSONObject *json;
 		if (!file_is_dir(lib))
 		{
-			error_exit("Packaged .c3l are not supported yet.");
+			json = resolve_zip_library(lib, &lib);
 		}
-		const char *manifest_path = file_append_path(lib, MANIFEST_FILE);
+		else
-		char *read = file_read_all(manifest_path, &size);
-		json_init_string(&parser, read, &malloc_arena);
-		JSONObject *json = json_parse(&parser);
-		if (parser.error_message)
 		{
-			error_exit("Error on line %d reading '%s':'%s'", parser.line, manifest_path, parser.error_message);
+			const char *manifest_path = file_append_path(lib, MANIFEST_FILE);
+			size_t size;
+			json = read_manifest(lib, file_read_all(manifest_path, &size));
 		}
 		if (lib_count == MAX_LIB_DIRS * 2) error_exit("Too many libraries added, exceeded %d.", MAX_LIB_DIRS * 2);
 		libraries[lib_count++] = add_library(json, lib);

src/utils/common.h

@@ -58,3 +58,10 @@
 #define __unused
 #endif
 
+#ifdef __GNUC__
+#define PACK( __Declaration__ ) __Declaration__ __attribute__((__packed__))
+#endif
+
+#ifdef _MSC_VER
+#define PACK( __Declaration__ ) __pragma( pack(push, 1) ) __Declaration__ __pragma( pack(pop))
+#endif

src/utils/file_utils.c

@@ -87,6 +87,23 @@ bool dir_make(const char *path)
 #endif
 }
 
+bool dir_make_recursive(char *path)
+{
+	size_t len = strlen(path);
+	for (size_t i = len; i > 1; i--)
+	{
+		char c = path[i];
+		if (c == '\\' || c == '/')
+		{
+			path[i] = '\0';
+			dir_make_recursive(path);
+			path[i] = c;
+			break;
+		}
+	}
+	return dir_make(path);
+}
+
 bool dir_change(const char *path)
 {
 #if (_MSC_VER)
@@ -105,6 +122,22 @@ static inline bool is_path_separator(char c)
 #endif
 }
 
+const char *filename(const char *path)
+{
+	// Find the filename.
+	for (size_t j = strlen(path); j > 0; j--)
+	{
+		switch (path[j - 1])
+		{
+			case '/':
+			case '\\':
+				return &path[j];
+			default:
+				break;
+		}
+	}
+	return path;
+}
 
 /**
  * Split a file into path + filename, allocating memory for them and returning them in

src/utils/lib.h

@@ -60,8 +60,10 @@ uint16_t *win_utf8to16(const char *name);
 char *win_utf16to8(const uint16_t *name);
 // Use as if it was mkdir(..., 0755) == 0
 bool dir_make(const char *path);
+bool dir_make_recursive(char *path);
 // Use as if it was chdir(...) == 0
 bool dir_change(const char *path);
+const char *filename(const char *path);
 bool file_namesplit(const char *path, char** filename_ptr, char** directory_ptr);
 const char* file_expand_path(const char* path);
 const char* find_lib_dir(void);
@@ -580,3 +582,27 @@ static inline bool char_is_letter_(char c)
 	}
 }
 
+#define ZIP_MAX_NAME 512
+
+typedef struct
+{
+	char name[ZIP_MAX_NAME];
+	size_t offset;
+	size_t uncompressed_size;
+	size_t compressed_size;
+	uint32_t file_crc32;
+	int compression_method;
+} ZipFile;
+
+typedef struct
+{
+	long offset;
+	int files;
+	int current_file;
+	FILE *file;
+} ZipDirIterator;
+
+const char *zip_dir_iterator(FILE *zip, ZipDirIterator *iterator);
+const char *zip_dir_iterator_next(ZipDirIterator *iterator, ZipFile *file);
+const char *zip_file_read(FILE *zip, ZipFile *file, void **buffer_ptr);
+const char *zip_file_write(FILE *zip, ZipFile *file, const char *dir, bool overwrite);

src/utils/unzipper.c

@@ -0,0 +1,344 @@
+/**
+ * This code is indebted to the
+ * JUnzip library by Joonas Pihlajamaa (firstname.lastname@iki.fi).
+ */
+#include "lib.h"
+#include "miniz.h"
+
+#define ZIP_BUFFER_SIZE 65536
+#define FILE_OUTBUF_LEN 65536
+
+uint8_t internal_buffer[ZIP_BUFFER_SIZE]; // limits maximum zip descriptor size
+uint8_t file_out_buffer[FILE_OUTBUF_LEN];
+
+typedef PACK(struct
+{
+    uint32_t signature; // 0x04034B50
+    uint16_t version_needed_to_extract; // unsupported
+    uint16_t general_purpose_bit_flag; // unsupported
+    uint16_t compression_method;
+    uint16_t last_mod_file_time;
+    uint16_t last_mod_file_date;
+    uint32_t crc32;
+    uint32_t compressed_size;
+    uint32_t uncompressed_size;
+    uint16_t filename_len;
+    uint16_t extra_field_len; // unsupported
+}) JZLocalFileHeader;
+
+typedef PACK(struct
+{
+    uint32_t signature; // 0x02014B50
+    uint16_t version_made_by; // unsupported
+    uint16_t version_needed_to_extract; // unsupported
+    uint16_t general_purpose_bit_flag; // unsupported
+    uint16_t compression_method;
+    uint16_t last_mod_file_time;
+    uint16_t last_mod_file_date;
+    uint32_t crc32;
+    uint32_t compressed_size;
+    uint32_t uncompressed_size;
+    uint16_t file_name_len;
+    uint16_t extra_field_len; // unsupported
+    uint16_t comment_len; // unsupported
+    uint16_t disk_number_start; // unsupported
+    uint16_t internal_file_attributes; // unsupported
+    uint32_t external_file_attributes; // unsupported
+    uint32_t relative_offset_of_local_header;
+}) ZipGlobalFileHeader;
+
+
+typedef PACK(struct
+{
+    uint32_t signature; // 0x06054b50
+    uint16_t disk_number; // unsupported
+    uint16_t central_dir_disk_number; // unsupported
+    uint16_t num_entries_this_disk; // unsupported
+    uint16_t num_entries;
+    uint32_t central_dir_size;
+    uint32_t central_dir_offset;
+    uint16_t zip_comment_len;
+    // Followed by .ZIP file comment (variable size)
+}) ZipEndRecord;
+
+INLINE bool read_all(FILE *file, void *buffer, size_t len)
+{
+	size_t read = fread(buffer, 1, len, file);
+	return read == len;
+}
+
+const char *zip_dir_iterator(FILE *zip, ZipDirIterator *iterator)
+{
+	if (fseek(zip, 0, SEEK_END)) return "Couldn't move to end of .c3l file!";
+
+	long file_size = ftell(zip);
+	if (file_size <= sizeof(ZipEndRecord)) return "Too small to be a .c3l";
+
+	long read_bytes = file_size < sizeof(internal_buffer) ? file_size : sizeof(internal_buffer);
+
+	if (fseek(zip, file_size - read_bytes, SEEK_SET)) return "Cannot seek in .c3l file";
+
+	if (!read_all(zip, internal_buffer, read_bytes)) return "Couldn't read end of .c3l file";
+
+	// Naively assume signature can only be found in one place...
+	ZipEndRecord *er;
+	long i = 0;
+	for (i = read_bytes - sizeof(ZipEndRecord); i >= 0; i--)
+	{
+		er = (ZipEndRecord *)(internal_buffer + i);
+		if (er->signature == 0x06054B50) break;
+	}
+
+	if (i < 0) return "End record signature not found in .c3l file";
+
+	ZipEndRecord record;
+	memcpy(&record, er, sizeof(ZipEndRecord));
+
+	if (record.disk_number || record.central_dir_disk_number ||
+			record.num_entries != record.num_entries_this_disk)
+	{
+		return "Unsupported .c3l structure";
+	}
+
+	iterator->offset = record.central_dir_offset;
+	iterator->files = record.num_entries;
+	iterator->current_file = 0;
+	iterator->file = zip;
+	return NULL;
+}
+
+const char *zip_dir_iterator_next(ZipDirIterator *iterator, ZipFile *file)
+{
+	assert(iterator->current_file < iterator->files);
+	iterator->current_file++;
+	FILE *zip = iterator->file;
+	if (fseek(zip, iterator->offset, SEEK_SET)) return "Cannot seek in c3l file!";
+	ZipGlobalFileHeader file_header;
+	if (!read_all(zip, &file_header, sizeof(ZipGlobalFileHeader)))
+	{
+		return str_printf("Couldn't read file header %d!", iterator->current_file);
+	}
+
+	if (file_header.signature != 0x02014B50)
+	{
+		return str_printf("Invalid file header signature %d!", iterator->current_file);
+	}
+
+	if (ZIP_MAX_NAME < file_header.file_name_len + 1)
+	{
+		return str_printf("Filename too long %d", iterator->current_file);
+	}
+	if (!read_all(zip, file->name, file_header.file_name_len))
+	{
+		return str_printf("Couldn't read filename %d!", iterator->current_file);
+	}
+
+	file->name[file_header.file_name_len] = '\0';
+	if (fseek(zip, file_header.extra_field_len, SEEK_CUR) ||
+		fseek(zip, file_header.comment_len, SEEK_CUR))
+	{
+		return str_printf("Couldn't skip extra field or file comment %s", file->name);
+	}
+	if (file_header.compression_method != 0 && file_header.compression_method != 8)
+	{
+		return str_printf("Illegal compression method '%s'", file->name);
+	}
+	if (file_header.compression_method == 0 &&
+	    (file_header.compressed_size != file_header.uncompressed_size))
+	{
+		return str_printf("Invalid compression '%s'", file->name);
+	}
+	file->uncompressed_size = file_header.uncompressed_size;
+	file->compressed_size = file_header.compressed_size;
+	file->offset = file_header.relative_offset_of_local_header + sizeof(JZLocalFileHeader) - 4;
+	file->file_crc32 = file_header.crc32;
+	file->compression_method = file_header.compression_method;
+	iterator->offset = ftell(zip);
+	return NULL;
+}
+
+INLINE const char *zip_prepare_zip_for_read(FILE *zip, ZipFile *file)
+{
+	if (fseek(zip, file->offset, SEEK_SET)) return "Failed to search in file.";
+
+	uint16_t field1;
+	uint16_t field2;
+	if (!read_all(zip, &field1, 2)) return "Failed to read name len";
+	if (!read_all(zip, &field2, 2)) return "Failed to read extra len";
+	if (fseek(zip, field1 + field2, SEEK_CUR)) return "Failed to skip len";
+	return NULL;
+}
+
+const char *zip_file_read(FILE *zip, ZipFile *file, void **buffer_ptr)
+{
+	const char *error = zip_prepare_zip_for_read(zip, file);
+	if (error) return error;
+
+	unsigned char *bytes = MALLOC(file->uncompressed_size);
+	*buffer_ptr = bytes;
+
+	// Uncompressed
+	if (file->compression_method == 0)
+	{
+		if (!read_all(zip, bytes, file->uncompressed_size) || ferror(zip)) return "Failed to read data.";
+		return NULL;
+	}
+
+	// Only deflate supported.
+	assert(file->compression_method == 8 && "Should already be checked.");
+
+	// Deflate - using zlib
+	z_stream strm = { .zalloc = Z_NULL, .zfree = Z_NULL, .opaque = Z_NULL, .avail_in = 0, .next_in = Z_NULL };
+
+	// Use inflateInit2 with negative window bits to indicate raw data
+	if (inflateInit2(&strm, -MAX_WBITS) != Z_OK) return "Failed to init zlib";
+
+	// Inflate compressed data
+	long compressed_left = file->compressed_size;
+	long uncompressed_left = file->uncompressed_size;
+	while (compressed_left && uncompressed_left)
+	{
+		size_t to_read = ZIP_BUFFER_SIZE < compressed_left ? ZIP_BUFFER_SIZE : compressed_left;
+		strm.avail_in = fread(internal_buffer, 1, to_read, zip);
+		if (strm.avail_in == 0 || ferror(zip))
+		{
+			inflateEnd(&strm);
+			return "Failed to read zip";
+		}
+		strm.next_in = internal_buffer;
+		strm.avail_out = uncompressed_left;
+		strm.next_out = bytes;
+
+		compressed_left -= strm.avail_in;
+
+		switch (inflate(&strm, Z_NO_FLUSH))
+		{
+			case Z_STREAM_ERROR:
+				return "Unexpected inflate error";
+			case Z_NEED_DICT:
+			case Z_DATA_ERROR:
+				return "Inflate data error";
+			case Z_MEM_ERROR:
+				return "Inflate memory error";
+			case Z_STREAM_END:
+				goto END;
+			default:
+				break;
+		}
+		bytes += uncompressed_left - strm.avail_out;
+		uncompressed_left = strm.avail_out;
+	}
+END:
+	inflateEnd(&strm);
+	return NULL;
+}
+
+const char *zip_file_write(FILE *zip, ZipFile *file, const char *dir, bool overwrite)
+{
+	const char *error = zip_prepare_zip_for_read(zip, file);
+	if (error) return error;
+
+	char *file_name;
+	char *dir_path;
+	if (!file_namesplit(file->name, &file_name, &dir_path)) return "Failed to split file name";
+
+	if (dir_path)
+	{
+		char *new_path = str_printf("%s/%s", dir, dir_path);
+		dir_make_recursive(new_path);
+		dir = new_path;
+	}
+
+	const char *out_file_name = file_append_path(dir, file_name);
+	if (!overwrite && file_exists(out_file_name)) return NULL;
+
+	FILE *f = fopen(out_file_name, "wb");
+	if (!f) return "Failed to open file output path.";
+
+	// Uncompressed
+	if (file->compression_method == 0)
+	{
+		size_t left_to_read = file->uncompressed_size;
+		while (left_to_read)
+		{
+			size_t amount_to_read = left_to_read < ZIP_BUFFER_SIZE ? left_to_read : ZIP_BUFFER_SIZE;
+			assert(amount_to_read > 0);
+			if (!read_all(zip, internal_buffer, amount_to_read))
+			{
+				fclose(f);
+				return "Failed to read data";
+			}
+			size_t written = fwrite(internal_buffer, 1, amount_to_read, f);
+			if (written != amount_to_read)
+			{
+				fclose(f);
+				return "Failed to write";
+			}
+		}
+		return NULL;
+	}
+
+	// Only deflate supported.
+	assert(file->compression_method == 8 && "Should already be checked.");
+
+	// Deflate - using zlib
+	z_stream strm = { .zalloc = Z_NULL, .zfree = Z_NULL, .opaque = Z_NULL, .avail_in = 0, .next_in = Z_NULL };
+
+	// Use inflateInit2 with negative window bits to indicate raw data
+	if (inflateInit2(&strm, -MAX_WBITS) != Z_OK) return "Failed to init zlib";
+
+	// Inflate compressed data
+	long compressed_left = file->compressed_size;
+	while (compressed_left)
+	{
+		size_t to_read = ZIP_BUFFER_SIZE < compressed_left ? ZIP_BUFFER_SIZE : compressed_left;
+		strm.avail_in = fread(internal_buffer, 1, to_read, zip);
+		if (strm.avail_in == 0 || ferror(zip))
+		{
+			inflateEnd(&strm);
+			fclose(f);
+			return "Failed to read zip";
+		}
+
+		strm.next_in = internal_buffer;
+		while (strm.avail_in)
+		{
+			strm.avail_out = FILE_OUTBUF_LEN;
+			strm.next_out = file_out_buffer;
+
+			bool stream_end = false;
+			switch (inflate(&strm, Z_NO_FLUSH))
+			{
+				case Z_STREAM_ERROR:
+					fclose(f);
+					return "Unexpected inflate error";
+				case Z_NEED_DICT:
+				case Z_DATA_ERROR:
+					fclose(f);
+					return "Inflate data error";
+				case Z_MEM_ERROR:
+					fclose(f);
+					return "Inflate memory error";
+				case Z_STREAM_END:
+					stream_end = true;
+					FALLTHROUGH;
+				default:
+					break;
+			}
+
+			size_t to_write = FILE_OUTBUF_LEN - strm.avail_out;
+
+			if (to_write > 0 && to_write != fwrite(file_out_buffer, 1, to_write, f))
+			{
+				fclose(f);
+				return "Failed to write";
+			}
+			if (stream_end) break;
+		}
+		compressed_left = file->compressed_size - strm.total_in;
+	}
+	fclose(f);
+	inflateEnd(&strm);
+	return NULL;
+}
+

src/version.h

@@ -1 +1 @@
-#define COMPILER_VERSION "0.4.1"
+#define COMPILER_VERSION "0.4.2"