C3L zip support. Version bump.

src/compiler/libraries.c

@@ -142,6 +142,71 @@ static void add_library_dependency(Library *library, Library **library_list, siz
 	}
 }
 
+INLINE void zip_check_err(const char *lib, const char *error)
+{
+	if (error) error_exit("Malformed compressed '%s' library: %s.", lib, error);
+}
+
+INLINE JSONObject* read_manifest(const char *lib, const char *manifest_data)
+{
+	JsonParser parser;
+	json_init_string(&parser, manifest_data, &malloc_arena);
+	JSONObject *json = json_parse(&parser);
+	if (parser.error_message)
+	{
+		error_exit("Error on line %d reading '%s':'%s'", parser.line, lib, parser.error_message);
+	}
+	return json;
+}
+
+static inline JSONObject *resolve_zip_library(const char *lib, const char **resulting_library)
+{
+	FILE *f = fopen(lib, "rb");
+	if (!f) error_exit("Failed to open library '%s' for reading.", lib);
+	ZipDirIterator iterator;
+	const char *error;
+
+	// Find the manifest.
+	ZipFile file;
+	zip_check_err(lib, zip_dir_iterator(f, &iterator));
+	do
+	{
+		if (iterator.current_file >= iterator.files) error_exit("Missing manifest in '%s'.", lib);
+		zip_check_err(lib, zip_dir_iterator_next(&iterator, &file));
+		if (strcmp(file.name, MANIFEST_FILE) == 0) break;
+	} while (1);
+
+	// Read the manifest.
+	char *manifest_data;
+	zip_check_err(lib, zip_file_read(f, &file, (void**)&manifest_data));
+
+	// Parse the JSON
+	JSONObject *json = read_manifest(lib, manifest_data);
+
+	// Create the directory for the temporary files.
+	const char *lib_name = filename(lib);
+	scratch_buffer_clear();
+	scratch_buffer_append(active_target.build_dir ? active_target.build_dir : "_temp_build");
+	scratch_buffer_printf("/_c3l/%s_%x/", lib_name, file.file_crc32);
+	const char *lib_dir = scratch_buffer_copy();
+	dir_make_recursive(scratch_buffer_to_string());
+	scratch_buffer_append_char('/');
+	char *dir = scratch_buffer_to_string();
+
+	// Iterate through all files.
+	zip_check_err(lib, zip_dir_iterator(f, &iterator));
+	while (iterator.current_file < iterator.files)
+	{
+		zip_check_err(lib, zip_dir_iterator_next(&iterator, &file));
+		if (file.uncompressed_size == 0 || file.name[0] == '.') continue;
+		// Copy file.
+		zip_file_write(f, &file, dir, false);
+	}
+	fclose(f);
+	*resulting_library = lib_dir;
+	return json;
+
+}
 void resolve_libraries(void)
 {
 	static const char *c3lib_suffix = ".c3l";
@@ -155,19 +220,17 @@ void resolve_libraries(void)
 	size_t lib_count = 0;
 	VECEACH(c3_libs, i)
 	{
-		size_t size;
 		const char *lib = c3_libs[i];
+		JSONObject *json;
 		if (!file_is_dir(lib))
 		{
-			error_exit("Packaged .c3l are not supported yet.");
+			json = resolve_zip_library(lib, &lib);
 		}
-		const char *manifest_path = file_append_path(lib, MANIFEST_FILE);
-		char *read = file_read_all(manifest_path, &size);
-		json_init_string(&parser, read, &malloc_arena);
-		JSONObject *json = json_parse(&parser);
-		if (parser.error_message)
+		else
 		{
-			error_exit("Error on line %d reading '%s':'%s'", parser.line, manifest_path, parser.error_message);
+			const char *manifest_path = file_append_path(lib, MANIFEST_FILE);
+			size_t size;
+			json = read_manifest(lib, file_read_all(manifest_path, &size));
 		}
 		if (lib_count == MAX_LIB_DIRS * 2) error_exit("Too many libraries added, exceeded %d.", MAX_LIB_DIRS * 2);
 		libraries[lib_count++] = add_library(json, lib);

src/utils/common.h

@@ -58,3 +58,10 @@
 #define __unused
 #endif
 
+#ifdef __GNUC__
+#define PACK( __Declaration__ ) __Declaration__ __attribute__((__packed__))
+#endif
+
+#ifdef _MSC_VER
+#define PACK( __Declaration__ ) __pragma( pack(push, 1) ) __Declaration__ __pragma( pack(pop))
+#endif

src/utils/file_utils.c

@@ -87,6 +87,23 @@ bool dir_make(const char *path)
 #endif
 }
 
+bool dir_make_recursive(char *path)
+{
+	size_t len = strlen(path);
+	for (size_t i = len; i > 1; i--)
+	{
+		char c = path[i];
+		if (c == '\\' || c == '/')
+		{
+			path[i] = '\0';
+			dir_make_recursive(path);
+			path[i] = c;
+			break;
+		}
+	}
+	return dir_make(path);
+}
+
 bool dir_change(const char *path)
 {
 #if (_MSC_VER)
@@ -105,6 +122,22 @@ static inline bool is_path_separator(char c)
 #endif
 }
 
+const char *filename(const char *path)
+{
+	// Find the filename.
+	for (size_t j = strlen(path); j > 0; j--)
+	{
+		switch (path[j - 1])
+		{
+			case '/':
+			case '\\':
+				return &path[j];
+			default:
+				break;
+		}
+	}
+	return path;
+}
 
 /**
  * Split a file into path + filename, allocating memory for them and returning them in

src/utils/lib.h

@@ -60,8 +60,10 @@ uint16_t *win_utf8to16(const char *name);
 char *win_utf16to8(const uint16_t *name);
 // Use as if it was mkdir(..., 0755) == 0
 bool dir_make(const char *path);
+bool dir_make_recursive(char *path);
 // Use as if it was chdir(...) == 0
 bool dir_change(const char *path);
+const char *filename(const char *path);
 bool file_namesplit(const char *path, char** filename_ptr, char** directory_ptr);
 const char* file_expand_path(const char* path);
 const char* find_lib_dir(void);
@@ -580,3 +582,27 @@ static inline bool char_is_letter_(char c)
 	}
 }
 
+#define ZIP_MAX_NAME 512
+
+typedef struct
+{
+	char name[ZIP_MAX_NAME];
+	size_t offset;
+	size_t uncompressed_size;
+	size_t compressed_size;
+	uint32_t file_crc32;
+	int compression_method;
+} ZipFile;
+
+typedef struct
+{
+	long offset;
+	int files;
+	int current_file;
+	FILE *file;
+} ZipDirIterator;
+
+const char *zip_dir_iterator(FILE *zip, ZipDirIterator *iterator);
+const char *zip_dir_iterator_next(ZipDirIterator *iterator, ZipFile *file);
+const char *zip_file_read(FILE *zip, ZipFile *file, void **buffer_ptr);
+const char *zip_file_write(FILE *zip, ZipFile *file, const char *dir, bool overwrite);

src/utils/unzipper.c

@@ -0,0 +1,344 @@
+/**
+ * This code is indebted to the
+ * JUnzip library by Joonas Pihlajamaa (firstname.lastname@iki.fi).
+ */
+#include "lib.h"
+#include "miniz.h"
+
+#define ZIP_BUFFER_SIZE 65536
+#define FILE_OUTBUF_LEN 65536
+
+uint8_t internal_buffer[ZIP_BUFFER_SIZE]; // limits maximum zip descriptor size
+uint8_t file_out_buffer[FILE_OUTBUF_LEN];
+
+typedef PACK(struct
+{
+    uint32_t signature; // 0x04034B50
+    uint16_t version_needed_to_extract; // unsupported
+    uint16_t general_purpose_bit_flag; // unsupported
+    uint16_t compression_method;
+    uint16_t last_mod_file_time;
+    uint16_t last_mod_file_date;
+    uint32_t crc32;
+    uint32_t compressed_size;
+    uint32_t uncompressed_size;
+    uint16_t filename_len;
+    uint16_t extra_field_len; // unsupported
+}) JZLocalFileHeader;
+
+typedef PACK(struct
+{
+    uint32_t signature; // 0x02014B50
+    uint16_t version_made_by; // unsupported
+    uint16_t version_needed_to_extract; // unsupported
+    uint16_t general_purpose_bit_flag; // unsupported
+    uint16_t compression_method;
+    uint16_t last_mod_file_time;
+    uint16_t last_mod_file_date;
+    uint32_t crc32;
+    uint32_t compressed_size;
+    uint32_t uncompressed_size;
+    uint16_t file_name_len;
+    uint16_t extra_field_len; // unsupported
+    uint16_t comment_len; // unsupported
+    uint16_t disk_number_start; // unsupported
+    uint16_t internal_file_attributes; // unsupported
+    uint32_t external_file_attributes; // unsupported
+    uint32_t relative_offset_of_local_header;
+}) ZipGlobalFileHeader;
+
+
+typedef PACK(struct
+{
+    uint32_t signature; // 0x06054b50
+    uint16_t disk_number; // unsupported
+    uint16_t central_dir_disk_number; // unsupported
+    uint16_t num_entries_this_disk; // unsupported
+    uint16_t num_entries;
+    uint32_t central_dir_size;
+    uint32_t central_dir_offset;
+    uint16_t zip_comment_len;
+    // Followed by .ZIP file comment (variable size)
+}) ZipEndRecord;
+
+INLINE bool read_all(FILE *file, void *buffer, size_t len)
+{
+	size_t read = fread(buffer, 1, len, file);
+	return read == len;
+}
+
+const char *zip_dir_iterator(FILE *zip, ZipDirIterator *iterator)
+{
+	if (fseek(zip, 0, SEEK_END)) return "Couldn't move to end of .c3l file!";
+
+	long file_size = ftell(zip);
+	if (file_size <= sizeof(ZipEndRecord)) return "Too small to be a .c3l";
+
+	long read_bytes = file_size < sizeof(internal_buffer) ? file_size : sizeof(internal_buffer);
+
+	if (fseek(zip, file_size - read_bytes, SEEK_SET)) return "Cannot seek in .c3l file";
+
+	if (!read_all(zip, internal_buffer, read_bytes)) return "Couldn't read end of .c3l file";
+
+	// Naively assume signature can only be found in one place...
+	ZipEndRecord *er;
+	long i = 0;
+	for (i = read_bytes - sizeof(ZipEndRecord); i >= 0; i--)
+	{
+		er = (ZipEndRecord *)(internal_buffer + i);
+		if (er->signature == 0x06054B50) break;
+	}
+
+	if (i < 0) return "End record signature not found in .c3l file";
+
+	ZipEndRecord record;
+	memcpy(&record, er, sizeof(ZipEndRecord));
+
+	if (record.disk_number || record.central_dir_disk_number ||
+			record.num_entries != record.num_entries_this_disk)
+	{
+		return "Unsupported .c3l structure";
+	}
+
+	iterator->offset = record.central_dir_offset;
+	iterator->files = record.num_entries;
+	iterator->current_file = 0;
+	iterator->file = zip;
+	return NULL;
+}
+
+const char *zip_dir_iterator_next(ZipDirIterator *iterator, ZipFile *file)
+{
+	assert(iterator->current_file < iterator->files);
+	iterator->current_file++;
+	FILE *zip = iterator->file;
+	if (fseek(zip, iterator->offset, SEEK_SET)) return "Cannot seek in c3l file!";
+	ZipGlobalFileHeader file_header;
+	if (!read_all(zip, &file_header, sizeof(ZipGlobalFileHeader)))
+	{
+		return str_printf("Couldn't read file header %d!", iterator->current_file);
+	}
+
+	if (file_header.signature != 0x02014B50)
+	{
+		return str_printf("Invalid file header signature %d!", iterator->current_file);
+	}
+
+	if (ZIP_MAX_NAME < file_header.file_name_len + 1)
+	{
+		return str_printf("Filename too long %d", iterator->current_file);
+	}
+	if (!read_all(zip, file->name, file_header.file_name_len))
+	{
+		return str_printf("Couldn't read filename %d!", iterator->current_file);
+	}
+
+	file->name[file_header.file_name_len] = '\0';
+	if (fseek(zip, file_header.extra_field_len, SEEK_CUR) ||
+		fseek(zip, file_header.comment_len, SEEK_CUR))
+	{
+		return str_printf("Couldn't skip extra field or file comment %s", file->name);
+	}
+	if (file_header.compression_method != 0 && file_header.compression_method != 8)
+	{
+		return str_printf("Illegal compression method '%s'", file->name);
+	}
+	if (file_header.compression_method == 0 &&
+	    (file_header.compressed_size != file_header.uncompressed_size))
+	{
+		return str_printf("Invalid compression '%s'", file->name);
+	}
+	file->uncompressed_size = file_header.uncompressed_size;
+	file->compressed_size = file_header.compressed_size;
+	file->offset = file_header.relative_offset_of_local_header + sizeof(JZLocalFileHeader) - 4;
+	file->file_crc32 = file_header.crc32;
+	file->compression_method = file_header.compression_method;
+	iterator->offset = ftell(zip);
+	return NULL;
+}
+
+INLINE const char *zip_prepare_zip_for_read(FILE *zip, ZipFile *file)
+{
+	if (fseek(zip, file->offset, SEEK_SET)) return "Failed to search in file.";
+
+	uint16_t field1;
+	uint16_t field2;
+	if (!read_all(zip, &field1, 2)) return "Failed to read name len";
+	if (!read_all(zip, &field2, 2)) return "Failed to read extra len";
+	if (fseek(zip, field1 + field2, SEEK_CUR)) return "Failed to skip len";
+	return NULL;
+}
+
+const char *zip_file_read(FILE *zip, ZipFile *file, void **buffer_ptr)
+{
+	const char *error = zip_prepare_zip_for_read(zip, file);
+	if (error) return error;
+
+	unsigned char *bytes = MALLOC(file->uncompressed_size);
+	*buffer_ptr = bytes;
+
+	// Uncompressed
+	if (file->compression_method == 0)
+	{
+		if (!read_all(zip, bytes, file->uncompressed_size) || ferror(zip)) return "Failed to read data.";
+		return NULL;
+	}
+
+	// Only deflate supported.
+	assert(file->compression_method == 8 && "Should already be checked.");
+
+	// Deflate - using zlib
+	z_stream strm = { .zalloc = Z_NULL, .zfree = Z_NULL, .opaque = Z_NULL, .avail_in = 0, .next_in = Z_NULL };
+
+	// Use inflateInit2 with negative window bits to indicate raw data
+	if (inflateInit2(&strm, -MAX_WBITS) != Z_OK) return "Failed to init zlib";
+
+	// Inflate compressed data
+	long compressed_left = file->compressed_size;
+	long uncompressed_left = file->uncompressed_size;
+	while (compressed_left && uncompressed_left)
+	{
+		size_t to_read = ZIP_BUFFER_SIZE < compressed_left ? ZIP_BUFFER_SIZE : compressed_left;
+		strm.avail_in = fread(internal_buffer, 1, to_read, zip);
+		if (strm.avail_in == 0 || ferror(zip))
+		{
+			inflateEnd(&strm);
+			return "Failed to read zip";
+		}
+		strm.next_in = internal_buffer;
+		strm.avail_out = uncompressed_left;
+		strm.next_out = bytes;
+
+		compressed_left -= strm.avail_in;
+
+		switch (inflate(&strm, Z_NO_FLUSH))
+		{
+			case Z_STREAM_ERROR:
+				return "Unexpected inflate error";
+			case Z_NEED_DICT:
+			case Z_DATA_ERROR:
+				return "Inflate data error";
+			case Z_MEM_ERROR:
+				return "Inflate memory error";
+			case Z_STREAM_END:
+				goto END;
+			default:
+				break;
+		}
+		bytes += uncompressed_left - strm.avail_out;
+		uncompressed_left = strm.avail_out;
+	}
+END:
+	inflateEnd(&strm);
+	return NULL;
+}
+
+const char *zip_file_write(FILE *zip, ZipFile *file, const char *dir, bool overwrite)
+{
+	const char *error = zip_prepare_zip_for_read(zip, file);
+	if (error) return error;
+
+	char *file_name;
+	char *dir_path;
+	if (!file_namesplit(file->name, &file_name, &dir_path)) return "Failed to split file name";
+
+	if (dir_path)
+	{
+		char *new_path = str_printf("%s/%s", dir, dir_path);
+		dir_make_recursive(new_path);
+		dir = new_path;
+	}
+
+	const char *out_file_name = file_append_path(dir, file_name);
+	if (!overwrite && file_exists(out_file_name)) return NULL;
+
+	FILE *f = fopen(out_file_name, "wb");
+	if (!f) return "Failed to open file output path.";
+
+	// Uncompressed
+	if (file->compression_method == 0)
+	{
+		size_t left_to_read = file->uncompressed_size;
+		while (left_to_read)
+		{
+			size_t amount_to_read = left_to_read < ZIP_BUFFER_SIZE ? left_to_read : ZIP_BUFFER_SIZE;
+			assert(amount_to_read > 0);
+			if (!read_all(zip, internal_buffer, amount_to_read))
+			{
+				fclose(f);
+				return "Failed to read data";
+			}
+			size_t written = fwrite(internal_buffer, 1, amount_to_read, f);
+			if (written != amount_to_read)
+			{
+				fclose(f);
+				return "Failed to write";
+			}
+		}
+		return NULL;
+	}
+
+	// Only deflate supported.
+	assert(file->compression_method == 8 && "Should already be checked.");
+
+	// Deflate - using zlib
+	z_stream strm = { .zalloc = Z_NULL, .zfree = Z_NULL, .opaque = Z_NULL, .avail_in = 0, .next_in = Z_NULL };
+
+	// Use inflateInit2 with negative window bits to indicate raw data
+	if (inflateInit2(&strm, -MAX_WBITS) != Z_OK) return "Failed to init zlib";
+
+	// Inflate compressed data
+	long compressed_left = file->compressed_size;
+	while (compressed_left)
+	{
+		size_t to_read = ZIP_BUFFER_SIZE < compressed_left ? ZIP_BUFFER_SIZE : compressed_left;
+		strm.avail_in = fread(internal_buffer, 1, to_read, zip);
+		if (strm.avail_in == 0 || ferror(zip))
+		{
+			inflateEnd(&strm);
+			fclose(f);
+			return "Failed to read zip";
+		}
+
+		strm.next_in = internal_buffer;
+		while (strm.avail_in)
+		{
+			strm.avail_out = FILE_OUTBUF_LEN;
+			strm.next_out = file_out_buffer;
+
+			bool stream_end = false;
+			switch (inflate(&strm, Z_NO_FLUSH))
+			{
+				case Z_STREAM_ERROR:
+					fclose(f);
+					return "Unexpected inflate error";
+				case Z_NEED_DICT:
+				case Z_DATA_ERROR:
+					fclose(f);
+					return "Inflate data error";
+				case Z_MEM_ERROR:
+					fclose(f);
+					return "Inflate memory error";
+				case Z_STREAM_END:
+					stream_end = true;
+					FALLTHROUGH;
+				default:
+					break;
+			}
+
+			size_t to_write = FILE_OUTBUF_LEN - strm.avail_out;
+
+			if (to_write > 0 && to_write != fwrite(file_out_buffer, 1, to_write, f))
+			{
+				fclose(f);
+				return "Failed to write";
+			}
+			if (stream_end) break;
+		}
+		compressed_left = file->compressed_size - strm.total_in;
+	}
+	fclose(f);
+	inflateEnd(&strm);
+	return NULL;
+}
+

src/version.h

@@ -1 +1 @@
-#define COMPILER_VERSION "0.4.1"
+#define COMPILER_VERSION "0.4.2"