From d61482dffcf5381b4e842a622e5ecd3afbdee711 Mon Sep 17 00:00:00 2001
From: Pierre Curto <pierre.curto@gmail.com>
Date: Tue, 12 Sep 2023 13:49:52 +0200
Subject: [PATCH] fix Object.free (#982)

* lib/std/collections: add HashMap.@each_entry()
* lib/std/json: fix Object.free() when object is a map
* lib/std/collections: fix allocator use in Object.{set,set_at,append}
* lib/std: add char.from_hex
* lib/std/collections: print arrays and objects compactly
* lib/std/io: fix Formatter.vprintf result
* lib/std/io/stream: rename module for ByteBuffer
* lib/std/io/stream: make Scanner a Stream reader
* lib/std/io: make std{in,err,out} return File* if no libc
---
 lib/std/ascii.c3                  |  4 +++
 lib/std/collections/map.c3        |  9 ++++++-
 lib/std/collections/object.c3     | 41 ++++++++++++++-----------------
 lib/std/io/formatter.c3           | 26 +++++++++++---------
 lib/std/io/io.c3                  | 12 ++++-----
 lib/std/io/stream/bytebuffer.c3   | 29 +++++++++++++++++++++-
 lib/std/io/stream/limitreader.c3  |  2 +-
 lib/std/io/stream/scanner.c3      | 35 +++++++++++++++++++++++---
 test/unit/stdlib/encoding/json.c3 |  9 +++++++
 test/unit/stdlib/io/bytebuffer.c3 |  4 +--
 test/unit/stdlib/io/scanner.c3    | 22 ++++++++++++++---
 11 files changed, 141 insertions(+), 52 deletions(-)

diff --git a/lib/std/ascii.c3 b/lib/std/ascii.c3
index 715d02ebb..ad5fe9036 100644
--- a/lib/std/ascii.c3
+++ b/lib/std/ascii.c3
@@ -53,6 +53,10 @@ fn bool char.is_blank(char c) => is_blank_m(c);
 fn bool char.is_cntrl(char c) => is_cntrl_m(c);
 fn char char.to_lower(char c) => (char)to_lower_m(c);
 fn char char.to_upper(char c) => (char)to_upper_m(c);
+/**
+ * @require c.is_xdigit()
+ **/
+fn char char.from_hex(char c) => c.is_digit() ? c - '0' : 10 + (c | 0x20) - 'a';
 
 fn bool uint.in_range(uint c, uint start, uint len) => in_range_m(c, start, len);
 fn bool uint.is_lower(uint c) => is_lower_m(c);
diff --git a/lib/std/collections/map.c3 b/lib/std/collections/map.c3
index c66221a9f..6de101579 100644
--- a/lib/std/collections/map.c3
+++ b/lib/std/collections/map.c3
@@ -209,6 +209,13 @@ fn Key[] HashMap.key_list(&map, Allocator* using = mem::heap())
 }
 
 macro HashMap.@each(map; @body(key, value))
+{
+	map.@each_entry(; Entry* entry) {
+		@body(entry.key, entry.value);
+	};
+}
+
+macro HashMap.@each_entry(map; @body(entry))
 {
 	if (map.count)
 	{
@@ -216,7 +223,7 @@ macro HashMap.@each(map; @body(key, value))
 		{
 			while (entry)
 			{
-				@body(entry.key, entry.value);
+				@body(entry);
 				entry = entry.next;
 			}
 		}
diff --git a/lib/std/collections/object.c3 b/lib/std/collections/object.c3
index 1c6efdea6..2b0cc39f0 100644
--- a/lib/std/collections/object.c3
+++ b/lib/std/collections/object.c3
@@ -43,10 +43,10 @@ fn usz! Object.to_format(&self, Formatter* formatter) @dynamic
 			usz n = formatter.printf("[")!;
 			foreach (i, ol : self.array)
 			{
-				n += formatter.printf(i == 0 ? " " : ", ")!;
+				if (i > 0) n += formatter.printf(",")!;
 				n += ol.to_format(formatter)!;
 			}
-			n += formatter.printf(" ]")!;
+			n += formatter.printf("]")!;
 			return n;
 		case ObjectInternalMap:
 			usz n = formatter.printf("{")!;
@@ -54,12 +54,12 @@ fn usz! Object.to_format(&self, Formatter* formatter) @dynamic
 			{
 				foreach (i, key : self.map.key_tlist())
 				{
-					n += formatter.printf(i == 0 ? " " : ", ")!;
-					n += formatter.printf(`"%s": `, key)!;
+					if (i > 0) n += formatter.printf(",")!;
+					n += formatter.printf(`"%s":`, key)!;
 					n += self.map.get(key).to_format(formatter)!;
 				}
 			};
-			n += formatter.printf(" }")!;
+			n += formatter.printf("}")!;
 			return n;
 		default:
 			switch (self.type.kindof)
@@ -104,7 +104,7 @@ macro Object* new_enum(e, Allocator* using = mem::heap())
 	return o;
 }
 
-fn Object* new_float(double f, Allocator* using = mem::current_allocator())
+fn Object* new_float(double f, Allocator* using = mem::heap())
 {
 	Object* o = malloc(Object, .using = using);
 	*o = { .f = f, .allocator = using, .type = double.typeid };
@@ -139,14 +139,9 @@ fn void Object.free(&self)
 			}
 			self.array.free();
 		case ObjectInternalMap:
-			@pool()
-			{
-				foreach (key : self.map.key_tlist())
-				{
-					(void)self.map.get(key).free();
-					free(key, .using = self.allocator);
-				}
-				self.map.free();
+			self.map.@each_entry(; ObjectInternalMapEntry* entry) {
+				free(entry.key, .using = self.allocator);
+				entry.value.free();
 			};
 		default:
 			break;
@@ -204,17 +199,17 @@ fn void Object.set_object(&self, String key, Object* new_object) @private
 	self.map.set(key.copy(self.map.allocator), new_object);
 }
 
-macro Object* object_from_value(value) @private
+macro Object* Object.object_from_value(&self, value) @private
 {
 	var $Type = $typeof(value);
 
 	$switch
 		$case types::is_int($Type):
-			return new_int(value);
+			return new_int(value, self.allocator);
 		$case types::is_float($Type):
-			return new_float(value);
+			return new_float(value, self.allocator);
 		$case $Type.typeid == String.typeid:
-			return new_string(value);
+			return new_string(value, self.allocator);
 		$case $Type.typeid == bool.typeid:
 			return new_bool(value);
 		$case $Type.typeid == Object*.typeid:
@@ -223,7 +218,7 @@ macro Object* object_from_value(value) @private
 			if (value != null) return CastResult.TYPE_MISMATCH?;
 			return &NULL_OBJECT;
 		$case $checks(String s = value):
-			return new_string(value);
+			return new_string(value, self.allocator);
 		$default:
 			$error "Unsupported object type.";
 	$endswitch
@@ -232,7 +227,7 @@ macro Object* object_from_value(value) @private
 
 macro Object* Object.set(&self, String key, value)
 {
-	Object* val = object_from_value(value);
+	Object* val = self.object_from_value(value);
 	self.set_object(key, val);
 	return val;
 }
@@ -242,7 +237,7 @@ macro Object* Object.set(&self, String key, value)
  **/
 macro Object* Object.set_at(&self, usz index, String key, value)
 {
-	Object* val = object_from_value(value);
+	Object* val = self.object_from_value(value);
 	self.set_object_at(key, index, val);
 	return val;
 }
@@ -253,7 +248,7 @@ macro Object* Object.set_at(&self, usz index, String key, value)
  **/
 macro Object* Object.append(&self, value)
 {
-	Object* val = object_from_value(value);
+	Object* val = self.object_from_value(value);
 	self.append_object(val);
 	return val;
 }
@@ -477,7 +472,7 @@ fn double! Object.get_float_at(&self, usz index)
 fn Object* Object.get_or_create_obj(&self, String key)
 {
 	if (try obj = self.get(key) && !obj.is_null()) return obj;
-	Object* container = new_obj();
+	Object* container = new_obj(self.allocator);
 	self.set(key, container);
 	return container;
 }
diff --git a/lib/std/io/formatter.c3 b/lib/std/io/formatter.c3
index 49281c5ea..c8d4cb52b 100644
--- a/lib/std/io/formatter.c3
+++ b/lib/std/io/formatter.c3
@@ -293,6 +293,7 @@ fn usz! Formatter.vprintf(&self, String format, any[] anys)
 		// use null output function
 		self.out_fn = &out_null_fn;
 	}
+	usz total_len;
 	usz format_len = format.len;
 	usz variant_index = 0;
 	for (usz i = 0; i < format_len; i++)
@@ -302,7 +303,7 @@ fn usz! Formatter.vprintf(&self, String format, any[] anys)
 		if (c != '%')
 		{
 			// no
-			self.out(c)!;
+			total_len += self.out(c)!;
 			continue;
 		}
 		i++;
@@ -310,7 +311,7 @@ fn usz! Formatter.vprintf(&self, String format, any[] anys)
 		c = format[i];
 		if (c == '%')
 		{
-			self.out(c)!;
+			total_len += self.out(c)!;
 			continue;
 		}
 		// evaluate flags
@@ -377,42 +378,43 @@ fn usz! Formatter.vprintf(&self, String format, any[] anys)
 				self.flags.uppercase = true;
 				nextcase;
 			case 'a':
-				self.atoa(float_from_any(current)!!)!;
+				total_len += self.atoa(float_from_any(current)!!)!;
 				continue;
 			case 'F' :
 				self.flags.uppercase = true;
 				nextcase;
 			case 'f':
-				self.ftoa(float_from_any(current)!!)!;
+				total_len += self.ftoa(float_from_any(current)!!)!;
 				continue;
 			case 'E':
 				self.flags.uppercase = true;
 				nextcase;
 			case 'e':
-				self.etoa(float_from_any(current)!!)!;
+				total_len += self.etoa(float_from_any(current)!!)!;
 				continue;
 			case 'G':
 				self.flags.uppercase = true;
 				nextcase;
 			case 'g':
-				self.gtoa(float_from_any(current)!!)!;
+				total_len += self.gtoa(float_from_any(current)!!)!;
 				continue;
 			case 'c':
-				self.out_char(current)!;
+				total_len += self.out_char(current)!;
 				continue;
 			case 's':
 				if (self.flags.left)
 				{
 					usz len = self.out_str(current)!;
-					self.pad(' ', self.width, len)!;
+					total_len += len;
+					total_len += self.pad(' ', self.width, len)!;
 					continue;
 				}
 				OutputFn out_fn = self.out_fn;
 				self.out_fn = (OutputFn)&out_null_fn;
 				usz len = self.out_str(current)!;
 				self.out_fn = out_fn;
-				self.pad(' ', self.width, len)!;
-				self.out_str(current)!;
+				total_len += self.pad(' ', self.width, len)!;
+				total_len += self.out_str(current)!;
 				continue;
 			case 'p':
 				self.flags.zeropad = true;
@@ -432,13 +434,13 @@ fn usz! Formatter.vprintf(&self, String format, any[] anys)
 		bool is_neg;
 		uint128 v = int_from_any(current, &is_neg)!!;
 
-		self.ntoa(v, is_neg, base)!;
+		total_len += self.ntoa(v, is_neg, base)!;
 	}
 	// termination
 	//	out((char)0, buffer, idx < maxlen ? idx : maxlen - 1U, maxlen);
 
 	// return written chars without terminating \0
-	return self.idx;
+	return total_len;
 }
 
 
diff --git a/lib/std/io/io.c3 b/lib/std/io/io.c3
index 73b0487bd..6753af57d 100644
--- a/lib/std/io/io.c3
+++ b/lib/std/io/io.c3
@@ -141,17 +141,17 @@ fn void putchar(char c) @inline
 	(void)stdout_file.putc(c);
 }
 
-fn File stdout()
+fn File* stdout()
 {
-	return stdout_file;
+	return &stdout_file;
 }
 
-fn File stderr()
+fn File* stderr()
 {
-	return stderr_file;
+	return &stderr_file;
 }
 
-fn File stdin()
+fn File* stdin()
 {
-	return stdin_file;
+	return &stdin_file;
 }
diff --git a/lib/std/io/stream/bytebuffer.c3 b/lib/std/io/stream/bytebuffer.c3
index ab0614fde..fe4874451 100644
--- a/lib/std/io/stream/bytebuffer.c3
+++ b/lib/std/io/stream/bytebuffer.c3
@@ -30,9 +30,19 @@ fn ByteBuffer*! ByteBuffer.tinit(&self, usz max_read, usz initial_capacity = 16)
 	return self.init(max_read, initial_capacity, mem::temp())!;
 }
 
+/**
+ * @require buf.len > 0
+ * @require self.bytes.len == 0 "Buffer already initialized."
+ **/
+fn ByteBuffer*! ByteBuffer.init_with_buffer(&self, char[] buf)
+{
+	*self = { .stream.fns = &BYTEBUFFER_INTERFACE, .max_read = buf.len, .bytes = buf };
+	return self;
+}
+
 fn void! ByteBuffer.free(&self)
 {
-	self.allocator.free(self.bytes)!;
+	if (self.allocator) self.allocator.free(self.bytes)!;
 	*self = {};
 }
 
@@ -104,6 +114,23 @@ fn void! ByteBuffer.pushback_byte(&self)
 	self.has_last = false;
 }
 
+fn void! ByteBuffer.seek(&self, isz offset, Seek seek)
+{
+	switch (seek)
+	{
+		case SET:
+			if (offset < 0 || offset > self.write_idx) return IoError.INVALID_POSITION?;
+			self.read_idx = offset;
+		case CURSOR:
+			if ((offset < 0 && self.read_idx < -offset) ||
+				(offset > 0 && self.read_idx + offset > self.write_idx)) return IoError.INVALID_POSITION?;
+			self.read_idx += offset;
+		case END:
+			if (offset < 0 || offset > self.write_idx) return IoError.INVALID_POSITION?;
+			self.read_idx = self.write_idx - offset;
+	}
+}
+
 fn usz! ByteBuffer.available(&self) @inline
 {
 	return self.write_idx - self.read_idx;
diff --git a/lib/std/io/stream/limitreader.c3 b/lib/std/io/stream/limitreader.c3
index 87ba463de..1f9a6a895 100644
--- a/lib/std/io/stream/limitreader.c3
+++ b/lib/std/io/stream/limitreader.c3
@@ -1,4 +1,4 @@
-module std::io::stream;
+module std::io;
 
 struct LimitReader
 {
diff --git a/lib/std/io/stream/scanner.c3 b/lib/std/io/stream/scanner.c3
index cfd7b0a06..dd524d4ca 100644
--- a/lib/std/io/stream/scanner.c3
+++ b/lib/std/io/stream/scanner.c3
@@ -2,7 +2,8 @@ module std::io;
 
 struct Scanner
 {
-	Stream* stream;
+	inline Stream stream;
+	Stream* wrapped_stream;
 	char[] buf;
 	usz pattern_idx;
 	usz read_idx;
@@ -20,9 +21,14 @@ struct Scanner
  **/
 fn void Scanner.init(&self, Stream* stream, char[] buffer)
 {
-	*self = { .stream = stream, .buf = buffer };
+	*self = { .stream.fns = &SCANNER_INTERFACE, .wrapped_stream = stream, .buf = buffer };
 }
 
+const StreamInterface SCANNER_INTERFACE = {
+	.read_fn = (ReadStreamFn)&Scanner.read,
+	.read_byte_fn = (ReadByteStreamFn)&Scanner.read_byte,
+};
+
 /**
  * Return and clear any remaining unscanned data.
  **/
@@ -85,7 +91,7 @@ macro usz! Scanner.find(&self, buf, pattern) @private
 
 macro usz! Scanner.refill(&self, buf) @private
 {
-	usz! n = self.stream.read(buf);
+	usz! n = self.wrapped_stream.read(buf);
 	if (catch err = n)
 	{
 		case IoError.EOF:
@@ -94,4 +100,27 @@ macro usz! Scanner.refill(&self, buf) @private
 			return err?;
 	}
 	return n;
+}
+
+fn usz! Scanner.read(&self, char[] bytes)
+{
+	usz n;
+	if (self.pattern_idx < self.read_idx)
+	{
+		n = min(bytes.len, self.read_idx - self.pattern_idx);
+		bytes[:n] = self.buf[self.pattern_idx:n];
+		self.pattern_idx += n;
+		bytes = bytes[n..];
+	}
+	n += self.wrapped_stream.read(bytes)!;
+	return n;
+}
+
+fn char! Scanner.read_byte(&self)
+{
+	if (self.pattern_idx < self.read_idx)
+	{
+		return self.buf[self.pattern_idx++];
+	}
+	return self.wrapped_stream.read_byte();
 }
\ No newline at end of file
diff --git a/test/unit/stdlib/encoding/json.c3 b/test/unit/stdlib/encoding/json.c3
index 8ec6b7444..67fdf58e6 100644
--- a/test/unit/stdlib/encoding/json.c3
+++ b/test/unit/stdlib/encoding/json.c3
@@ -8,6 +8,7 @@ fn void! simple_test()
 	ByteReader reader;
 	reader.init(`{ "b": 123, "c": [ { "d": 66 }, null, "hello", false, { "id": "xyz" } ] }`);
 	Object* o = json::parse(&reader)!;
+	defer o.free();
 	assert(o.get_int("b")! == 123);
 	assert(o.get("c").get_len()! == 5);
 	assert(o.get("c").get_at(0).get_int("d")! == 66);
@@ -16,3 +17,11 @@ fn void! simple_test()
 	assert(o.get("c").get_bool_at(3)! == false);
 	assert(o.get("c").get_at(4).get_string("id")! == "xyz");
 }
+
+fn void! simple_test2()
+{
+	ByteReader reader;
+	reader.init(`{"jsonrpc":"2.0","id":null,"method":"initialize"}`);
+	Object* o = json::parse(&reader)!;
+	defer o.free();
+}
\ No newline at end of file
diff --git a/test/unit/stdlib/io/bytebuffer.c3 b/test/unit/stdlib/io/bytebuffer.c3
index 055a882ad..61747d4a4 100644
--- a/test/unit/stdlib/io/bytebuffer.c3
+++ b/test/unit/stdlib/io/bytebuffer.c3
@@ -1,7 +1,7 @@
-module bytebuffer_test @test;
+module std::io::bytebuffer @test;
 import std::io;
 
-fn void! write_read_test()
+fn void! write_read()
 {
 	ByteBuffer buffer;
 	buffer.init(0)!;
diff --git a/test/unit/stdlib/io/scanner.c3 b/test/unit/stdlib/io/scanner.c3
index e8671ff0f..a4ae1a515 100644
--- a/test/unit/stdlib/io/scanner.c3
+++ b/test/unit/stdlib/io/scanner.c3
@@ -1,6 +1,5 @@
-module scanner_test @test;
+module std::io @test;
 import std::collections::list;
-import std::io;
 
 def Results = List(<String>);
 
@@ -11,7 +10,7 @@ struct ScanTest
     String left_over;
 }
 
-fn void! test_scanner()
+fn void! scanner()
 {
     ScanTest[] tcases = {
         {"aa,,bb", {"aa"}, "bb"},
@@ -48,4 +47,21 @@ fn void! test_scanner()
         String left_over = (String)fl;
         assert(left_over == tc.left_over, "%s -> %s", tc.in, left_over);
     }
+}
+
+fn void! scanner_as_reader()
+{
+	ByteReader br;
+	br.init("Lorem ipsum sit.");
+	Scanner sc;
+	char[8] buffer;
+	sc.init(&br, buffer[..]);
+
+	sc.scan(" ")!;
+
+	char[16] res;
+	usz n = sc.read(&res)!;
+	String str = (String)res[:n];
+
+	assert(str == "ipsum sit.", "got '%s'; want 'ipsum sit.'", str);
 }
\ No newline at end of file