From 16bbc5a02658a57ca310e2d98bd39fa5468886de Mon Sep 17 00:00:00 2001
From: Christoffer Lerno <christoffer@aegik.com>
Date: Fri, 20 Dec 2024 13:18:40 +0100
Subject: [PATCH] Add "tokenizer" to String.

---
 lib/std/core/string.c3 | 33 ++++++++++++++++++++++-----------
 releasenotes.md        |  2 ++
 2 files changed, 24 insertions(+), 11 deletions(-)

diff --git a/lib/std/core/string.c3 b/lib/std/core/string.c3
index aa17a0cb3..1c764d601 100644
--- a/lib/std/core/string.c3
+++ b/lib/std/core/string.c3
@@ -719,7 +719,12 @@ fn float! String.to_float(s) => s.to_real(float);
 
 fn Splitter String.splitter(self, String split)
 {
-	return Splitter { self, split, 0 };
+	return { .string = self, .split = split };
+}
+
+fn Splitter String.tokenize(self, String split)
+{
+	return { .string = self, .split = split, .tokenize = true };
 }
 
 struct Splitter
@@ -727,6 +732,8 @@ struct Splitter
 	String string;
 	String split;
 	usz current;
+	bool tokenize;
+	int last_index;
 }
 
 fn void Splitter.reset(&self)
@@ -736,18 +743,22 @@ fn void Splitter.reset(&self)
 
 fn String! Splitter.next(&self)
 {
-	usz len = self.string.len;
-	usz current = self.current;
-	if (current >= len) return IteratorResult.NO_MORE_ELEMENT?;
-	String remaining = self.string[current..];
-	usz! next = remaining.index_of(self.split);
-	if (try next)
+	while (true)
 	{
-		defer self.current = current + next + self.split.len;
-		return remaining[:next];
+		usz len = self.string.len;
+		usz current = self.current;
+		if (current >= len) return IteratorResult.NO_MORE_ELEMENT?;
+		String remaining = self.string[current..];
+		usz! next = remaining.index_of(self.split);
+		if (try next)
+		{
+			self.current = current + next + self.split.len;
+			if (!next && self.tokenize) continue;
+			return remaining[:next];
+		}
+		self.current = len;
+		return remaining;
 	}
-	self.current = len;
-	return remaining;
 }
 
 macro String new_struct_to_str(x, Allocator allocator = allocator::heap())
diff --git a/releasenotes.md b/releasenotes.md
index 373b3b761..ffd7a4640 100644
--- a/releasenotes.md
+++ b/releasenotes.md
@@ -9,10 +9,12 @@ None
 - Fix case trying to initialize a `char[*]*` from a String.
 - Fix Map & HashMap `put_all_for_create` not copying all elements, causing `init_from_map` to create incomplete copy.
 - Fix bug when a macro calling an extern function was called in another module also declaring and calling the same function. #1690
+
 ### Stdlib changes
 - Increase BitWriter.write_bits limit up to 32 bits.
 - Updates to `Slice2d`, like `get_xy` and others.
 - Added `iter()` `value_iter()` and `key_iter()` to HashMap.
+- Add "tokenizer" to String.
 
 ## 0.6.5 Change list