shishantbiswas/c3c
1
1
Fork 0
mirror of https://github.com/c3lang/c3c.git synced 2026-02-27 12:01:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add "tokenizer" to String.

Browse Source
This commit is contained in:
Christoffer Lerno
2024-12-20 13:18:40 +01:00
parent 627f10cd18
commit 16bbc5a026
2 changed files with 24 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
lib/std/core/string.c3
View File

@@ -719,7 +719,12 @@ fn float! String.to_float(s) => s.to_real(float);
fn Splitter String.splitter(self, String split)
{
return Splitter { self, split, 0 };
return { .string = self, .split = split };
}
fn Splitter String.tokenize(self, String split)
{
return { .string = self, .split = split, .tokenize = true };
}
struct Splitter
@@ -727,6 +732,8 @@ struct Splitter
String string;
String split;
usz current;
bool tokenize;
int last_index;
}
fn void Splitter.reset(&self)
@@ -736,18 +743,22 @@ fn void Splitter.reset(&self)
fn String! Splitter.next(&self)
{
usz len = self.string.len;
usz current = self.current;
if (current >= len) return IteratorResult.NO_MORE_ELEMENT?;
String remaining = self.string[current..];
usz! next = remaining.index_of(self.split);
if (try next)
while (true)
{
defer self.current = current + next + self.split.len;
return remaining[:next];
usz len = self.string.len;
usz current = self.current;
if (current >= len) return IteratorResult.NO_MORE_ELEMENT?;
String remaining = self.string[current..];
usz! next = remaining.index_of(self.split);
if (try next)
{
self.current = current + next + self.split.len;
if (!next && self.tokenize) continue;
return remaining[:next];
}
self.current = len;
return remaining;
}
self.current = len;
return remaining;
}
macro String new_struct_to_str(x, Allocator allocator = allocator::heap())