New test runner

lib/std/core/runtime_test.c3

@@ -2,11 +2,38 @@
 // Use of this source code is governed by the MIT license
 // a copy of which can be found in the LICENSE_STDLIB file.
 module std::core::runtime;
+import std::core::test @public;
 import libc, std::time, std::io, std::sort;
+import std::os::env;
 
 def TestFn = fn void!() @if($$OLD_TEST);
 def TestFn = fn void() @if(!$$OLD_TEST);
 
+TestContext* test_context @private;
+
+struct TestContext
+{
+	JmpBuf buf;
+	// Allows filtering test cased or modules by substring, e.g. 'foo::', 'foo::test_add'
+	String test_filter;
+	// Triggers debugger breakpoint when assert or test:: checks failed
+	bool breakpoint_on_assert;
+
+	// internal state
+	bool assert_print_backtrace;
+	bool has_ansi_codes;
+	bool is_in_panic;
+	String current_test_name;
+	TestFn setup_fn;
+	TestFn teardown_fn;
+
+	char* error_buffer;
+	usz error_buffer_capacity;
+	File fake_stdout;
+	File orig_stdout;
+	File orig_stderr;
+}
+
 struct TestUnit
 {
 	String name;
@@ -25,11 +52,6 @@ fn TestUnit[] test_collection_create(Allocator allocator = allocator::heap())
 	return tests;
 }
 
-struct TestContext
-{
-	JmpBuf buf;
-}
-
 // Sort the tests by their name in ascending order.
 fn int cmp_test_unit(TestUnit a, TestUnit b)
 {
@@ -44,82 +66,175 @@ fn int cmp_test_unit(TestUnit a, TestUnit b)
 	return (int)(an - bn);
 }
 
-TestContext* test_context @private;
-
-fn void test_panic(String message, String file, String function, uint line)
+fn bool terminal_has_ansi_codes() @local
 {
-	io::printn("[error]");
-	io::print("\n Error: ");
-	io::print(message);
-	io::printn();
-	io::printfn("  - in %s %s:%s.\n", function, file, line);
+	@pool()
+	{
+		if (try v = env::get_var_temp("TERM"))
+		{
+			if (v.contains("xterm") || v.contains("vt100") || v.contains("screen")) return true;
+		}
+		$if env::WIN32 || env::NO_LIBC:
+			return false;
+		$else
+			return io::stdout().isatty();
+		$endif
+	};
+}
+
+fn void test_panic(String message, String file, String function, uint line) @local
+{
+	if (test_context.is_in_panic) return;
+	test_context.is_in_panic = true;
+
+	unmute_output(true);
+	(void)io::stdout().flush();
+	if (test_context.assert_print_backtrace)
+	{
+		$if env::NATIVE_STACKTRACE:
+    		builtin::print_backtrace(message, 0);
+		$endif
+	}
+	io::printf("\nTest failed ^^^ ( %s:%s ) %s\n",  file, line, message);
+	test_context.assert_print_backtrace = true;
+
+	if (test_context.breakpoint_on_assert)
+	{
+		breakpoint();
+	}
+
+	if (test_context.teardown_fn)
+	{
+		test_context.teardown_fn();
+	}
+
+	test_context.is_in_panic = false;
 	libc::longjmp(&test_context.buf, 1);
 }
 
-fn bool run_tests(TestUnit[] tests) @if($$OLD_TEST)
+fn void mute_output() @local
 {
-	usz max_name;
-	foreach (&unit : tests)
-	{
-		if (max_name < unit.name.len) max_name = unit.name.len;
-	}
-	quicksort(tests, &cmp_test_unit);
+	if (!test_context.fake_stdout.file) return;
+	assert(!test_context.orig_stderr.file);
+	assert(!test_context.orig_stdout.file);
 
-	TestContext context;
-	test_context = &context;
+	File* stdout = io::stdout();
+	File* stderr = io::stderr();
 
-	PanicFn old_panic = builtin::panic;
-	defer builtin::panic = old_panic;
-	builtin::panic = &test_panic;
-	int tests_passed = 0;
-	int test_count = tests.len;
-	DString name = dstring::temp_with_capacity(64);
-	usz len = max_name + 9;
-	name.append_repeat('-', len / 2);
-	name.append(" TESTS ");
-	name.append_repeat('-', len - len / 2);
-	io::printn(name);
-	name.clear();
-	foreach(unit : tests)
-	{
-		defer name.clear();
-		name.appendf("Testing %s ", unit.name);
-		name.append_repeat('.', max_name - unit.name.len + 2);
-		io::printf("%s ", name.str_view());
-		(void)io::stdout().flush();
-		if (libc::setjmp(&context.buf) == 0)
-		{
-			if (catch err = unit.func())
-			{
-				io::printfn("[failed] Failed due to: %s", err);
-				continue;
-			}
-			io::printn("[ok]");
-			tests_passed++;
-		}
-	}
-	io::printfn("\n%d test%s run.\n", test_count, test_count > 1 ? "s" : "");
-	io::printfn("Test Result: %s. %d passed, %d failed.",
-		tests_passed < test_count ? "FAILED" : "ok", tests_passed, test_count - tests_passed);
-	return test_count == tests_passed;
+	test_context.orig_stderr = *stderr;
+	test_context.orig_stdout = *stdout;
+
+	*stderr = test_context.fake_stdout;
+	*stdout = test_context.fake_stdout;
+
+	(void)test_context.fake_stdout.seek(0, Seek.SET)!!;
 }
 
-fn bool run_tests(TestUnit[] tests) @if(!$$OLD_TEST)
+fn void unmute_output(bool has_error) @local
+{
+	if (!test_context.fake_stdout.file)
+	{
+		return;
+	}
+	assert(test_context.orig_stderr.file);
+	assert(test_context.orig_stdout.file);
+
+	File* stdout = io::stdout();
+	File* stderr = io::stderr();
+
+	*stderr = test_context.orig_stderr;
+	*stdout = test_context.orig_stdout;
+	test_context.orig_stderr.file = null;
+	test_context.orig_stdout.file = null;
+
+	usz log_size = test_context.fake_stdout.seek(0, Seek.CURSOR)!!;
+	if (has_error)
+	{
+		io::printn(test_context.has_ansi_codes ? "[\e[0;31mFAIL\e[0m]" : "[FAIL]");
+	}
+
+	if (has_error && log_size > 0)
+	{
+		test_context.fake_stdout.write_byte('\n')!!;
+		test_context.fake_stdout.write_byte('\0')!!;
+		(void)test_context.fake_stdout.seek(0, Seek.SET)!!;
+
+		io::printfn("\n========== TEST LOG ============");
+		io::printfn("%s\n", test_context.current_test_name);
+		while (try c = test_context.fake_stdout.read_byte())
+		{
+			if (@unlikely(c == '\0'))
+			{
+				// ignore junk from previous tests 
+				break;
+			}
+			libc::putchar(c);
+		}
+		io::printf("========== TEST END ============");
+	}
+	(void)stdout.flush();
+}
+
+fn bool run_tests(String[] args, TestUnit[] tests) @private
 {
 	usz max_name;
+	bool sort_tests = true;
 	foreach (&unit : tests)
 	{
 		if (max_name < unit.name.len) max_name = unit.name.len;
 	}
-	quicksort(tests, &cmp_test_unit);
-
-	TestContext context;
+	TestContext context =
+	{
+		.assert_print_backtrace = true,
+		.breakpoint_on_assert = false,
+		.test_filter = "",
+		.has_ansi_codes = terminal_has_ansi_codes(),
+	};
+	for (int i = 1; i < args.len; i++)
+	{
+		switch (args[i])
+		{
+			case "breakpoint":
+				context.breakpoint_on_assert = true;
+			case "nosort":
+				sort_tests = false;
+			case "noansi":
+				context.has_ansi_codes = false;
+			case "useansi":
+				context.has_ansi_codes = true;
+			case "filter":
+				if (i == args.len - 1)
+				{
+					io::printn("Invalid arguments to test runner.");
+					return false;
+				}
+				context.test_filter = args[i + 1];
+				i++;
+			default:
+				io::printfn("Unknown argument: %s", args[i]);
+		}
+	}
 	test_context = &context;
 
+	if (sort_tests)
+	{
+		quicksort(tests, &cmp_test_unit);
+	}
+
+	// Buffer for hijacking the output
+	$if (!env::NO_LIBC):
+		test_context.fake_stdout.file = libc::tmpfile();
+	$endif
+	if (test_context.fake_stdout.file == null)
+	{
+		io::print("Failed to hijack stdout, tests will print everything");
+	}
+
 	PanicFn old_panic = builtin::panic;
 	defer builtin::panic = old_panic;
 	builtin::panic = &test_panic;
 	int tests_passed = 0;
+	int tests_skipped = 0;
 	int test_count = tests.len;
 	DString name = dstring::temp_with_capacity(64);
 	usz len = max_name + 9;
@@ -130,28 +245,71 @@ fn bool run_tests(TestUnit[] tests) @if(!$$OLD_TEST)
 	name.clear();
 	foreach(unit : tests)
 	{
+		if (test_context.test_filter && !unit.name.contains(test_context.test_filter))
+		{
+			tests_skipped++;
+			continue;
+		}
+		test_context.setup_fn = null;
+		test_context.teardown_fn = null;
+		test_context.current_test_name = unit.name;
+
 		defer name.clear();
 		name.appendf("Testing %s ", unit.name);
 		name.append_repeat('.', max_name - unit.name.len + 2);
 		io::printf("%s ", name.str_view());
-		(void)io::stdout().flush();
+
 		if (libc::setjmp(&context.buf) == 0)
 		{
-			unit.func();
-			io::printn("[ok]");
+			mute_output();
+			$if(!$$OLD_TEST):
+				unit.func();
+			$else
+				if (catch err = unit.func())
+				{
+					io::printf("[FAIL] Failed due to: %s", err);
+					continue;
+				}
+			$endif
+			unmute_output(false); // all good, discard output
+
+
+			io::printfn(test_context.has_ansi_codes ? "[\e[0;32mPASS\e[0m]" : "[PASS]");
 			tests_passed++;
+
+			if (test_context.teardown_fn)
+			{
+				test_context.teardown_fn();
+			}
 		}
 	}
-	io::printfn("\n%d test%s run.\n", test_count, test_count > 1 ? "s" : "");
-	io::printfn("Test Result: %s. %d passed, %d failed.",
-		tests_passed < test_count ? "FAILED" : "ok", tests_passed, test_count - tests_passed);
-	return test_count == tests_passed;
+	io::printfn("\n%d test%s run.\n", test_count-tests_skipped, test_count > 1 ? "s" : "");
+
+	int n_failed = test_count - tests_passed - tests_skipped;
+	io::printf("Test Result: %s%s%s: ",
+				test_context.has_ansi_codes ? (n_failed ? "\e[0;31m" : "\e[0;32m") : "",
+				n_failed ? "FAILED" : "PASSED",
+				test_context.has_ansi_codes ? "\e[0m" : "",
+				);
+
+	io::printfn("%d passed, %d failed, %d skipped.",
+				tests_passed,
+				n_failed,
+				tests_skipped);
+
+	// cleanup fake_stdout file
+	if (test_context.fake_stdout.file) libc::fclose(test_context.fake_stdout.file);
+	test_context.fake_stdout.file = null;
+
+	return n_failed == 0;
 }
 
 fn bool default_test_runner(String[] args)
 {
 	@pool()
 	{
-		return run_tests(test_collection_create(allocator::temp()));
+		assert(test_context == null, "test suite is already running");
+		return run_tests(args, test_collection_create(allocator::temp()));
 	};
 }
+

lib/std/core/test.c3

@@ -0,0 +1,223 @@
+<*
+Unit test module
+
+This module provides a toolset of macros for running unit test checks 
+
+Example:
+```c3
+module sample::m;
+import std::io;
+
+fault MathError
+{
+	DIVISION_BY_ZERO
+}
+
+fn double! divide(int a, int b)
+{
+	if (b == 0) return MathError.DIVISION_BY_ZERO?;
+	return (double)(a) / (double)(b);
+}
+
+fn void! test_div() @test
+{
+	test::is_equal(2, divide(6, 3)!);
+	test::is_not_equal(1, 2);
+	test::is_almost_equal(m::divide(1, 3)!, 0.333, places: 3);
+	test::is_greater_equal(3, 3);
+	test::is_greater(2, divide(3, 3)!);
+	test::is_less(2, 3);
+	test::is_less_equal(2, 3);
+	test::@check(2 == 2, "divide: %d", divide(6, 3)!);
+	test::@error(m::divide(3, 0), MathError.DIVISION_BY_ZERO);
+}
+
+```
+*>
+// Copyright (c) 2025 Alex Veden <i@alexveden.com>. All rights reserved.
+// Use of this source code is governed by the MIT license
+// a copy of which can be found in the LICENSE_STDLIB file.
+module std::core::test;
+import std::core::runtime @public;
+import std::math, std::io, libc;
+
+<*
+ Initializes test case context.
+
+ @param setup_fn `initializer function for test case`
+ @param teardown_fn `cleanup function for test context (may be null)`
+
+ @require runtime::test_context != null "Only allowed in @test functions"
+ @require setup_fn != null "setup_fn must always be set"
+*>
+macro setup(TestFn setup_fn, TestFn teardown_fn = null)
+{
+	runtime::test_context.setup_fn = setup_fn;
+	runtime::test_context.teardown_fn = teardown_fn;
+	runtime::test_context.setup_fn();
+}
+
+<*
+ Checks condition and fails assertion if not true
+
+ @param #condition `any boolean condition, will be expanded by text`
+ @param format `printf compatible format`
+ @param args `vargs for format`
+ @require runtime::test_context != null "Only allowed in @test functions"
+*>
+macro @check(#condition, String format = "", args...)
+{
+	if (!#condition)
+	{
+		@stack_mem(512; Allocator allocator)
+		{
+			DString s;
+			s.new_init(allocator: allocator);
+			s.appendf("check `%s` failed. ", $stringify(#condition));
+			s.appendf(format, ...args);
+			print_panicf(s.str_view());
+		};
+	}
+}
+
+<*
+ Check if function returns specific error
+
+ @param #funcresult `result of function execution`
+ @param error_expected `expected error of function execution`
+ @require runtime::test_context != null "Only allowed in @test functions"
+*>
+macro @error(#funcresult, anyfault error_expected)
+{
+	if (catch err = #funcresult)
+	{
+		if (err != error_expected)
+		{
+			print_panicf("`%s` expected to return error [%s], got [%s]",
+				$stringify(#funcresult), error_expected, err);
+		}
+		return;
+	}
+	print_panicf("`%s` error [%s] was not returned.", $stringify(#funcresult), error_expected);
+}
+
+<*
+ Check if left == right
+
+ @param left `left argument of any comparable type`
+ @param right `right argument of any comparable type`
+ @require runtime::test_context != null "Only allowed in @test functions"
+*>
+macro is_equal(left, right)
+{
+	if (!equals(left, right))
+	{
+		print_panicf("`%s` != `%s`", left, right);
+	}
+}
+
+<*
+ Check left floating point value is approximately equals to right value
+
+ @param places `number of decimal places to compare (default: 7)`
+ @param delta `minimal allowed difference (overrides places parameter)`
+ @param equal_nan `allows comparing nan values, if left and right both nans result is ok`
+
+ @require places > 0, places <= 20 "too many decimal places"
+ @require delta >= 0, delta <= 1 "delta must be a small number"
+ @require runtime::test_context != null "Only allowed in @test functions"
+*>
+fn void is_almost_equal(double left, double right, uint places = 7, double delta = 0, bool equal_nan = true)
+{
+	double diff = left - right;
+	double eps = delta;
+	if (eps == 0) eps = 1.0 / math::pow(10.0, places);
+
+	if (!math::is_approx(left, right, eps))
+	{
+		if (equal_nan && math::is_nan(left) && math::is_nan(right)) return;
+		print_panicf("Not almost equal: `%s` !~~ `%s` delta=%e diff: %e", left, right, eps, diff);
+	}
+}
+
+<*
+ Check if left != right
+
+ @param left `left argument of any comparable type`
+ @param right `right argument of any comparable type`
+ @require runtime::test_context != null "Only allowed in @test functions"
+*>
+macro void is_not_equal(left, right)
+{
+	if (equals(left, right))
+	{
+		print_panicf("`%s` == `%s`", left, right);
+	}
+}
+
+<*
+ Check if left > right
+
+ @param left `left argument of any comparable type`
+ @param right `right argument of any comparable type`
+ @require runtime::test_context != null "Only allowed in @test functions"
+*>
+macro is_greater(left, right)
+{
+	if (!builtin::greater(left, right))
+	{
+		print_panicf("`%s` <= `%s`", left, right);
+	}
+}
+
+<*
+ Check if left >= right
+
+ @param left `left argument of any comparable type`
+ @param right `right argument of any comparable type`
+ @require runtime::test_context != null "Only allowed in @test functions"
+*>
+macro is_greater_equal(left, right)
+{
+	if (!builtin::greater_eq(left, right))
+	{
+		print_panicf("`%s` < `%s`", left, right);
+	}
+}
+
+<*
+ Check if left < right
+
+ @param left `left argument of any comparable type`
+ @param right `right argument of any comparable type`
+ @require runtime::test_context != null "Only allowed in @test functions"
+*>
+macro is_less(left, right)
+{
+	if (!builtin::less(left, right))
+	{
+		print_panicf("`%s` >= `%s`", left, right);
+	}
+}
+
+<*
+ Check if left <= right
+
+ @param left `left argument of any comparable type`
+ @param right `right argument of any comparable type`
+ @require runtime::test_context != null "Only allowed in @test functions"
+*>
+macro is_less_equal(left, right)
+{
+	if (!builtin::less_eq(left, right))
+	{
+		print_panicf("`%s` > `%s`", left, right);
+	}
+}
+
+macro void print_panicf(format, ...) @local
+{
+	runtime::test_context.assert_print_backtrace = false;
+	builtin::panicf(format, $$FILE, $$FUNC, $$LINE, $vasplat);
+}
+

lib/std/io/file.c3

@@ -130,6 +130,15 @@ fn usz! File.write(&self, char[] buffer) @dynamic
 	return os::native_fwrite(self.file, buffer);
 }
 
+fn Fd File.fd(self) @if(env::LIBC)
+{
+	return libc::fileno(self.file);
+}
+
+fn bool File.isatty(self) @if(env::LIBC)
+{
+	return libc::isatty(self.fd()) > 0;
+}
 
 fn char! File.read_byte(&self) @dynamic
 {

lib/std/io/io.c3

@@ -227,7 +227,14 @@ fn void! out_putstream_fn(void* data, char c) @private
 
 fn void! out_putchar_fn(void* data @unused, char c) @private
 {
-	libc::putchar(c);
+	$if env::TESTING:
+		// HACK: this is used for the purpose of unit test output hijacking
+		File* stdout = io::stdout();
+		assert(stdout.file);
+		libc::fputc(c, stdout.file);
+	$else
+		libc::putchar(c);
+	$endif
 }
 
 <*
@@ -263,7 +270,7 @@ fn usz! printfn(String format, args...) @maydiscard
 	Formatter formatter;
 	formatter.init(&out_putchar_fn);
 	usz! len = formatter.vprintf(format, args);
-	putchar('\n');
+	out_putchar_fn(null, '\n')!;
 	io::stdout().flush()!;
 	return len + 1;
 }

lib/std/libc/libc.c3

@@ -172,6 +172,7 @@ extern fn usz strxfrm(char* dest, ZString src, usz n);
 extern fn CInt system(ZString str);
 extern fn Time_t timegm(Tm *timeptr) @if(!env::WIN32);
 extern fn ZString tmpnam(ZString str);
+extern fn CFile tmpfile();
 extern fn CInt ungetc(CInt c, CFile stream);
 extern fn CInt unsetenv(ZString name);
 extern fn isz write(Fd fd, void* buffer, usz count) @if(!env::WIN32);