From 0c33b78a2f97a1cbe12c2cdf97595020f101dad2 Mon Sep 17 00:00:00 2001 From: Aleksandr Vedeneev Date: Wed, 26 Feb 2025 03:49:21 +0400 Subject: [PATCH] Test runner args #1967 (#1988) * harmonized testrun arguments with c3c --test-* naming added --test-quiet option added --test-noleak to disable tracking allocator mem leak detection added --test-nocapture - tests can print out everything as they run * Move changes to lib7 --------- Co-authored-by: Christoffer Lerno --- lib/std/core/runtime_test.c3 | 73 ++++++++++++++++++++++------------- lib7/std/core/runtime_test.c3 | 73 ++++++++++++++++++++++------------- src/build/build.h | 3 ++ src/build/build_options.c | 18 +++++++++ src/build/builder.c | 13 ++++--- 5 files changed, 123 insertions(+), 57 deletions(-) diff --git a/lib/std/core/runtime_test.c3 b/lib/std/core/runtime_test.c3 index ded27fb18..4d2476eb5 100644 --- a/lib/std/core/runtime_test.c3 +++ b/lib/std/core/runtime_test.c3 @@ -24,6 +24,8 @@ struct TestContext bool assert_print_backtrace; bool has_ansi_codes; bool is_in_panic; + bool is_quiet_mode; + bool is_no_capture; String current_test_name; TestFn setup_fn; TestFn teardown_fn; @@ -117,7 +119,7 @@ fn void test_panic(String message, String file, String function, uint line) @loc fn void mute_output() @local { - if (!test_context.fake_stdout.file) return; + if (test_context.is_no_capture || !test_context.fake_stdout.file) return; File* stdout = io::stdout(); File* stderr = io::stderr(); *stderr = test_context.fake_stdout; @@ -127,7 +129,7 @@ fn void mute_output() @local fn void unmute_output(bool has_error) @local { - if (!test_context.fake_stdout.file) return; + if (test_context.is_no_capture || !test_context.fake_stdout.file) return; File* stdout = io::stdout(); File* stderr = io::stderr(); @@ -138,6 +140,7 @@ fn void unmute_output(bool has_error) @local usz log_size = test_context.fake_stdout.seek(0, Seek.CURSOR)!!; if (has_error) { + io::printf("\nTesting %s ", test_context.current_test_name); io::printn(test_context.has_ansi_codes ? "[\e[0;31mFAIL\e[0m]" : "[FAIL]"); } @@ -153,7 +156,7 @@ fn void unmute_output(bool has_error) @local { if (@unlikely(c == '\0')) { - // ignore junk from previous tests + // ignore junk from previous tests break; } libc::putchar(c); @@ -167,6 +170,7 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private { usz max_name; bool sort_tests = true; + bool check_leaks = true; foreach (&unit : tests) { if (max_name < unit.name.len) max_name = unit.name.len; @@ -185,15 +189,21 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private { switch (args[i]) { - case "breakpoint": + case "--test-breakpoint": context.breakpoint_on_assert = true; - case "nosort": + case "--test-nosort": sort_tests = false; - case "noansi": + case "--test-noleak": + check_leaks = false; + case "--test-nocapture": + context.is_no_capture = true; + case "--noansi": context.has_ansi_codes = false; - case "useansi": + case "--useansi": context.has_ansi_codes = true; - case "filter": + case "--test-quiet": + context.is_quiet_mode = true; + case "--test-filter": if (i == args.len - 1) { io::printn("Invalid arguments to test runner."); @@ -232,7 +242,7 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private name.append_repeat('-', len / 2); name.append(" TESTS "); name.append_repeat('-', len - len / 2); - io::printn(name); + if (!context.is_quiet_mode) io::printn(name); name.clear(); TempState temp_state = mem::temp_push(); defer mem::temp_pop(temp_state); @@ -251,7 +261,14 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private defer name.clear(); name.appendf("Testing %s ", unit.name); name.append_repeat('.', max_name - unit.name.len + 2); - io::printf("%s ", name.str_view()); + if (context.is_quiet_mode) + { + io::print("."); + } + else + { + io::printf("%s ", name.str_view()); + } (void)io::stdout().flush(); TrackingAllocator mem; @@ -260,7 +277,7 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private { mute_output(); mem.clear(); - allocator::thread_allocator = &mem; + if (check_leaks) allocator::thread_allocator = &mem; $if(!$$OLD_TEST): unit.func(); $else @@ -271,22 +288,26 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private } $endif // track cleanup that may take place in teardown_fn - if (test_context.teardown_fn) + if (context.teardown_fn) { - test_context.teardown_fn(); + context.teardown_fn(); } - allocator::thread_allocator = context.stored.allocator; + if (check_leaks) allocator::thread_allocator = context.stored.allocator; unmute_output(false); // all good, discard output if (mem.has_leaks()) - { - io::print(test_context.has_ansi_codes ? "[\e[0;31mFAIL\e[0m]" : "[FAIL]"); - io::printn(" LEAKS DETECTED!"); - mem.print_report(); - } - else - { - io::printfn(test_context.has_ansi_codes ? "[\e[0;32mPASS\e[0m]" : "[PASS]"); + { + if(context.is_quiet_mode) io::printf("\n%s ", context.current_test_name); + io::print(context.has_ansi_codes ? "[\e[0;31mFAIL\e[0m]" : "[FAIL]"); + io::printn(" LEAKS DETECTED!"); + mem.print_report(); + } + else + { + if (!context.is_quiet_mode) + { + io::printfn(context.has_ansi_codes ? "[\e[0;32mPASS\e[0m]" : "[PASS]"); + } tests_passed++; } } @@ -296,9 +317,9 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private int n_failed = test_count - tests_passed - tests_skipped; io::printf("Test Result: %s%s%s: ", - test_context.has_ansi_codes ? (n_failed ? "\e[0;31m" : "\e[0;32m") : "", + context.has_ansi_codes ? (n_failed ? "\e[0;31m" : "\e[0;32m") : "", n_failed ? "FAILED" : "PASSED", - test_context.has_ansi_codes ? "\e[0m" : "", + context.has_ansi_codes ? "\e[0m" : "", ); io::printfn("%d passed, %d failed, %d skipped.", @@ -307,8 +328,8 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private tests_skipped); // cleanup fake_stdout file - if (test_context.fake_stdout.file) libc::fclose(test_context.fake_stdout.file); - test_context.fake_stdout.file = null; + if (context.fake_stdout.file) libc::fclose(context.fake_stdout.file); + context.fake_stdout.file = null; return n_failed == 0; } diff --git a/lib7/std/core/runtime_test.c3 b/lib7/std/core/runtime_test.c3 index ce5a678ae..00d275e8f 100644 --- a/lib7/std/core/runtime_test.c3 +++ b/lib7/std/core/runtime_test.c3 @@ -24,6 +24,8 @@ struct TestContext bool assert_print_backtrace; bool has_ansi_codes; bool is_in_panic; + bool is_quiet_mode; + bool is_no_capture; String current_test_name; TestFn setup_fn; TestFn teardown_fn; @@ -118,7 +120,7 @@ fn void test_panic(String message, String file, String function, uint line) @loc fn void mute_output() @local { - if (!test_context.fake_stdout.file) return; + if (test_context.is_no_capture || !test_context.fake_stdout.file) return; File* stdout = io::stdout(); File* stderr = io::stderr(); *stderr = test_context.fake_stdout; @@ -128,7 +130,7 @@ fn void mute_output() @local fn void unmute_output(bool has_error) @local { - if (!test_context.fake_stdout.file) return; + if (test_context.is_no_capture || !test_context.fake_stdout.file) return; File* stdout = io::stdout(); File* stderr = io::stderr(); @@ -139,6 +141,7 @@ fn void unmute_output(bool has_error) @local usz log_size = test_context.fake_stdout.seek(0, Seek.CURSOR)!!; if (has_error) { + io::printf("\nTesting %s ", test_context.current_test_name); io::printn(test_context.has_ansi_codes ? "[\e[0;31mFAIL\e[0m]" : "[FAIL]"); } @@ -154,7 +157,7 @@ fn void unmute_output(bool has_error) @local { if (@unlikely(c == '\0')) { - // ignore junk from previous tests + // ignore junk from previous tests break; } libc::putchar(c); @@ -168,6 +171,7 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private { usz max_name; bool sort_tests = true; + bool check_leaks = true; foreach (&unit : tests) { if (max_name < unit.name.len) max_name = unit.name.len; @@ -186,15 +190,21 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private { switch (args[i]) { - case "breakpoint": + case "--test-breakpoint": context.breakpoint_on_assert = true; - case "nosort": + case "--test-nosort": sort_tests = false; - case "noansi": + case "--test-noleak": + check_leaks = false; + case "--test-nocapture": + context.is_no_capture = true; + case "--noansi": context.has_ansi_codes = false; - case "useansi": + case "--useansi": context.has_ansi_codes = true; - case "filter": + case "--test-quiet": + context.is_quiet_mode = true; + case "--test-filter": if (i == args.len - 1) { io::printn("Invalid arguments to test runner."); @@ -233,7 +243,7 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private name.append_repeat('-', len / 2); name.append(" TESTS "); name.append_repeat('-', len - len / 2); - io::printn(name); + if (!context.is_quiet_mode) io::printn(name); name.clear(); TempState temp_state = mem::temp_push(); defer mem::temp_pop(temp_state); @@ -252,7 +262,14 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private defer name.clear(); name.appendf("Testing %s ", unit.name); name.append_repeat('.', max_name - unit.name.len + 2); - io::printf("%s ", name.str_view()); + if (context.is_quiet_mode) + { + io::print("."); + } + else + { + io::printf("%s ", name.str_view()); + } (void)io::stdout().flush(); TrackingAllocator mem; @@ -261,7 +278,7 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private { mute_output(); mem.clear(); - allocator::thread_allocator = &mem; + if (check_leaks) allocator::thread_allocator = &mem; $if(!$$OLD_TEST): unit.func(); $else @@ -272,22 +289,26 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private } $endif // track cleanup that may take place in teardown_fn - if (test_context.teardown_fn) + if (context.teardown_fn) { - test_context.teardown_fn(); + context.teardown_fn(); } - allocator::thread_allocator = context.stored.allocator; + if (check_leaks) allocator::thread_allocator = context.stored.allocator; unmute_output(false); // all good, discard output if (mem.has_leaks()) - { - io::print(test_context.has_ansi_codes ? "[\e[0;31mFAIL\e[0m]" : "[FAIL]"); - io::printn(" LEAKS DETECTED!"); - mem.print_report(); - } - else - { - io::printfn(test_context.has_ansi_codes ? "[\e[0;32mPASS\e[0m]" : "[PASS]"); + { + if (context.is_quiet_mode) io::printf("\n%s ", context.current_test_name); + io::print(context.has_ansi_codes ? "[\e[0;31mFAIL\e[0m]" : "[FAIL]"); + io::printn(" LEAKS DETECTED!"); + mem.print_report(); + } + else + { + if (!context.is_quiet_mode) + { + io::printfn(context.has_ansi_codes ? "[\e[0;32mPASS\e[0m]" : "[PASS]"); + } tests_passed++; } } @@ -297,9 +318,9 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private int n_failed = test_count - tests_passed - tests_skipped; io::printf("Test Result: %s%s%s: ", - test_context.has_ansi_codes ? (n_failed ? "\e[0;31m" : "\e[0;32m") : "", + context.has_ansi_codes ? (n_failed ? "\e[0;31m" : "\e[0;32m") : "", n_failed ? "FAILED" : "PASSED", - test_context.has_ansi_codes ? "\e[0m" : "", + context.has_ansi_codes ? "\e[0m" : "", ); io::printfn("%d passed, %d failed, %d skipped.", @@ -308,8 +329,8 @@ fn bool run_tests(String[] args, TestUnit[] tests) @private tests_skipped); // cleanup fake_stdout file - if (test_context.fake_stdout.file) libc::fclose(test_context.fake_stdout.file); - test_context.fake_stdout.file = null; + if (context.fake_stdout.file) libc::fclose(context.fake_stdout.file); + context.fake_stdout.file = null; return n_failed == 0; } diff --git a/src/build/build.h b/src/build/build.h index 7845d2ad0..785f36873 100644 --- a/src/build/build.h +++ b/src/build/build.h @@ -505,7 +505,10 @@ typedef struct BuildOptions_ ValidationLevel validation_level; Ansi ansi; bool test_breakpoint; + bool test_quiet; bool test_nosort; + bool test_noleak; + bool test_nocapture; const char *custom_linker_path; uint32_t symtab_size; unsigned version; diff --git a/src/build/build_options.c b/src/build/build_options.c index 1606cac48..b96bd5598 100644 --- a/src/build/build_options.c +++ b/src/build/build_options.c @@ -137,6 +137,9 @@ static void usage(bool full) print_opt("--test-filter ", "Set a filter when running tests, running only matching tests."); print_opt("--test-breakpoint", "When running tests, trigger a breakpoint on failure."); print_opt("--test-nosort", "Do not sort tests."); + print_opt("--test-noleak", "Disable tracking allocator and memory leak detection for tests"); + print_opt("--test-nocapture", "Disable test stdout capturing, all tests can print as they run"); + print_opt("--test-quiet", "Run tests without printing full names, printing output only on failure"); } PRINTF(""); print_opt("-l ", "Link with the static or dynamic library provided."); @@ -724,6 +727,21 @@ static void parse_option(BuildOptions *options) options->test_breakpoint = true; return; } + if (match_longopt("test-noleak")) + { + options->test_noleak = true; + return; + } + if (match_longopt("test-nocapture")) + { + options->test_nocapture = true; + return; + } + if (match_longopt("test-quiet")) + { + options->test_quiet = true; + return; + } if (match_longopt("test-nosort")) { options->test_nosort = true; diff --git a/src/build/builder.c b/src/build/builder.c index 77b7b2e0f..b263d15ec 100644 --- a/src/build/builder.c +++ b/src/build/builder.c @@ -282,21 +282,24 @@ static void update_build_target_from_options(BuildTarget *target, BuildOptions * switch (options->ansi) { case ANSI_ON: - vec_add(target->args, "useansi"); + vec_add(target->args, "--useansi"); break; case ANSI_OFF: - vec_add(target->args, "noansi"); + vec_add(target->args, "--noansi"); break; default: break; } if (options->test_filter) { - vec_add(target->args, "filter"); + vec_add(target->args, "--test-filter"); vec_add(target->args, options->test_filter); } - if (options->test_breakpoint) vec_add(target->args, "breakpoint"); - if (options->test_nosort) vec_add(target->args, "nosort"); + if (options->test_breakpoint) vec_add(target->args, "--test-breakpoint"); + if (options->test_nosort) vec_add(target->args, "--test-nosort"); + if (options->test_quiet) vec_add(target->args, "--test-quiet"); + if (options->test_noleak) vec_add(target->args, "--test-noleak"); + if (options->test_nocapture) vec_add(target->args, "--test-nocapture"); break; case COMMAND_RUN: case COMMAND_COMPILE_RUN: