From d6485ca08b16d7c909094ffd13e17ce231418473 Mon Sep 17 00:00:00 2001 From: Christoffer Lerno Date: Wed, 19 Feb 2025 02:23:51 +0100 Subject: [PATCH] Test new tester script. --- .github/workflows/main.yml | 19 ++- README.md | 2 +- nix/default.nix | 2 +- test/src/{tester.c3 => test_suite_runner.c3} | 156 +++++++++++------- .../test_suite/statements/dead_statements.c3t | 4 +- 5 files changed, 113 insertions(+), 70 deletions(-) rename test/src/{tester.c3 => test_suite_runner.c3} (68%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0569f5210..3a63f6b3f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -95,8 +95,7 @@ jobs: - name: run compiler tests run: | cd test - python3.exe src/tester.py ..\build\${{ matrix.build_type }}\c3c.exe test_suite/ - + ..\build\${{ matrix.build_type }}\c3c.exe compile-run -O1 src/test_suite_runner.c3 --enable-new-generics -- ..\build\${{ matrix.build_type }}\c3c.exe test_suite/ - name: Test python script run: | @@ -144,6 +143,7 @@ jobs: run: | cd resources ../build/c3c compile-run --print-linking examples/hello_world_many.c3 + ../build/c3c compile-run --print-linking examples/process.c3 ../build/c3c compile-run --print-linking examples/time.c3 ../build/c3c compile-run --print-linking examples/fannkuch-redux.c3 ../build/c3c compile-run --print-linking examples/contextfree/boolerr.c3 @@ -168,8 +168,8 @@ jobs: - name: run compiler tests run: | cd test - python3 src/tester.py ../build/c3c.exe test_suite/ - + ../build/c3c.exe compile --target windows-x64 -O1 src/test_suite_runner.c3 --enable-new-generics + ./test_suite_runner.exe ../build/c3c.exe test_suite/ build-msys2-clang: runs-on: windows-latest @@ -220,7 +220,7 @@ jobs: - name: run compiler tests run: | cd test - python3 src/tester.py ../build/c3c.exe test_suite/ + ../build/c3c.exe compile-run -O1 src/test_suite_runner.c3 --enable-new-generics build-linux: runs-on: ubuntu-22.04 @@ -381,7 +381,7 @@ jobs: - name: run compiler tests run: | cd test - python3 src/tester.py ../build/c3c test_suite/ + ../build/c3c compile-run -O1 src/test_suite_runner.c3 --enable-new-generics -- ../build/c3c test_suite/ - name: bundle_output if: matrix.llvm_version == env.LLVM_RELEASE_VERSION_LINUX @@ -502,7 +502,7 @@ jobs: - name: run compiler tests run: | cd test - python3 src/tester.py ../build/c3c test_suite/ + ../build/c3c compile-run -O1 src/test_suite_runner.c3 --enable-new-generics -- ../build/c3c test_suite/ - name: bundle_output if: matrix.llvm_version == env.LLVM_RELEASE_VERSION_UBUNTU20 @@ -606,7 +606,7 @@ jobs: - name: run compiler tests run: | cd test - python3 src/tester.py ../build/c3c test_suite/ + ../build/c3c compile-run -O1 src/test_suite_runner.c3 --enable-new-generics -- ../build/c3c test_suite/ build-mac: runs-on: macos-latest @@ -686,7 +686,8 @@ jobs: - name: run compiler tests run: | cd test - python3 src/tester.py ../build/c3c test_suite/ + ../build/c3c compile -O1 src/test_suite_runner.c3 --enable-new-generics + ./test_suite_runner ../build/c3c test_suite/ - name: bundle_output if: matrix.llvm_version == env.LLVM_RELEASE_VERSION_MAC diff --git a/README.md b/README.md index 36f499638..2b096121f 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ fn void main() ### Current status -The current stable version of the compiler is **version 0.6.6**. +The current stable version of the compiler is **version 0.6.7**. The upcoming 0.6.x releases will focus on expanding the standard library. Follow the issues [here](https://github.com/c3lang/c3c/issues). diff --git a/nix/default.nix b/nix/default.nix index 573ec49b6..c7dc61f9d 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -61,7 +61,7 @@ in llvmPackages.stdenv.mkDerivation (finalAttrs: { checkPhase = '' runHook preCheck ( cd ../resources/testproject; ../../build/c3c build --trust=full ) - ( cd ../test; python src/tester.py ../build/c3c test_suite ) + ( cd ../test; ../build/c3c compile-run -O1 src/test_suite_runner.c3 --enable-new-generics -- ../build/c3c test_suite ) runHook postCheck ''; diff --git a/test/src/tester.c3 b/test/src/test_suite_runner.c3 similarity index 68% rename from test/src/tester.c3 rename to test/src/test_suite_runner.c3 index 15b92c234..9563a853c 100644 --- a/test/src/tester.c3 +++ b/test/src/test_suite_runner.c3 @@ -17,6 +17,7 @@ int skip_count; int success_count; Path start_cwd; Path test_dir; +bool print_to_file; fn void arg_error_exit(String fmt, args...) @noreturn { @@ -34,9 +35,8 @@ fn void error_exit(String fmt, args...) @noreturn fn void main(String[] args) { appname = args[0]; - args = { "foo", "../cmake-build-debug-latest/c3c", "test_suite/" }; if (args.len < 3 || args.len > 4) usage(); - + print_to_file = !io::stdout().isatty(); // Check compiler path start_cwd = path::temp_cwd()!!; test_dir = start_cwd.temp_append("_c3test")!!; @@ -45,7 +45,7 @@ fn void main(String[] args) if (!path::is_file(path)) { - error_exit("Error: Invalid path to compiler: %s", path.path_string); + error_exit("Error: Invalid path to compiler: %s (%s relative to %s)", path.path_string, args[1], start_cwd); } compiler_path = path; @@ -82,9 +82,9 @@ struct RunFile String name; File file; int line_offset; - List{String} expected_lines; - List{Error} warnings; - List{Error} errors; + List {String} expected_lines; + List {Error} warnings; + List {Error} errors; bool is_llvm; } @@ -128,46 +128,61 @@ struct RunSettings List{RunFile*} files; } -fn bool check_line(RunSettings*, String type, String file, String line, String col, String message) +fn bool check_line(RunSettings* settings, String type, String file, String line_str, String col, String message) => @pool() { + int line = line_str.to_int()!!; + String basename = path::temp_new(file).basename()!!; + foreach (f : settings.files) + { + if (f.name != basename) continue; + List{Error}* list; + switch (type) + { + case "Error": + list = &f.errors; + break; + case "Warning": + list = &f.warnings; + break; + default: + error_exit("FAILED - Unknown error type '%s'", type); + } + foreach (i, err : *list) + { + if (line != err.line) continue; + if (!message.contains(err.text)) return false; + list.remove_at(i); + return true; + } + } return false; -/* - def check_line(self, typ, file, line, col, message): - map_ = {} - if typ == 'Error': - map_ = self.errors - elif typ == 'Warning': - map_ = self.warnings - else: - self.exit_error("Unknown type: " + typ) - file = os.path.basename(file) - key = file + ":" + line - value = map_.get(key) - if value is None: - return False - if value in message: - del map_[key] - return True - else: - return False -*/ } + fn bool parse_result(DString out, RunSettings settings) { // Inefficient, fix. bool success = true; + int errors = 0; foreach (line : out.str_view().tsplit("\n")) { if (!line) continue; String[] parts = line.tsplit("|", 5); if (parts.len != 5) { - error_exit("Unexpected error: %s", out); + io::printn("FAILED - Unexpected response from compiler:"); + io::printn("Output: ----------------------------------------------------------"); + io::print(out); + io::printn("------------------------------------------------------------------"); + return false; } if (!check_line(&settings, ...parts[:5])) { - parts[0].convert_ascii_to_lower(); - io::printf("Unexpected %s in %s line %s:", parts[0], path::temp_new(parts[1]).basename()!!, parts[2]); + if (success) + { + io::printn("FAILED\n\n Unexpected compilation errors:"); + io::printn(" ------------------------------"); + } + io::printf(" %d. %s at %s:%s: ", ++errors, parts[0], path::temp_new(parts[1]).basename()!!, parts[2]); io::printfn(`"%s"`, parts[4]); success = false; } @@ -177,11 +192,17 @@ fn bool parse_result(DString out, RunSettings settings) { if (file.errors.len()) { + if (success) io::printn("FAILED - Missing errors"); + if (!not_found_errors) + { + io::printn(); + io::printn(" Errors that never occurred:"); + io::printn(" ---------------------------"); + } success = false; - if (!not_found_errors) io::printn("Errors that never occurred:"); foreach (i, &item : file.errors) { - io::printfn(`%d. %s %d expected: "%s"`, ++not_found_errors, file.name, item.line, item.text); + io::printfn(` %d. %s:%d expected: "%s"`, ++not_found_errors, file.name, item.line, item.text); } } } @@ -189,14 +210,21 @@ fn bool parse_result(DString out, RunSettings settings) { if (file.warnings.len()) { + if (success) io::printn("FAILED - Missing warnings"); success = false; - if (!not_found_warnings) io::printn("Warnings that never occurred:"); + if (!not_found_warnings) + { + if (!success) io::printn(); + io::printn(" Warnings that never occurred:"); + io::printn(" -----------------------------"); + } foreach (i, &item : file.warnings) { - io::printfn(`%d. %s %d expected: "%s"`, ++not_found_warnings, file.name, item.line, item.text); + io::printfn(` %d. %s:%d expected: "%s"`, ++not_found_errors, file.name, item.line, item.text); } } } + if (!success) io::printn(); return success; } @@ -213,17 +241,17 @@ fn void parse_trailing_directive(int line_number, String line, RunSettings* sett line = line[6..].trim(); settings.current_file.errors.push({ line_number, line }); default: - error_exit("Unknown trailing directive '%s'", line); + error_exit("FAILED - Unknown trailing directive '%s'", line); } } -fn void parse_header_directive(int line_no, String line, RunSettings* settings, bool is_single) +fn void parse_header_directive(int* line_no, String line, RunSettings* settings, bool is_single) { line = line[4..].trim(); switch { case line.starts_with("error:"): line = line[6..].trim(); - settings.current_file.errors.push({ line_no, line }); + settings.current_file.errors.push({ *line_no, line }); case line.starts_with("safe:"): settings.safe = line[5..].trim() == "yes"; case line.starts_with("debuginfo:"): @@ -235,34 +263,43 @@ fn void parse_header_directive(int line_no, String line, RunSettings* settings, case line.starts_with("deprecation:"): settings.no_deprecation = line[12..].trim() == "no"; case line.starts_with("file:"): - if (is_single) error_exit("'file' directive only allowed with .c3t"); + if (is_single) error_exit("FAILED - 'file' directive only allowed with .c3t"); settings.current_file.close(); line = line[5..].trim(); RunFile* file = settings.current_file = create_file(line)!!; + *line_no = 1; settings.files.push(file); settings.current_file = file; case line.starts_with("expect:"): - if (is_single) error_exit("'expect' directive only allowed with .c3t"); + if (is_single) error_exit("FAILED - 'expect' directive only allowed with .c3t"); line = line[7..].trim(); settings.current_file.close(); RunFile* file = settings.current_file = create_file(line, llvm: true)!!; settings.files.push(file); default: - io::printfn("Unknown header directive '%s'", line); + io::printfn("FAILED - Unknown header directive '%s'", line); libc::exit(1); } } fn void test_file(Path file_path) { - io::printfn("Checking %s", file_path); + test_count++; + if (print_to_file) + { + io::printf("- %d/%d %s: ", test_count, test_count - success_count - 1, file_path); + } + else + { + io::printf("- %d/%d Compiling: %s ", test_count, test_count - success_count - 1, file_path); + } + (void)io::stdout().flush(); static int count = 1; bool single; (void)path::rmtree(test_dir); if (@catch(path::mkdir(test_dir))) { - io::printfn("Failed to create temp test directory '%s'.", test_dir); - libc::exit(1); + error_exit("FAILED - Failed to create temp test directory '%s'.", test_dir); } switch (file_path.extension() ?? "") { @@ -271,14 +308,12 @@ fn void test_file(Path file_path) case "c3t": single = false; default: - io::printfn("Unexpected file name '%s', expected a file with a '.c3' or '.c3t' suffix.", file_path.str_view()); - libc::exit(1); + error_exit("FAILED - Unexpected file name '%s', expected a file with a '.c3' or '.c3t' suffix.", file_path.str_view()); } File! f = file::open_path(file_path, "rb"); if (catch f) { - io::printfn("Error: Failed to open '%s'.", file_path); - libc::exit(1); + error_exit("FAILED - Failed to open '%s'.", file_path); } defer (void)f.close(); RunSettings settings; @@ -286,24 +321,22 @@ fn void test_file(Path file_path) settings.files.temp_init(); settings.current_file = create_file(file_path.basename()[..^(single ? 4 : 5)].tconcat(".c3"))!!; settings.files.push(settings.current_file); - int line_no = 0; + int line_no = 1; while (try line = io::treadline(&f)) { if (line.starts_with("// #") || line.starts_with("/* #")) { - parse_header_directive(line_no, line, &settings, single); + parse_header_directive(&line_no, line, &settings, single); continue; } else if (line.contains("// #")) { parse_trailing_directive(line_no, line, &settings, single); - continue; } settings.current_file.add_line(line); line_no++; } settings.current_file.close(); - test_count += 1; DString cmdline = dstring::temp_new(compiler_path.str_view()); cmdline.append(" compile-only --test "); foreach (file : settings.files) @@ -322,16 +355,16 @@ fn void test_file(Path file_path) { cmdline.appendf("%s ", opt); } - io::printfn("Dir: %s", test_dir); path::chdir(test_dir)!!; - io::printn(cmdline); - SubProcess compilation = process::create(cmdline.str_view().trim().tsplit(" "), { .search_user_path = true })!!; + SubProcess compilation = process::create(cmdline.str_view().trim().tsplit(" "), { .search_user_path, .no_window, .inherit_environment })!!; + defer compilation.destroy(); CInt result = compilation.join()!!; DString out = dstring::temp_new(); io::copy_to(&&compilation.stderr(), &out)!!; if (result != 0 && result != 1) { - io::printfn("Error(%s): ", result, out); + (void)io::copy_to(&&compilation.stdout(), &out); + io::printfn("FAILED - Error(%s): ", result, out); return; } if (!parse_result(out, settings)) return; @@ -340,7 +373,7 @@ fn void test_file(Path file_path) if (!file.is_llvm) continue; if (!file::exists(file.name)) { - io::printfn("Did not compile file %s.", file.name); + io::printfn("FAILED - Did not compile file %s.", file.name); return; } File file_ll = file::open(file.name, "rb")!!; @@ -357,7 +390,7 @@ fn void test_file(Path file_path) } if (try next) { - io::printfn(`%s did not contain: "%s"`, file.name, next); + io::printfn(`FAILED - %s did not contain: "%s"`, file.name, next); io::printfn("\n\n\n---------------------------------------------------> %s\n\n", file.name); (void)file_ll.seek(0); (void)io::printn((String)io::read_new_fully(&file_ll, allocator: allocator::temp())); @@ -365,6 +398,15 @@ fn void test_file(Path file_path) return; } } + if (print_to_file) + { + io::print("Passed."); + io::printn(); + } + else + { + for (int i = 0; i < 200; i++) io::print("\b \b"); + } success_count++; } diff --git a/test/test_suite/statements/dead_statements.c3t b/test/test_suite/statements/dead_statements.c3t index a7a716a44..341c5cb4f 100644 --- a/test/test_suite/statements/dead_statements.c3t +++ b/test/test_suite/statements/dead_statements.c3t @@ -6,8 +6,8 @@ import std::collections::list; fn void! load_corpus2(String code, String path) @local { - for(;;) io::printfn("hi"); // #warning: This code will never execute - path::Path p = path::temp_new(path)!; + for(;;) io::printfn("hi"); + path::Path p = path::temp_new(path)!; // #warning: This code will never execute if (!path::exists(p)) {