From 5bd21c10b6855a2cb97aa7825adc08c59c9fdef3 Mon Sep 17 00:00:00 2001 From: Pierre Curto Date: Wed, 16 Aug 2023 12:28:07 +0200 Subject: [PATCH] improve tests (#932) * test: fix warnings generated by Python's interpreter Signed-off-by: Pierre Curto * lib/std/core/runtime: sort tests to run; improve tests output Signed-off-by: Pierre Curto --------- Signed-off-by: Pierre Curto --- lib/std/core/runtime.c3 | 66 ++++++++++++------ test/src/tester.py | 112 +++++++++++++++++++------------ test/unit/stdlib/core/runtime.c3 | 37 ++++++++++ 3 files changed, 152 insertions(+), 63 deletions(-) create mode 100644 test/unit/stdlib/core/runtime.c3 diff --git a/lib/std/core/runtime.c3 b/lib/std/core/runtime.c3 index c80918d18..b03ce632f 100644 --- a/lib/std/core/runtime.c3 +++ b/lib/std/core/runtime.c3 @@ -2,6 +2,7 @@ // Use of this source code is governed by the MIT license // a copy of which can be found in the LICENSE_STDLIB file. module std::core::runtime; +import libc; struct StackTrace { @@ -40,7 +41,25 @@ fn TestRunner test_runner_create() }; } -import libc; +struct TestUnit +{ + String name; + TestFn func; +} + +// Sort the tests by their name in ascending order. +fn int cmp_unit(TestUnit a, TestUnit b) +{ + usz an = a.name.len; + usz bn = b.name.len; + if (an > bn) @swap(a, b); + foreach (i, ac : a.name) + { + char bc = b.name[i]; + if (ac != bc) return an > bn ? bc - ac : ac - bc; + } + return (int)(an - bn); +} TestRunner* current_runner @private; @@ -54,23 +73,40 @@ fn void test_panic(String message, String file, String function, uint line) libc::longjmp(¤t_runner.buf, 1); } -fn bool TestRunner.run(TestRunner* runner) +fn bool TestRunner.run(&self) { - current_runner = runner; + assert(self.test_names.len == self.test_fns.len); + TestUnit[] units = malloc(TestUnit, self.test_names.len); + defer free(units); + + usz max_name; + foreach (i, name: self.test_names) + { + units[i] = { name, self.test_fns[i] }; + max_name = max(max_name, name.len); + } + quicksort(units, &cmp_unit); + + current_runner = self; PanicFn old_panic = builtin::panic; defer builtin::panic = old_panic; builtin::panic = &test_panic; int tests_passed = 0; - int tests = runner.test_names.len; + int tests = units.len; io::printn("----- TESTS -----"); - foreach(i, String name : runner.test_names) + DString name; + name.tinit(); + foreach(unit : units) { - io::printf("Testing %s ... ", name); + defer name.clear(); + name.printf("Testing %s ", unit.name); + name.append_repeat('.', max_name - unit.name.len + 2); + io::printf("%s ", name.as_str()); CallstackElement* stack = $$stacktrace(); if (stack) stack.prev = null; - if (libc::setjmp(&runner.buf) == 0) + if (libc::setjmp(&self.buf) == 0) { - if (catch err = runner.test_fns[i]()) + if (catch err = unit.func()) { io::printfn("[failed] Failed due to: %s", err); continue; @@ -79,17 +115,9 @@ fn bool TestRunner.run(TestRunner* runner) tests_passed++; } } - io::printfn("\n%d test(s) run.\n", tests); - io::print("Test Result: "); - if (tests_passed < tests) - { - io::print("FAILED"); - } - else - { - io::print("ok"); - } - io::printfn(". %d passed, %d failed.", tests_passed, tests - tests_passed); + io::printfn("\n%d test%s run.\n", tests, tests > 1 ? "s" : ""); + io::printfn("Test Result: %s. %d passed, %d failed.", + tests_passed < tests ? "FAILED" : "ok", tests_passed, tests - tests_passed); return tests == tests_passed; } diff --git a/test/src/tester.py b/test/src/tester.py index 7689b7414..fd9bc6da6 100644 --- a/test/src/tester.py +++ b/test/src/tester.py @@ -1,8 +1,13 @@ #!/usr/bin/python -import os, sys, shutil, subprocess, tempfile +import os +import shutil +import subprocess +import sys +import tempfile TEST_DIR = tempfile.mkdtemp().replace('\\', '/') + '/c3test/' + class Config: run_skipped = False cwd = "." @@ -10,6 +15,7 @@ class Config: numsuccess = 0 numskipped = 0 + class File: def __init__(self, filepath): with open(filepath, encoding='utf8') as reader: @@ -17,6 +23,7 @@ class File: self.filepath = filepath self.filename = os.path.basename(filepath) + class TargetFile: def __init__(self, filepath, is_target, line_offset): self.is_target = is_target @@ -30,7 +37,8 @@ class TargetFile: self.filename = os.path.basename(filepath) def close(self): - if self.file: self.file.close() + if self.file: + self.file.close() self.file = None def write(self, line): @@ -40,6 +48,7 @@ class TargetFile: else: self.expected_lines.append(line) + class Issues: def __init__(self, conf, file, single): self.conf = conf @@ -66,22 +75,24 @@ class Issues: exit(-1) def set_failed(self): - if not self.has_errors: print(" Failed.") + if not self.has_errors: + print(" Failed.") self.has_errors = True - def check_line(self, type, file, line, message): - map = {} - if type == 'Error': - map = self.errors - elif type == 'Warning': - map = self.warnings + def check_line(self, typ, file, line, message): + map_ = {} + if typ == 'Error': + map_ = self.errors + elif typ == 'Warning': + map_ = self.warnings else: - self.exit_error("Unknown type: " + type) + self.exit_error("Unknown type: " + typ) key = file + ":" + line - value = map.get(key) - if value == None: return False + value = map_.get(key) + if value is None: + return False if value in message: - del map[key] + del map_[key] return True else: return False @@ -89,7 +100,8 @@ class Issues: def parse_result(self, lines): for line in lines: parts = line.split('|', maxsplit=4) - if len(parts) != 4: self.exit_error("Illegal error result: " + line); + if len(parts) != 4: + self.exit_error("Illegal error result: " + line) if not self.check_line(parts[0], parts[1], parts[2], parts[3]): self.set_failed() print("Unexpected " + parts[0].lower() + " in " + parts[1] + " line " + parts[2] + ":", end="") @@ -107,18 +119,19 @@ class Issues: os.chdir(TEST_DIR) target = "" debug = "-g0 " - if (self.arch): + if self.arch: target = " --target " + self.arch - if (self.debuginfo): + if self.debuginfo: debug = "-g " opts = "" for opt in self.opts: opts += ' ' + opt - if (self.safe): + if self.safe: opts += " --safe" else: opts += " --fast" - code = subprocess.run(self.conf.compiler + target + ' -O0 ' + opts + ' ' + debug + args, universal_newlines=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + code = subprocess.run(self.conf.compiler + target + ' -O0 ' + opts + ' ' + debug + args, + universal_newlines=True, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) os.chdir(self.conf.cwd) if code.returncode != 0 and code.returncode != 1: self.set_failed() @@ -127,7 +140,6 @@ class Issues: return self.parse_result(code.stderr.splitlines(keepends=False)) - def parse_single(self): self.current_file = TargetFile(TEST_DIR + self.sourcefile.filename, True, 1) lines = len(self.sourcefile.content) @@ -139,7 +151,8 @@ class Issues: self.line += 1 self.current_file.close() - print("- " + str(self.conf.numtests) + "/" + str(self.conf.numtests - self.conf.numsuccess - 1) + " " + self.sourcefile.filepath + ":", end="") + print("- " + str(self.conf.numtests) + "/" + str( + self.conf.numtests - self.conf.numsuccess - 1) + " " + self.sourcefile.filepath + ":", end="") self.compile("--test compile " + self.current_file.filepath) if not self.has_errors: self.conf.numsuccess += 1 @@ -147,26 +160,26 @@ class Issues: def parse_header_directive(self, line): line = line[4:].strip() - if (line.startswith("safe:")): + if line.startswith("safe:"): self.safe = line[5:].strip() == "yes" return - if (line.startswith("debuginfo:")): + if line.startswith("debuginfo:"): self.debuginfo = line[10:].strip() == "yes" return - if (line.startswith("opt:")): + if line.startswith("opt:"): self.opts.append(line[4:].strip()) return - if (line.startswith("target:")): + if line.startswith("target:"): self.arch = line[7:].strip() return - if (line.startswith("file:")): + if line.startswith("file:"): if self.current_file: self.current_file.close() line = line[5:].strip() self.current_file = TargetFile(TEST_DIR + line, True, -self.line) self.files.append(self.current_file) return - elif (line.startswith("expect:")): + elif line.startswith("expect:"): line = line[7:].strip() if self.current_file: self.current_file.close() @@ -178,12 +191,12 @@ class Issues: def parse_trailing_directive(self, line): line = line.split('// #', 2)[1].strip() - if (line.startswith("warning:")): + if line.startswith("warning:"): print("TODO" + line) exit(-1) - elif (line.startswith("target:")): + elif line.startswith("target:"): self.arch = line[7:].strip() - elif (line.startswith("error:")): + elif line.startswith("error:"): line = line[6:].strip() self.errors[self.current_file.filename + ":%d" % (self.line + self.current_file.line_offset)] = line else: @@ -209,15 +222,16 @@ class Issues: self.current_file.close() self.current_file = None - print("- " + str(self.conf.numtests) + "/" + str(self.conf.numtests - self.conf.numsuccess - 1) + " " + self.sourcefile.filepath + ":", end="") + print("- " + str(self.conf.numtests) + "/" + str( + self.conf.numtests - self.conf.numsuccess - 1) + " " + self.sourcefile.filepath + ":", end="") files_to_compile = "" for file in self.files: if file.is_target: files_to_compile += " " + file.filepath - self.compile("--test compile " + files_to_compile) - if self.has_errors: return + if self.has_errors: + return for file in self.files: if not file.is_target: @@ -249,7 +263,7 @@ class Issues: current_line += 1 searched_line += 1 continue - if next_line != None and next_line in lines[current_line]: + if next_line is not None and next_line in lines[current_line]: current_line += 1 searched_line += 2 continue @@ -259,13 +273,16 @@ class Issues: print(" Passed.") def parse(self): - if len(self.sourcefile.content) == 0: self.exit_error("File was empty") + if len(self.sourcefile.content) == 0: + self.exit_error("File was empty") is_skip = self.sourcefile.content[0].startswith("// #skip") if is_skip != self.skip: - print("- " + str(self.conf.numtests) + "/" + str(self.conf.numtests - self.conf.numsuccess - 1) + " " + self.sourcefile.filepath + ": *SKIPPED*") + print("- " + str(self.conf.numtests) + "/" + str( + self.conf.numtests - self.conf.numsuccess - 1) + " " + self.sourcefile.filepath + ": *SKIPPED*") self.conf.numskipped += 1 return - if is_skip: self.line += 1 + if is_skip: + self.line += 1 if self.single: self.parse_single() else: @@ -273,12 +290,13 @@ class Issues: def usage(): - print("Usage: " + sys.argv[0] + " [-s]") + print("Usage: " + sys.argv[0] + " [-s]") print('') print('Options:') print(" -s, --skipped only run skipped tests") exit(-1) + def handle_file(filepath, conf): if filepath.endswith('.c3'): single = True @@ -288,7 +306,7 @@ def handle_file(filepath, conf): return shutil.rmtree(TEST_DIR, ignore_errors=True) - os.mkdir(TEST_DIR, mode = 0o777) + os.mkdir(TEST_DIR, mode=0o777) conf.numtests += 1 @@ -296,7 +314,6 @@ def handle_file(filepath, conf): issues.parse() - def handle_dir(filepath, conf): for file in os.listdir(filepath): file = filepath + "/" + file @@ -305,19 +322,23 @@ def handle_dir(filepath, conf): elif os.path.isfile(file): handle_file(file, conf) + def main(): args = len(sys.argv) conf = Config() - if args < 3 or args > 4: usage() + if args < 3 or args > 4: + usage() conf.compiler = os.getcwd() + "/" + sys.argv[1] if not os.path.isfile(conf.compiler): print("Error: Invalid path to compiler: " + conf.compiler) usage() if args == 4: - if (sys.argv[3] != '-s' and sys.argv[3] != '--skipped'): usage() + if sys.argv[3] != '-s' and sys.argv[3] != '--skipped': + usage() conf.run_skipped = True filepath = sys.argv[2] - if filepath.endswith('/'): filepath = filepath[:-1] + if filepath.endswith('/'): + filepath = filepath[:-1] conf.cwd = os.getcwd() if os.path.isfile(filepath): handle_file(filepath, conf) @@ -327,9 +348,12 @@ def main(): print("Error: Invalid path to tests: " + filepath) usage() - print("Found %d tests: %.1f%% (%d / %d) passed (%d skipped)." % (conf.numtests, 100 * conf.numsuccess / max(1, conf.numtests - conf.numskipped), conf.numsuccess, conf.numtests - conf.numskipped, conf.numskipped)) - if (conf.numsuccess != conf.numtests - conf.numskipped): + print("Found %d tests: %.1f%% (%d / %d) passed (%d skipped)." % ( + conf.numtests, 100 * conf.numsuccess / max(1, conf.numtests - conf.numskipped), conf.numsuccess, + conf.numtests - conf.numskipped, conf.numskipped)) + if conf.numsuccess != conf.numtests - conf.numskipped: exit(-1) exit(0) + main() diff --git a/test/unit/stdlib/core/runtime.c3 b/test/unit/stdlib/core/runtime.c3 new file mode 100644 index 000000000..c6f14b35f --- /dev/null +++ b/test/unit/stdlib/core/runtime.c3 @@ -0,0 +1,37 @@ +module std::core::runtime_test; +import std::sort; + +fn void! cmp_unit() @test +{ + TestUnit[] list = { + { .name = "http::url_test::url_query" }, + { .name = "http::url_test::url_init" }, + { .name = "http::url_test::url_decode" }, + { .name = "text_test::test_render_notag" }, + { .name = "text_test::test_render_tag1" }, + { .name = "text_test::test_render_template_iter" }, + { .name = "http::header_test::header_scan" }, + { .name = "http::header_test::header" }, + { .name = "stringmap_test::test_map" }, + { .name = "text_test::test_render_template" }, + }; + quicksort(list, &runtime::cmp_unit); + + String[] want = { + "http::header_test::header", + "http::header_test::header_scan", + "http::url_test::url_decode", + "http::url_test::url_init", + "http::url_test::url_query", + "stringmap_test::test_map", + "text_test::test_render_notag", + "text_test::test_render_tag1", + "text_test::test_render_template", + "text_test::test_render_template_iter", + }; + assert(list.len == want.len); + foreach (i, l : list) + { + assert(l.name == want[i], "got %s; want %s", l.name, want[i]); + } +} \ No newline at end of file