From efb492eaceb845ababc15f9c64b47d4a44e3bea5 Mon Sep 17 00:00:00 2001 From: Dmitry Atamanov Date: Tue, 29 Aug 2023 15:19:30 +0500 Subject: [PATCH] Add simple benchmark runner. --- lib/std/core/runtime.c3 | 133 +++++++++++++++++++++++++++---- resources/examples/benchmarks.c3 | 29 +++++++ test/unit/stdlib/core/runtime.c3 | 2 +- 3 files changed, 149 insertions(+), 15 deletions(-) create mode 100644 resources/examples/benchmarks.c3 diff --git a/lib/std/core/runtime.c3 b/lib/std/core/runtime.c3 index 11cfaced1..b7de0bdeb 100644 --- a/lib/std/core/runtime.c3 +++ b/lib/std/core/runtime.c3 @@ -25,6 +25,123 @@ struct SubArrayStruct } def BenchmarkFn = fn void!(); + +struct BenchmarkUnit +{ + String name; + BenchmarkFn func; +} + +fn BenchmarkUnit[] benchmark_collection_create(Allocator* using = mem::heap()) +{ + BenchmarkFn[] fns = $$BENCHMARK_FNS; + String[] names = $$BENCHMARK_NAMES; + BenchmarkUnit[] benchmarks = malloc(BenchmarkUnit, names.len, .using = using); + foreach (i, benchmark : fns) + { + benchmarks[i] = { names[i], fns[i] }; + } + return benchmarks; +} + +const DEFAULT_BENCHMARK_WARMUP_ITERATIONS = 3; +const DEFAULT_BENCHMARK_MAX_ITERATIONS = 10000; + +uint benchmark_warmup_iterations @private = DEFAULT_BENCHMARK_WARMUP_ITERATIONS; +uint benchmark_max_iterations @private = DEFAULT_BENCHMARK_MAX_ITERATIONS; + +fn void set_benchmark_warmup_iterations(uint value) @builtin +{ + benchmark_warmup_iterations = value; +} + +fn void set_benchmark_max_iterations(uint value) @builtin +{ + assert(value > 0); + benchmark_max_iterations = value; +} + +fn bool run_benchmarks(BenchmarkUnit[] benchmarks) +{ + int benchmarks_passed = 0; + int benchmark_count = benchmarks.len; + usz max_name; + + foreach (&unit : benchmarks) + { + if (max_name < unit.name.len) max_name = unit.name.len; + } + + usz len = max_name + 9; + + DString name; + + name.tinit(); + name.append_repeat('-', len / 2); + name.append(" BENCHMARKS "); + name.append_repeat('-', len - len / 2); + + io::printn(name); + + name.clear(); + + long sys_clock_started; + long sys_clock_finished; + long sys_clocks; + Clock clock; + anyfault err; + + foreach(unit : benchmarks) + { + defer name.clear(); + name.printf("Benchmarking %s ", unit.name); + name.append_repeat('.', max_name - unit.name.len + 2); + io::printf("%s ", name.as_str()); + + for(uint i = 0; i < benchmark_warmup_iterations; i++) + { + err = unit.func() @inline; + @volatile_load(err); + } + + clock = std::time::clock::now(); + sys_clock_started = $$sysclock(); + + for(uint i = 0; i < benchmark_max_iterations; i++) + { + err = unit.func() @inline; + @volatile_load(err); + } + + sys_clock_finished = $$sysclock(); + NanoDuration nano_seconds = clock.mark(); + sys_clocks = sys_clock_finished - sys_clock_started; + + if (err) + { + io::printfn("[failed] Failed due to: %s", err); + continue; + } + + io::printfn("[ok] %.2f ns, %.2f CPU's clocks", (float)nano_seconds / benchmark_max_iterations, (float)sys_clocks / benchmark_max_iterations); + benchmarks_passed++; + } + + io::printfn("\n%d benchmark%s run.\n", benchmark_count, benchmark_count > 1 ? "s" : ""); + io::printfn("Benchmarks Result: %s. %d passed, %d failed.", + benchmarks_passed < benchmark_count ? "FAILED" : "ok", benchmarks_passed, benchmark_count - benchmarks_passed); + + return benchmark_count == benchmarks_passed; +} + +fn bool __run_default_benchmark_runner() +{ + @pool() + { + return run_benchmarks(benchmark_collection_create(mem::temp())); + }; +} + def TestFn = fn void!(); struct TestUnit @@ -50,9 +167,8 @@ struct TestContext JmpBuf buf; } - // Sort the tests by their name in ascending order. -fn int cmp_unit(TestUnit a, TestUnit b) +fn int cmp_test_unit(TestUnit a, TestUnit b) { usz an = a.name.len; usz bn = b.name.len; @@ -84,7 +200,7 @@ fn bool run_tests(TestUnit[] tests) { if (max_name < unit.name.len) max_name = unit.name.len; } - quicksort(tests, &cmp_unit); + quicksort(tests, &cmp_test_unit); TestContext context; test_context = &context; @@ -135,17 +251,6 @@ fn bool __run_default_test_runner() }; } -fn bool __run_default_benchmark_runner() -{ - BenchmarkFn[] fns = $$BENCHMARK_FNS; - String[] names = $$BENCHMARK_NAMES; - foreach (i, benchmark : fns) - { - io::printfn("Benchmark: %s %x.", names[i], fns[i]); - } - return true; -} - module std::core::runtime @if(WASM_NOLIBC); extern fn void __wasm_call_ctors(); diff --git a/resources/examples/benchmarks.c3 b/resources/examples/benchmarks.c3 new file mode 100644 index 000000000..6b55caef1 --- /dev/null +++ b/resources/examples/benchmarks.c3 @@ -0,0 +1,29 @@ +import std::thread; + +fn void! bench1() @benchmark +{ + return std::thread::sleep_ms(1); +} + +fn void! bench123456789() @benchmark +{ + return std::thread::sleep_ms(2); +} + +fn void! long_name_bench() @benchmark +{ + return std::thread::sleep_ms(3); +} + +fn void! very_long_name_bench() @benchmark +{ + return std::thread::sleep_ms(10); +} + +static initialize +{ + set_benchmark_warmup_iterations(5); + set_benchmark_max_iterations(1000); +} + + diff --git a/test/unit/stdlib/core/runtime.c3 b/test/unit/stdlib/core/runtime.c3 index c6f14b35f..20fd0f285 100644 --- a/test/unit/stdlib/core/runtime.c3 +++ b/test/unit/stdlib/core/runtime.c3 @@ -15,7 +15,7 @@ fn void! cmp_unit() @test { .name = "stringmap_test::test_map" }, { .name = "text_test::test_render_template" }, }; - quicksort(list, &runtime::cmp_unit); + quicksort(list, &runtime::cmp_test_unit); String[] want = { "http::header_test::header",