From bbc9bffdada67ad86890408a2dd1f4d4fdc630e9 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Thu, 11 Sep 2025 17:19:33 +0200 Subject: [PATCH 1/8] feat(core): add native __codspeed_root_frame__ function This is not used yet, but keeping the implementation just in case. If untouched for a long time, do not hesitate to remove --- packages/core/binding.gyp | 1 + packages/core/src/native_core/index.ts | 3 +++ .../core/src/native_core/instruments/hooks.ts | 6 +++++ .../native_core/instruments/hooks_wrapper.cc | 23 +++++++++++++++++++ .../src/native_core/linux_perf/linux_perf.cc | 2 +- 5 files changed, 34 insertions(+), 1 deletion(-) diff --git a/packages/core/binding.gyp b/packages/core/binding.gyp index fb60fa44..b04df35c 100644 --- a/packages/core/binding.gyp +++ b/packages/core/binding.gyp @@ -9,6 +9,7 @@ "-fno-exceptions" ], "cflags": [ + "-g", "-Wno-maybe-uninitialized", "-Wno-unused-variable", "-Wno-unused-parameter", diff --git a/packages/core/src/native_core/index.ts b/packages/core/src/native_core/index.ts index f56417be..5b35f6a3 100644 --- a/packages/core/src/native_core/index.ts +++ b/packages/core/src/native_core/index.ts @@ -46,6 +46,9 @@ try { setIntegration: (_name: string, _version: string) => { return 0; }, + __codspeed_root_frame__: (callback: () => T): T => { + return callback(); + }, }, isBound: false, }; diff --git a/packages/core/src/native_core/instruments/hooks.ts b/packages/core/src/native_core/instruments/hooks.ts index e71457e1..3d321a5f 100644 --- a/packages/core/src/native_core/instruments/hooks.ts +++ b/packages/core/src/native_core/instruments/hooks.ts @@ -31,4 +31,10 @@ export interface InstrumentHooks { * @returns 0 on success, non-zero on error */ setIntegration(name: string, version: string): number; + + /** + * Execute a callback function with __codspeed_root_frame__ in its stack trace + * @param callback Function to execute + */ + __codspeed_root_frame__(callback: () => T): T; } diff --git a/packages/core/src/native_core/instruments/hooks_wrapper.cc b/packages/core/src/native_core/instruments/hooks_wrapper.cc index f760b370..3b6013bf 100644 --- a/packages/core/src/native_core/instruments/hooks_wrapper.cc +++ b/packages/core/src/native_core/instruments/hooks_wrapper.cc @@ -81,6 +81,27 @@ Napi::Number SetIntegration(const Napi::CallbackInfo &info) { return Napi::Number::New(env, result); } +Napi::Value __attribute__ ((noinline)) __codspeed_root_frame__(const Napi::CallbackInfo &info) { + Napi::Env env = info.Env(); + + if (info.Length() != 1) { + Napi::TypeError::New(env, "Expected 1 argument: callback function") + .ThrowAsJavaScriptException(); + return env.Undefined(); + } + + if (!info[0].IsFunction()) { + Napi::TypeError::New(env, "Expected function argument") + .ThrowAsJavaScriptException(); + return env.Undefined(); + } + + Napi::Function callback = info[0].As(); + Napi::Value result = callback.Call(env.Global(), {}); + + return result; +} + Napi::Object Initialize(Napi::Env env, Napi::Object exports) { Napi::Object instrumentHooksObj = Napi::Object::New(env); @@ -96,6 +117,8 @@ Napi::Object Initialize(Napi::Env env, Napi::Object exports) { Napi::Function::New(env, SetExecutedBenchmark)); instrumentHooksObj.Set(Napi::String::New(env, "setIntegration"), Napi::Function::New(env, SetIntegration)); + instrumentHooksObj.Set(Napi::String::New(env, "__codspeed_root_frame__"), + Napi::Function::New(env, __codspeed_root_frame__)); exports.Set(Napi::String::New(env, "InstrumentHooks"), instrumentHooksObj); diff --git a/packages/core/src/native_core/linux_perf/linux_perf.cc b/packages/core/src/native_core/linux_perf/linux_perf.cc index 31ace3f8..40e5cbec 100644 --- a/packages/core/src/native_core/linux_perf/linux_perf.cc +++ b/packages/core/src/native_core/linux_perf/linux_perf.cc @@ -39,4 +39,4 @@ Napi::Value LinuxPerf::Stop(const Napi::CallbackInfo &info) { return Napi::Boolean::New(info.Env(), false); } -} // namespace codspeed_native \ No newline at end of file +} // namespace codspeed_native From bf330c3123c234107ba402a3190c73bdab7b2891 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Thu, 11 Sep 2025 17:21:36 +0200 Subject: [PATCH 2/8] refactor(tinybench-plugin): call setupCore no matter the codspeed mode if not disabled --- packages/tinybench-plugin/src/index.ts | 2 ++ packages/tinybench-plugin/src/instrumented.ts | 2 -- packages/tinybench-plugin/tests/index.integ.test.ts | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/tinybench-plugin/src/index.ts b/packages/tinybench-plugin/src/index.ts index 329575ef..e6bbf91a 100644 --- a/packages/tinybench-plugin/src/index.ts +++ b/packages/tinybench-plugin/src/index.ts @@ -3,6 +3,7 @@ import { getGitDir, InstrumentHooks, mongoMeasurement, + setupCore, SetupInstrumentsRequestBody, SetupInstrumentsResponse, tryIntrospect, @@ -22,6 +23,7 @@ export function withCodSpeed(bench: Bench): Bench { if (codspeedRunnerMode === "disabled") { return bench; } + setupCore(); const rootCallingFile = getCallingFile(); diff --git a/packages/tinybench-plugin/src/instrumented.ts b/packages/tinybench-plugin/src/instrumented.ts index 80955247..66633e60 100644 --- a/packages/tinybench-plugin/src/instrumented.ts +++ b/packages/tinybench-plugin/src/instrumented.ts @@ -2,7 +2,6 @@ import { InstrumentHooks, mongoMeasurement, optimizeFunction, - setupCore, teardownCore, } from "@codspeed/core"; import { Bench, Fn, FnOptions } from "tinybench"; @@ -18,7 +17,6 @@ export function runInstrumentedBench( console.log( `[CodSpeed] running with @codspeed/tinybench v${__VERSION__} (instrumented mode)` ); - setupCore(); for (const task of bench.tasks) { const uri = getTaskUri(bench, task.name, rootCallingFile); diff --git a/packages/tinybench-plugin/tests/index.integ.test.ts b/packages/tinybench-plugin/tests/index.integ.test.ts index fe467d7b..1b513703 100644 --- a/packages/tinybench-plugin/tests/index.integ.test.ts +++ b/packages/tinybench-plugin/tests/index.integ.test.ts @@ -205,7 +205,7 @@ describe("Benchmark.Suite", () => { expect(afterAll).toHaveBeenCalledTimes(2); }); - it("should call setupCore and teardownCore only once after run()", async () => { + it("should call setupCore and teardownCore only once", async () => { mockCore.InstrumentHooks.isInstrumented.mockReturnValue(true); const bench = withCodSpeed(new Bench()) .add("RegExp", function () { @@ -215,7 +215,7 @@ describe("Benchmark.Suite", () => { /o/.test("Hello World!"); }); - expect(mockCore.setupCore).not.toHaveBeenCalled(); + expect(mockCore.setupCore).toHaveBeenCalledTimes(1); expect(mockCore.teardownCore).not.toHaveBeenCalled(); await bench.run(); From 04d3abfb5b0ff5eca3b1121dfb275a2565a5b988 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Thu, 11 Sep 2025 17:29:26 +0200 Subject: [PATCH 3/8] feat(tinybench-plugin): add support for perf profiling Support is still far from perfect for async/heavy code. --- .gitignore | 3 + packages/tinybench-plugin/benches/sample.ts | 4 +- packages/tinybench-plugin/benches/timing.ts | 6 +- packages/tinybench-plugin/src/walltime.ts | 100 ++++++++++++++++++-- 4 files changed, 101 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 4abc61c4..3dd48dfe 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,9 @@ yarn-error.log* lerna-debug.log* .pnpm-debug.log* +# JIT dumps +jit-*.dump + # Diagnostic reports (https://nodejs.org/api/report.html) report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json diff --git a/packages/tinybench-plugin/benches/sample.ts b/packages/tinybench-plugin/benches/sample.ts index 19633d62..4cc40ecf 100644 --- a/packages/tinybench-plugin/benches/sample.ts +++ b/packages/tinybench-plugin/benches/sample.ts @@ -35,7 +35,7 @@ bench }); (async () => { - await bench.run(); + bench.runSync(); console.table(bench.table()); const timingBench = withCodSpeed( @@ -44,6 +44,6 @@ bench registerTimingBenchmarks(timingBench); - await timingBench.run(); + timingBench.runSync(); console.table(timingBench.table()); })(); diff --git a/packages/tinybench-plugin/benches/timing.ts b/packages/tinybench-plugin/benches/timing.ts index b9a139c6..79130428 100644 --- a/packages/tinybench-plugin/benches/timing.ts +++ b/packages/tinybench-plugin/benches/timing.ts @@ -8,15 +8,15 @@ const busySleep = (ms: number): void => { }; export function registerTimingBenchmarks(bench: Bench) { - bench.add("wait 1ms", async () => { + bench.add("wait 1ms", () => { busySleep(1); }); - bench.add("wait 500ms", async () => { + bench.add("wait 500ms", () => { busySleep(500); }); - bench.add("wait 1sec", async () => { + bench.add("wait 1sec", () => { busySleep(1000); }); } diff --git a/packages/tinybench-plugin/src/walltime.ts b/packages/tinybench-plugin/src/walltime.ts index 2c22c539..22aa5097 100644 --- a/packages/tinybench-plugin/src/walltime.ts +++ b/packages/tinybench-plugin/src/walltime.ts @@ -1,5 +1,6 @@ import { calculateQuantiles, + InstrumentHooks, mongoMeasurement, msToNs, msToS, @@ -7,7 +8,7 @@ import { type Benchmark, type BenchmarkStats, } from "@codspeed/core"; -import { Bench, TaskResult } from "tinybench"; +import { Bench, Fn, TaskResult } from "tinybench"; import { getTaskUri } from "./uri"; declare const __VERSION__: string; @@ -28,21 +29,27 @@ export function runWalltimeBench(bench: Bench, rootCallingFile: string): void { const benchmarks: Benchmark[] = []; - // Run the bench naturally to collect TaskResult data - const results = []; - // Collect and report walltime data for (const task of bench.tasks) { const uri = getTaskUri(bench, task.name, rootCallingFile); + // Override the function under test to add a static frame + const { fn } = task as unknown as { fn: Fn }; + async function __codspeed_root_frame__() { + await fn(); + } + (task as any).fn = __codspeed_root_frame__; + // run the warmup of the task right before its actual run if (bench.opts.warmup) { await task.warmup(); } + await mongoMeasurement.start(uri); - const taskResult = await task.run(); + InstrumentHooks.startBenchmark(); + await task.run(); + InstrumentHooks.stopBenchmark(); await mongoMeasurement.stop(uri); - results.push(taskResult); if (task.result) { // Convert tinybench result to BenchmarkStats format @@ -67,8 +74,87 @@ export function runWalltimeBench(bench: Bench, rootCallingFile: string): void { }; benchmarks.push(benchmark); + console.log(` ✔ Collected walltime data for ${uri}`); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + } else { + console.warn(` ⚠ No result data available for ${uri}`); + } + } + + // Write results to JSON file using core function + if (benchmarks.length > 0) { + writeWalltimeResults(benchmarks); + } + + console.log( + `[CodSpeed] Done collecting walltime data for ${bench.tasks.length} benches.` + ); + // Restore our custom run method + bench.run = originalRun; + + return bench.tasks; + }; + + bench.runSync = () => { + console.log( + `[CodSpeed] running with @codspeed/tinybench v${__VERSION__} (walltime mode)` + ); + + // Store the original run method before we override it + const originalRun = bench.run; + + // Temporarily restore the original run to get actual benchmark results + const benchProto = Object.getPrototypeOf(bench); + const prototypeRun = benchProto.run; + bench.run = prototypeRun; + + const benchmarks: Benchmark[] = []; + + // Collect and report walltime data + for (const task of bench.tasks) { + const uri = getTaskUri(bench, task.name, rootCallingFile); + + // run the warmup of the task right before its actual run + if (bench.opts.warmup) { + task.warmup(); + } + // Override the function under test to add a static frame + const { fn } = task as unknown as { fn: Fn }; + function __codspeed_root_frame__() { + fn(); + } + (task as any).fn = __codspeed_root_frame__; + + InstrumentHooks.startBenchmark(); + task.runSync(); + InstrumentHooks.stopBenchmark(); + + if (task.result) { + // Convert tinybench result to BenchmarkStats format + const stats = convertTinybenchResultToBenchmarkStats( + task.result, + bench.opts.warmup ? bench.opts.warmupIterations ?? 0 : 0 + ); + + const benchmark: Benchmark = { + name: task.name, + uri, + config: { + max_rounds: bench.opts.iterations ?? null, + max_time_ns: bench.opts.time ? msToNs(bench.opts.time) : null, + min_round_time_ns: null, // tinybench does not have an option for this + warmup_time_ns: + bench.opts.warmup && bench.opts.warmupTime + ? msToNs(bench.opts.warmupTime) + : null, + }, + stats, + }; + + benchmarks.push(benchmark); console.log(` ✔ Collected walltime data for ${uri}`); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); } else { console.warn(` ⚠ No result data available for ${uri}`); } @@ -85,7 +171,7 @@ export function runWalltimeBench(bench: Bench, rootCallingFile: string): void { // Restore our custom run method bench.run = originalRun; - return results; + return bench.tasks; }; } From 7edb085cfabed3cdfeaaa392417e47fd969931f3 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Thu, 11 Sep 2025 17:30:53 +0200 Subject: [PATCH 4/8] feat(core): add a warning in results.json about walltime profiling code --- packages/core/src/walltime/index.ts | 50 ++++++++++++++++------- packages/core/src/walltime/interfaces.ts | 1 + packages/tinybench-plugin/src/walltime.ts | 2 +- 3 files changed, 38 insertions(+), 15 deletions(-) diff --git a/packages/core/src/walltime/index.ts b/packages/core/src/walltime/index.ts index 5f06d671..7632fa80 100644 --- a/packages/core/src/walltime/index.ts +++ b/packages/core/src/walltime/index.ts @@ -8,19 +8,34 @@ export function getProfileFolder(): string | null { return process.env.CODSPEED_PROFILE_FOLDER || null; } -export function writeWalltimeResults(benchmarks: Benchmark[]) { +export function writeWalltimeResults( + benchmarks: Benchmark[], + asyncWarning = false +): void { const profileFolder = getProfileFolder(); - let resultPath: string; - - if (profileFolder) { - const resultsDir = path.join(profileFolder, "results"); - fs.mkdirSync(resultsDir, { recursive: true }); - resultPath = path.join(resultsDir, `${process.pid}.json`); - } else { - // Fallback: write to .codspeed in current working directory - const codspeedDir = path.join(process.cwd(), ".codspeed"); - fs.mkdirSync(codspeedDir, { recursive: true }); - resultPath = path.join(codspeedDir, `results_${Date.now()}.json`); + + const resultDir = (() => { + if (profileFolder) { + return path.join(profileFolder, "results"); + } else { + // Fallback: write to .codspeed in current working directory + return path.join(process.cwd(), ".codspeed"); + } + })(); + fs.mkdirSync(resultDir, { recursive: true }); + const resultPath = path.join(resultDir, `${process.pid}.json`); + + // Check if file already exists and merge benchmarks + let existingBenchmarks: Benchmark[] = []; + if (fs.existsSync(resultPath)) { + try { + const existingData = JSON.parse( + fs.readFileSync(resultPath, "utf-8") + ) as ResultData; + existingBenchmarks = existingData.benchmarks || []; + } catch (error) { + console.warn(`[CodSpeed] Failed to read existing results file: ${error}`); + } } const data: ResultData = { @@ -30,11 +45,18 @@ export function writeWalltimeResults(benchmarks: Benchmark[]) { pid: process.pid, }, instrument: { type: "walltime" }, - benchmarks: benchmarks, + benchmarks: [...existingBenchmarks, ...benchmarks], + metadata: asyncWarning + ? { + async_warning: "Profiling is inaccurate due to async operations", + } + : undefined, }; fs.writeFileSync(resultPath, JSON.stringify(data, null, 2)); - console.log(`[CodSpeed] Results written to ${resultPath}`); + console.log( + `[CodSpeed] Results written to ${resultPath} (${data.benchmarks.length} total benchmarks)` + ); } export * from "./interfaces"; diff --git a/packages/core/src/walltime/interfaces.ts b/packages/core/src/walltime/interfaces.ts index 126a442a..a0518837 100644 --- a/packages/core/src/walltime/interfaces.ts +++ b/packages/core/src/walltime/interfaces.ts @@ -40,4 +40,5 @@ export interface ResultData { }; instrument: { type: "walltime" }; benchmarks: Benchmark[]; + metadata?: Record; } diff --git a/packages/tinybench-plugin/src/walltime.ts b/packages/tinybench-plugin/src/walltime.ts index 22aa5097..cdddce63 100644 --- a/packages/tinybench-plugin/src/walltime.ts +++ b/packages/tinybench-plugin/src/walltime.ts @@ -83,7 +83,7 @@ export function runWalltimeBench(bench: Bench, rootCallingFile: string): void { // Write results to JSON file using core function if (benchmarks.length > 0) { - writeWalltimeResults(benchmarks); + writeWalltimeResults(benchmarks, true); } console.log( From 725aee4c916d91c9474d3bb8f3d572fee0476490 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Fri, 12 Sep 2025 11:44:45 +0200 Subject: [PATCH 5/8] feat(tinybench-plugin): support bench runSync in instrumented mode --- packages/tinybench-plugin/src/instrumented.ts | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/packages/tinybench-plugin/src/instrumented.ts b/packages/tinybench-plugin/src/instrumented.ts index 66633e60..7d4a158f 100644 --- a/packages/tinybench-plugin/src/instrumented.ts +++ b/packages/tinybench-plugin/src/instrumented.ts @@ -63,4 +63,55 @@ export function runInstrumentedBench( console.log(`[CodSpeed] Done running ${bench.tasks.length} benches.`); return bench.tasks; }; + + bench.runSync = () => { + console.log( + `[CodSpeed] running with @codspeed/tinybench v${__VERSION__} (instrumented mode)` + ); + + for (const task of bench.tasks) { + const uri = getTaskUri(bench, task.name, rootCallingFile); + + // Access private fields + const { fnOpts, fn } = task as unknown as { fnOpts?: FnOptions; fn: Fn }; + + // Call beforeAll hook if it exists + fnOpts?.beforeAll?.call(task, "run"); + + // run optimizations + optimizeFunction(async () => { + fnOpts?.beforeEach?.call(task, "run"); + fn(); + fnOpts?.afterEach?.call(task, "run"); + }); + + // run instrumented benchmark + fnOpts?.beforeEach?.call(task, "run"); + + // await mongoMeasurement.start(uri); + global.gc?.(); + (function __codspeed_root_frame__() { + InstrumentHooks.startBenchmark(); + fn(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + })(); + mongoMeasurement.stop(uri); + + fnOpts?.afterEach?.call(task, "run"); + + fnOpts?.afterAll?.call(task, "run"); + + // print results + console.log( + ` ✔ ${ + InstrumentHooks.isInstrumented() ? "Measured" : "Checked" + } ${uri}` + ); + } + + teardownCore(); + console.log(`[CodSpeed] Done running ${bench.tasks.length} benches.`); + return bench.tasks; + }; } From ab65f70c6a119a9d429ddff74647afcc38553fac Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Fri, 12 Sep 2025 12:05:14 +0200 Subject: [PATCH 6/8] chore: fix eslint warnings --- packages/core/src/native_core/index.ts | 2 ++ packages/tinybench-plugin/src/walltime.ts | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/packages/core/src/native_core/index.ts b/packages/core/src/native_core/index.ts index 5b35f6a3..9cbb4a90 100644 --- a/packages/core/src/native_core/index.ts +++ b/packages/core/src/native_core/index.ts @@ -40,9 +40,11 @@ try { stopBenchmark: () => { return 0; }, + // eslint-disable-next-line @typescript-eslint/no-unused-vars setExecutedBenchmark: (_pid: number, _uri: string) => { return 0; }, + // eslint-disable-next-line @typescript-eslint/no-unused-vars setIntegration: (_name: string, _version: string) => { return 0; }, diff --git a/packages/tinybench-plugin/src/walltime.ts b/packages/tinybench-plugin/src/walltime.ts index cdddce63..6ff932db 100644 --- a/packages/tinybench-plugin/src/walltime.ts +++ b/packages/tinybench-plugin/src/walltime.ts @@ -35,9 +35,11 @@ export function runWalltimeBench(bench: Bench, rootCallingFile: string): void { // Override the function under test to add a static frame const { fn } = task as unknown as { fn: Fn }; + // eslint-disable-next-line no-inner-declarations async function __codspeed_root_frame__() { await fn(); } + // eslint-disable-next-line @typescript-eslint/no-explicit-any (task as any).fn = __codspeed_root_frame__; // run the warmup of the task right before its actual run @@ -121,9 +123,11 @@ export function runWalltimeBench(bench: Bench, rootCallingFile: string): void { // Override the function under test to add a static frame const { fn } = task as unknown as { fn: Fn }; + // eslint-disable-next-line no-inner-declarations function __codspeed_root_frame__() { fn(); } + // eslint-disable-next-line @typescript-eslint/no-explicit-any (task as any).fn = __codspeed_root_frame__; InstrumentHooks.startBenchmark(); From d3e40bbac60fd4669bea5294073fbb8022199116 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Mon, 15 Sep 2025 14:46:06 +0200 Subject: [PATCH 7/8] refactor(tinybench-plugin): commonize logic between `run` and `runSync` --- packages/tinybench-plugin/src/instrumented.ts | 142 +++++------ packages/tinybench-plugin/src/walltime.ts | 231 ++++++++---------- 2 files changed, 169 insertions(+), 204 deletions(-) diff --git a/packages/tinybench-plugin/src/instrumented.ts b/packages/tinybench-plugin/src/instrumented.ts index 7d4a158f..e4a33953 100644 --- a/packages/tinybench-plugin/src/instrumented.ts +++ b/packages/tinybench-plugin/src/instrumented.ts @@ -4,7 +4,7 @@ import { optimizeFunction, teardownCore, } from "@codspeed/core"; -import { Bench, Fn, FnOptions } from "tinybench"; +import { Bench, Fn, FnOptions, Task } from "tinybench"; import { getTaskUri } from "./uri"; declare const __VERSION__: string; @@ -13,103 +13,89 @@ export function runInstrumentedBench( bench: Bench, rootCallingFile: string ): void { - bench.run = async () => { - console.log( - `[CodSpeed] running with @codspeed/tinybench v${__VERSION__} (instrumented mode)` - ); - - for (const task of bench.tasks) { - const uri = getTaskUri(bench, task.name, rootCallingFile); + const runTaskAsync = async (task: Task, uri: string): Promise => { + const { fnOpts, fn } = task as unknown as { fnOpts?: FnOptions; fn: Fn }; - // Access private fields - const { fnOpts, fn } = task as unknown as { fnOpts?: FnOptions; fn: Fn }; + await fnOpts?.beforeAll?.call(task, "run"); + await optimizeFunction(async () => { + await fnOpts?.beforeEach?.call(task, "run"); + await fn(); + await fnOpts?.afterEach?.call(task, "run"); + }); + await fnOpts?.beforeEach?.call(task, "run"); + await mongoMeasurement.start(uri); - // Call beforeAll hook if it exists - await fnOpts?.beforeAll?.call(task, "run"); + await (async function __codspeed_root_frame__() { + global.gc?.(); + InstrumentHooks.startBenchmark(); + await fn(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + })(); + + await mongoMeasurement.stop(uri); + await fnOpts?.afterEach?.call(task, "run"); + await fnOpts?.afterAll?.call(task, "run"); + }; - // run optimizations - await optimizeFunction(async () => { - await fnOpts?.beforeEach?.call(task, "run"); - await fn(); - await fnOpts?.afterEach?.call(task, "run"); - }); + // Sync task runner + const runTaskSync = (task: Task, uri: string): void => { + const { fnOpts, fn } = task as unknown as { fnOpts?: FnOptions; fn: Fn }; - // run instrumented benchmark - await fnOpts?.beforeEach?.call(task, "run"); + fnOpts?.beforeAll?.call(task, "run"); + fnOpts?.beforeEach?.call(task, "run"); - await mongoMeasurement.start(uri); + (function __codspeed_root_frame__() { global.gc?.(); - await (async function __codspeed_root_frame__() { - InstrumentHooks.startBenchmark(); - await fn(); - InstrumentHooks.stopBenchmark(); - InstrumentHooks.setExecutedBenchmark(process.pid, uri); - })(); - await mongoMeasurement.stop(uri); - - await fnOpts?.afterEach?.call(task, "run"); + InstrumentHooks.startBenchmark(); + fn(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + })(); + + fnOpts?.afterEach?.call(task, "run"); + fnOpts?.afterAll?.call(task, "run"); + }; - await fnOpts?.afterAll?.call(task, "run"); + bench.run = async () => { + logStart(); - // print results - console.log( - ` ✔ ${ - InstrumentHooks.isInstrumented() ? "Measured" : "Checked" - } ${uri}` - ); + for (const task of bench.tasks) { + const uri = getTaskUri(bench, task.name, rootCallingFile); + await runTaskAsync(task, uri); + logTaskCompletion(uri); } - teardownCore(); - console.log(`[CodSpeed] Done running ${bench.tasks.length} benches.`); - return bench.tasks; + return logEnd(); }; bench.runSync = () => { - console.log( - `[CodSpeed] running with @codspeed/tinybench v${__VERSION__} (instrumented mode)` - ); + logStart(); for (const task of bench.tasks) { const uri = getTaskUri(bench, task.name, rootCallingFile); + runTaskSync(task, uri); + logTaskCompletion(uri); + } - // Access private fields - const { fnOpts, fn } = task as unknown as { fnOpts?: FnOptions; fn: Fn }; - - // Call beforeAll hook if it exists - fnOpts?.beforeAll?.call(task, "run"); - - // run optimizations - optimizeFunction(async () => { - fnOpts?.beforeEach?.call(task, "run"); - fn(); - fnOpts?.afterEach?.call(task, "run"); - }); + return logEnd(); + }; - // run instrumented benchmark - fnOpts?.beforeEach?.call(task, "run"); + const logStart = () => { + console.log( + `[CodSpeed] running with @codspeed/tinybench v${__VERSION__} (instrumented mode)` + ); + }; - // await mongoMeasurement.start(uri); - global.gc?.(); - (function __codspeed_root_frame__() { - InstrumentHooks.startBenchmark(); - fn(); - InstrumentHooks.stopBenchmark(); - InstrumentHooks.setExecutedBenchmark(process.pid, uri); - })(); - mongoMeasurement.stop(uri); - - fnOpts?.afterEach?.call(task, "run"); - - fnOpts?.afterAll?.call(task, "run"); - - // print results - console.log( - ` ✔ ${ - InstrumentHooks.isInstrumented() ? "Measured" : "Checked" - } ${uri}` - ); - } + const logTaskCompletion = (uri: string) => { + console.log( + ` ✔ ${ + InstrumentHooks.isInstrumented() ? "Measured" : "Checked" + } ${uri}` + ); + }; + const logEnd = () => { teardownCore(); console.log(`[CodSpeed] Done running ${bench.tasks.length} benches.`); return bench.tasks; diff --git a/packages/tinybench-plugin/src/walltime.ts b/packages/tinybench-plugin/src/walltime.ts index 6ff932db..7418eb29 100644 --- a/packages/tinybench-plugin/src/walltime.ts +++ b/packages/tinybench-plugin/src/walltime.ts @@ -5,42 +5,25 @@ import { msToNs, msToS, writeWalltimeResults, - type Benchmark, + type Benchmark as CodspeedBenchmark, type BenchmarkStats, } from "@codspeed/core"; -import { Bench, Fn, TaskResult } from "tinybench"; +import { Bench, Fn, Task, TaskResult } from "tinybench"; import { getTaskUri } from "./uri"; declare const __VERSION__: string; export function runWalltimeBench(bench: Bench, rootCallingFile: string): void { bench.run = async () => { - console.log( - `[CodSpeed] running with @codspeed/tinybench v${__VERSION__} (walltime mode)` - ); - - // Store the original run method before we override it - const originalRun = bench.run; - - // Temporarily restore the original run to get actual benchmark results - const benchProto = Object.getPrototypeOf(bench); - const prototypeRun = benchProto.run; - bench.run = prototypeRun; - - const benchmarks: Benchmark[] = []; + logStart(); + const codspeedBenchmarks: CodspeedBenchmark[] = []; // Collect and report walltime data for (const task of bench.tasks) { const uri = getTaskUri(bench, task.name, rootCallingFile); // Override the function under test to add a static frame - const { fn } = task as unknown as { fn: Fn }; - // eslint-disable-next-line no-inner-declarations - async function __codspeed_root_frame__() { - await fn(); - } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (task as any).fn = __codspeed_root_frame__; + wrapTaskFunction(task, true); // run the warmup of the task right before its actual run if (bench.opts.warmup) { @@ -53,130 +36,126 @@ export function runWalltimeBench(bench: Bench, rootCallingFile: string): void { InstrumentHooks.stopBenchmark(); await mongoMeasurement.stop(uri); - if (task.result) { - // Convert tinybench result to BenchmarkStats format - const stats = convertTinybenchResultToBenchmarkStats( - task.result, - bench.opts.warmup ? bench.opts.warmupIterations ?? 0 : 0 - ); - - const benchmark: Benchmark = { - name: task.name, - uri, - config: { - max_rounds: bench.opts.iterations ?? null, - max_time_ns: bench.opts.time ? msToNs(bench.opts.time) : null, - min_round_time_ns: null, // tinybench does not have an option for this - warmup_time_ns: - bench.opts.warmup && bench.opts.warmupTime - ? msToNs(bench.opts.warmupTime) - : null, - }, - stats, - }; - - benchmarks.push(benchmark); - console.log(` ✔ Collected walltime data for ${uri}`); - InstrumentHooks.setExecutedBenchmark(process.pid, uri); - } else { - console.warn(` ⚠ No result data available for ${uri}`); - } - } - - // Write results to JSON file using core function - if (benchmarks.length > 0) { - writeWalltimeResults(benchmarks, true); + registerCodspeedBenchmarkFromTask( + codspeedBenchmarks, + task, + bench, + rootCallingFile + ); } - console.log( - `[CodSpeed] Done collecting walltime data for ${bench.tasks.length} benches.` - ); - // Restore our custom run method - bench.run = originalRun; - - return bench.tasks; + return finalizeWalltimeRun(bench, codspeedBenchmarks, true); }; bench.runSync = () => { - console.log( - `[CodSpeed] running with @codspeed/tinybench v${__VERSION__} (walltime mode)` - ); + logStart(); + const codspeedBenchmarks: CodspeedBenchmark[] = []; - // Store the original run method before we override it - const originalRun = bench.run; - - // Temporarily restore the original run to get actual benchmark results - const benchProto = Object.getPrototypeOf(bench); - const prototypeRun = benchProto.run; - bench.run = prototypeRun; - - const benchmarks: Benchmark[] = []; - - // Collect and report walltime data for (const task of bench.tasks) { - const uri = getTaskUri(bench, task.name, rootCallingFile); + // Override the function under test to add a static frame + wrapTaskFunction(task, false); - // run the warmup of the task right before its actual run if (bench.opts.warmup) { task.warmup(); } - // Override the function under test to add a static frame - const { fn } = task as unknown as { fn: Fn }; - // eslint-disable-next-line no-inner-declarations - function __codspeed_root_frame__() { - fn(); - } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (task as any).fn = __codspeed_root_frame__; - InstrumentHooks.startBenchmark(); task.runSync(); InstrumentHooks.stopBenchmark(); - if (task.result) { - // Convert tinybench result to BenchmarkStats format - const stats = convertTinybenchResultToBenchmarkStats( - task.result, - bench.opts.warmup ? bench.opts.warmupIterations ?? 0 : 0 - ); - - const benchmark: Benchmark = { - name: task.name, - uri, - config: { - max_rounds: bench.opts.iterations ?? null, - max_time_ns: bench.opts.time ? msToNs(bench.opts.time) : null, - min_round_time_ns: null, // tinybench does not have an option for this - warmup_time_ns: - bench.opts.warmup && bench.opts.warmupTime - ? msToNs(bench.opts.warmupTime) - : null, - }, - stats, - }; - - benchmarks.push(benchmark); - console.log(` ✔ Collected walltime data for ${uri}`); - InstrumentHooks.setExecutedBenchmark(process.pid, uri); - } else { - console.warn(` ⚠ No result data available for ${uri}`); - } + registerCodspeedBenchmarkFromTask( + codspeedBenchmarks, + task, + bench, + rootCallingFile + ); } - // Write results to JSON file using core function - if (benchmarks.length > 0) { - writeWalltimeResults(benchmarks); - } + return finalizeWalltimeRun(bench, codspeedBenchmarks, false); + }; +} - console.log( - `[CodSpeed] Done collecting walltime data for ${bench.tasks.length} benches.` - ); - // Restore our custom run method - bench.run = originalRun; +function logStart() { + console.log( + `[CodSpeed] running with @codspeed/tinybench v${__VERSION__} (walltime mode)` + ); +} - return bench.tasks; - }; +const TINYBENCH_WARMUP_DEFAULT = 16; + +function registerCodspeedBenchmarkFromTask( + codspeedBenchmarks: CodspeedBenchmark[], + task: Task, + bench: Bench, + rootCallingFile: string +): void { + const uri = getTaskUri(bench, task.name, rootCallingFile); + + if (!task.result) { + console.warn(` ⚠ No result data available for ${uri}`); + return; + } + + const warmupIterations = bench.opts.warmup + ? bench.opts.warmupIterations ?? TINYBENCH_WARMUP_DEFAULT + : 0; + const stats = convertTinybenchResultToBenchmarkStats( + task.result, + warmupIterations + ); + + codspeedBenchmarks.push({ + name: task.name, + uri, + config: { + max_rounds: bench.opts.iterations ?? null, + max_time_ns: bench.opts.time ? msToNs(bench.opts.time) : null, + min_round_time_ns: null, // tinybench does not have an option for this + warmup_time_ns: + bench.opts.warmup && bench.opts.warmupTime + ? msToNs(bench.opts.warmupTime) + : null, + }, + stats, + }); + + console.log(` ✔ Collected walltime data for ${uri}`); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); +} + +function wrapTaskFunction(task: Task, isAsync: boolean): void { + const { fn } = task as unknown as { fn: Fn }; + if (isAsync) { + // eslint-disable-next-line no-inner-declarations + async function __codspeed_root_frame__() { + await fn(); + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (task as any).fn = __codspeed_root_frame__; + } else { + // eslint-disable-next-line no-inner-declarations + function __codspeed_root_frame__() { + fn(); + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (task as any).fn = __codspeed_root_frame__; + } +} + +function finalizeWalltimeRun( + bench: Bench, + benchmarks: CodspeedBenchmark[], + isAsync: boolean +) { + // Write results to JSON file using core function + if (benchmarks.length > 0) { + writeWalltimeResults(benchmarks, isAsync); + } + + console.log( + `[CodSpeed] Done collecting walltime data for ${bench.tasks.length} benches.` + ); + return bench.tasks; } function convertTinybenchResultToBenchmarkStats( From e31ee8b5d9c6d9a309eb11c407198e580f4825b7 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Wed, 17 Sep 2025 11:09:05 +0200 Subject: [PATCH 8/8] refactor(tinybench): share structure across walltime and instrumented --- packages/tinybench-plugin/src/index.ts | 10 +- .../tinybench-plugin/src/index.unit.test.ts | 14 +- packages/tinybench-plugin/src/instrumented.ts | 111 ++++----- packages/tinybench-plugin/src/shared.ts | 89 +++++++ packages/tinybench-plugin/src/walltime.ts | 222 ++++++++---------- .../__snapshots__/index.integ.test.ts.snap | 8 +- .../tests/index.integ.test.ts | 2 +- 7 files changed, 249 insertions(+), 207 deletions(-) create mode 100644 packages/tinybench-plugin/src/shared.ts diff --git a/packages/tinybench-plugin/src/index.ts b/packages/tinybench-plugin/src/index.ts index e6bbf91a..06241a91 100644 --- a/packages/tinybench-plugin/src/index.ts +++ b/packages/tinybench-plugin/src/index.ts @@ -3,7 +3,6 @@ import { getGitDir, InstrumentHooks, mongoMeasurement, - setupCore, SetupInstrumentsRequestBody, SetupInstrumentsResponse, tryIntrospect, @@ -12,9 +11,9 @@ import path from "path"; import { get as getStackTrace } from "stack-trace"; import { Bench } from "tinybench"; import { fileURLToPath } from "url"; -import { runInstrumentedBench } from "./instrumented"; +import { setupCodspeedInstrumentedBench } from "./instrumented"; import { getOrCreateUriMap } from "./uri"; -import { runWalltimeBench } from "./walltime"; +import { setupCodspeedWalltimeBench } from "./walltime"; tryIntrospect(); @@ -23,7 +22,6 @@ export function withCodSpeed(bench: Bench): Bench { if (codspeedRunnerMode === "disabled") { return bench; } - setupCore(); const rootCallingFile = getCallingFile(); @@ -42,9 +40,9 @@ export function withCodSpeed(bench: Bench): Bench { }; if (codspeedRunnerMode === "instrumented") { - runInstrumentedBench(bench, rootCallingFile); + setupCodspeedInstrumentedBench(bench, rootCallingFile); } else if (codspeedRunnerMode === "walltime") { - runWalltimeBench(bench, rootCallingFile); + setupCodspeedWalltimeBench(bench, rootCallingFile); } return bench; diff --git a/packages/tinybench-plugin/src/index.unit.test.ts b/packages/tinybench-plugin/src/index.unit.test.ts index 7bb6954b..0cf303d8 100644 --- a/packages/tinybench-plugin/src/index.unit.test.ts +++ b/packages/tinybench-plugin/src/index.unit.test.ts @@ -3,7 +3,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { withCodSpeed } from "."; const mockInstrumented = vi.hoisted(() => ({ - runInstrumentedBench: vi.fn(), + setupCodspeedInstrumentedBench: vi.fn(), })); vi.mock("./instrumented", () => ({ @@ -11,7 +11,7 @@ vi.mock("./instrumented", () => ({ })); const mockWalltime = vi.hoisted(() => ({ - runWalltimeBench: vi.fn(), + setupCodspeedWalltimeBench: vi.fn(), })); vi.mock("./walltime", () => ({ @@ -44,8 +44,8 @@ describe("withCodSpeed behavior without different codspeed modes", () => { withCodSpeed(new Bench()); - expect(mockInstrumented.runInstrumentedBench).toHaveBeenCalled(); - expect(mockWalltime.runWalltimeBench).not.toHaveBeenCalled(); + expect(mockInstrumented.setupCodspeedInstrumentedBench).toHaveBeenCalled(); + expect(mockWalltime.setupCodspeedWalltimeBench).not.toHaveBeenCalled(); }); it("should run in walltime mode when CODSPEED_RUNNER_MODE=walltime", async () => { @@ -54,7 +54,9 @@ describe("withCodSpeed behavior without different codspeed modes", () => { withCodSpeed(new Bench()); - expect(mockInstrumented.runInstrumentedBench).not.toHaveBeenCalled(); - expect(mockWalltime.runWalltimeBench).toHaveBeenCalled(); + expect( + mockInstrumented.setupCodspeedInstrumentedBench + ).not.toHaveBeenCalled(); + expect(mockWalltime.setupCodspeedWalltimeBench).toHaveBeenCalled(); }); }); diff --git a/packages/tinybench-plugin/src/instrumented.ts b/packages/tinybench-plugin/src/instrumented.ts index e4a33953..7462aa87 100644 --- a/packages/tinybench-plugin/src/instrumented.ts +++ b/packages/tinybench-plugin/src/instrumented.ts @@ -2,18 +2,40 @@ import { InstrumentHooks, mongoMeasurement, optimizeFunction, - teardownCore, } from "@codspeed/core"; import { Bench, Fn, FnOptions, Task } from "tinybench"; -import { getTaskUri } from "./uri"; +import { BaseBenchRunner } from "./shared"; -declare const __VERSION__: string; - -export function runInstrumentedBench( +export function setupCodspeedInstrumentedBench( bench: Bench, rootCallingFile: string ): void { - const runTaskAsync = async (task: Task, uri: string): Promise => { + const runner = new InstrumentedBenchRunner(bench, rootCallingFile); + runner.setupBenchMethods(); +} + +class InstrumentedBenchRunner extends BaseBenchRunner { + protected getModeName(): string { + return "instrumented mode"; + } + + private taskCompletionMessage() { + return InstrumentHooks.isInstrumented() ? "Measured" : "Checked"; + } + + private wrapFunctionWithFrame(fn: Fn, isAsync: boolean): Fn { + if (isAsync) { + return async function __codspeed_root_frame__() { + await fn(); + }; + } else { + return function __codspeed_root_frame__() { + fn(); + }; + } + } + + protected async runTaskAsync(task: Task, uri: string): Promise { const { fnOpts, fn } = task as unknown as { fnOpts?: FnOptions; fn: Fn }; await fnOpts?.beforeAll?.call(task, "run"); @@ -25,79 +47,38 @@ export function runInstrumentedBench( await fnOpts?.beforeEach?.call(task, "run"); await mongoMeasurement.start(uri); - await (async function __codspeed_root_frame__() { - global.gc?.(); - InstrumentHooks.startBenchmark(); - await fn(); - InstrumentHooks.stopBenchmark(); - InstrumentHooks.setExecutedBenchmark(process.pid, uri); - })(); + global.gc?.(); + await this.wrapWithInstrumentHooksAsync( + this.wrapFunctionWithFrame(fn, true), + uri + ); await mongoMeasurement.stop(uri); await fnOpts?.afterEach?.call(task, "run"); await fnOpts?.afterAll?.call(task, "run"); - }; - // Sync task runner - const runTaskSync = (task: Task, uri: string): void => { + this.logTaskCompletion(uri, this.taskCompletionMessage()); + } + + protected runTaskSync(task: Task, uri: string): void { const { fnOpts, fn } = task as unknown as { fnOpts?: FnOptions; fn: Fn }; fnOpts?.beforeAll?.call(task, "run"); fnOpts?.beforeEach?.call(task, "run"); - (function __codspeed_root_frame__() { - global.gc?.(); - InstrumentHooks.startBenchmark(); - fn(); - InstrumentHooks.stopBenchmark(); - InstrumentHooks.setExecutedBenchmark(process.pid, uri); - })(); + this.wrapWithInstrumentHooks(this.wrapFunctionWithFrame(fn, false), uri); fnOpts?.afterEach?.call(task, "run"); fnOpts?.afterAll?.call(task, "run"); - }; - - bench.run = async () => { - logStart(); - for (const task of bench.tasks) { - const uri = getTaskUri(bench, task.name, rootCallingFile); - await runTaskAsync(task, uri); - logTaskCompletion(uri); - } - - return logEnd(); - }; - - bench.runSync = () => { - logStart(); - - for (const task of bench.tasks) { - const uri = getTaskUri(bench, task.name, rootCallingFile); - runTaskSync(task, uri); - logTaskCompletion(uri); - } + this.logTaskCompletion(uri, this.taskCompletionMessage()); + } - return logEnd(); - }; - - const logStart = () => { - console.log( - `[CodSpeed] running with @codspeed/tinybench v${__VERSION__} (instrumented mode)` - ); - }; - - const logTaskCompletion = (uri: string) => { - console.log( - ` ✔ ${ - InstrumentHooks.isInstrumented() ? "Measured" : "Checked" - } ${uri}` - ); - }; + protected finalizeAsyncRun(): Task[] { + return this.finalizeBenchRun(); + } - const logEnd = () => { - teardownCore(); - console.log(`[CodSpeed] Done running ${bench.tasks.length} benches.`); - return bench.tasks; - }; + protected finalizeSyncRun(): Task[] { + return this.finalizeBenchRun(); + } } diff --git a/packages/tinybench-plugin/src/shared.ts b/packages/tinybench-plugin/src/shared.ts new file mode 100644 index 00000000..bc38e3b6 --- /dev/null +++ b/packages/tinybench-plugin/src/shared.ts @@ -0,0 +1,89 @@ +import { InstrumentHooks, setupCore, teardownCore } from "@codspeed/core"; +import { Bench, Fn, Task } from "tinybench"; +import { getTaskUri } from "./uri"; + +declare const __VERSION__: string; + +export abstract class BaseBenchRunner { + protected bench: Bench; + protected rootCallingFile: string; + + constructor(bench: Bench, rootCallingFile: string) { + this.bench = bench; + this.rootCallingFile = rootCallingFile; + } + + private setupBenchRun(): void { + setupCore(); + this.logStart(); + } + + private logStart(): void { + console.log( + `[CodSpeed] running with @codspeed/tinybench v${__VERSION__} (${this.getModeName()})` + ); + } + + protected getTaskUri(task: Task): string { + return getTaskUri(this.bench, task.name, this.rootCallingFile); + } + + protected logTaskCompletion(uri: string, status: string): void { + console.log(`[CodSpeed] ${status} ${uri}`); + } + + protected finalizeBenchRun(): Task[] { + teardownCore(); + console.log(`[CodSpeed] Done running ${this.bench.tasks.length} benches.`); + return this.bench.tasks; + } + + protected wrapWithInstrumentHooks(fn: () => T, uri: string): T { + InstrumentHooks.startBenchmark(); + const result = fn(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + return result; + } + + protected async wrapWithInstrumentHooksAsync( + fn: Fn, + uri: string + ): Promise { + InstrumentHooks.startBenchmark(); + const result = await fn(); + InstrumentHooks.stopBenchmark(); + InstrumentHooks.setExecutedBenchmark(process.pid, uri); + return result; + } + + protected abstract getModeName(): string; + protected abstract runTaskAsync(task: Task, uri: string): Promise; + protected abstract runTaskSync(task: Task, uri: string): void; + protected abstract finalizeAsyncRun(): Task[]; + protected abstract finalizeSyncRun(): Task[]; + + public setupBenchMethods(): void { + this.bench.run = async () => { + this.setupBenchRun(); + + for (const task of this.bench.tasks) { + const uri = this.getTaskUri(task); + await this.runTaskAsync(task, uri); + } + + return this.finalizeAsyncRun(); + }; + + this.bench.runSync = () => { + this.setupBenchRun(); + + for (const task of this.bench.tasks) { + const uri = this.getTaskUri(task); + this.runTaskSync(task, uri); + } + + return this.finalizeSyncRun(); + }; + } +} diff --git a/packages/tinybench-plugin/src/walltime.ts b/packages/tinybench-plugin/src/walltime.ts index 7418eb29..0c86e1b3 100644 --- a/packages/tinybench-plugin/src/walltime.ts +++ b/packages/tinybench-plugin/src/walltime.ts @@ -1,6 +1,5 @@ import { calculateQuantiles, - InstrumentHooks, mongoMeasurement, msToNs, msToS, @@ -9,155 +8,128 @@ import { type BenchmarkStats, } from "@codspeed/core"; import { Bench, Fn, Task, TaskResult } from "tinybench"; -import { getTaskUri } from "./uri"; +import { BaseBenchRunner } from "./shared"; -declare const __VERSION__: string; - -export function runWalltimeBench(bench: Bench, rootCallingFile: string): void { - bench.run = async () => { - logStart(); - const codspeedBenchmarks: CodspeedBenchmark[] = []; +export function setupCodspeedWalltimeBench( + bench: Bench, + rootCallingFile: string +): void { + const runner = new WalltimeBenchRunner(bench, rootCallingFile); + runner.setupBenchMethods(); +} - // Collect and report walltime data - for (const task of bench.tasks) { - const uri = getTaskUri(bench, task.name, rootCallingFile); +class WalltimeBenchRunner extends BaseBenchRunner { + private codspeedBenchmarks: CodspeedBenchmark[] = []; - // Override the function under test to add a static frame - wrapTaskFunction(task, true); + protected getModeName(): string { + return "walltime mode"; + } - // run the warmup of the task right before its actual run - if (bench.opts.warmup) { - await task.warmup(); - } + protected async runTaskAsync(task: Task, uri: string): Promise { + // Override the function under test to add a static frame + this.wrapTaskFunction(task, true); - await mongoMeasurement.start(uri); - InstrumentHooks.startBenchmark(); - await task.run(); - InstrumentHooks.stopBenchmark(); - await mongoMeasurement.stop(uri); - - registerCodspeedBenchmarkFromTask( - codspeedBenchmarks, - task, - bench, - rootCallingFile - ); + // run the warmup of the task right before its actual run + if (this.bench.opts.warmup) { + await task.warmup(); } - return finalizeWalltimeRun(bench, codspeedBenchmarks, true); - }; - - bench.runSync = () => { - logStart(); - const codspeedBenchmarks: CodspeedBenchmark[] = []; + await mongoMeasurement.start(uri); + await this.wrapWithInstrumentHooksAsync(() => task.run(), uri); + await mongoMeasurement.stop(uri); - for (const task of bench.tasks) { - // Override the function under test to add a static frame - wrapTaskFunction(task, false); - - if (bench.opts.warmup) { - task.warmup(); - } + this.registerCodspeedBenchmarkFromTask(task); + } - InstrumentHooks.startBenchmark(); - task.runSync(); - InstrumentHooks.stopBenchmark(); + protected runTaskSync(task: Task, uri: string): void { + // Override the function under test to add a static frame + this.wrapTaskFunction(task, false); - registerCodspeedBenchmarkFromTask( - codspeedBenchmarks, - task, - bench, - rootCallingFile - ); + if (this.bench.opts.warmup) { + task.warmup(); } - return finalizeWalltimeRun(bench, codspeedBenchmarks, false); - }; -} + this.wrapWithInstrumentHooks(() => task.runSync(), uri); -function logStart() { - console.log( - `[CodSpeed] running with @codspeed/tinybench v${__VERSION__} (walltime mode)` - ); -} + this.registerCodspeedBenchmarkFromTask(task); + } -const TINYBENCH_WARMUP_DEFAULT = 16; + protected finalizeAsyncRun(): Task[] { + return this.finalizeWalltimeRun(true); + } -function registerCodspeedBenchmarkFromTask( - codspeedBenchmarks: CodspeedBenchmark[], - task: Task, - bench: Bench, - rootCallingFile: string -): void { - const uri = getTaskUri(bench, task.name, rootCallingFile); + protected finalizeSyncRun(): Task[] { + return this.finalizeWalltimeRun(false); + } - if (!task.result) { - console.warn(` ⚠ No result data available for ${uri}`); - return; + private wrapTaskFunction(task: Task, isAsync: boolean): void { + const { fn } = task as unknown as { fn: Fn }; + if (isAsync) { + // eslint-disable-next-line no-inner-declarations + async function __codspeed_root_frame__() { + await fn(); + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (task as any).fn = __codspeed_root_frame__; + } else { + // eslint-disable-next-line no-inner-declarations + function __codspeed_root_frame__() { + fn(); + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (task as any).fn = __codspeed_root_frame__; + } } - const warmupIterations = bench.opts.warmup - ? bench.opts.warmupIterations ?? TINYBENCH_WARMUP_DEFAULT - : 0; - const stats = convertTinybenchResultToBenchmarkStats( - task.result, - warmupIterations - ); - - codspeedBenchmarks.push({ - name: task.name, - uri, - config: { - max_rounds: bench.opts.iterations ?? null, - max_time_ns: bench.opts.time ? msToNs(bench.opts.time) : null, - min_round_time_ns: null, // tinybench does not have an option for this - warmup_time_ns: - bench.opts.warmup && bench.opts.warmupTime - ? msToNs(bench.opts.warmupTime) - : null, - }, - stats, - }); - - console.log(` ✔ Collected walltime data for ${uri}`); - InstrumentHooks.setExecutedBenchmark(process.pid, uri); -} + private registerCodspeedBenchmarkFromTask(task: Task): void { + const uri = this.getTaskUri(task); -function wrapTaskFunction(task: Task, isAsync: boolean): void { - const { fn } = task as unknown as { fn: Fn }; - if (isAsync) { - // eslint-disable-next-line no-inner-declarations - async function __codspeed_root_frame__() { - await fn(); - } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (task as any).fn = __codspeed_root_frame__; - } else { - // eslint-disable-next-line no-inner-declarations - function __codspeed_root_frame__() { - fn(); + if (!task.result) { + console.warn(` ⚠ No result data available for ${uri}`); + return; } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - (task as any).fn = __codspeed_root_frame__; - } -} -function finalizeWalltimeRun( - bench: Bench, - benchmarks: CodspeedBenchmark[], - isAsync: boolean -) { - // Write results to JSON file using core function - if (benchmarks.length > 0) { - writeWalltimeResults(benchmarks, isAsync); + const warmupIterations = this.bench.opts.warmup + ? this.bench.opts.warmupIterations ?? TINYBENCH_WARMUP_DEFAULT + : 0; + const stats = convertTinybenchResultToBenchmarkStats( + task.result, + warmupIterations + ); + + this.codspeedBenchmarks.push({ + name: task.name, + uri, + config: { + max_rounds: this.bench.opts.iterations ?? null, + max_time_ns: this.bench.opts.time ? msToNs(this.bench.opts.time) : null, + min_round_time_ns: null, // tinybench does not have an option for this + warmup_time_ns: + this.bench.opts.warmup && this.bench.opts.warmupTime + ? msToNs(this.bench.opts.warmupTime) + : null, + }, + stats, + }); + + this.logTaskCompletion(uri, "Collected walltime data for"); } - console.log( - `[CodSpeed] Done collecting walltime data for ${bench.tasks.length} benches.` - ); - return bench.tasks; + private finalizeWalltimeRun(isAsync: boolean): Task[] { + // Write results to JSON file using core function + if (this.codspeedBenchmarks.length > 0) { + writeWalltimeResults(this.codspeedBenchmarks, isAsync); + } + + console.log( + `[CodSpeed] Done collecting walltime data for ${this.bench.tasks.length} benches.` + ); + return this.bench.tasks; + } } +const TINYBENCH_WARMUP_DEFAULT = 16; + function convertTinybenchResultToBenchmarkStats( result: TaskResult, warmupIterations: number diff --git a/packages/tinybench-plugin/tests/__snapshots__/index.integ.test.ts.snap b/packages/tinybench-plugin/tests/__snapshots__/index.integ.test.ts.snap index 5a41b12c..bed46639 100644 --- a/packages/tinybench-plugin/tests/__snapshots__/index.integ.test.ts.snap +++ b/packages/tinybench-plugin/tests/__snapshots__/index.integ.test.ts.snap @@ -7,10 +7,10 @@ exports[`Benchmark.Suite > check console output(instrumented=%p) false 1`] = ` "[CodSpeed] running with @codspeed/tinybench v1.0.0 (instrumented mode)", ], [ - " ✔ Checked packages/tinybench-plugin/tests/index.integ.test.ts::RegExp", + "[CodSpeed] Checked packages/tinybench-plugin/tests/index.integ.test.ts::RegExp", ], [ - " ✔ Checked packages/tinybench-plugin/tests/index.integ.test.ts::RegExp2", + "[CodSpeed] Checked packages/tinybench-plugin/tests/index.integ.test.ts::RegExp2", ], [ "[CodSpeed] Done running 2 benches.", @@ -24,10 +24,10 @@ exports[`Benchmark.Suite > check console output(instrumented=%p) true 1`] = ` { "log": [ [ - " ✔ Measured packages/tinybench-plugin/tests/index.integ.test.ts::RegExp", + "[CodSpeed] Measured packages/tinybench-plugin/tests/index.integ.test.ts::RegExp", ], [ - " ✔ Measured packages/tinybench-plugin/tests/index.integ.test.ts::RegExp2", + "[CodSpeed] Measured packages/tinybench-plugin/tests/index.integ.test.ts::RegExp2", ], [ "[CodSpeed] Done running 2 benches.", diff --git a/packages/tinybench-plugin/tests/index.integ.test.ts b/packages/tinybench-plugin/tests/index.integ.test.ts index 1b513703..b0d44751 100644 --- a/packages/tinybench-plugin/tests/index.integ.test.ts +++ b/packages/tinybench-plugin/tests/index.integ.test.ts @@ -215,7 +215,7 @@ describe("Benchmark.Suite", () => { /o/.test("Hello World!"); }); - expect(mockCore.setupCore).toHaveBeenCalledTimes(1); + expect(mockCore.setupCore).not.toHaveBeenCalled(); expect(mockCore.teardownCore).not.toHaveBeenCalled(); await bench.run();