From a3d2755c4b88c4bdbf58925cc83524c8f9ab0c5b Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Thu, 28 Aug 2025 14:24:13 +0200 Subject: [PATCH 1/5] chore: bump version to 5.0.0 --- lerna.json | 6 ++++-- packages/benchmark.js-plugin/package.json | 4 ++-- packages/core/package.json | 2 +- packages/tinybench-plugin/package.json | 4 ++-- packages/vitest-plugin/package.json | 4 ++-- pnpm-lock.yaml | 6 +++--- 6 files changed, 14 insertions(+), 12 deletions(-) diff --git a/lerna.json b/lerna.json index d9d77fcd..9819cb20 100644 --- a/lerna.json +++ b/lerna.json @@ -1,7 +1,9 @@ { "npmClient": "pnpm", "useWorkspaces": true, - "packages": ["packages/*"], + "packages": [ + "packages/*" + ], "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "4.0.1" + "version": "5.0.0" } diff --git a/packages/benchmark.js-plugin/package.json b/packages/benchmark.js-plugin/package.json index a99ed43c..9b7ca25b 100644 --- a/packages/benchmark.js-plugin/package.json +++ b/packages/benchmark.js-plugin/package.json @@ -1,6 +1,6 @@ { "name": "@codspeed/benchmark.js-plugin", - "version": "4.0.1", + "version": "5.0.0", "description": "Benchmark.js compatibility layer for CodSpeed", "keywords": [ "codspeed", @@ -27,7 +27,7 @@ "jest-mock-extended": "^3.0.4" }, "dependencies": { - "@codspeed/core": "workspace:^4.0.1", + "@codspeed/core": "workspace:^5.0.0", "lodash": "^4.17.10", "stack-trace": "1.0.0-pre2" }, diff --git a/packages/core/package.json b/packages/core/package.json index ad5dba6a..1fc2345f 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@codspeed/core", - "version": "4.0.1", + "version": "5.0.0", "description": "The core Node library used to integrate with Codspeed runners", "keywords": [ "codspeed", diff --git a/packages/tinybench-plugin/package.json b/packages/tinybench-plugin/package.json index db43c514..f5c41f57 100644 --- a/packages/tinybench-plugin/package.json +++ b/packages/tinybench-plugin/package.json @@ -1,6 +1,6 @@ { "name": "@codspeed/tinybench-plugin", - "version": "4.0.1", + "version": "5.0.0", "description": "tinybench compatibility layer for CodSpeed", "keywords": [ "codspeed", @@ -31,7 +31,7 @@ "vitest": "^3.2.4" }, "dependencies": { - "@codspeed/core": "workspace:^4.0.1", + "@codspeed/core": "workspace:^5.0.0", "stack-trace": "1.0.0-pre2" }, "peerDependencies": { diff --git a/packages/vitest-plugin/package.json b/packages/vitest-plugin/package.json index f26c53e7..3a4c336c 100644 --- a/packages/vitest-plugin/package.json +++ b/packages/vitest-plugin/package.json @@ -1,6 +1,6 @@ { "name": "@codspeed/vitest-plugin", - "version": "4.0.1", + "version": "5.0.0", "description": "vitest plugin for CodSpeed", "keywords": [ "codspeed", @@ -28,7 +28,7 @@ "bench": "vitest bench" }, "dependencies": { - "@codspeed/core": "workspace:^4.0.1" + "@codspeed/core": "workspace:^5.0.0" }, "peerDependencies": { "vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5b8bb92e..98e6f532 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -213,7 +213,7 @@ importers: packages/benchmark.js-plugin: dependencies: '@codspeed/core': - specifier: workspace:^4.0.1 + specifier: workspace:^5.0.0 version: link:../core lodash: specifier: ^4.17.10 @@ -275,7 +275,7 @@ importers: packages/tinybench-plugin: dependencies: '@codspeed/core': - specifier: workspace:^4.0.1 + specifier: workspace:^5.0.0 version: link:../core stack-trace: specifier: 1.0.0-pre2 @@ -297,7 +297,7 @@ importers: packages/vitest-plugin: dependencies: '@codspeed/core': - specifier: workspace:^4.0.1 + specifier: workspace:^5.0.0 version: link:../core devDependencies: '@total-typescript/shoehorn': From 102e9c1df25edb1ea059e3ee4454b8ce50cc02a4 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Fri, 19 Sep 2025 10:26:32 +0200 Subject: [PATCH 2/5] ci: pin action to v3 and remove moon's --concurency usage v3 was the action version of the previous base job. We want to do changes one by one, and v4 (current main) does not work without the concurrency change. This just brings the concurrency change that we will need to set the action version to the latest (v4) again. --- .github/workflows/ci.yml | 4 ++-- .github/workflows/codspeed.yml | 13 ++++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b26f64db..699e530f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,7 @@ jobs: - name: Run benchmarks with tinybench-plugin # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v2` - uses: CodSpeedHQ/action@main + uses: CodSpeedHQ/action@v3 with: run: pnpm --filter ${{ matrix.example }} bench-tinybench env: @@ -70,7 +70,7 @@ jobs: CODSPEED_DEBUG: true - name: Run benchmarks with benchmark.js-plugin # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v2` - uses: CodSpeedHQ/action@main + uses: CodSpeedHQ/action@v3 with: run: pnpm --filter ${{ matrix.example }} bench-benchmark-js env: diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 1695e38b..d0c77b2d 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -25,10 +25,12 @@ jobs: - name: Run benchmarks # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v2` - uses: CodSpeedHQ/action@main + uses: CodSpeedHQ/action@v3 with: run: | - pnpm moon run --concurrency 1 :bench + pnpm moon run tinybench-plugin:bench + pnpm moon run vitest-plugin:bench + pnpm moon run benchmark.js-plugin:bench pnpm --workspace-concurrency 1 -r bench-tinybench pnpm --workspace-concurrency 1 -r bench-benchmark-js pnpm --workspace-concurrency 1 -r bench-vitest @@ -51,10 +53,11 @@ jobs: - name: Run benchmarks # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v2` - uses: CodSpeedHQ/action@main + uses: CodSpeedHQ/action@v3 with: - # Only tinybench supports walltime for now run: | - pnpm moon run --concurrency 1 :bench + pnpm moon run tinybench-plugin:bench + pnpm moon run vitest-plugin:bench + pnpm moon run benchmark.js-plugin:bench pnpm --workspace-concurrency 1 -r bench-tinybench pnpm --workspace-concurrency 1 -r bench-vitest From c3ddfff2dc84d7eecd5d7a42c1a4b262f2f4e0f9 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Fri, 19 Sep 2025 11:07:38 +0200 Subject: [PATCH 3/5] ci: re-follow main version for action to use v4 Enables perf by default on walltime runs, and requires explicit mode. --- .github/workflows/ci.yml | 6 ++++-- .github/workflows/codspeed.yml | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 699e530f..720726b7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,16 +62,18 @@ jobs: - name: Run benchmarks with tinybench-plugin # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v2` - uses: CodSpeedHQ/action@v3 + uses: CodSpeedHQ/action@main with: + mode: instrumentation run: pnpm --filter ${{ matrix.example }} bench-tinybench env: CODSPEED_SKIP_UPLOAD: true CODSPEED_DEBUG: true - name: Run benchmarks with benchmark.js-plugin # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v2` - uses: CodSpeedHQ/action@v3 + uses: CodSpeedHQ/action@main with: + mode: instrumentation run: pnpm --filter ${{ matrix.example }} bench-benchmark-js env: CODSPEED_SKIP_UPLOAD: true diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index d0c77b2d..9c002e5f 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -25,8 +25,9 @@ jobs: - name: Run benchmarks # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v2` - uses: CodSpeedHQ/action@v3 + uses: CodSpeedHQ/action@main with: + mode: instrumentation run: | pnpm moon run tinybench-plugin:bench pnpm moon run vitest-plugin:bench @@ -53,8 +54,9 @@ jobs: - name: Run benchmarks # use version from `main` branch to always test the latest version, in real projects, use a tag, like `@v2` - uses: CodSpeedHQ/action@v3 + uses: CodSpeedHQ/action@main with: + mode: walltime run: | pnpm moon run tinybench-plugin:bench pnpm moon run vitest-plugin:bench From 81f0dc9f93de801fe713d4b33c14091686b88eb6 Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Fri, 19 Sep 2025 11:45:59 +0200 Subject: [PATCH 4/5] fix(core): split node flags to be more minimalistic in walltime These flags will be tweaked in the PR adding proper perf management. --- packages/core/src/introspection.ts | 41 +++++++++++-------- .../vitest-plugin/src/__tests__/index.test.ts | 4 +- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/packages/core/src/introspection.ts b/packages/core/src/introspection.ts index 1be58ab1..ce965922 100644 --- a/packages/core/src/introspection.ts +++ b/packages/core/src/introspection.ts @@ -1,28 +1,35 @@ import { writeFileSync } from "fs"; +import { getCodspeedRunnerMode } from "."; const CUSTOM_INTROSPECTION_EXIT_CODE = 0; export const getV8Flags = () => { const nodeVersionMajor = parseInt(process.version.slice(1).split(".")[0]); + const codspeedRunnerMode = getCodspeedRunnerMode(); - const flags = [ - "--hash-seed=1", - "--random-seed=1", - "--no-opt", - "--predictable", - "--predictable-gc-schedule", - "--interpreted-frames-native-stack", - "--allow-natives-syntax", - "--expose-gc", - "--no-concurrent-sweeping", - "--max-old-space-size=4096", - ]; - if (nodeVersionMajor < 18) { - flags.push("--no-randomize-hashes"); - } - if (nodeVersionMajor < 20) { - flags.push("--no-scavenge-task"); + const flags = ["--interpreted-frames-native-stack", "--allow-natives-syntax"]; + + if (codspeedRunnerMode === "instrumented") { + flags.push( + ...[ + "--hash-seed=1", + "--random-seed=1", + "--no-opt", + "--predictable", + "--predictable-gc-schedule", + "--expose-gc", + "--no-concurrent-sweeping", + "--max-old-space-size=4096", + ] + ); + if (nodeVersionMajor < 18) { + flags.push("--no-randomize-hashes"); + } + if (nodeVersionMajor < 20) { + flags.push("--no-scavenge-task"); + } } + return flags; }; diff --git a/packages/vitest-plugin/src/__tests__/index.test.ts b/packages/vitest-plugin/src/__tests__/index.test.ts index 290b0dde..b0322819 100644 --- a/packages/vitest-plugin/src/__tests__/index.test.ts +++ b/packages/vitest-plugin/src/__tests__/index.test.ts @@ -93,13 +93,13 @@ describe("codSpeedPlugin", () => { poolOptions: { forks: { execArgv: [ + "--interpreted-frames-native-stack", + "--allow-natives-syntax", "--hash-seed=1", "--random-seed=1", "--no-opt", "--predictable", "--predictable-gc-schedule", - "--interpreted-frames-native-stack", - "--allow-natives-syntax", "--expose-gc", "--no-concurrent-sweeping", "--max-old-space-size=4096", From 016737521ca68df61e20cfad94ede886ed89ef8c Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Fri, 26 Sep 2025 10:35:57 +0200 Subject: [PATCH 5/5] chore: just make a run with the __codspeed_root_frame__ --- packages/tinybench-plugin/src/walltime.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/tinybench-plugin/src/walltime.ts b/packages/tinybench-plugin/src/walltime.ts index 2c22c539..28933b5b 100644 --- a/packages/tinybench-plugin/src/walltime.ts +++ b/packages/tinybench-plugin/src/walltime.ts @@ -7,7 +7,7 @@ import { type Benchmark, type BenchmarkStats, } from "@codspeed/core"; -import { Bench, TaskResult } from "tinybench"; +import { Bench, Fn, TaskResult } from "tinybench"; import { getTaskUri } from "./uri"; declare const __VERSION__: string; @@ -35,6 +35,14 @@ export function runWalltimeBench(bench: Bench, rootCallingFile: string): void { for (const task of bench.tasks) { const uri = getTaskUri(bench, task.name, rootCallingFile); + const { fn } = task as unknown as { fn: Fn }; + // eslint-disable-next-line no-inner-declarations + async function __codspeed_root_frame__() { + await fn(); + } + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (task as any).fn = __codspeed_root_frame__; + // run the warmup of the task right before its actual run if (bench.opts.warmup) { await task.warmup();