Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions examples/with-telemetry/bench/benchmark.js.bench.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
import { withCodSpeed } from "@codspeed/benchmark.js-plugin";
import Benchmark from "benchmark";
import {
legacyLog,
logWithoutSpan,
noTelemetry,
parseAndSum,
} from "../src/workloads";

const suite = withCodSpeed(new Benchmark.Suite());

suite
.add("parse_and_sum", {
defer: true,
fn: (deferred: Benchmark.Deferred) => {
parseAndSum().then(() => deferred.resolve());
},
})
.add("log_without_span", () => {
logWithoutSpan();
})
.add("legacy_log", () => {
legacyLog();
})
.add("no_telemetry", () => {
noTelemetry();
})
.on("cycle", (event: Benchmark.Event) => {
console.log(String(event.target));
});

await suite.run({ async: true });
27 changes: 27 additions & 0 deletions examples/with-telemetry/bench/tinybench.bench.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
import { withCodSpeed } from "@codspeed/tinybench-plugin";
import { Bench } from "tinybench";
import {
legacyLog,
logWithoutSpan,
noTelemetry,
parseAndSum,
} from "../src/workloads";

const bench = withCodSpeed(new Bench());

bench
.add("parse_and_sum", async () => {
await parseAndSum();
})
.add("log_without_span", () => {
logWithoutSpan();
})
.add("legacy_log", () => {
legacyLog();
})
.add("no_telemetry", () => {
noTelemetry();
});

await bench.run();
console.table(bench.table());
26 changes: 26 additions & 0 deletions examples/with-telemetry/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
{
"name": "with-telemetry",
"private": true,
"type": "module",
"scripts": {
"bench-benchmark-js": "node --import tsx bench/benchmark.js.bench.ts",
"bench-tinybench": "node --import tsx bench/tinybench.bench.ts",
"bench-vitest": "vitest bench --run",
"typecheck": "tsc --noEmit --pretty"
},
"dependencies": {
"@opentelemetry/api": "^1.10.0",
"@opentelemetry/api-logs": "^0.223.0"
},
"devDependencies": {
"@codspeed/benchmark.js-plugin": "workspace:*",
"@codspeed/tinybench-plugin": "workspace:*",
"@codspeed/vitest-plugin": "workspace:*",
"@types/benchmark": "^2.1.2",
"benchmark": "^2.1.4",
"tinybench": "^4.0.1",
"tsx": "^4.23.12",
"typescript": "^5.1.3",
"vitest": "^5.0.0"
}
}
33 changes: 33 additions & 0 deletions examples/with-telemetry/src/workloads.bench.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import { describe, test } from "vitest";
import {
legacyLog,
logWithoutSpan,
noTelemetry,
parseAndSum,
} from "./workloads";

describe("telemetry", () => {
test("parse_and_sum", async ({ bench }) => {
await bench("parse_and_sum", async () => {
await parseAndSum();
}).run();
});

test("log_without_span", async ({ bench }) => {
await bench("log_without_span", () => {
logWithoutSpan();
}).run();
});

test("legacy_log", async ({ bench }) => {
await bench("legacy_log", () => {
legacyLog();
}).run();
});

test("no_telemetry", async ({ bench }) => {
await bench("no_telemetry", () => {
noTelemetry();
}).run();
});
});
84 changes: 84 additions & 0 deletions examples/with-telemetry/src/workloads.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
/**
* Workloads emitting telemetry in different ways: nested spans with logs in
* them, a log outside of any span, a log through the legacy logs API, or
* nothing at all.
*/
import { logs, SeverityNumber, trace } from "@opentelemetry/api";
import { logs as legacyLogs } from "@opentelemetry/api-logs";

const tracer = trace.getTracer("with-telemetry");
const logger = logs.getLogger("with-telemetry");
const legacyLogger = legacyLogs.getLogger("with-telemetry-legacy");

function parse(input: string): number[] {
return tracer.startActiveSpan("parse", (span) => {
span.setAttribute("input_len", input.length);
const values = input
.split(",")
.map(Number)
.filter((value) => !Number.isNaN(value));
logger.emit({
severityNumber: SeverityNumber.INFO,
severityText: "INFO",
body: "parsed values",
attributes: { count: values.length },
});
span.end();
return values;
});
}

async function sum(values: number[]): Promise<number> {
return tracer.startActiveSpan("sum", async (span) => {
// Awaited, so that the span must follow the async context
await Promise.resolve();
const total = values.reduce((acc, value) => acc + value, 0);
span.end();
return total;
});
}

/** Nested spans, with logs in them, across an `await`. */
export async function parseAndSum(): Promise<number> {
return tracer.startActiveSpan("parse_and_sum", async (span) => {
const values = parse("1,2,x,3");
if (values.length < 4) {
logger.emit({
severityNumber: SeverityNumber.WARN,
severityText: "WARN",
body: "skipped invalid fields",
attributes: { skipped: 4 - values.length },
});
}
const total = await sum(values);
span.end();
return total;
});
}

/** A log outside of any span. */
export function logWithoutSpan(): number {
const values = [1, 2, 3];
logger.emit({
severityNumber: SeverityNumber.INFO,
severityText: "INFO",
body: "summed without a span",
attributes: { count: values.length },
});
return values.reduce((acc, value) => acc + value, 0);
}

/** A log through `@opentelemetry/api-logs`, which older instrumentations use. */
export function legacyLog(): number {
const values = [1, 2, 3];
legacyLogger.emit({
severityText: "INFO",
body: `summing ${values.length} values, logged with the legacy logs API`,
});
return values.reduce((acc, value) => acc + value, 0);
}

/** No telemetry at all. */
export function noTelemetry(): number {
return [1, 2, 3].reduce((acc, value) => acc + value, 0);
}
13 changes: 13 additions & 0 deletions examples/with-telemetry/tsconfig.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
{
"compilerOptions": {
"lib": ["es2023"],
"module": "ESNext",
"verbatimModuleSyntax": true,
"target": "es2022",
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"moduleResolution": "Node"
}
}
11 changes: 11 additions & 0 deletions examples/with-telemetry/vitest.config.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
import codspeedPlugin from "@codspeed/vitest-plugin";
import { defineConfig } from "vitest/config";

export default defineConfig({
plugins: [codspeedPlugin()],
test: {
benchmark: {
exclude: ["**/bench/**/*", "**/node_modules/**/*"],
},
},
});
11 changes: 8 additions & 3 deletions packages/benchmark.js-plugin/src/index.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import {
flushTelemetry,
getCallingFile,
InstrumentHooks,
mongoMeasurement,
Expand All @@ -9,6 +10,7 @@ import {
SetupInstrumentsResponse,
teardownCore,
tryIntrospect,
withRoundMarkers,
wrapWithRootFrame,
wrapWithRootFrameSync,
} from "@codspeed/core";
Expand Down Expand Up @@ -186,7 +188,7 @@ async function runBenchmarks({
await (bench.fn as CallableFunction)();
};
} else {
benchPayload = bench.fn as CallableFunction;
benchPayload = bench.fn as () => unknown;
}

if (typeof bench.options.setup === "function") {
Expand All @@ -197,19 +199,21 @@ async function runBenchmarks({
await optimizeFunction(benchPayload);
await mongoMeasurement.start(uri);
global.gc?.();
const measuredPayload = withRoundMarkers(benchPayload);
await wrapWithRootFrame(async () => {
InstrumentHooks.startBenchmark();
await benchPayload();
await measuredPayload();
InstrumentHooks.stopBenchmark();
InstrumentHooks.setExecutedBenchmark(process.pid, uri);
})();
await mongoMeasurement.stop(uri);
} else {
optimizeFunctionSync(benchPayload);
await mongoMeasurement.start(uri);
const measuredPayload = withRoundMarkers(benchPayload);
wrapWithRootFrameSync(() => {
InstrumentHooks.startBenchmark();
benchPayload();
measuredPayload();
InstrumentHooks.stopBenchmark();
InstrumentHooks.setExecutedBenchmark(process.pid, uri);
})();
Expand All @@ -223,6 +227,7 @@ async function runBenchmarks({
console.log(` ✔ Measured ${uri}`);
benchmarkCompletedListeners.forEach((listener) => listener());
}
await flushTelemetry();
teardownCore();
console.log(`[CodSpeed] Done running ${benches.length} benches.`);
}
Expand Down
1 change: 1 addition & 0 deletions packages/benchmark.js-plugin/tests/index.integ.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ jest.mock("@codspeed/core", () => {
mockCore.getCallingFile = actual.getCallingFile;
mockCore.wrapWithRootFrame = actual.wrapWithRootFrame;
mockCore.wrapWithRootFrameSync = actual.wrapWithRootFrameSync;
mockCore.withRoundMarkers = actual.withRoundMarkers;
return mockCore;
});

Expand Down
8 changes: 8 additions & 0 deletions packages/core/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,14 @@
"prebuildify": "^6.0.1"
},
"dependencies": {
"@opentelemetry/api": "^1.10.0",
"@opentelemetry/api-logs": "^0.223.0",
"@opentelemetry/context-async-hooks": "^3.1.0",
"@opentelemetry/exporter-logs-otlp-proto": "^0.301.0",
"@opentelemetry/exporter-trace-otlp-proto": "^0.301.0",
"@opentelemetry/resources": "^3.1.0",
"@opentelemetry/sdk-logs": "^3.1.0",
"@opentelemetry/sdk-trace": "^3.1.0",
Comment on lines +44 to +49

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Supported Node.js installs fail

The new OpenTelemetry SDK packages require Node.js >=22.15.0, but this repository still supports earlier Node.js 22 releases. If a user's package manager enforces dependency engine requirements, these regular dependencies prevent them from installing any CodSpeed plugin—even when they never enable telemetry. Use compatible dependencies or explicitly raise the supported minimum.

Prompt To Fix With AI
This is a comment left during a code review.
Path: packages/core/package.json
Line: 44-49

Comment:
**Supported Node.js installs fail**

The new OpenTelemetry SDK packages require Node.js `>=22.15.0`, but this repository still supports earlier Node.js 22 releases. If a user's package manager enforces dependency engine requirements, these regular dependencies prevent them from installing any CodSpeed plugin—even when they never enable telemetry. Use compatible dependencies or explicitly raise the supported minimum.

---

For each issue above, determine whether it is valid and should be fixed. If so, fix it directly.

Fix in Claude Code Fix in Codex

"axios": "^1.21.0",
"form-data": "^4.0.6",
"node-gyp-build": "^4.6.0",
Expand Down
5 changes: 4 additions & 1 deletion packages/core/rollup.config.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,10 @@ export default defineConfig([
sourcemap: true,
},
],
plugins: declarationsPlugin({ compilerOptions: { composite: false } }),
plugins: declarationsPlugin({
// pnpm links the dependencies of a package next to its real path
compilerOptions: { composite: false, preserveSymlinks: false },
}),
},
{
input: entrypoint,
Expand Down
7 changes: 7 additions & 0 deletions packages/core/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { MongoMeasurement } from "./mongoMeasurement";
import native_core from "./native_core";
import { warnOnUnsupportedNodeVersion } from "./nodeVersion";
import { getCodspeedRunnerMode } from "./runnerMode";
import { setupTelemetry } from "./telemetry";

declare const __VERSION__: string;

Expand All @@ -12,6 +13,11 @@ export const isBound = native_core.isBound;

export const mongoMeasurement = new MongoMeasurement();

// On import, before the benchmarked modules that the plugins load next get their
// tracers and loggers: those obtained before through another copy of the
// OpenTelemetry API, such as its ESM build, would never export.
setupTelemetry();

export const setupCore = () => {
warnOnUnsupportedNodeVersion();

Expand Down Expand Up @@ -64,6 +70,7 @@ export { optimizeFunction, optimizeFunctionSync } from "./optimization";
export { wrapWithRootFrame, wrapWithRootFrameSync } from "./rootFrame";
export { getCodspeedRunnerMode, getInstrumentMode } from "./runnerMode";
export type { InstrumentMode } from "./runnerMode";
export { flushTelemetry, withRoundMarkers } from "./telemetry";
export * from "./utils";
export * from "./walltime";
export const InstrumentHooks = native_core.InstrumentHooks;
Expand Down
9 changes: 8 additions & 1 deletion packages/core/src/introspection.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { writeFileSync } from "fs";
import path from "path";

import { getInstrumentMode } from "./runnerMode";
import { getCodspeedRunnerMode, getInstrumentMode } from "./runnerMode";

const CUSTOM_INTROSPECTION_EXIT_CODE = 0;

Expand All @@ -22,6 +22,13 @@ export const getV8Flags = () => {

const flags = ["--interpreted-frames-native-stack", "--allow-natives-syntax"];

// The spans and logs keep the timings of the code as it runs in production,
// so V8 compiles and collects garbage as it does by default.
if (getCodspeedRunnerMode() === "telemetry") {
flags.push(...getSeedFlags(nodeVersionMajor));
return flags;
}

switch (instrumentMode) {
case "analysis": {
flags.push(
Expand Down
Loading
Loading