From d2057118f07271a450d21c32c778ab6fd35dee62 Mon Sep 17 00:00:00 2001 From: Luc Varoqui Date: Fri, 13 Nov 2026 17:05:39 +0100 Subject: [PATCH 1/3] feat: export the telemetry of the benchmarks over OTLP Add a `telemetry` runner mode. In this mode, the runner sets `CODSPEED_OTEL_ENDPOINT`, and the core sends the OpenTelemetry spans and logs of the benchmarked code over OTLP/HTTP to that receiver, so that CodSpeed can show the telemetry of each benchmark. The core sets up the export when it is imported, before the benchmarked modules get their tracers and loggers. It registers global tracer and logger providers and an `AsyncLocalStorage` context manager, which keeps the active span across async calls. It loads the OpenTelemetry SDK only when the endpoint is set, so the other modes never load it. If the benchmarked code already registered its own providers, the core keeps them and warns that their telemetry is not exported. Each span and log record also carries its times on the clock of the benchmark markers (`codspeed.monotonic_start_ns`, `codspeed.monotonic_end_ns`, `codspeed.monotonic_time_ns`). Trace and span ids come from `crypto`, because the benchmarks run with a seeded `Math.random()`. The mode runs each benchmark once, like simulation, but with the default V8 flags, so the timings stay close to production. The tinybench, benchmark.js and Vitest plugins wrap each benchmarked call with `withRoundMarkers`, which sends the start and end of the call as a round of the benchmark. The runner uses these rounds to find the benchmark of each span and log. The plugins also flush the export once their benchmarks are done, since Vitest may kill its worker before the process exits. With Vitest, a setup file imports the core in each worker, ahead of the benchmark files. Add the `examples/with-telemetry` project, which runs the same workloads with each of the three plugins. Closes COD-4149 Co-Authored-By: Claude Opus 5.5 --- .../bench/benchmark.js.bench.ts | 32 ++ .../with-telemetry/bench/tinybench.bench.ts | 27 ++ examples/with-telemetry/package.json | 26 ++ .../with-telemetry/src/workloads.bench.ts | 33 ++ examples/with-telemetry/src/workloads.ts | 84 +++++ examples/with-telemetry/tsconfig.json | 13 + examples/with-telemetry/vitest.config.ts | 11 + packages/benchmark.js-plugin/src/index.ts | 11 +- .../tests/index.integ.test.ts | 1 + packages/core/package.json | 8 + packages/core/rollup.config.mjs | 5 +- packages/core/src/index.ts | 7 + packages/core/src/introspection.ts | 9 +- packages/core/src/runnerMode.ts | 13 +- packages/core/src/telemetry.ts | 297 ++++++++++++++++++ packages/core/tests/telemetry.integ.test.ts | 116 +++++++ packages/tinybench-plugin/src/analysis.ts | 5 +- packages/tinybench-plugin/src/shared.ts | 8 +- packages/vitest-plugin/rollup.config.mjs | 6 + packages/vitest-plugin/src/index.ts | 4 + packages/vitest-plugin/src/legacy/analysis.ts | 7 +- packages/vitest-plugin/src/telemetrySetup.ts | 4 + packages/vitest-plugin/src/v5/provider.ts | 10 +- pnpm-lock.yaml | 218 ++++++++++++- 24 files changed, 935 insertions(+), 20 deletions(-) create mode 100644 examples/with-telemetry/bench/benchmark.js.bench.ts create mode 100644 examples/with-telemetry/bench/tinybench.bench.ts create mode 100644 examples/with-telemetry/package.json create mode 100644 examples/with-telemetry/src/workloads.bench.ts create mode 100644 examples/with-telemetry/src/workloads.ts create mode 100644 examples/with-telemetry/tsconfig.json create mode 100644 examples/with-telemetry/vitest.config.ts create mode 100644 packages/core/src/telemetry.ts create mode 100644 packages/core/tests/telemetry.integ.test.ts create mode 100644 packages/vitest-plugin/src/telemetrySetup.ts diff --git a/examples/with-telemetry/bench/benchmark.js.bench.ts b/examples/with-telemetry/bench/benchmark.js.bench.ts new file mode 100644 index 00000000..9ec77c69 --- /dev/null +++ b/examples/with-telemetry/bench/benchmark.js.bench.ts @@ -0,0 +1,32 @@ +import { withCodSpeed } from "@codspeed/benchmark.js-plugin"; +import Benchmark from "benchmark"; +import { + legacyLog, + logWithoutSpan, + noTelemetry, + parseAndSum, +} from "../src/workloads"; + +const suite = withCodSpeed(new Benchmark.Suite()); + +suite + .add("parse_and_sum", { + defer: true, + fn: (deferred: Benchmark.Deferred) => { + parseAndSum().then(() => deferred.resolve()); + }, + }) + .add("log_without_span", () => { + logWithoutSpan(); + }) + .add("legacy_log", () => { + legacyLog(); + }) + .add("no_telemetry", () => { + noTelemetry(); + }) + .on("cycle", (event: Benchmark.Event) => { + console.log(String(event.target)); + }); + +await suite.run({ async: true }); diff --git a/examples/with-telemetry/bench/tinybench.bench.ts b/examples/with-telemetry/bench/tinybench.bench.ts new file mode 100644 index 00000000..b98f4715 --- /dev/null +++ b/examples/with-telemetry/bench/tinybench.bench.ts @@ -0,0 +1,27 @@ +import { withCodSpeed } from "@codspeed/tinybench-plugin"; +import { Bench } from "tinybench"; +import { + legacyLog, + logWithoutSpan, + noTelemetry, + parseAndSum, +} from "../src/workloads"; + +const bench = withCodSpeed(new Bench()); + +bench + .add("parse_and_sum", async () => { + await parseAndSum(); + }) + .add("log_without_span", () => { + logWithoutSpan(); + }) + .add("legacy_log", () => { + legacyLog(); + }) + .add("no_telemetry", () => { + noTelemetry(); + }); + +await bench.run(); +console.table(bench.table()); diff --git a/examples/with-telemetry/package.json b/examples/with-telemetry/package.json new file mode 100644 index 00000000..9b45757b --- /dev/null +++ b/examples/with-telemetry/package.json @@ -0,0 +1,26 @@ +{ + "name": "with-telemetry", + "private": true, + "type": "module", + "scripts": { + "bench-benchmark-js": "node --import tsx bench/benchmark.js.bench.ts", + "bench-tinybench": "node --import tsx bench/tinybench.bench.ts", + "bench-vitest": "vitest bench --run", + "typecheck": "tsc --noEmit --pretty" + }, + "dependencies": { + "@opentelemetry/api": "^1.10.0", + "@opentelemetry/api-logs": "^0.223.0" + }, + "devDependencies": { + "@codspeed/benchmark.js-plugin": "workspace:*", + "@codspeed/tinybench-plugin": "workspace:*", + "@codspeed/vitest-plugin": "workspace:*", + "@types/benchmark": "^2.1.2", + "benchmark": "^2.1.4", + "tinybench": "^4.0.1", + "tsx": "^4.23.12", + "typescript": "^5.1.3", + "vitest": "^5.0.0" + } +} diff --git a/examples/with-telemetry/src/workloads.bench.ts b/examples/with-telemetry/src/workloads.bench.ts new file mode 100644 index 00000000..afbfa56e --- /dev/null +++ b/examples/with-telemetry/src/workloads.bench.ts @@ -0,0 +1,33 @@ +import { describe, test } from "vitest"; +import { + legacyLog, + logWithoutSpan, + noTelemetry, + parseAndSum, +} from "./workloads"; + +describe("telemetry", () => { + test("parse_and_sum", async ({ bench }) => { + await bench("parse_and_sum", async () => { + await parseAndSum(); + }).run(); + }); + + test("log_without_span", async ({ bench }) => { + await bench("log_without_span", () => { + logWithoutSpan(); + }).run(); + }); + + test("legacy_log", async ({ bench }) => { + await bench("legacy_log", () => { + legacyLog(); + }).run(); + }); + + test("no_telemetry", async ({ bench }) => { + await bench("no_telemetry", () => { + noTelemetry(); + }).run(); + }); +}); diff --git a/examples/with-telemetry/src/workloads.ts b/examples/with-telemetry/src/workloads.ts new file mode 100644 index 00000000..fa35f029 --- /dev/null +++ b/examples/with-telemetry/src/workloads.ts @@ -0,0 +1,84 @@ +/** + * Workloads emitting telemetry in different ways: nested spans with logs in + * them, a log outside of any span, a log through the legacy logs API, or + * nothing at all. + */ +import { logs, SeverityNumber, trace } from "@opentelemetry/api"; +import { logs as legacyLogs } from "@opentelemetry/api-logs"; + +const tracer = trace.getTracer("with-telemetry"); +const logger = logs.getLogger("with-telemetry"); +const legacyLogger = legacyLogs.getLogger("with-telemetry-legacy"); + +function parse(input: string): number[] { + return tracer.startActiveSpan("parse", (span) => { + span.setAttribute("input_len", input.length); + const values = input + .split(",") + .map(Number) + .filter((value) => !Number.isNaN(value)); + logger.emit({ + severityNumber: SeverityNumber.INFO, + severityText: "INFO", + body: "parsed values", + attributes: { count: values.length }, + }); + span.end(); + return values; + }); +} + +async function sum(values: number[]): Promise { + return tracer.startActiveSpan("sum", async (span) => { + // Awaited, so that the span must follow the async context + await Promise.resolve(); + const total = values.reduce((acc, value) => acc + value, 0); + span.end(); + return total; + }); +} + +/** Nested spans, with logs in them, across an `await`. */ +export async function parseAndSum(): Promise { + return tracer.startActiveSpan("parse_and_sum", async (span) => { + const values = parse("1,2,x,3"); + if (values.length < 4) { + logger.emit({ + severityNumber: SeverityNumber.WARN, + severityText: "WARN", + body: "skipped invalid fields", + attributes: { skipped: 4 - values.length }, + }); + } + const total = await sum(values); + span.end(); + return total; + }); +} + +/** A log outside of any span. */ +export function logWithoutSpan(): number { + const values = [1, 2, 3]; + logger.emit({ + severityNumber: SeverityNumber.INFO, + severityText: "INFO", + body: "summed without a span", + attributes: { count: values.length }, + }); + return values.reduce((acc, value) => acc + value, 0); +} + +/** A log through `@opentelemetry/api-logs`, which older instrumentations use. */ +export function legacyLog(): number { + const values = [1, 2, 3]; + legacyLogger.emit({ + severityText: "INFO", + body: `summing ${values.length} values, logged with the legacy logs API`, + }); + return values.reduce((acc, value) => acc + value, 0); +} + +/** No telemetry at all. */ +export function noTelemetry(): number { + return [1, 2, 3].reduce((acc, value) => acc + value, 0); +} diff --git a/examples/with-telemetry/tsconfig.json b/examples/with-telemetry/tsconfig.json new file mode 100644 index 00000000..ace1de03 --- /dev/null +++ b/examples/with-telemetry/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "lib": ["es2023"], + "module": "ESNext", + "verbatimModuleSyntax": true, + "target": "es2022", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "moduleResolution": "Node" + } +} diff --git a/examples/with-telemetry/vitest.config.ts b/examples/with-telemetry/vitest.config.ts new file mode 100644 index 00000000..9ac7dd68 --- /dev/null +++ b/examples/with-telemetry/vitest.config.ts @@ -0,0 +1,11 @@ +import codspeedPlugin from "@codspeed/vitest-plugin"; +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + plugins: [codspeedPlugin()], + test: { + benchmark: { + exclude: ["**/bench/**/*", "**/node_modules/**/*"], + }, + }, +}); diff --git a/packages/benchmark.js-plugin/src/index.ts b/packages/benchmark.js-plugin/src/index.ts index fa69a35a..79115a50 100644 --- a/packages/benchmark.js-plugin/src/index.ts +++ b/packages/benchmark.js-plugin/src/index.ts @@ -1,4 +1,5 @@ import { + flushTelemetry, getCallingFile, InstrumentHooks, mongoMeasurement, @@ -9,6 +10,7 @@ import { SetupInstrumentsResponse, teardownCore, tryIntrospect, + withRoundMarkers, wrapWithRootFrame, wrapWithRootFrameSync, } from "@codspeed/core"; @@ -186,7 +188,7 @@ async function runBenchmarks({ await (bench.fn as CallableFunction)(); }; } else { - benchPayload = bench.fn as CallableFunction; + benchPayload = bench.fn as () => unknown; } if (typeof bench.options.setup === "function") { @@ -197,9 +199,10 @@ async function runBenchmarks({ await optimizeFunction(benchPayload); await mongoMeasurement.start(uri); global.gc?.(); + const measuredPayload = withRoundMarkers(benchPayload); await wrapWithRootFrame(async () => { InstrumentHooks.startBenchmark(); - await benchPayload(); + await measuredPayload(); InstrumentHooks.stopBenchmark(); InstrumentHooks.setExecutedBenchmark(process.pid, uri); })(); @@ -207,9 +210,10 @@ async function runBenchmarks({ } else { optimizeFunctionSync(benchPayload); await mongoMeasurement.start(uri); + const measuredPayload = withRoundMarkers(benchPayload); wrapWithRootFrameSync(() => { InstrumentHooks.startBenchmark(); - benchPayload(); + measuredPayload(); InstrumentHooks.stopBenchmark(); InstrumentHooks.setExecutedBenchmark(process.pid, uri); })(); @@ -223,6 +227,7 @@ async function runBenchmarks({ console.log(` ✔ Measured ${uri}`); benchmarkCompletedListeners.forEach((listener) => listener()); } + await flushTelemetry(); teardownCore(); console.log(`[CodSpeed] Done running ${benches.length} benches.`); } diff --git a/packages/benchmark.js-plugin/tests/index.integ.test.ts b/packages/benchmark.js-plugin/tests/index.integ.test.ts index f3e21034..351c0e89 100644 --- a/packages/benchmark.js-plugin/tests/index.integ.test.ts +++ b/packages/benchmark.js-plugin/tests/index.integ.test.ts @@ -13,6 +13,7 @@ jest.mock("@codspeed/core", () => { mockCore.getCallingFile = actual.getCallingFile; mockCore.wrapWithRootFrame = actual.wrapWithRootFrame; mockCore.wrapWithRootFrameSync = actual.wrapWithRootFrameSync; + mockCore.withRoundMarkers = actual.withRoundMarkers; return mockCore; }); diff --git a/packages/core/package.json b/packages/core/package.json index e504c765..a84e2d5d 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -39,6 +39,14 @@ "prebuildify": "^6.0.1" }, "dependencies": { + "@opentelemetry/api": "^1.10.0", + "@opentelemetry/api-logs": "^0.223.0", + "@opentelemetry/context-async-hooks": "^3.1.0", + "@opentelemetry/exporter-logs-otlp-proto": "^0.301.0", + "@opentelemetry/exporter-trace-otlp-proto": "^0.301.0", + "@opentelemetry/resources": "^3.1.0", + "@opentelemetry/sdk-logs": "^3.1.0", + "@opentelemetry/sdk-trace": "^3.1.0", "axios": "^1.21.0", "form-data": "^4.0.6", "node-gyp-build": "^4.6.0", diff --git a/packages/core/rollup.config.mjs b/packages/core/rollup.config.mjs index b7b3a87d..4e7d55a9 100644 --- a/packages/core/rollup.config.mjs +++ b/packages/core/rollup.config.mjs @@ -14,7 +14,10 @@ export default defineConfig([ sourcemap: true, }, ], - plugins: declarationsPlugin({ compilerOptions: { composite: false } }), + plugins: declarationsPlugin({ + // pnpm links the dependencies of a package next to its real path + compilerOptions: { composite: false, preserveSymlinks: false }, + }), }, { input: entrypoint, diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 8ceb582f..12cff68b 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -3,6 +3,7 @@ import { MongoMeasurement } from "./mongoMeasurement"; import native_core from "./native_core"; import { warnOnUnsupportedNodeVersion } from "./nodeVersion"; import { getCodspeedRunnerMode } from "./runnerMode"; +import { setupTelemetry } from "./telemetry"; declare const __VERSION__: string; @@ -12,6 +13,11 @@ export const isBound = native_core.isBound; export const mongoMeasurement = new MongoMeasurement(); +// On import, before the benchmarked modules that the plugins load next get their +// tracers and loggers: those obtained before through another copy of the +// OpenTelemetry API, such as its ESM build, would never export. +setupTelemetry(); + export const setupCore = () => { warnOnUnsupportedNodeVersion(); @@ -64,6 +70,7 @@ export { optimizeFunction, optimizeFunctionSync } from "./optimization"; export { wrapWithRootFrame, wrapWithRootFrameSync } from "./rootFrame"; export { getCodspeedRunnerMode, getInstrumentMode } from "./runnerMode"; export type { InstrumentMode } from "./runnerMode"; +export { flushTelemetry, withRoundMarkers } from "./telemetry"; export * from "./utils"; export * from "./walltime"; export const InstrumentHooks = native_core.InstrumentHooks; diff --git a/packages/core/src/introspection.ts b/packages/core/src/introspection.ts index eeb26010..dfd222ee 100644 --- a/packages/core/src/introspection.ts +++ b/packages/core/src/introspection.ts @@ -1,7 +1,7 @@ import { writeFileSync } from "fs"; import path from "path"; -import { getInstrumentMode } from "./runnerMode"; +import { getCodspeedRunnerMode, getInstrumentMode } from "./runnerMode"; const CUSTOM_INTROSPECTION_EXIT_CODE = 0; @@ -22,6 +22,13 @@ export const getV8Flags = () => { const flags = ["--interpreted-frames-native-stack", "--allow-natives-syntax"]; + // The spans and logs keep the timings of the code as it runs in production, + // so V8 compiles and collects garbage as it does by default. + if (getCodspeedRunnerMode() === "telemetry") { + flags.push(...getSeedFlags(nodeVersionMajor)); + return flags; + } + switch (instrumentMode) { case "analysis": { flags.push( diff --git a/packages/core/src/runnerMode.ts b/packages/core/src/runnerMode.ts index f7d8f046..dddbf837 100644 --- a/packages/core/src/runnerMode.ts +++ b/packages/core/src/runnerMode.ts @@ -2,7 +2,8 @@ export type CodSpeedRunnerMode = | "disabled" | "simulation" | "memory" - | "walltime"; + | "walltime" + | "telemetry"; export type InstrumentMode = "disabled" | "analysis" | "walltime"; @@ -23,6 +24,8 @@ export function getCodspeedRunnerMode(): CodSpeedRunnerMode { return "memory"; } else if (codspeedRunnerMode === "walltime") { return "walltime"; + } else if (codspeedRunnerMode === "telemetry") { + return "telemetry"; } console.warn( @@ -33,8 +36,12 @@ export function getCodspeedRunnerMode(): CodSpeedRunnerMode { export function getInstrumentMode(): InstrumentMode { const runnerMode = getCodspeedRunnerMode(); - // Both "simulation" and "memory" map to "analysis" instrument mode - if (runnerMode === "simulation" || runnerMode === "memory") { + // These modes run each benchmark once + if ( + runnerMode === "simulation" || + runnerMode === "memory" || + runnerMode === "telemetry" + ) { return "analysis"; } return runnerMode; // "disabled" or "walltime" diff --git a/packages/core/src/telemetry.ts b/packages/core/src/telemetry.ts new file mode 100644 index 00000000..c0759026 --- /dev/null +++ b/packages/core/src/telemetry.ts @@ -0,0 +1,297 @@ +/** + * Export of the OpenTelemetry spans and logs emitted by the benchmarks, over + * OTLP/HTTP, to the receiver at `CODSPEED_OTEL_ENDPOINT`. + * + * Everything is a no-op unless the variable is set. The OpenTelemetry SDK is + * only loaded then, so that the measured runs don't load it. + * + * Spans and log records keep the wall clock times of the SDK. They also carry + * their times on the clock of the benchmark markers: spans as their + * `codspeed.monotonic_start_ns` and `codspeed.monotonic_end_ns` attributes, log + * records as their `codspeed.monotonic_time_ns` attribute. + */ + +import type * as Api from "@opentelemetry/api"; +import type * as ApiLogs from "@opentelemetry/api-logs"; +import type * as ContextAsyncHooks from "@opentelemetry/context-async-hooks"; +import type * as LogExporter from "@opentelemetry/exporter-logs-otlp-proto"; +import type * as TraceExporter from "@opentelemetry/exporter-trace-otlp-proto"; +import type * as Resources from "@opentelemetry/resources"; +import type * as SdkLogs from "@opentelemetry/sdk-logs"; +import type * as SdkTrace from "@opentelemetry/sdk-trace"; +import { randomBytes } from "crypto"; +import path from "path"; +import native_core from "./native_core"; +import { getCodspeedRunnerMode } from "./runnerMode"; + +/** Base URL of the OTLP/HTTP receiver, e.g. `http://localhost:4318`. */ +const ENDPOINT_ENV = "CODSPEED_OTEL_ENDPOINT"; + +const MONOTONIC_START_ATTRIBUTE = "codspeed.monotonic_start_ns"; +const MONOTONIC_END_ATTRIBUTE = "codspeed.monotonic_end_ns"; +const MONOTONIC_TIME_ATTRIBUTE = "codspeed.monotonic_time_ns"; + +/** + * Ended spans, and log records, queued for export, past which they are + * dropped. Larger than the SDK default of 2048, which a benchmark emitting on + * every call fills faster than the exporter drains it. + */ +const MAX_QUEUE_SIZE = 1 << 16; + +/** Spans or log records sent per export request. */ +const EXPORT_BATCH_SIZE = 4096; + +interface Providers { + tracer: SdkTrace.TracerProvider; + logger: SdkLogs.LoggerProvider; +} + +/** `undefined` until set up, `null` when nothing is exported. */ +let providers: Providers | null | undefined; + +/** + * The time on the clock of the benchmark markers, in nanoseconds. + * + * OTLP has no unsigned integers: the clock fits in the integer range of a + * double for the first 104 days of uptime. + */ +function monotonicNow(): number { + return Number(native_core.InstrumentHooks.currentTimestamp()); +} + +/** + * Registers global OpenTelemetry tracer and logger providers exporting to the + * receiver at `CODSPEED_OTEL_ENDPOINT`, and a context manager propagating the + * active span across async calls. Does nothing when the variable is unset, or + * when called again. + * + * Providers already registered by the benchmarked code are kept: what they + * emit isn't exported to CodSpeed. + */ +export function setupTelemetry(): void { + if (providers !== undefined) { + return; + } + providers = null; + + const endpoint = process.env[ENDPOINT_ENV]?.replace(/\/+$/, ""); + if (!endpoint) { + return; + } + try { + providers = registerProviders(endpoint); + } catch (error) { + console.warn( + `[CodSpeed] failed to set up the OpenTelemetry export: ${error}`, + ); + return; + } + // The event loop keeps running as long as the shutdown sends requests. + process.once("beforeExit", () => { + void shutdownTelemetry(); + }); +} + +/** + * Exports the spans and log records ended so far. Call it before the process + * may exit without emptying its event loop, e.g. in a worker its pool kills. + */ +export async function flushTelemetry(): Promise { + if (!providers) { + return; + } + const results = await Promise.allSettled([ + providers.tracer.forceFlush(), + providers.logger.forceFlush(), + ]); + warnOnFailures(results, "flush"); +} + +async function shutdownTelemetry(): Promise { + if (!providers) { + return; + } + const { tracer, logger } = providers; + providers = null; + const results = await Promise.allSettled([ + tracer.shutdown(), + logger.shutdown(), + ]); + warnOnFailures(results, "shut down"); +} + +function warnOnFailures( + results: PromiseSettledResult[], + action: string, +): void { + for (const result of results) { + if (result.status === "rejected") { + console.warn( + `[CodSpeed] failed to ${action} the OpenTelemetry export: ${result.reason}`, + ); + } + } +} + +/* eslint-disable @typescript-eslint/no-require-imports */ +function registerProviders(endpoint: string): Providers { + const api = require("@opentelemetry/api") as typeof Api; + const apiLogs = require("@opentelemetry/api-logs") as typeof ApiLogs; + const { AsyncLocalStorageContextManager } = + require("@opentelemetry/context-async-hooks") as typeof ContextAsyncHooks; + const { OTLPLogExporter } = + require("@opentelemetry/exporter-logs-otlp-proto") as typeof LogExporter; + const { OTLPTraceExporter } = + require("@opentelemetry/exporter-trace-otlp-proto") as typeof TraceExporter; + const { resourceFromAttributes } = + require("@opentelemetry/resources") as typeof Resources; + const { BatchLogRecordProcessor, LoggerProvider } = + require("@opentelemetry/sdk-logs") as typeof SdkLogs; + const { BatchSpanProcessor, TracerProvider } = + require("@opentelemetry/sdk-trace") as typeof SdkTrace; + /* eslint-enable @typescript-eslint/no-require-imports */ + + const resource = resourceFromAttributes({ "service.name": serviceName() }); + const tracer = new TracerProvider({ + resource, + idGenerator: new CryptoIdGenerator(), + spanProcessors: [ + // Before the batch processor, which exports the span as it ends. + new MonotonicTimestampSpanProcessor(), + new BatchSpanProcessor({ + exporter: new OTLPTraceExporter({ url: `${endpoint}/v1/traces` }), + maxQueueSize: MAX_QUEUE_SIZE, + maxExportBatchSize: EXPORT_BATCH_SIZE, + }), + ], + }); + const logger = new LoggerProvider({ + resource, + processors: [ + new MonotonicTimeLogRecordProcessor(), + new BatchLogRecordProcessor({ + exporter: new OTLPLogExporter({ url: `${endpoint}/v1/logs` }), + maxQueueSize: MAX_QUEUE_SIZE, + maxExportBatchSize: EXPORT_BATCH_SIZE, + }), + ], + }); + + const contextManager = new AsyncLocalStorageContextManager().enable(); + if (!api.context.setGlobalContextManager(contextManager)) { + contextManager.disable(); + } + if (!api.trace.setGlobalTracerProvider(tracer)) { + console.warn( + "[CodSpeed] an OpenTelemetry tracer provider is already registered, its spans are not exported to CodSpeed", + ); + } + // The logs API moved into `@opentelemetry/api`, with a global provider of its + // own: instrumentations still emitting through `@opentelemetry/api-logs` read + // the provider registered there. + if (!api.logs.setGlobalLoggerProvider(logger)) { + console.warn( + "[CodSpeed] an OpenTelemetry logger provider is already registered, its logs are not exported to CodSpeed", + ); + } + apiLogs.logs.setGlobalLoggerProvider(logger); + + return { tracer, logger }; +} + +/** + * The SDK generates ids from `Math.random()`, which the benchmarks run with a + * fixed seed: every benchmark process would generate the same trace ids. + */ +class CryptoIdGenerator implements SdkTrace.IdGenerator { + generateTraceId(): string { + return randomBytes(16).toString("hex"); + } + + generateSpanId(): string { + return randomBytes(8).toString("hex"); + } +} + +/** + * Sets the times a span starts and ends at on the clock of the benchmark + * markers, as the SDK takes its own times. + */ +class MonotonicTimestampSpanProcessor implements SdkTrace.SpanProcessor { + onStart(span: SdkTrace.Span): void { + span.setAttribute(MONOTONIC_START_ATTRIBUTE, monotonicNow()); + } + + /** Called before the span is ended, when its attributes can still be set. */ + onEnding(span: SdkTrace.Span): void { + span.setAttribute(MONOTONIC_END_ATTRIBUTE, monotonicNow()); + } + + onEnd(): void {} + + async forceFlush(): Promise {} + + async shutdown(): Promise {} +} + +/** Sets the time a log record is emitted at on the clock of the benchmark markers. */ +class MonotonicTimeLogRecordProcessor implements SdkLogs.LogRecordProcessor { + onEmit(logRecord: SdkLogs.ReadWriteLogRecord): void { + logRecord.setAttribute(MONOTONIC_TIME_ATTRIBUTE, monotonicNow()); + } + + async forceFlush(): Promise {} + + async shutdown(): Promise {} +} + +/** + * `OTEL_SERVICE_NAME` when set, else the package running the benchmarks, else + * the entry script. + */ +function serviceName(): string { + const fromEnv = process.env.OTEL_SERVICE_NAME || process.env.npm_package_name; + if (fromEnv) { + return fromEnv; + } + const script = process.argv[1]; + return script + ? path.basename(script, path.extname(script)) + : "codspeed-benchmark"; +} + +/** + * In telemetry mode, wraps the benchmarked function to time each call on the + * clock of the benchmark markers, sent as the bounds of a round of the current + * benchmark. Telemetry emitted in the round is attributed to the benchmark. + * + * Call the returned function between `startBenchmark()` and `stopBenchmark()`: + * the round markers must fall within the sample markers those send. Otherwise, + * returns the function unchanged, keeping the measured code as it is. + */ +export function withRoundMarkers(fn: () => T): () => T { + if (getCodspeedRunnerMode() !== "telemetry") { + return fn; + } + const hooks = native_core.InstrumentHooks; + const addRound = (start: bigint) => { + const end = hooks.currentTimestamp(); + hooks.addMarker(process.pid, hooks.MARKER_TYPE_BENCHMARK_START, start); + hooks.addMarker(process.pid, hooks.MARKER_TYPE_BENCHMARK_END, end); + }; + return () => { + const start = hooks.currentTimestamp(); + let result: T; + try { + result = fn(); + } catch (error) { + addRound(start); + throw error; + } + if (result instanceof Promise) { + return result.finally(() => addRound(start)) as T; + } + addRound(start); + return result; + }; +} diff --git a/packages/core/tests/telemetry.integ.test.ts b/packages/core/tests/telemetry.integ.test.ts new file mode 100644 index 00000000..efd581b7 --- /dev/null +++ b/packages/core/tests/telemetry.integ.test.ts @@ -0,0 +1,116 @@ +import { execFile } from "child_process"; +import http from "http"; +import { AddressInfo } from "net"; +import path from "path"; +import { promisify } from "util"; + +const execFileAsync = promisify(execFile); + +const corePath = path.resolve(__dirname, ".."); + +/** Emits a span with a child and a log in it, then flushes the export. */ +const EMIT_SCRIPT = ` +const core = require(${JSON.stringify(corePath)}); +const { logs, trace } = require("@opentelemetry/api"); +const tracer = trace.getTracer("test"); +tracer.startActiveSpan("parent", async (parent) => { + await tracer.startActiveSpan("child", async (child) => { + await Promise.resolve(); + logs.getLogger("test").emit({ body: "in child" }); + child.end(); + }); + parent.end(); + await core.flushTelemetry(); + const loaded = Object.keys(require.cache).some((file) => + file.includes("@opentelemetry/sdk-trace"), + ); + console.log(JSON.stringify({ loaded })); +}); +`; + +interface ReceivedRequest { + url: string; + contentType: string | undefined; + body: Buffer; +} + +async function startReceiver(): Promise<{ + endpoint: string; + requests: ReceivedRequest[]; + close: () => Promise; +}> { + const requests: ReceivedRequest[] = []; + const server = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (chunk: Buffer) => chunks.push(chunk)); + req.on("end", () => { + requests.push({ + url: req.url ?? "", + contentType: req.headers["content-type"], + body: Buffer.concat(chunks), + }); + res.writeHead(200, { "content-type": "application/x-protobuf" }); + res.end(); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address() as AddressInfo; + return { + endpoint: `http://127.0.0.1:${port}/`, + requests, + close: () => new Promise((resolve) => server.close(() => resolve())), + }; +} + +async function runEmitScript(env: NodeJS.ProcessEnv): Promise { + const { stdout } = await execFileAsync( + process.execPath, + ["-e", EMIT_SCRIPT], + { cwd: corePath, env: { ...process.env, ...env } }, + ); + const lastLine = stdout.trim().split("\n").pop() ?? ""; + return (JSON.parse(lastLine) as { loaded: boolean }).loaded; +} + +describe("telemetry", () => { + it("exports spans and logs with their monotonic times", async () => { + const receiver = await startReceiver(); + try { + const loaded = await runEmitScript({ + CODSPEED_ENV: "runner", + CODSPEED_RUNNER_MODE: "telemetry", + CODSPEED_OTEL_ENDPOINT: receiver.endpoint, + }); + expect(loaded).toBe(true); + + const traces = receiver.requests.filter((r) => r.url === "/v1/traces"); + const logs = receiver.requests.filter((r) => r.url === "/v1/logs"); + expect(traces).toHaveLength(1); + expect(logs).toHaveLength(1); + for (const request of [...traces, ...logs]) { + expect(request.contentType).toBe("application/x-protobuf"); + } + + const traceBody = traces[0].body.toString("latin1"); + expect(traceBody).toContain("parent"); + expect(traceBody).toContain("child"); + expect(traceBody).toContain("codspeed.monotonic_start_ns"); + expect(traceBody).toContain("codspeed.monotonic_end_ns"); + + const logBody = logs[0].body.toString("latin1"); + expect(logBody).toContain("in child"); + expect(logBody).toContain("codspeed.monotonic_time_ns"); + } finally { + await receiver.close(); + } + }); + + it("doesn't load the SDK without an endpoint", async () => { + const loaded = await runEmitScript({ + CODSPEED_ENV: "runner", + CODSPEED_RUNNER_MODE: "simulation", + CODSPEED_OTEL_ENDPOINT: "", + }); + expect(loaded).toBe(false); + }); +}); diff --git a/packages/tinybench-plugin/src/analysis.ts b/packages/tinybench-plugin/src/analysis.ts index 6ebb991d..84c7ef5a 100644 --- a/packages/tinybench-plugin/src/analysis.ts +++ b/packages/tinybench-plugin/src/analysis.ts @@ -4,6 +4,7 @@ import { mongoMeasurement, optimizeFunction, optimizeFunctionSync, + withRoundMarkers, wrapWithRootFrame, wrapWithRootFrameSync, } from "@codspeed/core"; @@ -41,7 +42,7 @@ class AnalysisBenchRunner extends BaseBenchRunner { await mongoMeasurement.start(uri); global.gc?.(); - const rootFrame = wrapWithRootFrame(fn); + const rootFrame = wrapWithRootFrame(withRoundMarkers(fn)); InstrumentHooks.startBenchmark(); try { await rootFrame(); @@ -71,7 +72,7 @@ class AnalysisBenchRunner extends BaseBenchRunner { callSyncHook(task, fnOpts, "beforeEach"); global.gc?.(); - const rootFrame = wrapWithRootFrameSync(fn); + const rootFrame = wrapWithRootFrameSync(withRoundMarkers(fn)); InstrumentHooks.startBenchmark(); try { rootFrame(); diff --git a/packages/tinybench-plugin/src/shared.ts b/packages/tinybench-plugin/src/shared.ts index 2d86eced..ccea7ff3 100644 --- a/packages/tinybench-plugin/src/shared.ts +++ b/packages/tinybench-plugin/src/shared.ts @@ -1,4 +1,9 @@ -import { setupCore, teardownCore, type IsolationMode } from "@codspeed/core"; +import { + flushTelemetry, + setupCore, + teardownCore, + type IsolationMode, +} from "@codspeed/core"; import { Bench, Task } from "tinybench"; import { CapturedTaskData, getTaskData } from "./taskData"; import { getTaskUri } from "./uri"; @@ -87,6 +92,7 @@ export abstract class BaseBenchRunner { const uri = this.getTaskUri(task); await this.runTaskAsync(task, uri); } + await flushTelemetry(); return this.finalizeAsyncRun(); }; diff --git a/packages/vitest-plugin/rollup.config.mjs b/packages/vitest-plugin/rollup.config.mjs index b469682d..136087ca 100644 --- a/packages/vitest-plugin/rollup.config.mjs +++ b/packages/vitest-plugin/rollup.config.mjs @@ -23,6 +23,12 @@ export default defineConfig([ plugins: jsPlugins(pkg.version), external: ["@codspeed/core", /^vitest/], }, + { + input: "src/telemetrySetup.ts", + output: { file: "dist/telemetrySetup.mjs", format: "es" }, + plugins: jsPlugins(pkg.version), + external: ["@codspeed/core", /^vitest/], + }, // Vitest imports these by file path (as `test.runner` or `benchmark.provider`), so // each needs its own output; `resolveFile` expects it at the same path as in `src/`. { diff --git a/packages/vitest-plugin/src/index.ts b/packages/vitest-plugin/src/index.ts index 99ddc66b..a8d6bc4c 100644 --- a/packages/vitest-plugin/src/index.ts +++ b/packages/vitest-plugin/src/index.ts @@ -1,4 +1,5 @@ import { + getCodspeedRunnerMode, getInstrumentMode, getV8Flags, InstrumentHooks, @@ -53,6 +54,9 @@ export default function codspeedPlugin(options: WalltimeOptions = {}): Plugin { test: { pool: "forks", globalSetup: [resolveFile("globalSetup")], + ...(getCodspeedRunnerMode() === "telemetry" && { + setupFiles: [resolveFile("telemetrySetup")], + }), ...backend.getBenchmarkTestConfig( getV8Flags(), resolveFile, diff --git a/packages/vitest-plugin/src/legacy/analysis.ts b/packages/vitest-plugin/src/legacy/analysis.ts index f71f0c0a..ae72f15e 100644 --- a/packages/vitest-plugin/src/legacy/analysis.ts +++ b/packages/vitest-plugin/src/legacy/analysis.ts @@ -1,10 +1,12 @@ import { + flushTelemetry, InstrumentHooks, logDebug, mongoMeasurement, optimizeFunction, setupCore, teardownCore, + withRoundMarkers, wrapWithRootFrame, } from "@codspeed/core"; import { type RunnerTestSuite } from "vitest"; @@ -62,10 +64,11 @@ async function runAnalysisBench( await bench.setup(task, "run"); await callSuiteHook(suite, benchmark, "beforeEach"); await mongoMeasurement.start(uri); + const measuredFn = withRoundMarkers(fn); global.gc?.(); await wrapWithRootFrame(async () => { InstrumentHooks.startBenchmark(); - await fn(); + await measuredFn(); InstrumentHooks.stopBenchmark(); InstrumentHooks.setExecutedBenchmark(process.pid, uri); })(); @@ -111,6 +114,8 @@ export class AnalysisRunner extends NodeBenchmarkRunner { await runAnalysisBenchmarkSuite(suite, await this.importTinybench()); logCodSpeed(`running suite ${suite.name} done`); + // Vitest may kill the worker once its benchmarks are done. + await flushTelemetry(); teardownCore(); } } diff --git a/packages/vitest-plugin/src/telemetrySetup.ts b/packages/vitest-plugin/src/telemetrySetup.ts new file mode 100644 index 00000000..4d87ddef --- /dev/null +++ b/packages/vitest-plugin/src/telemetrySetup.ts @@ -0,0 +1,4 @@ +// Importing the core registers the OpenTelemetry providers, which must happen +// before the benchmark files get their tracers and loggers. Vitest imports the +// setup files ahead of each test file, in the worker running it. +import "@codspeed/core"; diff --git a/packages/vitest-plugin/src/v5/provider.ts b/packages/vitest-plugin/src/v5/provider.ts index 67461416..49bd5f29 100644 --- a/packages/vitest-plugin/src/v5/provider.ts +++ b/packages/vitest-plugin/src/v5/provider.ts @@ -1,4 +1,5 @@ import { + flushTelemetry, getGitDir, getInstrumentMode, InstrumentHooks, @@ -6,6 +7,7 @@ import { setupCore, teardownCore, walltimeIsolation, + withRoundMarkers, wrapWithRootFrame, wrapWithRootFrameSync, writeForkSamples, @@ -167,6 +169,8 @@ async function runAnalysis( console.log(`[CodSpeed] ${label} ${uri}`); results.push(analysisResult(name)); } + // Vitest may kill the worker once its benchmarks are done. + await flushTelemetry(); return results; } @@ -191,10 +195,11 @@ async function runAnalysisTask( }); await fnOpts?.beforeEach?.("run"); + const measuredFn = withRoundMarkers(fn); global.gc?.(); await wrapWithRootFrame(async () => { InstrumentHooks.startBenchmark(); - await fn(); + await measuredFn(); InstrumentHooks.stopBenchmark(); InstrumentHooks.setExecutedBenchmark(process.pid, uri); })(); @@ -212,10 +217,11 @@ async function runAnalysisTask( }); await fnOpts?.beforeEach?.("run"); + const measuredFn = withRoundMarkers(fn); global.gc?.(); wrapWithRootFrameSync(() => { InstrumentHooks.startBenchmark(); - fn(); + measuredFn(); InstrumentHooks.stopBenchmark(); InstrumentHooks.setExecutedBenchmark(process.pid, uri); })(); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1a3c492d..8d9e3546 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -111,6 +111,43 @@ importers: specifier: ^4.0.1 version: 4.0.1 + examples/with-telemetry: + dependencies: + '@opentelemetry/api': + specifier: ^1.10.0 + version: 1.10.0 + '@opentelemetry/api-logs': + specifier: ^0.223.0 + version: 0.223.0 + devDependencies: + '@codspeed/benchmark.js-plugin': + specifier: workspace:* + version: link:../../packages/benchmark.js-plugin + '@codspeed/tinybench-plugin': + specifier: workspace:* + version: link:../../packages/tinybench-plugin + '@codspeed/vitest-plugin': + specifier: workspace:* + version: link:../../packages/vitest-plugin + '@types/benchmark': + specifier: ^2.1.2 + version: 2.1.2 + benchmark: + specifier: ^2.1.4 + version: 2.1.4 + tinybench: + specifier: ^4.0.1 + version: 4.0.1 + tsx: + specifier: ^4.23.12 + version: 4.23.12 + typescript: + specifier: ^5.1.3 + version: 5.8.3 + vitest: + specifier: ^5.0.0 + version: 5.0.2(@opentelemetry/api@1.10.0)(@types/node@26.4.1)(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) + examples/with-tinybench-v5: devDependencies: '@codspeed/tinybench-plugin': @@ -193,7 +230,7 @@ importers: version: 5.1.3 vitest: specifier: ^5.0.0 - version: 5.0.2(@types/node@26.4.1)(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) + version: 5.0.2(@opentelemetry/api@1.10.0)(@types/node@26.4.1)(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) examples/with-typescript-simple-cjs: devDependencies: @@ -270,6 +307,30 @@ importers: packages/core: dependencies: + '@opentelemetry/api': + specifier: ^1.10.0 + version: 1.10.0 + '@opentelemetry/api-logs': + specifier: ^0.223.0 + version: 0.223.0 + '@opentelemetry/context-async-hooks': + specifier: ^3.1.0 + version: 3.1.0(@opentelemetry/api@1.10.0) + '@opentelemetry/exporter-logs-otlp-proto': + specifier: ^0.301.0 + version: 0.301.0(@opentelemetry/api@1.10.0) + '@opentelemetry/exporter-trace-otlp-proto': + specifier: ^0.301.0 + version: 0.301.0(@opentelemetry/api@1.10.0) + '@opentelemetry/resources': + specifier: ^3.1.0 + version: 3.1.0(@opentelemetry/api@1.10.0) + '@opentelemetry/sdk-logs': + specifier: ^3.1.0 + version: 3.1.0(@opentelemetry/api@1.10.0) + '@opentelemetry/sdk-trace': + specifier: ^3.1.0 + version: 3.1.0(@opentelemetry/api@1.10.0) axios: specifier: ^1.21.0 version: 1.21.0 @@ -351,7 +412,7 @@ importers: version: 8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0) vitest: specifier: ^5.0.0 - version: 5.0.2(@types/node@26.4.1)(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) + version: 5.0.2(@opentelemetry/api@1.10.0)(@types/node@26.4.1)(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) packages: @@ -2033,6 +2094,78 @@ packages: '@octokit/types@13.10.0': resolution: {integrity: sha512-ifLaO34EbbPj0Xgro4G5lP5asESjwHracYJvVaPIyXMuiuXLlhic3S47cBdTb+jfODkTE5YtGCLt3Ay3+J97sA==} + '@opentelemetry/api-logs@0.223.0': + resolution: {integrity: sha512-Oepm6etVPFCn0VWB1t833iOTgz1eYBDUPxcrOR/XvwzD1+Ag9VzSPSrrKAFNJ/haatj+8CXbG47pZXvdNQKDJw==} + engines: {node: '>=8.0.0'} + + '@opentelemetry/api@1.10.0': + resolution: {integrity: sha512-lMkyes8iHrG+mk6WH7X70MdCLI2RS0FwNBY1PyQ8UemaijE2m5mv5A0Niy3HtbYTFrgkUEQjFMXmSAk2hQDuPQ==} + engines: {node: '>=8.0.0'} + + '@opentelemetry/context-async-hooks@3.1.0': + resolution: {integrity: sha512-IEgk4jBcLY1yLej9dFSJJWEIkUZTRsfBkJ0p/M8hpFKYBMYNzB8wwoziACFRiFW8mgpyBTRCjqjnd/IV5OoNLg==} + engines: {node: '>=22.15.0'} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.11.0' + + '@opentelemetry/core@3.1.0': + resolution: {integrity: sha512-TS/6jqU5gilzjvjGnsTEKVFF/0MMAtBc2psxaX39IGCBYLat83/iHxYBWn+O6WB+pnyB0SwiH46rvTEp/OeYxA==} + engines: {node: '>=22.15.0'} + peerDependencies: + '@opentelemetry/api': '>=1.10.0 <1.11.0' + + '@opentelemetry/exporter-logs-otlp-proto@0.301.0': + resolution: {integrity: sha512-KQR2KbQcFJHPbTknEXqpdcSyv9UT2xfLaXGNwcxQnjl9QBtSlZSup23AnkOlWuEplQ6aVaaPxSp2a2hfjMA3iQ==} + engines: {node: '>=22.15.0'} + peerDependencies: + '@opentelemetry/api': ^1.10.0 + + '@opentelemetry/exporter-trace-otlp-proto@0.301.0': + resolution: {integrity: sha512-GcW0pOtxbw1IIWiMxu7eBQkpxMuvW7PrcZzG84UwHILzu/tW+HeFu6+kOVQTGsADYod2YNFCgpueXkLFdQ/BSw==} + engines: {node: '>=22.15.0'} + peerDependencies: + '@opentelemetry/api': ^1.10.0 + + '@opentelemetry/otlp-exporter-base@0.301.0': + resolution: {integrity: sha512-X5qmkSNvJMkUWnOsIZ1lg2VkcJkRTuXMvlACngE7EkxKWkfhgjq1GXd/qp8kCATokAq0RfNHtMFIZJ18FRjqKg==} + engines: {node: '>=22.15.0'} + peerDependencies: + '@opentelemetry/api': ^1.10.0 + + '@opentelemetry/otlp-transformer@0.301.0': + resolution: {integrity: sha512-lxFBIG8mhAnFouNxJNpT+6zxn7lrqwej9fPUWbrxuzldYXimesH8p5xwZYPHlapRMlGt2eMBvzMsvGnJUlJMgg==} + engines: {node: '>=22.15.0'} + peerDependencies: + '@opentelemetry/api': '>=1.10.0 <1.11.0' + + '@opentelemetry/resources@3.1.0': + resolution: {integrity: sha512-51dObapLj5PFRoQ+V4G4pZuQhzgFw/fC4m6F6Qa7rhjkrPyldroR34uOhL1vQqbpR02WdRh0JmsxyNlh2YNNvg==} + engines: {node: '>=22.15.0'} + peerDependencies: + '@opentelemetry/api': '>=1.10.0 <1.11.0' + + '@opentelemetry/sdk-logs@3.1.0': + resolution: {integrity: sha512-/WSmZkeGXQjSazW538YsD+3/aiQzZYA81pAddH2+7+VBxuTJl04W9c0xLke+m9jSTfuZaCIS4fbzeSUKKIzc4Q==} + engines: {node: '>=22.15.0'} + peerDependencies: + '@opentelemetry/api': '>=1.10.0 <1.11.0' + + '@opentelemetry/sdk-metrics@3.1.0': + resolution: {integrity: sha512-eU7MfYDxGPAJhA8bKYTkAFpmKrKMbgyKZQoBCKu+L9rJ3SKK3wfN3Vuv1/Kx90nLMwQW0931Zf04bO9LKYbcFQ==} + engines: {node: '>=22.15.0'} + peerDependencies: + '@opentelemetry/api': '>=1.10.0 <1.11.0' + + '@opentelemetry/sdk-trace@3.1.0': + resolution: {integrity: sha512-MlK9KhpKp4Gkz7o9OjwdWgSLCms1mr5SSDykqwaBq4NUsB6IhM4ihXpC6fZek3LGuCB8KIueQAPpcgU1Q3UYkg==} + engines: {node: '>=22.15.0'} + peerDependencies: + '@opentelemetry/api': '>=1.10.0 <1.11.0' + + '@opentelemetry/semantic-conventions@1.44.0': + resolution: {integrity: sha512-E3JkOlMBi3N8OGGkbl7tFyqk2MAS6/lySRZBK3Mu7AzjF1fT8oIChH9ChE48SUWzDn9Kdwta+G/p5LJ4xW4CLQ==} + engines: {node: '>=14'} + '@oxc-project/types@0.147.0': resolution: {integrity: sha512-IJ3s6ltHLp45S0bh7phkX+gJO7A1Wuz2EaqpAhb8WjqDwbzMiWKHhyyT42tskaWjEYXtHtVCPpnBJVT9+dcRLg==} @@ -3949,13 +4082,13 @@ packages: git-raw-commits@2.0.11: resolution: {integrity: sha512-VnctFhw+xfj8Va1xtfEqCUD2XDrbAPSJx+hSrE5K7fGdjZruW7XV+QOrN7LF/RJyvspRiD2I0asWsxFp0ya26A==} engines: {node: '>=10'} - deprecated: This package is no longer maintained. For the JavaScript API, please use @conventional-changelog/git-client instead. + deprecated: Deprecated and no longer maintained. Use @conventional-changelog/git-client instead. hasBin: true git-raw-commits@3.0.0: resolution: {integrity: sha512-b5OHmZ3vAgGrDn/X0kS+9qCfNKWe4K/jFnhwzVWWg0/k5eLa3060tZShrRg8Dja5kPc+YjS0Gc6y7cRr44Lpjw==} engines: {node: '>=14'} - deprecated: This package is no longer maintained. For the JavaScript API, please use @conventional-changelog/git-client instead. + deprecated: Deprecated and no longer maintained. Use @conventional-changelog/git-client instead. hasBin: true git-remote-origin-url@2.0.0: @@ -3965,7 +4098,7 @@ packages: git-semver-tags@5.0.1: resolution: {integrity: sha512-hIvOeZwRbQ+7YEUmCkHqo8FOLQZCEn18yevLHADlFPZY02KJGsu5FZt9YW/lybfK2uhWFI7Qg/07LekJiTv7iA==} engines: {node: '>=14'} - deprecated: This package is no longer maintained. For the JavaScript API, please use @conventional-changelog/git-client instead. + deprecated: Deprecated and no longer maintained. Use @conventional-changelog/git-client instead. hasBin: true git-up@7.0.0: @@ -9021,6 +9154,78 @@ snapshots: dependencies: '@octokit/openapi-types': 24.2.0 + '@opentelemetry/api-logs@0.223.0': + dependencies: + '@opentelemetry/api': 1.10.0 + + '@opentelemetry/api@1.10.0': {} + + '@opentelemetry/context-async-hooks@3.1.0(@opentelemetry/api@1.10.0)': + dependencies: + '@opentelemetry/api': 1.10.0 + + '@opentelemetry/core@3.1.0(@opentelemetry/api@1.10.0)': + dependencies: + '@opentelemetry/api': 1.10.0 + '@opentelemetry/semantic-conventions': 1.44.0 + + '@opentelemetry/exporter-logs-otlp-proto@0.301.0(@opentelemetry/api@1.10.0)': + dependencies: + '@opentelemetry/api': 1.10.0 + '@opentelemetry/otlp-exporter-base': 0.301.0(@opentelemetry/api@1.10.0) + '@opentelemetry/otlp-transformer': 0.301.0(@opentelemetry/api@1.10.0) + '@opentelemetry/sdk-logs': 3.1.0(@opentelemetry/api@1.10.0) + + '@opentelemetry/exporter-trace-otlp-proto@0.301.0(@opentelemetry/api@1.10.0)': + dependencies: + '@opentelemetry/api': 1.10.0 + '@opentelemetry/otlp-exporter-base': 0.301.0(@opentelemetry/api@1.10.0) + '@opentelemetry/otlp-transformer': 0.301.0(@opentelemetry/api@1.10.0) + '@opentelemetry/sdk-trace': 3.1.0(@opentelemetry/api@1.10.0) + + '@opentelemetry/otlp-exporter-base@0.301.0(@opentelemetry/api@1.10.0)': + dependencies: + '@opentelemetry/api': 1.10.0 + '@opentelemetry/core': 3.1.0(@opentelemetry/api@1.10.0) + '@opentelemetry/otlp-transformer': 0.301.0(@opentelemetry/api@1.10.0) + + '@opentelemetry/otlp-transformer@0.301.0(@opentelemetry/api@1.10.0)': + dependencies: + '@opentelemetry/api': 1.10.0 + '@opentelemetry/core': 3.1.0(@opentelemetry/api@1.10.0) + '@opentelemetry/resources': 3.1.0(@opentelemetry/api@1.10.0) + '@opentelemetry/sdk-logs': 3.1.0(@opentelemetry/api@1.10.0) + '@opentelemetry/sdk-metrics': 3.1.0(@opentelemetry/api@1.10.0) + '@opentelemetry/sdk-trace': 3.1.0(@opentelemetry/api@1.10.0) + + '@opentelemetry/resources@3.1.0(@opentelemetry/api@1.10.0)': + dependencies: + '@opentelemetry/api': 1.10.0 + '@opentelemetry/core': 3.1.0(@opentelemetry/api@1.10.0) + '@opentelemetry/semantic-conventions': 1.44.0 + + '@opentelemetry/sdk-logs@3.1.0(@opentelemetry/api@1.10.0)': + dependencies: + '@opentelemetry/api': 1.10.0 + '@opentelemetry/core': 3.1.0(@opentelemetry/api@1.10.0) + '@opentelemetry/resources': 3.1.0(@opentelemetry/api@1.10.0) + '@opentelemetry/semantic-conventions': 1.44.0 + + '@opentelemetry/sdk-metrics@3.1.0(@opentelemetry/api@1.10.0)': + dependencies: + '@opentelemetry/api': 1.10.0 + '@opentelemetry/core': 3.1.0(@opentelemetry/api@1.10.0) + '@opentelemetry/resources': 3.1.0(@opentelemetry/api@1.10.0) + + '@opentelemetry/sdk-trace@3.1.0(@opentelemetry/api@1.10.0)': + dependencies: + '@opentelemetry/api': 1.10.0 + '@opentelemetry/core': 3.1.0(@opentelemetry/api@1.10.0) + '@opentelemetry/resources': 3.1.0(@opentelemetry/api@1.10.0) + '@opentelemetry/semantic-conventions': 1.44.0 + + '@opentelemetry/semantic-conventions@1.44.0': {} + '@oxc-project/types@0.147.0': {} '@pkgjs/parseargs@0.11.0': @@ -14407,7 +14612,7 @@ snapshots: - tsx - yaml - vitest@5.0.2(@types/node@26.4.1)(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)): + vitest@5.0.2(@opentelemetry/api@1.10.0)(@types/node@26.4.1)(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)): dependencies: '@types/chai': 5.2.2 '@vitest/mocker': 5.0.2(vite@8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0)) @@ -14424,6 +14629,7 @@ snapshots: vite: 8.2.2(@types/node@26.4.1)(esbuild@0.28.2)(tsx@4.23.12)(yaml@2.9.0) why-is-node-running: 3.2.2 optionalDependencies: + '@opentelemetry/api': 1.10.0 '@types/node': 26.4.1 transitivePeerDependencies: - msw From df10575f2b17a5266164f44aee0c43cc9d8f4ecd Mon Sep 17 00:00:00 2001 From: Luc Varoqui Date: Mon, 16 Nov 2026 15:42:39 +0100 Subject: [PATCH 2/3] fix: end the round after thenables and flush the telemetry of failed runs `withRoundMarkers` only waited for native promises. A benchmark returning another thenable, such as a database query, got its end marker as soon as it returned, so the spans and logs it emitted afterwards fell outside its round. It now waits for any object with a `then` method, and returns a native promise, which the plugins only await. The Vitest runners skipped `flushTelemetry()` when a benchmark or a hook threw, so Vitest could stop the worker with spans and logs still queued, including those that explain the failure. They now flush in a `finally` block. Refs COD-4149 Co-Authored-By: Claude Opus 5.5 --- packages/core/src/telemetry.ts | 14 ++++++++++++-- packages/vitest-plugin/src/legacy/analysis.ts | 9 ++++++--- packages/vitest-plugin/src/v5/provider.ts | 17 ++++++++++------- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/packages/core/src/telemetry.ts b/packages/core/src/telemetry.ts index c0759026..f71bc39c 100644 --- a/packages/core/src/telemetry.ts +++ b/packages/core/src/telemetry.ts @@ -288,10 +288,20 @@ export function withRoundMarkers(fn: () => T): () => T { addRound(start); throw error; } - if (result instanceof Promise) { - return result.finally(() => addRound(start)) as T; + if (isPromiseLike(result)) { + // Covers the thenables that aren't native promises, such as database + // queries. The caller gets a native promise, which it only awaits. + return Promise.resolve(result).finally(() => addRound(start)) as T; } addRound(start); return result; }; } + +function isPromiseLike(value: unknown): value is PromiseLike { + return ( + (typeof value === "object" || typeof value === "function") && + value !== null && + typeof (value as { then?: unknown }).then === "function" + ); +} diff --git a/packages/vitest-plugin/src/legacy/analysis.ts b/packages/vitest-plugin/src/legacy/analysis.ts index ae72f15e..2ba264aa 100644 --- a/packages/vitest-plugin/src/legacy/analysis.ts +++ b/packages/vitest-plugin/src/legacy/analysis.ts @@ -111,11 +111,14 @@ export class AnalysisRunner extends NodeBenchmarkRunner { patchRootSuiteWithFullFilePath(suite); logCodSpeed(`running suite ${suite.name}`); - await runAnalysisBenchmarkSuite(suite, await this.importTinybench()); + try { + await runAnalysisBenchmarkSuite(suite, await this.importTinybench()); + } finally { + // Vitest may kill the worker once its benchmarks are done, or failed. + await flushTelemetry(); + } logCodSpeed(`running suite ${suite.name} done`); - // Vitest may kill the worker once its benchmarks are done. - await flushTelemetry(); teardownCore(); } } diff --git a/packages/vitest-plugin/src/v5/provider.ts b/packages/vitest-plugin/src/v5/provider.ts index 49bd5f29..8b74d5d1 100644 --- a/packages/vitest-plugin/src/v5/provider.ts +++ b/packages/vitest-plugin/src/v5/provider.ts @@ -163,14 +163,17 @@ async function runAnalysis( ): Promise { const label = InstrumentHooks.isInstrumented() ? "Measured" : "Checked"; const results: BenchResult[] = []; - for (const { name, fn, fnOpts } of registrations) { - const uri = getUri(name); - await runAnalysisTask(fn, fnOpts, uri); - console.log(`[CodSpeed] ${label} ${uri}`); - results.push(analysisResult(name)); + try { + for (const { name, fn, fnOpts } of registrations) { + const uri = getUri(name); + await runAnalysisTask(fn, fnOpts, uri); + console.log(`[CodSpeed] ${label} ${uri}`); + results.push(analysisResult(name)); + } + } finally { + // Vitest may kill the worker once its benchmarks are done, or failed. + await flushTelemetry(); } - // Vitest may kill the worker once its benchmarks are done. - await flushTelemetry(); return results; } From 5fc9260db286da61435881915463976d6415db22 Mon Sep 17 00:00:00 2001 From: Luc Varoqui Date: Mon, 16 Nov 2026 15:42:40 +0100 Subject: [PATCH 3/3] test(core): check the links and times of the exported telemetry The integration test only searched the exported protobuf bytes for names and attribute keys: it would pass with zero timestamps, unlinked spans or a log missing its span. Split the building of the providers out of `registerProviders` into `buildProviders`, which takes the span and log exporters, and test it with the SDK's in-memory exporters. The test checks that the child span links to its parent, that a log links to its span, and that the monotonic times of the spans and logs are in order. The subprocess test still covers the OTLP/HTTP export. Refs COD-4149 Co-Authored-By: Claude Opus 5.5 --- packages/core/src/telemetry.ts | 63 ++++++++++------ packages/core/tests/telemetry.integ.test.ts | 81 +++++++++++++++++++++ 2 files changed, 122 insertions(+), 22 deletions(-) diff --git a/packages/core/src/telemetry.ts b/packages/core/src/telemetry.ts index f71bc39c..ce2153b7 100644 --- a/packages/core/src/telemetry.ts +++ b/packages/core/src/telemetry.ts @@ -143,6 +143,45 @@ function registerProviders(endpoint: string): Providers { require("@opentelemetry/exporter-logs-otlp-proto") as typeof LogExporter; const { OTLPTraceExporter } = require("@opentelemetry/exporter-trace-otlp-proto") as typeof TraceExporter; + /* eslint-enable @typescript-eslint/no-require-imports */ + + const { tracer, logger } = buildProviders( + new OTLPTraceExporter({ url: `${endpoint}/v1/traces` }), + new OTLPLogExporter({ url: `${endpoint}/v1/logs` }), + ); + + const contextManager = new AsyncLocalStorageContextManager().enable(); + if (!api.context.setGlobalContextManager(contextManager)) { + contextManager.disable(); + } + if (!api.trace.setGlobalTracerProvider(tracer)) { + console.warn( + "[CodSpeed] an OpenTelemetry tracer provider is already registered, its spans are not exported to CodSpeed", + ); + } + // The logs API moved into `@opentelemetry/api`, with a global provider of its + // own: instrumentations still emitting through `@opentelemetry/api-logs` read + // the provider registered there. + if (!api.logs.setGlobalLoggerProvider(logger)) { + console.warn( + "[CodSpeed] an OpenTelemetry logger provider is already registered, its logs are not exported to CodSpeed", + ); + } + apiLogs.logs.setGlobalLoggerProvider(logger); + + return { tracer, logger }; +} + +/** + * Builds the tracer and logger providers that send the spans to `spanExporter` + * and the log records to `logExporter`, with their times on the clock of the + * benchmark markers. Doesn't register them. + */ +export function buildProviders( + spanExporter: SdkTrace.SpanExporter, + logExporter: SdkLogs.LogRecordExporter, +): Providers { + /* eslint-disable @typescript-eslint/no-require-imports */ const { resourceFromAttributes } = require("@opentelemetry/resources") as typeof Resources; const { BatchLogRecordProcessor, LoggerProvider } = @@ -159,7 +198,7 @@ function registerProviders(endpoint: string): Providers { // Before the batch processor, which exports the span as it ends. new MonotonicTimestampSpanProcessor(), new BatchSpanProcessor({ - exporter: new OTLPTraceExporter({ url: `${endpoint}/v1/traces` }), + exporter: spanExporter, maxQueueSize: MAX_QUEUE_SIZE, maxExportBatchSize: EXPORT_BATCH_SIZE, }), @@ -170,32 +209,12 @@ function registerProviders(endpoint: string): Providers { processors: [ new MonotonicTimeLogRecordProcessor(), new BatchLogRecordProcessor({ - exporter: new OTLPLogExporter({ url: `${endpoint}/v1/logs` }), + exporter: logExporter, maxQueueSize: MAX_QUEUE_SIZE, maxExportBatchSize: EXPORT_BATCH_SIZE, }), ], }); - - const contextManager = new AsyncLocalStorageContextManager().enable(); - if (!api.context.setGlobalContextManager(contextManager)) { - contextManager.disable(); - } - if (!api.trace.setGlobalTracerProvider(tracer)) { - console.warn( - "[CodSpeed] an OpenTelemetry tracer provider is already registered, its spans are not exported to CodSpeed", - ); - } - // The logs API moved into `@opentelemetry/api`, with a global provider of its - // own: instrumentations still emitting through `@opentelemetry/api-logs` read - // the provider registered there. - if (!api.logs.setGlobalLoggerProvider(logger)) { - console.warn( - "[CodSpeed] an OpenTelemetry logger provider is already registered, its logs are not exported to CodSpeed", - ); - } - apiLogs.logs.setGlobalLoggerProvider(logger); - return { tracer, logger }; } diff --git a/packages/core/tests/telemetry.integ.test.ts b/packages/core/tests/telemetry.integ.test.ts index efd581b7..af8a0918 100644 --- a/packages/core/tests/telemetry.integ.test.ts +++ b/packages/core/tests/telemetry.integ.test.ts @@ -1,8 +1,22 @@ +import { context } from "@opentelemetry/api"; +import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks"; +import { InMemoryLogRecordExporter } from "@opentelemetry/sdk-logs"; +import { InMemorySpanExporter } from "@opentelemetry/sdk-trace"; import { execFile } from "child_process"; import http from "http"; import { AddressInfo } from "net"; import path from "path"; import { promisify } from "util"; +import { buildProviders } from "../src/telemetry"; + +// The addon only loads from the built package. The clock of the markers is +// monotonic, like this one. +jest.mock("../src/native_core", () => ({ + __esModule: true, + default: { + InstrumentHooks: { currentTimestamp: () => process.hrtime.bigint() }, + }, +})); const execFileAsync = promisify(execFile); @@ -114,3 +128,70 @@ describe("telemetry", () => { expect(loaded).toBe(false); }); }); + +describe("buildProviders", () => { + beforeAll(() => { + context.setGlobalContextManager( + new AsyncLocalStorageContextManager().enable(), + ); + }); + + afterAll(() => { + context.disable(); + }); + + it("links the spans and logs, with their monotonic times in order", async () => { + const spanExporter = new InMemorySpanExporter(); + const logExporter = new InMemoryLogRecordExporter(); + const providers = buildProviders(spanExporter, logExporter); + const tracer = providers.tracer.getTracer("test"); + const logger = providers.logger.getLogger("test"); + const now = () => Number(process.hrtime.bigint()); + + const before = now(); + await tracer.startActiveSpan("parent", async (parent) => { + await tracer.startActiveSpan("child", async (child) => { + await Promise.resolve(); + logger.emit({ body: "in child" }); + child.end(); + }); + parent.end(); + }); + logger.emit({ body: "outside any span" }); + const after = now(); + await providers.tracer.forceFlush(); + await providers.logger.forceFlush(); + + const spans = spanExporter.getFinishedSpans(); + const parent = spans.find((span) => span.name === "parent"); + const child = spans.find((span) => span.name === "child"); + if (parent === undefined || child === undefined) { + throw new Error(`expected a parent and a child span, got ${spans}`); + } + expect(child.spanContext().traceId).toBe(parent.spanContext().traceId); + expect(child.parentSpanContext?.spanId).toBe(parent.spanContext().spanId); + + const startOf = (span: typeof parent) => + span.attributes["codspeed.monotonic_start_ns"] as number; + const endOf = (span: typeof parent) => + span.attributes["codspeed.monotonic_end_ns"] as number; + expect(before).toBeLessThanOrEqual(startOf(parent)); + expect(startOf(parent)).toBeLessThanOrEqual(startOf(child)); + expect(startOf(child)).toBeLessThanOrEqual(endOf(child)); + expect(endOf(child)).toBeLessThanOrEqual(endOf(parent)); + expect(endOf(parent)).toBeLessThanOrEqual(after); + + const [inChild, outside] = logExporter.getFinishedLogRecords(); + expect(inChild.body).toBe("in child"); + expect(inChild.spanContext?.traceId).toBe(child.spanContext().traceId); + expect(inChild.spanContext?.spanId).toBe(child.spanContext().spanId); + const logTime = inChild.attributes["codspeed.monotonic_time_ns"] as number; + expect(startOf(child)).toBeLessThanOrEqual(logTime); + expect(logTime).toBeLessThanOrEqual(endOf(child)); + + expect(outside.body).toBe("outside any span"); + expect(outside.spanContext).toBeUndefined(); + const outsideTime = outside.attributes["codspeed.monotonic_time_ns"]; + expect(outsideTime).toBeGreaterThanOrEqual(endOf(parent)); + }); +});