diff --git a/.changeset/cuddly-boats-press.md b/.changeset/cuddly-boats-press.md new file mode 100644 index 0000000000..7d44263936 --- /dev/null +++ b/.changeset/cuddly-boats-press.md @@ -0,0 +1,5 @@ +--- +"trigger.dev": patch +--- + +Add external log exporters and fix missing external trace exporters in deployed tasks diff --git a/packages/cli-v3/src/entryPoints/dev-run-worker.ts b/packages/cli-v3/src/entryPoints/dev-run-worker.ts index 66ed02d0db..f3994cd3c7 100644 --- a/packages/cli-v3/src/entryPoints/dev-run-worker.ts +++ b/packages/cli-v3/src/entryPoints/dev-run-worker.ts @@ -164,6 +164,7 @@ async function bootstrap() { url: env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318", instrumentations: config.telemetry?.instrumentations ?? config.instrumentations ?? [], exporters: config.telemetry?.exporters ?? [], + logExporters: config.telemetry?.logExporters ?? [], diagLogLevel: (env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none", forceFlushTimeoutMillis: 30_000, }); @@ -174,7 +175,8 @@ async function bootstrap() { const tracer = new TriggerTracer({ tracer: otelTracer, logger: otelLogger }); const consoleInterceptor = new ConsoleInterceptor( otelLogger, - typeof config.enableConsoleLogging === "boolean" ? config.enableConsoleLogging : true + typeof config.enableConsoleLogging === "boolean" ? config.enableConsoleLogging : true, + typeof config.disableConsoleInterceptor === "boolean" ? config.disableConsoleInterceptor : false ); const configLogLevel = triggerLogLevel ?? config.logLevel ?? "info"; diff --git a/packages/cli-v3/src/entryPoints/managed-run-worker.ts b/packages/cli-v3/src/entryPoints/managed-run-worker.ts index d1dc76a27b..b0ff027ec4 100644 --- a/packages/cli-v3/src/entryPoints/managed-run-worker.ts +++ b/packages/cli-v3/src/entryPoints/managed-run-worker.ts @@ -163,6 +163,8 @@ async function bootstrap() { instrumentations: config.instrumentations ?? [], diagLogLevel: (env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none", forceFlushTimeoutMillis: 30_000, + exporters: config.telemetry?.exporters ?? [], + logExporters: config.telemetry?.logExporters ?? [], }); const otelTracer: Tracer = tracingSDK.getTracer("trigger-dev-worker", VERSION); @@ -171,7 +173,8 @@ async function bootstrap() { const tracer = new TriggerTracer({ tracer: otelTracer, logger: otelLogger }); const consoleInterceptor = new ConsoleInterceptor( otelLogger, - typeof config.enableConsoleLogging === "boolean" ? config.enableConsoleLogging : true + typeof config.enableConsoleLogging === "boolean" ? config.enableConsoleLogging : true, + typeof config.disableConsoleInterceptor === "boolean" ? config.disableConsoleInterceptor : false ); const configLogLevel = triggerLogLevel ?? config.logLevel ?? "info"; diff --git a/packages/core/src/v3/config.ts b/packages/core/src/v3/config.ts index 9be80decc6..a98b8f03d5 100644 --- a/packages/core/src/v3/config.ts +++ b/packages/core/src/v3/config.ts @@ -11,6 +11,7 @@ import type { } from "./index.js"; import type { LogLevel } from "./logger/taskLogger.js"; import type { MachinePresetName } from "./schemas/common.js"; +import { LogRecordExporter } from "@opentelemetry/sdk-logs"; export type CompatibilityFlag = "run_engine_v2"; @@ -80,6 +81,13 @@ export type TriggerConfig = { * @see https://trigger.dev/docs/config/config-file#exporters */ exporters?: Array; + + /** + * Log exporters to use for OpenTelemetry. This is useful if you want to add custom log exporters to your tasks. + * + * @see https://trigger.dev/docs/config/config-file#exporters + */ + logExporters?: Array; }; /** @@ -131,6 +139,11 @@ export type TriggerConfig = { */ enableConsoleLogging?: boolean; + /** + * Disable the console interceptor. This will prevent logs from being sent to the trigger.dev backend. + */ + disableConsoleInterceptor?: boolean; + build?: { /** * Add custom conditions to the esbuild build. For example, if you are importing `ai/rsc`, you'll need to add "react-server" condition. diff --git a/packages/core/src/v3/consoleInterceptor.ts b/packages/core/src/v3/consoleInterceptor.ts index 82b34429f3..2d809ffbeb 100644 --- a/packages/core/src/v3/consoleInterceptor.ts +++ b/packages/core/src/v3/consoleInterceptor.ts @@ -10,12 +10,17 @@ import { clock } from "./clock-api.js"; export class ConsoleInterceptor { constructor( private readonly logger: logsAPI.Logger, - private readonly sendToStdIO: boolean + private readonly sendToStdIO: boolean, + private readonly interceptingDisabled: boolean ) {} // Intercept the console and send logs to the OpenTelemetry logger // during the execution of the callback async intercept(console: Console, callback: () => Promise): Promise { + if (this.interceptingDisabled) { + return await callback(); + } + // Save the original console methods const originalConsole = { log: console.log, diff --git a/packages/core/src/v3/otel/tracingSDK.ts b/packages/core/src/v3/otel/tracingSDK.ts index 171ccc440d..20a79b3b1b 100644 --- a/packages/core/src/v3/otel/tracingSDK.ts +++ b/packages/core/src/v3/otel/tracingSDK.ts @@ -1,4 +1,5 @@ import { DiagConsoleLogger, DiagLogLevel, TracerProvider, diag } from "@opentelemetry/api"; +import { RandomIdGenerator } from "@opentelemetry/sdk-trace-base"; import { logs } from "@opentelemetry/api-logs"; import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http"; import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; @@ -15,6 +16,8 @@ import { import { BatchLogRecordProcessor, LoggerProvider, + LogRecordExporter, + ReadableLogRecord, SimpleLogRecordProcessor, } from "@opentelemetry/sdk-logs"; import { @@ -87,9 +90,12 @@ export type TracingSDKConfig = { resource?: IResource; instrumentations?: Instrumentation[]; exporters?: SpanExporter[]; + logExporters?: LogRecordExporter[]; diagLogLevel?: TracingDiagnosticLogLevel; }; +const idGenerator = new RandomIdGenerator(); + export class TracingSDK { public readonly asyncResourceDetector = new AsyncResourceDetector(); private readonly _logProvider: LoggerProvider; @@ -158,7 +164,7 @@ export class TracingSDK { ) ); - const externalTraceId = crypto.randomUUID(); + const externalTraceId = idGenerator.generateTraceId(); for (const exporter of config.exporters ?? []) { traceProvider.addSpanProcessor( @@ -210,6 +216,28 @@ export class TracingSDK { ) ); + for (const externalLogExporter of config.logExporters ?? []) { + loggerProvider.addLogRecordProcessor( + getEnvVar("OTEL_BATCH_PROCESSING_ENABLED") === "1" + ? new BatchLogRecordProcessor( + new ExternalLogRecordExporterWrapper(externalLogExporter, externalTraceId), + { + maxExportBatchSize: parseInt(getEnvVar("OTEL_LOG_MAX_EXPORT_BATCH_SIZE") ?? "64"), + scheduledDelayMillis: parseInt( + getEnvVar("OTEL_LOG_SCHEDULED_DELAY_MILLIS") ?? "200" + ), + exportTimeoutMillis: parseInt( + getEnvVar("OTEL_LOG_EXPORT_TIMEOUT_MILLIS") ?? "30000" + ), + maxQueueSize: parseInt(getEnvVar("OTEL_LOG_MAX_QUEUE_SIZE") ?? "512"), + } + ) + : new SimpleLogRecordProcessor( + new ExternalLogRecordExporterWrapper(externalLogExporter, externalTraceId) + ) + ); + } + this._logProvider = loggerProvider; this._spanExporter = spanExporter; this._traceProvider = traceProvider; @@ -306,3 +334,50 @@ class ExternalSpanExporterWrapper { : Promise.resolve(); } } + +class ExternalLogRecordExporterWrapper { + constructor( + private underlyingExporter: LogRecordExporter, + private externalTraceId: string + ) {} + + export(logs: any[], resultCallback: (result: any) => void): void { + const modifiedLogs = logs.map(this.transformLogRecord.bind(this)); + + this.underlyingExporter.export(modifiedLogs, resultCallback); + } + + shutdown(): Promise { + return this.underlyingExporter.shutdown(); + } + + transformLogRecord(logRecord: ReadableLogRecord): ReadableLogRecord { + // If there's no spanContext, or if the externalTraceId is not set, return the original logRecord. + if (!logRecord.spanContext || !this.externalTraceId) { + return logRecord; + } + + // Capture externalTraceId for use within the proxy's scope. + const { externalTraceId } = this; + + return new Proxy(logRecord, { + get(target, prop, receiver) { + if (prop === "spanContext") { + // Intercept access to spanContext. + const originalSpanContext = target.spanContext; + // Ensure originalSpanContext exists (it should, due to the check above, but good for safety). + if (originalSpanContext) { + return { + ...originalSpanContext, + traceId: externalTraceId, // Override traceId. + }; + } + // Fallback if, for some reason, originalSpanContext is undefined here. + return undefined; + } + // For all other properties, defer to the original object. + return Reflect.get(target, prop, receiver); + }, + }); + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cc542efdfc..9ed8e79845 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1828,6 +1828,12 @@ importers: '@e2b/code-interpreter': specifier: ^1.1.0 version: 1.1.0 + '@opentelemetry/exporter-logs-otlp-http': + specifier: 0.52.1 + version: 0.52.1(@opentelemetry/api@1.9.0) + '@opentelemetry/exporter-trace-otlp-http': + specifier: 0.52.1 + version: 0.52.1(@opentelemetry/api@1.9.0) '@radix-ui/react-avatar': specifier: ^1.1.3 version: 1.1.3(@types/react-dom@19.0.4)(@types/react@19.0.12)(react-dom@19.0.0)(react@19.0.0) @@ -1866,7 +1872,7 @@ importers: version: 5.1.5 next: specifier: 15.2.4 - version: 15.2.4(@playwright/test@1.37.0)(react-dom@19.0.0)(react@19.0.0) + version: 15.2.4(@opentelemetry/api@1.9.0)(@playwright/test@1.37.0)(react-dom@19.0.0)(react@19.0.0) react: specifier: ^19.0.0 version: 19.0.0 @@ -1954,7 +1960,7 @@ importers: version: 5.1.5 next: specifier: 15.2.4 - version: 15.2.4(@playwright/test@1.37.0)(react-dom@19.0.0)(react@19.0.0) + version: 15.2.4(@opentelemetry/api@1.9.0)(@playwright/test@1.37.0)(react-dom@19.0.0)(react@19.0.0) react: specifier: ^19.0.0 version: 19.0.0 @@ -28612,7 +28618,7 @@ packages: - babel-plugin-macros dev: false - /next@15.2.4(@playwright/test@1.37.0)(react-dom@19.0.0)(react@19.0.0): + /next@15.2.4(@opentelemetry/api@1.9.0)(@playwright/test@1.37.0)(react-dom@19.0.0)(react@19.0.0): resolution: {integrity: sha512-VwL+LAaPSxEkd3lU2xWbgEOtrM8oedmyhBqaVNmgKB+GvZlCy9rgaEc+y2on0wv+l0oSFqLtYD6dcC1eAedUaQ==} engines: {node: ^18.18.0 || ^19.8.0 || >= 20.0.0} hasBin: true @@ -28634,6 +28640,7 @@ packages: optional: true dependencies: '@next/env': 15.2.4 + '@opentelemetry/api': 1.9.0 '@playwright/test': 1.37.0 '@swc/counter': 0.1.3 '@swc/helpers': 0.5.15 diff --git a/references/d3-chat/package.json b/references/d3-chat/package.json index d8de7d79c7..2cd49a089c 100644 --- a/references/d3-chat/package.json +++ b/references/d3-chat/package.json @@ -27,6 +27,8 @@ "@trigger.dev/python": "workspace:*", "@trigger.dev/react-hooks": "workspace:*", "@trigger.dev/sdk": "workspace:*", + "@opentelemetry/exporter-logs-otlp-http": "0.52.1", + "@opentelemetry/exporter-trace-otlp-http": "0.52.1", "@vercel/postgres": "^0.10.0", "ai": "4.2.5", "class-variance-authority": "^0.7.1", diff --git a/references/d3-chat/src/trigger/chat.ts b/references/d3-chat/src/trigger/chat.ts index cd7a6de71f..aed8ebec11 100644 --- a/references/d3-chat/src/trigger/chat.ts +++ b/references/d3-chat/src/trigger/chat.ts @@ -227,6 +227,8 @@ export const interruptibleChat = schemaTask({ prompt: z.string().describe("The prompt to chat with the AI"), }), run: async ({ prompt }, { signal }) => { + logger.info("interruptible-chat: starting", { prompt }); + const chunks: TextStreamPart<{}>[] = []; // 👇 This is a global onCancel hook, but it's inside of the run function diff --git a/references/d3-chat/trigger.config.ts b/references/d3-chat/trigger.config.ts index 745f065e19..603909666a 100644 --- a/references/d3-chat/trigger.config.ts +++ b/references/d3-chat/trigger.config.ts @@ -1,10 +1,32 @@ import { defineConfig } from "@trigger.dev/sdk"; import { pythonExtension } from "@trigger.dev/python/extension"; import { installPlaywrightChromium } from "./src/extensions/playwright"; +import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http"; +import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; export default defineConfig({ project: "proj_cdmymsrobxmcgjqzhdkq", dirs: ["./src/trigger"], + telemetry: { + logExporters: [ + new OTLPLogExporter({ + url: "https://api.axiom.co/v1/logs", + headers: { + Authorization: `Bearer ${process.env.AXIOM_TOKEN}`, + "X-Axiom-Dataset": "d3-chat-tester", + }, + }), + ], + exporters: [ + new OTLPTraceExporter({ + url: "https://api.axiom.co/v1/traces", + headers: { + Authorization: `Bearer ${process.env.AXIOM_TOKEN}`, + "X-Axiom-Dataset": "d3-chat-tester", + }, + }), + ], + }, maxDuration: 3600, build: { extensions: [