diff --git a/.changeset/little-trains-begin.md b/.changeset/little-trains-begin.md new file mode 100644 index 0000000000..11e4b8dc22 --- /dev/null +++ b/.changeset/little-trains-begin.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/python": patch +--- + +Introduced a new Python extension to enhance the build process. It now allows users to execute Python scripts with improved support and error handling. diff --git a/packages/build/README.md b/packages/build/README.md index d3ea2f35f9..fdf6403c2f 100644 --- a/packages/build/README.md +++ b/packages/build/README.md @@ -1,3 +1,3 @@ -# Official TypeScript SDK for Trigger.dev +# Official Build Package of Trigger.dev -View the full documentation for the [here](https://trigger.dev/docs) +View the full documentation [here](https://trigger.dev/docs) diff --git a/packages/python/CHANGELOG.md b/packages/python/CHANGELOG.md new file mode 100644 index 0000000000..abac457942 --- /dev/null +++ b/packages/python/CHANGELOG.md @@ -0,0 +1 @@ +# @trigger.dev/python diff --git a/packages/python/LICENSE b/packages/python/LICENSE new file mode 100644 index 0000000000..b448fb3800 --- /dev/null +++ b/packages/python/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Trigger.dev + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/python/README.md b/packages/python/README.md new file mode 100644 index 0000000000..7dd697b46c --- /dev/null +++ b/packages/python/README.md @@ -0,0 +1,102 @@ +# Python Extension for Trigger.dev + +The Python extension enhances Trigger.dev's build process by enabling limited support for executing Python scripts within your tasks. + +## Overview + +This extension introduces the pythonExtension build extension, which offers several key capabilities: + +- **Install Python Dependencies (Except in Dev):** Automatically installs Python and specified dependencies using pip. +- **Requirements File Support:** You can specify dependencies in a requirements.txt file. +- **Inline Requirements:** Define dependencies directly within your trigger.config.ts file using the requirements option. +- **Virtual Environment:** Creates a virtual environment (/opt/venv) inside containers to isolate Python dependencies. +- **Helper Functions:** Provides a variety of functions for executing Python code: + - run: Executes Python commands with proper environment setup. + - runInline: Executes inline Python code directly from Node. + - runScript: Executes standalone .py script files. +- **Custom Python Path:** In development, you can configure pythonBinaryPath to point to a custom Python installation. + +## Usage + +1. Add the extension to your trigger.config.ts file: + +```typescript +import { defineConfig } from "@trigger.dev/sdk/v3"; +import pythonExtension from "@trigger.dev/python/extension"; + +export default defineConfig({ + project: "", + build: { + extensions: [ + pythonExtension({ + requirementsFile: "./requirements.txt", // Optional: Path to your requirements file + pythonBinaryPath: path.join(rootDir, `.venv/bin/python`), // Optional: Custom Python binary path + scripts: ["my_script.py"], // List of Python scripts to include + }), + ], + }, +}); +``` + +2. (Optional) Create a requirements.txt file in your project root with the necessary Python dependencies. + +3. Execute Python scripts within your tasks using one of the provided functions: + +### Running a Python Script + +```typescript +import { task } from "@trigger.dev/sdk/v3"; +import python from "@trigger.dev/python"; + +export const myScript = task({ + id: "my-python-script", + run: async () => { + const result = await python.runScript("my_script.py", ["hello", "world"]); + return result.stdout; + }, +}); +``` + +### Running Inline Python Code + +```typescript +import { task } from "@trigger.dev/sdk/v3"; +import python from "@trigger.dev/python"; + +export const myTask = task({ + id: "to_datetime-task", + run: async () => { + const result = await python.runInline(` +import pandas as pd + +pandas.to_datetime("${+new Date() / 1000}") +`); + return result.stdout; + }, +}); +``` + +### Running Lower-Level Commands + +```typescript +import { task } from "@trigger.dev/sdk/v3"; +import python from "@trigger.dev/python"; + +export const pythonVersionTask = task({ + id: "python-version-task", + run: async () => { + const result = await python.run(["--version"]); + return result.stdout; // Expected output: Python 3.12.8 + }, +}); +``` + +## Limitations + +- This is a **partial implementation** and does not provide full Python support as an execution runtime for tasks. +- Only basic Python script execution is supported; scripts are not automatically copied to staging/production containers. +- Manual intervention may be required for installing and configuring binary dependencies in development environments. + +## Additional Information + +For more detailed documentation, visit the official docs at [Trigger.dev Documentation](https://trigger.dev/docs). diff --git a/packages/python/package.json b/packages/python/package.json new file mode 100644 index 0000000000..7d032b6c8a --- /dev/null +++ b/packages/python/package.json @@ -0,0 +1,93 @@ +{ + "name": "@trigger.dev/python", + "version": "3.3.16", + "description": "Python runtime and build extension for Trigger.dev", + "license": "MIT", + "publishConfig": { + "access": "public" + }, + "repository": { + "type": "git", + "url": "https://github.com/triggerdotdev/trigger.dev", + "directory": "packages/python" + }, + "type": "module", + "files": [ + "dist" + ], + "tshy": { + "selfLink": false, + "main": true, + "module": true, + "project": "./tsconfig.src.json", + "exports": { + "./package.json": "./package.json", + ".": "./src/index.ts", + "./extension": "./src/extension.ts" + }, + "sourceDialects": [ + "@triggerdotdev/source" + ] + }, + "typesVersions": { + "*": { + "extension": [ + "dist/commonjs/extension.d.ts" + ] + } + }, + "scripts": { + "clean": "rimraf dist", + "build": "tshy && pnpm run update-version", + "dev": "tshy --watch", + "typecheck": "tsc --noEmit -p tsconfig.src.json", + "update-version": "tsx ../../scripts/updateVersion.ts", + "check-exports": "attw --pack ." + }, + "dependencies": { + "@trigger.dev/build": "workspace:3.3.16", + "@trigger.dev/core": "workspace:3.3.16", + "@trigger.dev/sdk": "workspace:3.3.16", + "tinyexec": "^0.3.2" + }, + "devDependencies": { + "@types/node": "20.14.14", + "rimraf": "6.0.1", + "tshy": "^3.0.2", + "typescript": "^5.5.4", + "tsx": "4.17.0", + "esbuild": "^0.23.0", + "@arethetypeswrong/cli": "^0.15.4" + }, + "engines": { + "node": ">=18.20.0" + }, + "exports": { + "./package.json": "./package.json", + ".": { + "import": { + "@triggerdotdev/source": "./src/index.ts", + "types": "./dist/esm/index.d.ts", + "default": "./dist/esm/index.js" + }, + "require": { + "types": "./dist/commonjs/index.d.ts", + "default": "./dist/commonjs/index.js" + } + }, + "./extension": { + "import": { + "@triggerdotdev/source": "./src/extension.ts", + "types": "./dist/esm/extension.d.ts", + "default": "./dist/esm/extension.js" + }, + "require": { + "types": "./dist/commonjs/extension.d.ts", + "default": "./dist/commonjs/extension.js" + } + } + }, + "main": "./dist/commonjs/index.js", + "types": "./dist/commonjs/index.d.ts", + "module": "./dist/esm/index.js" +} diff --git a/packages/python/src/extension.ts b/packages/python/src/extension.ts new file mode 100644 index 0000000000..ea0c4140fb --- /dev/null +++ b/packages/python/src/extension.ts @@ -0,0 +1,119 @@ +import fs from "node:fs"; +import assert from "node:assert"; +import { additionalFiles } from "@trigger.dev/build/extensions/core"; +import { BuildManifest } from "@trigger.dev/core/v3"; +import { BuildContext, BuildExtension } from "@trigger.dev/core/v3/build"; + +export type PythonOptions = { + requirements?: string[]; + requirementsFile?: string; + /** + * [Dev-only] The path to the python binary. + * + * @remarks + * This option is typically used during local development or in specific testing environments + * where a particular Python installation needs to be targeted. It should point to the full path of the python executable. + * + * Example: `/usr/bin/python3` or `C:\\Python39\\python.exe` + */ + pythonBinaryPath?: string; + /** + * An array of glob patterns that specify which Python scripts are allowed to be executed. + * + * @remarks + * These scripts will be copied to the container during the build process. + */ + scripts?: string[]; +}; + +const splitAndCleanComments = (str: string) => + str + .split("\n") + .map((line) => line.trim()) + .filter((line) => line && !line.startsWith("#")); + +export function pythonExtension(options: PythonOptions = {}): BuildExtension { + return new PythonExtension(options); +} + +class PythonExtension implements BuildExtension { + public readonly name = "PythonExtension"; + + constructor(private options: PythonOptions = {}) { + assert( + !(this.options.requirements && this.options.requirementsFile), + "Cannot specify both requirements and requirementsFile" + ); + + if (this.options.requirementsFile) { + assert( + fs.existsSync(this.options.requirementsFile), + `Requirements file not found: ${this.options.requirementsFile}` + ); + this.options.requirements = splitAndCleanComments( + fs.readFileSync(this.options.requirementsFile, "utf-8") + ); + } + } + + async onBuildComplete(context: BuildContext, manifest: BuildManifest) { + await additionalFiles({ + files: this.options.scripts ?? [], + }).onBuildComplete!(context, manifest); + + if (context.target === "dev") { + if (this.options.pythonBinaryPath) { + process.env.PYTHON_BIN_PATH = this.options.pythonBinaryPath; + } + + return; + } + + context.logger.debug(`Adding ${this.name} to the build`); + + context.addLayer({ + id: "python-installation", + image: { + instructions: splitAndCleanComments(` + # Install Python + RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 python3-pip python3-venv && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + + # Set up Python environment + RUN python3 -m venv /opt/venv + ENV PATH="/opt/venv/bin:$PATH" + `), + }, + deploy: { + env: { + PYTHON_BIN_PATH: `/opt/venv/bin/python`, + }, + override: true, + }, + }); + + context.addLayer({ + id: "python-dependencies", + build: { + env: { + REQUIREMENTS_CONTENT: this.options.requirements?.join("\n") || "", + }, + }, + image: { + instructions: splitAndCleanComments(` + ARG REQUIREMENTS_CONTENT + RUN echo "$REQUIREMENTS_CONTENT" > requirements.txt + + # Install dependencies + RUN pip install --no-cache-dir -r requirements.txt + `), + }, + deploy: { + override: true, + }, + }); + } +} + +export default pythonExtension; diff --git a/packages/python/src/index.ts b/packages/python/src/index.ts new file mode 100644 index 0000000000..aefcfddbe4 --- /dev/null +++ b/packages/python/src/index.ts @@ -0,0 +1,63 @@ +import fs from "node:fs"; +import assert from "node:assert"; +import { logger } from "@trigger.dev/sdk/v3"; +import { x, Options as XOptions, Result } from "tinyexec"; + +export const run = async ( + scriptArgs: string[] = [], + options: Partial = {} +): Promise => { + const pythonBin = process.env.PYTHON_BIN_PATH || "python"; + + return await logger.trace("Python call", async (span) => { + span.addEvent("Properties", { + command: `${pythonBin} ${scriptArgs.join(" ")}`, + }); + + const result = await x(pythonBin, scriptArgs, { + ...options, + throwOnError: false, // Ensure errors are handled manually + }); + + span.addEvent("Output", { ...result }); + + if (result.exitCode !== 0) { + logger.error(result.stderr, { ...result }); + throw new Error(`Python command exited with non-zero code ${result.exitCode}`); + } + + return result; + }); +}; + +export const runScript = ( + scriptPath: string, + scriptArgs: string[] = [], + options: Partial = {} +) => { + assert(scriptPath, "Script path is required"); + assert(fs.existsSync(scriptPath), `Script does not exist: ${scriptPath}`); + + return run([scriptPath, ...scriptArgs], options); +}; + +export const runInline = async (scriptContent: string, options: Partial = {}) => { + assert(scriptContent, "Script content is required"); + + const tmpFile = `/tmp/script_${Date.now()}.py`; + await fs.promises.writeFile(tmpFile, scriptContent, { mode: 0o600 }); + + try { + return await runScript(tmpFile, [], options); + } finally { + try { + await fs.promises.unlink(tmpFile); + } catch (error) { + logger.warn(`Failed to clean up temporary file ${tmpFile}:`, { + error: (error as Error).stack || (error as Error).message, + }); + } + } +}; + +export default { run, runScript, runInline }; diff --git a/packages/python/tsconfig.json b/packages/python/tsconfig.json new file mode 100644 index 0000000000..16881b51b6 --- /dev/null +++ b/packages/python/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../.configs/tsconfig.base.json", + "references": [ + { + "path": "./tsconfig.src.json" + } + ] +} diff --git a/packages/python/tsconfig.src.json b/packages/python/tsconfig.src.json new file mode 100644 index 0000000000..db06c53317 --- /dev/null +++ b/packages/python/tsconfig.src.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "include": ["./src/**/*.ts"], + "compilerOptions": { + "isolatedDeclarations": false, + "composite": true, + "sourceMap": true, + "customConditions": ["@triggerdotdev/source"] + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index af762410b5..0802b03a8b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1417,6 +1417,43 @@ importers: specifier: ^1.6.0 version: 1.6.0(@types/node@20.14.14) + packages/python: + dependencies: + '@trigger.dev/build': + specifier: workspace:3.3.16 + version: link:../build + '@trigger.dev/core': + specifier: workspace:3.3.16 + version: link:../core + '@trigger.dev/sdk': + specifier: workspace:3.3.16 + version: link:../trigger-sdk + tinyexec: + specifier: ^0.3.2 + version: 0.3.2 + devDependencies: + '@arethetypeswrong/cli': + specifier: ^0.15.4 + version: 0.15.4 + '@types/node': + specifier: 20.14.14 + version: 20.14.14 + esbuild: + specifier: ^0.23.0 + version: 0.23.0 + rimraf: + specifier: 6.0.1 + version: 6.0.1 + tshy: + specifier: ^3.0.2 + version: 3.0.2 + tsx: + specifier: 4.17.0 + version: 4.17.0 + typescript: + specifier: ^5.5.4 + version: 5.5.4 + packages/react-hooks: dependencies: '@trigger.dev/core': @@ -1928,6 +1965,9 @@ importers: '@trigger.dev/build': specifier: workspace:* version: link:../../packages/build + '@trigger.dev/python': + specifier: workspace:* + version: link:../../packages/python '@types/email-reply-parser': specifier: ^1.4.2 version: 1.4.2 @@ -30668,6 +30708,10 @@ packages: resolution: {integrity: sha512-WiCJLEECkO18gwqIp6+hJg0//p23HXp4S+gGtAKu3mI2F2/sXC4FvHvXvB0zJVVaTPhx1/tOwdbRsa1sOBIKqQ==} dev: false + /tinyexec@0.3.2: + resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} + dev: false + /tinyglobby@0.2.10: resolution: {integrity: sha512-Zc+8eJlFMvgatPZTl6A9L/yht8QqdmUNtURHaKZLmKBE12hNPSrqNkUp2cs3M/UKmNVVAMFQYSjYIVHDjW5zew==} engines: {node: '>=12.0.0'} diff --git a/references/v3-catalog/package.json b/references/v3-catalog/package.json index c9de9cf913..5103a91cea 100644 --- a/references/v3-catalog/package.json +++ b/references/v3-catalog/package.json @@ -73,6 +73,7 @@ "@opentelemetry/sdk-trace-node": "^1.22.0", "@opentelemetry/semantic-conventions": "^1.22.0", "@trigger.dev/build": "workspace:*", + "@trigger.dev/python": "workspace:*", "@types/email-reply-parser": "^1.4.2", "@types/fluent-ffmpeg": "^2.1.26", "@types/node": "20.4.2", @@ -85,4 +86,4 @@ "tsconfig-paths": "^4.2.0", "typescript": "^5.5.4" } -} \ No newline at end of file +} diff --git a/references/v3-catalog/trigger.config.ts b/references/v3-catalog/trigger.config.ts index 9a24160445..b06685ade8 100644 --- a/references/v3-catalog/trigger.config.ts +++ b/references/v3-catalog/trigger.config.ts @@ -7,6 +7,7 @@ import { ffmpeg, syncEnvVars } from "@trigger.dev/build/extensions/core"; import { puppeteer } from "@trigger.dev/build/extensions/puppeteer"; import { prismaExtension } from "@trigger.dev/build/extensions/prisma"; import { emitDecoratorMetadata } from "@trigger.dev/build/extensions/typescript"; +import { pythonExtension } from "@trigger.dev/python/extension"; import { defineConfig } from "@trigger.dev/sdk/v3"; export { handleError } from "./src/handleError.js"; @@ -86,6 +87,7 @@ export default defineConfig({ value: secret.secretValue, })); }), + pythonExtension(), puppeteer(), ], external: ["re2"],