Merge pull request #1808 from triggerdotdev/re2-queue-indexing

re2: Queue indexing, queue trigger changes

Author: ericallam

.changeset/config.json

@@ -12,7 +12,13 @@
   "access": "public",
   "baseBranch": "main",
   "updateInternalDependencies": "patch",
-  "ignore": ["webapp", "supervisor", "coordinator", "docker-provider", "kubernetes-provider"],
+  "ignore": [
+    "webapp",
+    "coordinator",
+    "docker-provider",
+    "kubernetes-provider",
+    "supervisor"
+  ],
   "___experimentalUnsafeOptions_WILL_CHANGE_IN_PATCH": {
     "onlyUpdatePeerDependentsWhenOutOfRange": true
   }

.github/workflows/unit-tests.yml

@@ -38,7 +38,7 @@ jobs:
         run: pnpm run generate
 
       - name: 🧪 Run Webapp Unit Tests
-        run: pnpm run test --filter webapp
+        run: pnpm run test:webapp
         env:
           DATABASE_URL: postgresql://postgres:postgres@localhost:5432/postgres
           DIRECT_URL: postgresql://postgres:postgres@localhost:5432/postgres
@@ -47,7 +47,7 @@ jobs:
           ENCRYPTION_KEY: "secret"
 
       - name: 🧪 Run Package Unit Tests
-        run: pnpm run test --filter "@trigger.dev/*"
+        run: pnpm run test:packages
 
       - name: 🧪 Run Internal Unit Tests
-        run: pnpm run test --filter "@internal/*"
+        run: pnpm run test:internal

apps/webapp/app/components/navigation/SideMenu.tsx

@@ -175,6 +175,12 @@ export function SideMenu({
               to={v3EnvironmentPath(organization, project, environment)}
               data-action="tasks"
             />
+            <SideMenuItem
+              name="Runs"
+              icon={RunsIcon}
+              activeIconColor="text-teal-500"
+              to={v3RunsPath(organization, project, environment)}
+            />
             <SideMenuItem
               name="Batches"
               icon={Squares2X2Icon}
@@ -212,22 +218,6 @@ export function SideMenu({
             />
           </div>
 
-          <SideMenuSection title="Observability">
-            <SideMenuItem
-              name="Runs"
-              icon={RunsIcon}
-              activeIconColor="text-teal-500"
-              to={v3RunsPath(organization, project, environment)}
-            />
-            <SideMenuItem
-              name="Alerts"
-              icon={BellAlertIcon}
-              activeIconColor="text-red-500"
-              to={v3ProjectAlertsPath(organization, project, environment)}
-              data-action="alerts"
-            />
-          </SideMenuSection>
-
           <SideMenuSection title="Manage">
             <SideMenuItem
               name="API keys"
@@ -243,6 +233,13 @@ export function SideMenu({
               to={v3EnvironmentVariablesPath(organization, project, environment)}
               data-action="environment variables"
             />
+            <SideMenuItem
+              name="Alerts"
+              icon={BellAlertIcon}
+              activeIconColor="text-red-500"
+              to={v3ProjectAlertsPath(organization, project, environment)}
+              data-action="alerts"
+            />
             <SideMenuItem
               name="Project settings"
               icon={Cog8ToothIcon}

apps/webapp/app/components/primitives/Table.tsx

@@ -161,7 +161,12 @@ export const TableHeaderCell = forwardRef<HTMLTableCellElement, TableHeaderCellP
         {hiddenLabel ? (
           <span className="sr-only">{children}</span>
         ) : tooltip ? (
-          <div className="flex items-center gap-1">
+          <div
+            className={cn("flex items-center gap-1", {
+              "justify-center": alignment === "center",
+              "justify-end": alignment === "right",
+            })}
+          >
             {children}
             <InfoIconTooltip content={tooltip} contentClassName="normal-case tracking-normal" />
           </div>

apps/webapp/app/components/runs/v3/TaskPath.tsx

@@ -19,16 +19,16 @@ export function TaskPath({ filePath, functionName, className }: TaskPathProps) {
   );
 }
 
-type TaskFunctionNameProps = {
-  functionName: string;
+type TaskFileNameProps = {
+  fileName: string;
   variant?: InlineCodeVariant;
   className?: string;
 };
 
-export function TaskFunctionName({ variant, functionName, className }: TaskFunctionNameProps) {
+export function TaskFileName({ variant, fileName, className }: TaskFileNameProps) {
   return (
     <InlineCode variant={variant} className={cn("text-text-dimmed", className)}>
-      {`${functionName}()`}
+      {`${fileName}`}
     </InlineCode>
   );
 }

apps/webapp/app/components/runs/v3/TaskRunStatus.tsx

@@ -15,12 +15,14 @@ import { type TaskRunStatus } from "@trigger.dev/database";
 import assertNever from "assert-never";
 import { HourglassIcon } from "lucide-react";
 import { TimedOutIcon } from "~/assets/icons/TimedOutIcon";
+import { Callout } from "~/components/primitives/Callout";
 import { Spinner } from "~/components/primitives/Spinner";
 import { cn } from "~/utils/cn";
 
 export const allTaskRunStatuses = [
   "DELAYED",
   "WAITING_FOR_DEPLOY",
+  "PENDING_VERSION",
   "PENDING",
   "EXECUTING",
   "RETRYING_AFTER_FAILURE",
@@ -37,7 +39,7 @@ export const allTaskRunStatuses = [
 ] as const satisfies Readonly<Array<TaskRunStatus>>;
 
 export const filterableTaskRunStatuses = [
-  "WAITING_FOR_DEPLOY",
+  "PENDING_VERSION",
   "DELAYED",
   "PENDING",
   "WAITING_TO_RESUME",
@@ -56,7 +58,8 @@ export const filterableTaskRunStatuses = [
 const taskRunStatusDescriptions: Record<TaskRunStatus, string> = {
   DELAYED: "Task has been delayed and is waiting to be executed.",
   PENDING: "Task is waiting to be executed.",
-  WAITING_FOR_DEPLOY: "Task needs to be deployed first to start executing.",
+  PENDING_VERSION: "Run cannot execute until a version includes the task and queue.",
+  WAITING_FOR_DEPLOY: "Run cannot execute until a version includes the task and queue.",
   EXECUTING: "Task is currently being executed.",
   RETRYING_AFTER_FAILURE: "Task is being reattempted after a failure.",
   WAITING_TO_RESUME: `You have used a "wait" function. When the wait is complete, the task will resume execution.`,
@@ -73,6 +76,7 @@ const taskRunStatusDescriptions: Record<TaskRunStatus, string> = {
 
 export const QUEUED_STATUSES = [
   "PENDING",
+  "PENDING_VERSION",
   "WAITING_FOR_DEPLOY",
   "DELAYED",
 ] satisfies TaskRunStatus[];
@@ -104,6 +108,43 @@ export function TaskRunStatusCombo({
   );
 }
 
+const statusReasonsToDescription: Record<string, string> = {
+  NO_DEPLOYMENT: "No deployment or deployment image reference found for deployed run",
+  NO_WORKER: "No worker found for run",
+  TASK_NEVER_REGISTERED: "Task never registered",
+  QUEUE_NOT_FOUND: "Queue not found",
+  TASK_NOT_IN_LATEST: "Task not in latest version",
+  BACKGROUND_WORKER_MISMATCH: "Background worker mismatch",
+};
+
+export function TaskRunStatusReason({
+  status,
+  statusReason,
+}: {
+  status: TaskRunStatus;
+  statusReason?: string;
+}) {
+  if (status !== "PENDING_VERSION") {
+    return null;
+  }
+
+  if (!statusReason) {
+    return null;
+  }
+
+  const description = statusReasonsToDescription[statusReason];
+
+  if (!description) {
+    return null;
+  }
+
+  return (
+    <Callout to="https://trigger.dev/docs" variant="warning" className="text-sm">
+      {description}
+    </Callout>
+  );
+}
+
 export function TaskRunStatusLabel({ status }: { status: TaskRunStatus }) {
   return <span className={runStatusClassNameColor(status)}>{runStatusTitle(status)}</span>;
 }
@@ -120,6 +161,7 @@ export function TaskRunStatusIcon({
       return <ClockIcon className={cn(runStatusClassNameColor(status), className)} />;
     case "PENDING":
       return <RectangleStackIcon className={cn(runStatusClassNameColor(status), className)} />;
+    case "PENDING_VERSION":
     case "WAITING_FOR_DEPLOY":
       return <RectangleStackIcon className={cn(runStatusClassNameColor(status), className)} />;
     case "EXECUTING":
@@ -158,6 +200,7 @@ export function runStatusClassNameColor(status: TaskRunStatus): string {
     case "PENDING":
     case "DELAYED":
       return "text-charcoal-500";
+    case "PENDING_VERSION":
     case "WAITING_FOR_DEPLOY":
       return "text-amber-500";
     case "EXECUTING":
@@ -194,8 +237,9 @@ export function runStatusTitle(status: TaskRunStatus): string {
       return "Delayed";
     case "PENDING":
       return "Queued";
+    case "PENDING_VERSION":
     case "WAITING_FOR_DEPLOY":
-      return "Waiting for deploy";
+      return "Pending version";
     case "EXECUTING":
       return "Executing";
     case "WAITING_TO_RESUME":

apps/webapp/app/components/runs/v3/TaskRunsTable.tsx

@@ -330,7 +330,9 @@ export function TaskRunsTable({
                 <TableCell to={path} className="w-[1%]" actionClassName="pr-0 tabular-nums">
                   <div className="flex items-center gap-1">
                     <RectangleStackIcon className="size-4 text-text-dimmed" />
-                    {run.startedAt ? (
+                    {run.isPending ? (
+                      "–"
+                    ) : run.startedAt ? (
                       formatDuration(new Date(run.createdAt), new Date(run.startedAt), {
                         style: "short",
                       })

apps/webapp/app/database-types.ts

@@ -29,6 +29,7 @@ export const TaskRunAttemptStatus = {
 
 export const TaskRunStatus = {
   PENDING: "PENDING",
+  PENDING_VERSION: "PENDING_VERSION",
   WAITING_FOR_DEPLOY: "WAITING_FOR_DEPLOY",
   EXECUTING: "EXECUTING",
   WAITING_TO_RESUME: "WAITING_TO_RESUME",

apps/webapp/app/hooks/useFilterTasks.ts

@@ -4,7 +4,6 @@ type Task = {
   id: string;
   friendlyId: string;
   taskIdentifier: string;
-  exportName: string;
   filePath: string;
   triggerSource: string;
 };
@@ -17,10 +16,6 @@ export function useFilterTasks<T extends Task>({ tasks }: { tasks: T[] }) {
         return true;
       }
 
-      if (task.exportName.toLowerCase().includes(text.toLowerCase())) {
-        return true;
-      }
-
       if (task.filePath.toLowerCase().includes(text.toLowerCase())) {
         return true;
       }

apps/webapp/app/models/runtimeEnvironment.server.ts

@@ -1,4 +1,4 @@
-import { AuthenticatedEnvironment } from "@internal/testcontainers";
+import type { AuthenticatedEnvironment } from "@internal/run-engine";
 import type { Prisma, PrismaClientOrTransaction, RuntimeEnvironment } from "@trigger.dev/database";
 import { prisma } from "~/db.server";
 import { getUsername } from "~/utils/username";

apps/webapp/app/models/taskQueue.server.ts

@@ -1,4 +1,4 @@
-import { QueueOptions } from "@trigger.dev/core/v3/schemas";
+import { QueueManifest } from "@trigger.dev/core/v3/schemas";
 import { TaskQueue } from "@trigger.dev/database";
 import { prisma } from "~/db.server";
 
@@ -35,7 +35,7 @@ export async function findQueueInEnvironment(
     return;
   }
 
-  const queueConfig = QueueOptions.safeParse(task.queueConfig);
+  const queueConfig = QueueManifest.safeParse(task.queueConfig);
 
   if (queueConfig.success) {
     const taskQueueName = queueConfig.data.name

apps/webapp/app/models/taskRun.server.ts

@@ -125,6 +125,7 @@ export function batchTaskRunItemStatusForRunStatus(
     case TaskRunStatus.TIMED_OUT:
       return BatchTaskRunItemStatus.FAILED;
     case TaskRunStatus.PENDING:
+    case TaskRunStatus.PENDING_VERSION:
     case TaskRunStatus.WAITING_FOR_DEPLOY:
     case TaskRunStatus.WAITING_TO_RESUME:
     case TaskRunStatus.RETRYING_AFTER_FAILURE:

apps/webapp/app/presenters/v3/ApiRetrieveRunPresenter.server.ts

@@ -204,6 +204,9 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
       case "DELAYED": {
         return "DELAYED";
       }
+      case "PENDING_VERSION": {
+        return "PENDING_VERSION";
+      }
       case "WAITING_FOR_DEPLOY": {
         return "WAITING_FOR_DEPLOY";
       }
@@ -257,7 +260,11 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
   }
 
   static apiBooleanHelpersFromRunStatus(status: RunStatus) {
-    const isQueued = status === "QUEUED" || status === "WAITING_FOR_DEPLOY" || status === "DELAYED";
+    const isQueued =
+      status === "QUEUED" ||
+      status === "WAITING_FOR_DEPLOY" ||
+      status === "DELAYED" ||
+      status === "PENDING_VERSION";
     const isExecuting = status === "EXECUTING" || status === "REATTEMPTING" || status === "FROZEN";
     const isCompleted =
       status === "COMPLETED" ||

apps/webapp/app/presenters/v3/ApiRunListPresenter.server.ts

@@ -280,6 +280,9 @@ export class ApiRunListPresenter extends BasePresenter {
     switch (status) {
       case "DELAYED":
         return "DELAYED";
+      case "PENDING_VERSION": {
+        return "PENDING_VERSION";
+      }
       case "WAITING_FOR_DEPLOY": {
         return "WAITING_FOR_DEPLOY";
       }

apps/webapp/app/presenters/v3/DeploymentPresenter.server.ts

@@ -108,11 +108,10 @@ export class DeploymentPresenter {
             tasks: {
               select: {
                 slug: true,
-                exportName: true,
                 filePath: true,
               },
               orderBy: {
-                exportName: "asc",
+                slug: "asc",
               },
             },
             sdkVersion: true,

apps/webapp/app/presenters/v3/QueueListPresenter.server.ts

@@ -56,16 +56,19 @@ export class QueueListPresenter extends BasePresenter {
     const queues = await this._replica.taskQueue.findMany({
       where: {
         runtimeEnvironmentId: environment.id,
+        version: "V2",
       },
       select: {
         friendlyId: true,
         name: true,
+        orderableName: true,
         concurrencyLimit: true,
         type: true,
         paused: true,
+        releaseConcurrencyOnWaitpoint: true,
       },
       orderBy: {
-        name: "asc",
+        orderableName: "asc",
       },
       skip: (page - 1) * this.perPage,
       take: this.perPage,
@@ -92,6 +95,7 @@ export class QueueListPresenter extends BasePresenter {
         queued: results[0][queue.name] ?? 0,
         concurrencyLimit: queue.concurrencyLimit ?? null,
         paused: queue.paused,
+        releaseConcurrencyOnWaitpoint: queue.releaseConcurrencyOnWaitpoint,
       })
     );
   }

apps/webapp/app/presenters/v3/QueueRetrievePresenter.server.ts

@@ -76,6 +76,7 @@ export class QueueRetrievePresenter extends BasePresenter {
         queued: results[0]?.[queue.name] ?? 0,
         concurrencyLimit: queue.concurrencyLimit ?? null,
         paused: queue.paused,
+        releaseConcurrencyOnWaitpoint: queue.releaseConcurrencyOnWaitpoint,
       }),
     };
   }
@@ -105,6 +106,7 @@ export function toQueueItem(data: {
   queued: number;
   concurrencyLimit: number | null;
   paused: boolean;
+  releaseConcurrencyOnWaitpoint: boolean;
 }): QueueItem {
   return {
     id: data.friendlyId,
@@ -115,5 +117,6 @@ export function toQueueItem(data: {
     queued: data.queued,
     concurrencyLimit: data.concurrencyLimit,
     paused: data.paused,
+    releaseConcurrencyOnWaitpoint: data.releaseConcurrencyOnWaitpoint,
   };
 }