Skip to content

Commit 6d08842

Browse files
ericallamsamejr
ericallam
and
samejr
authored
feat: Add maxDuration to tasks (#1377)
* WIP * Get max duration working on deployed runs * Actually set the timed out runs to status = TIMED_OUT * The client status for TIMED_OUT is now MAX_DURATION_EXCEEDED * New TimedOutIcon * Added new timedout icon * Add ability to opt-out of maxDuration with timeout.None * MAX_DURATION_EXCEEDED -> TIMED_OUT * changeset * Improved styling for the status tooltip content --------- Co-authored-by: James Ritchie <[email protected]>
1 parent 665ccf8 commit 6d08842

File tree

55 files changed

+624
-87
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

55 files changed

+624
-87
lines changed

‎.changeset/tiny-forks-remember.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
"@trigger.dev/sdk": patch
3+
"trigger.dev": patch
4+
"@trigger.dev/core": patch
5+
---
6+
7+
Adding maxDuration to tasks to allow timing out runs after they exceed a certain number of seconds

‎apps/webapp/app/assets/icons/TimedOutIcon.tsx

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
export function TimedOutIcon({ className }: { className?: string }) {
2+
return (
3+
<svg
4+
className={className}
5+
width="20"
6+
height="20"
7+
viewBox="0 0 20 20"
8+
fill="none"
9+
xmlns="http://www.w3.org/2000/svg"
10+
>
11+
<path
12+
fillRule="evenodd"
13+
clipRule="evenodd"
14+
d="M9 2H8V3H9V4.07089C5.60771 4.55612 3 7.47353 3 11C3 14.866 6.13401 18 10 18C13.866 18 17 14.866 17 11C17 7.47353 14.3923 4.55612 11 4.07089V3H12V2H11H9ZM13.7603 3.36575C14.7218 2.40422 16.2807 2.40422 17.2422 3.36575C18.2038 4.32727 18.2038 5.8862 17.2422 6.84772L17.1462 6.94375C17.1251 6.96488 17.0908 6.96488 17.0697 6.94375L13.6642 3.53831C13.6431 3.51717 13.6431 3.4829 13.6642 3.46177L13.7603 3.36575ZM6.28876 3.58524C6.33584 3.53816 6.33584 3.46184 6.28876 3.41476L6.23971 3.36571C5.27819 2.40419 3.71925 2.40419 2.75773 3.36571C1.79621 4.32723 1.79621 5.88616 2.75773 6.84769L2.80678 6.89674C2.85386 6.94382 2.93019 6.94381 2.97726 6.89674L6.28876 3.58524ZM14.5858 17L16 15.5858L16.7071 16.2929C17.0976 16.6834 17.0976 17.3166 16.7071 17.7071C16.3166 18.0976 15.6834 18.0976 15.2929 17.7071L14.5858 17ZM5.42297 17L4.00875 15.5858L3.30165 16.2929C2.91112 16.6834 2.91112 17.3166 3.30165 17.7071C3.69217 18.0977 4.32534 18.0977 4.71586 17.7071L5.42297 17ZM6.29289 7.29289C6.68342 6.90237 7.31658 6.90237 7.70711 7.29289L10 9.58579L12.2929 7.29289C12.6834 6.90237 13.3166 6.90237 13.7071 7.29289C14.0976 7.68342 14.0976 8.31658 13.7071 8.70711L11.4142 11L13.7071 13.2929C14.0976 13.6834 14.0976 14.3166 13.7071 14.7071C13.3166 15.0976 12.6834 15.0976 12.2929 14.7071L10 12.4142L7.70711 14.7071C7.31658 15.0976 6.68342 15.0976 6.29289 14.7071C5.90237 14.3166 5.90237 13.6834 6.29289 13.2929L8.58579 11L6.29289 8.70711C5.90237 8.31658 5.90237 7.68342 6.29289 7.29289Z"
15+
fill="currentColor"
16+
/>
17+
</svg>
18+
);
19+
}

‎apps/webapp/app/components/primitives/Select.tsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -613,7 +613,7 @@ export function SelectPopover({
613613
"z-50 flex flex-col overflow-clip rounded border border-charcoal-700 bg-background-bright shadow-md outline-none animate-in fade-in-40",
614614
"min-w-[max(180px,calc(var(--popover-anchor-width)+0.5rem))]",
615615
"max-w-[min(480px,var(--popover-available-width))]",
616-
"max-h-[min(480px,var(--popover-available-height))]",
616+
"max-h-[min(520px,var(--popover-available-height))]",
617617
"origin-[var(--popover-transform-origin)]",
618618
className
619619
)}

‎apps/webapp/app/components/runs/v3/RunInspector.tsx

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -324,6 +324,12 @@ export function RunInspector({
324324
</Property.Value>
325325
</Property.Item>
326326
)}
327+
<Property.Item>
328+
<Property.Label>Max duration</Property.Label>
329+
<Property.Value>
330+
{run.maxDurationInSeconds ? `${run.maxDurationInSeconds}s` : "–"}
331+
</Property.Value>
332+
</Property.Item>
327333
<Property.Item>
328334
<Property.Label>Run invocation cost</Property.Label>
329335
<Property.Value>

‎apps/webapp/app/components/runs/v3/TaskRunStatus.tsx

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,14 @@ import {
88
NoSymbolIcon,
99
PauseCircleIcon,
1010
RectangleStackIcon,
11+
StopIcon,
1112
TrashIcon,
1213
XCircleIcon,
1314
} from "@heroicons/react/20/solid";
1415
import { TaskRunStatus } from "@trigger.dev/database";
1516
import assertNever from "assert-never";
1617
import { SnowflakeIcon } from "lucide-react";
18+
import { TimedOutIcon } from "~/assets/icons/TimedOutIcon";
1719
import { Spinner } from "~/components/primitives/Spinner";
1820
import { cn } from "~/utils/cn";
1921

@@ -27,6 +29,7 @@ export const allTaskRunStatuses = [
2729
"COMPLETED_SUCCESSFULLY",
2830
"CANCELED",
2931
"COMPLETED_WITH_ERRORS",
32+
"TIMED_OUT",
3033
"CRASHED",
3134
"PAUSED",
3235
"INTERRUPTED",
@@ -44,6 +47,7 @@ export const filterableTaskRunStatuses = [
4447
"COMPLETED_SUCCESSFULLY",
4548
"CANCELED",
4649
"COMPLETED_WITH_ERRORS",
50+
"TIMED_OUT",
4751
"CRASHED",
4852
"INTERRUPTED",
4953
"SYSTEM_FAILURE",
@@ -65,6 +69,7 @@ const taskRunStatusDescriptions: Record<TaskRunStatus, string> = {
6569
PAUSED: "Task has been paused by the user",
6670
CRASHED: "Task has crashed and won't be retried",
6771
EXPIRED: "Task has surpassed its ttl and won't be executed",
72+
TIMED_OUT: "Task has reached its maxDuration and has been stopped",
6873
};
6974

7075
export const QUEUED_STATUSES = [
@@ -140,6 +145,8 @@ export function TaskRunStatusIcon({
140145
return <FireIcon className={cn(runStatusClassNameColor(status), className)} />;
141146
case "EXPIRED":
142147
return <TrashIcon className={cn(runStatusClassNameColor(status), className)} />;
148+
case "TIMED_OUT":
149+
return <TimedOutIcon className={cn(runStatusClassNameColor(status), className)} />;
143150

144151
default: {
145152
assertNever(status);
@@ -174,6 +181,8 @@ export function runStatusClassNameColor(status: TaskRunStatus): string {
174181
return "text-error";
175182
case "CRASHED":
176183
return "text-error";
184+
case "TIMED_OUT":
185+
return "text-error";
177186
default: {
178187
assertNever(status);
179188
}
@@ -210,6 +219,8 @@ export function runStatusTitle(status: TaskRunStatus): string {
210219
return "Crashed";
211220
case "EXPIRED":
212221
return "Expired";
222+
case "TIMED_OUT":
223+
return "Timed out";
213224
default: {
214225
assertNever(status);
215226
}

‎apps/webapp/app/components/runs/v3/TaskRunsTable.tsx

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,13 @@ import { BeakerIcon, BookOpenIcon, CheckIcon } from "@heroicons/react/24/solid";
99
import { useLocation } from "@remix-run/react";
1010
import { formatDuration, formatDurationMilliseconds } from "@trigger.dev/core/v3";
1111
import { useCallback, useRef } from "react";
12+
import { Badge } from "~/components/primitives/Badge";
1213
import { Button, LinkButton } from "~/components/primitives/Buttons";
1314
import { Checkbox } from "~/components/primitives/Checkbox";
1415
import { Dialog, DialogTrigger } from "~/components/primitives/Dialog";
1516
import { Header3 } from "~/components/primitives/Headers";
1617
import { useSelectedItems } from "~/components/primitives/SelectedItemsProvider";
18+
import { SimpleTooltip } from "~/components/primitives/Tooltip";
1719
import { useEnvironments } from "~/hooks/useEnvironments";
1820
import { useFeatures } from "~/hooks/useFeatures";
1921
import { useOrganization } from "~/hooks/useOrganizations";
@@ -39,9 +41,12 @@ import {
3941
import { CancelRunDialog } from "./CancelRunDialog";
4042
import { LiveTimer } from "./LiveTimer";
4143
import { ReplayRunDialog } from "./ReplayRunDialog";
42-
import { TaskRunStatusCombo } from "./TaskRunStatus";
4344
import { RunTag } from "./RunTag";
44-
import { Badge } from "~/components/primitives/Badge";
45+
import {
46+
descriptionForTaskRunStatus,
47+
filterableTaskRunStatuses,
48+
TaskRunStatusCombo,
49+
} from "./TaskRunStatus";
4550

4651
type RunsTableProps = {
4752
total: number;
@@ -126,7 +131,27 @@ export function TaskRunsTable({
126131
<TableHeaderCell>Env</TableHeaderCell>
127132
<TableHeaderCell>Task</TableHeaderCell>
128133
<TableHeaderCell>Version</TableHeaderCell>
129-
<TableHeaderCell>Status</TableHeaderCell>
134+
<TableHeaderCell
135+
tooltip={
136+
<div className="flex flex-col divide-y divide-grid-dimmed">
137+
{filterableTaskRunStatuses.map((status) => (
138+
<div
139+
key={status}
140+
className="grid grid-cols-[8rem_1fr] gap-x-2 py-2 first:pt-1 last:pb-1"
141+
>
142+
<div className="mb-0.5 flex items-center gap-1.5 whitespace-nowrap">
143+
<TaskRunStatusCombo status={status} />
144+
</div>
145+
<Paragraph variant="extra-small" className="!text-wrap text-text-dimmed">
146+
{descriptionForTaskRunStatus(status)}
147+
</Paragraph>
148+
</div>
149+
))}
150+
</div>
151+
}
152+
>
153+
Status
154+
</TableHeaderCell>
130155
<TableHeaderCell>Started</TableHeaderCell>
131156
<TableHeaderCell
132157
colSpan={3}
@@ -287,7 +312,11 @@ export function TaskRunsTable({
287312
</TableCell>
288313
<TableCell to={path}>{run.version ?? "–"}</TableCell>
289314
<TableCell to={path}>
290-
<TaskRunStatusCombo status={run.status} />
315+
<SimpleTooltip
316+
content={descriptionForTaskRunStatus(run.status)}
317+
disableHoverableContent
318+
button={<TaskRunStatusCombo status={run.status} />}
319+
/>
291320
</TableCell>
292321
<TableCell to={path}>
293322
{run.startedAt ? <DateTime date={run.startedAt} /> : "–"}

‎apps/webapp/app/database-types.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ export const TaskRunStatus = {
4242
CRASHED: "CRASHED",
4343
DELAYED: "DELAYED",
4444
EXPIRED: "EXPIRED",
45+
TIMED_OUT: "TIMED_OUT",
4546
} as const satisfies Record<TaskRunStatusType, TaskRunStatusType>;
4647

4748
export const JobRunStatus = {

‎apps/webapp/app/models/taskRun.server.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ export function batchTaskRunItemStatusForRunStatus(
119119
case TaskRunStatus.SYSTEM_FAILURE:
120120
case TaskRunStatus.CRASHED:
121121
case TaskRunStatus.EXPIRED:
122+
case TaskRunStatus.TIMED_OUT:
122123
return BatchTaskRunItemStatus.FAILED;
123124
case TaskRunStatus.PENDING:
124125
case TaskRunStatus.WAITING_FOR_DEPLOY:

‎apps/webapp/app/presenters/v3/ApiRetrieveRunPresenter.server.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -270,6 +270,9 @@ export class ApiRetrieveRunPresenter extends BasePresenter {
270270
case "EXPIRED": {
271271
return "EXPIRED";
272272
}
273+
case "TIMED_OUT": {
274+
return "TIMED_OUT";
275+
}
273276
default: {
274277
assertNever(status);
275278
}

‎apps/webapp/app/presenters/v3/ApiRunListPresenter.server.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,9 @@ export class ApiRunListPresenter extends BasePresenter {
310310
case "EXPIRED": {
311311
return "EXPIRED";
312312
}
313+
case "TIMED_OUT": {
314+
return "TIMED_OUT";
315+
}
313316
default: {
314317
assertNever(status);
315318
}

‎apps/webapp/app/presenters/v3/SpanPresenter.server.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import { eventRepository } from "~/v3/eventRepository.server";
99
import { machinePresetFromName } from "~/v3/machinePresets.server";
1010
import { FINAL_ATTEMPT_STATUSES, isFinalRunStatus } from "~/v3/taskStatus";
1111
import { BasePresenter } from "./basePresenter.server";
12+
import { getMaxDuration } from "~/v3/utils/maxDuration";
1213

1314
type Result = Awaited<ReturnType<SpanPresenter["call"]>>;
1415
export type Span = NonNullable<NonNullable<Result>["span"]>;
@@ -69,6 +70,7 @@ export class SpanPresenter extends BasePresenter {
6970
taskIdentifier: true,
7071
friendlyId: true,
7172
isTest: true,
73+
maxDurationInSeconds: true,
7274
tags: {
7375
select: {
7476
name: true,
@@ -229,6 +231,7 @@ export class SpanPresenter extends BasePresenter {
229231
baseCostInCents: run.baseCostInCents,
230232
maxAttempts: run.maxAttempts ?? undefined,
231233
version: run.lockedToVersion?.version,
234+
maxDuration: run.maxDurationInSeconds ?? undefined,
232235
},
233236
queue: {
234237
name: run.queue,
@@ -307,6 +310,7 @@ export class SpanPresenter extends BasePresenter {
307310
},
308311
context: JSON.stringify(context, null, 2),
309312
metadata,
313+
maxDurationInSeconds: getMaxDuration(run.maxDurationInSeconds),
310314
};
311315
}
312316

‎apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.v3.$projectParam.runs.$runParam.spans.$spanParam/route.tsx

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -677,7 +677,12 @@ function RunBody({
677677
)}
678678
</Property.Value>
679679
</Property.Item>
680-
680+
<Property.Item>
681+
<Property.Label>Max duration</Property.Label>
682+
<Property.Value>
683+
{run.maxDurationInSeconds ? `${run.maxDurationInSeconds}s` : "–"}
684+
</Property.Value>
685+
</Property.Item>
681686
<Property.Item>
682687
<Property.Label>Run invocation cost</Property.Label>
683688
<Property.Value>

‎apps/webapp/app/v3/marqs/devQueueConsumer.server.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import {
2424
tracer,
2525
} from "../tracer.server";
2626
import { DevSubscriber, devPubSub } from "./devPubSub.server";
27+
import { getMaxDuration } from "../utils/maxDuration";
2728

2829
const MessageBody = z.discriminatedUnion("type", [
2930
z.object({
@@ -378,6 +379,10 @@ export class DevQueueConsumer {
378379
status: "EXECUTING",
379380
lockedToVersionId: backgroundWorker.id,
380381
startedAt: existingTaskRun.startedAt ?? new Date(),
382+
maxDurationInSeconds: getMaxDuration(
383+
existingTaskRun.maxDurationInSeconds,
384+
backgroundTask.maxDurationInSeconds
385+
),
381386
},
382387
include: {
383388
attempts: {

‎apps/webapp/app/v3/marqs/sharedQueueConsumer.server.ts

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ import { EnvironmentVariable } from "../environmentVariables/repository";
4444
import { machinePresetFromConfig } from "../machinePresets.server";
4545
import { env } from "~/env.server";
4646
import { isFinalAttemptStatus, isFinalRunStatus } from "../taskStatus";
47+
import { getMaxDuration } from "../utils/maxDuration";
4748

4849
const WithTraceContext = z.object({
4950
traceparent: z.string().optional(),
@@ -403,6 +404,10 @@ export class SharedQueueConsumer {
403404
startedAt: existingTaskRun.startedAt ?? new Date(),
404405
baseCostInCents: env.CENTS_PER_RUN,
405406
machinePreset: machinePresetFromConfig(backgroundTask.machineConfig ?? {}).name,
407+
maxDurationInSeconds: getMaxDuration(
408+
existingTaskRun.maxDurationInSeconds,
409+
backgroundTask.maxDurationInSeconds
410+
),
406411
},
407412
include: {
408413
runtimeEnvironment: true,
@@ -1067,6 +1072,7 @@ class SharedQueueTasks {
10671072
costInCents: taskRun.costInCents,
10681073
baseCostInCents: taskRun.baseCostInCents,
10691074
metadata,
1075+
maxDuration: taskRun.maxDurationInSeconds ?? undefined,
10701076
},
10711077
queue: {
10721078
id: queue.friendlyId,

‎apps/webapp/app/v3/requeueTaskRun.server.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ export class RequeueTaskRunService extends BaseService {
7070
case "COMPLETED_WITH_ERRORS":
7171
case "COMPLETED_SUCCESSFULLY":
7272
case "EXPIRED":
73+
case "TIMED_OUT":
7374
case "CANCELED": {
7475
logger.debug("[RequeueTaskRunService] Task run is completed", { taskRun });
7576

‎apps/webapp/app/v3/services/completeAttempt.server.ts

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -363,10 +363,16 @@ export class CompleteAttemptService extends BaseService {
363363
},
364364
});
365365

366+
const status =
367+
sanitizedError.type === "INTERNAL_ERROR" &&
368+
sanitizedError.code === "MAX_DURATION_EXCEEDED"
369+
? "TIMED_OUT"
370+
: "COMPLETED_WITH_ERRORS";
371+
366372
const finalizeService = new FinalizeTaskRunService();
367373
await finalizeService.call({
368374
id: taskRunAttempt.taskRunId,
369-
status: "COMPLETED_WITH_ERRORS",
375+
status,
370376
completedAt: new Date(),
371377
});
372378
}

‎apps/webapp/app/v3/services/createBackgroundWorker.server.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import { projectPubSub } from "./projectPubSub.server";
1515
import { RegisterNextTaskScheduleInstanceService } from "./registerNextTaskScheduleInstance.server";
1616
import cronstrue from "cronstrue";
1717
import { CheckScheduleService } from "./checkSchedule.server";
18+
import { clampMaxDuration } from "../utils/maxDuration";
1819

1920
export class CreateBackgroundWorkerService extends BaseService {
2021
public async call(
@@ -156,6 +157,7 @@ export async function createBackgroundTasks(
156157
machineConfig: task.machine,
157158
triggerSource: task.triggerSource === "schedule" ? "SCHEDULED" : "STANDARD",
158159
fileId: tasksToBackgroundFiles?.get(task.id) ?? null,
160+
maxDurationInSeconds: task.maxDuration ? clampMaxDuration(task.maxDuration) : null,
159161
},
160162
});
161163

‎apps/webapp/app/v3/services/createTaskRunAttempt.server.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,7 @@ export class CreateTaskRunAttemptService extends BaseService {
196196
maxAttempts: taskRun.maxAttempts ?? undefined,
197197
version: taskRun.lockedBy.worker.version,
198198
metadata,
199+
maxDuration: taskRun.maxDurationInSeconds ?? undefined,
199200
},
200201
queue: {
201202
id: queue.friendlyId,

‎apps/webapp/app/v3/services/triggerTask.server.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import { findCurrentWorkerFromEnvironment } from "../models/workerDeployment.ser
2323
import { handleMetadataPacket } from "~/utils/packets";
2424
import { ExpireEnqueuedRunService } from "./expireEnqueuedRun.server";
2525
import { guardQueueSizeLimitsForEnv } from "../queueSizeLimits.server";
26+
import { clampMaxDuration } from "../utils/maxDuration";
2627

2728
export type TriggerTaskServiceOptions = {
2829
idempotencyKey?: string;
@@ -373,6 +374,9 @@ export class TriggerTaskService extends BaseService {
373374
metadataType: metadataPacket?.dataType,
374375
seedMetadata: metadataPacket?.data,
375376
seedMetadataType: metadataPacket?.dataType,
377+
maxDurationInSeconds: body.options?.maxDuration
378+
? clampMaxDuration(body.options.maxDuration)
379+
: undefined,
376380
},
377381
});
378382

‎apps/webapp/app/v3/taskStatus.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ export const FINAL_RUN_STATUSES = [
4141
"SYSTEM_FAILURE",
4242
"EXPIRED",
4343
"CRASHED",
44+
"TIMED_OUT",
4445
] satisfies TaskRunStatus[];
4546

4647
export type FINAL_RUN_STATUSES = (typeof FINAL_RUN_STATUSES)[number];
@@ -96,6 +97,7 @@ export const FAILED_RUN_STATUSES = [
9697
"COMPLETED_WITH_ERRORS",
9798
"SYSTEM_FAILURE",
9899
"CRASHED",
100+
"TIMED_OUT",
99101
] satisfies TaskRunStatus[];
100102

101103
export function isFailedRunStatus(status: TaskRunStatus): boolean {

0 commit comments

Comments
 (0)