@@ -1,4 +1,5 @@
11import { resolveFailoverReasonFromError } from "../../agents/failover-error.js";
2+import { formatEmbeddedAgentExecutionPhase } from "../../agents/pi-embedded-runner/execution-phase.js";
23import type { CronConfig, CronRetryOn } from "../../config/types.cron.js";
34import type { HeartbeatRunResult } from "../../infra/heartbeat-wake.js";
45import {
@@ -109,6 +110,39 @@ type StartupCatchupPlan = {
109110deferredJobs: StartupDeferredJob[];
110111};
111112113+type CronAgentWatchdogState =
114+| "waiting_for_runner"
115+| "waiting_for_execution"
116+| "executing"
117+| "timed_out"
118+| "disposed";
119+120+type CronAgentPhaseWatchdogStage = "pre_execution" | "execution";
121+122+const CRON_AGENT_PHASE_WATCHDOG_STAGE = {
123+runner_entered: "pre_execution",
124+workspace: "pre_execution",
125+runtime_plugins: "pre_execution",
126+model_resolution: "pre_execution",
127+auth: "pre_execution",
128+context_engine: "pre_execution",
129+attempt_dispatch: "execution",
130+context_assembled: "execution",
131+turn_accepted: "execution",
132+process_spawned: "execution",
133+tool_execution_started: "execution",
134+assistant_output_started: "execution",
135+model_call_started: "execution",
136+} as const satisfies Record<CronAgentExecutionPhase, CronAgentPhaseWatchdogStage>;
137+138+type CronAgentWatchdog = {
139+start: () => void;
140+noteRunnerStarted: (info?: CronAgentExecutionStarted) => void;
141+notePhase: (info: CronAgentExecutionPhaseUpdate) => void;
142+activeExecution: () => CronAgentExecutionStarted | undefined;
143+dispose: () => void;
144+};
145+112146export async function executeJobCoreWithTimeout(
113147state: CronServiceState,
114148job: CronJob,
@@ -119,12 +153,6 @@ export async function executeJobCoreWithTimeout(
119153}
120154121155const runAbortController = new AbortController();
122-let timeoutId: NodeJS.Timeout | undefined;
123-let setupTimeoutId: NodeJS.Timeout | undefined;
124-let preExecutionTimeoutId: NodeJS.Timeout | undefined;
125-let activeExecution: CronAgentExecutionStarted | undefined;
126-let runnerStarted = false;
127-let executionStarted = false;
128156let timeoutReason: string | undefined;
129157const timeoutMarker = Symbol("cron-timeout");
130158let resolveTimeout: ((value: typeof timeoutMarker) => void) | undefined;
@@ -142,75 +170,16 @@ export async function executeJobCoreWithTimeout(
142170runAbortController.abort(reason);
143171resolveTimeout?.(timeoutMarker);
144172};
145-const startTimeout = () => {
146-if (!timeoutId) {
147-timeoutId = setTimeout(() => {
148-triggerTimeout(timeoutErrorMessage(activeExecution));
149-}, jobTimeoutMs);
150-}
151-};
152-const startSetupTimeout = () => {
153-if (setupTimeoutId || runnerStarted) {
154-return;
155-}
156-setupTimeoutId = setTimeout(() => {
157-if (!runnerStarted) {
158-triggerTimeout(setupTimeoutErrorMessage(activeExecution));
159-}
160-}, CRON_AGENT_SETUP_WATCHDOG_MS);
161-};
162-const clearSetupTimeout = () => {
163-if (!setupTimeoutId) {
164-return;
165-}
166-clearTimeout(setupTimeoutId);
167-setupTimeoutId = undefined;
168-};
169-const startPreExecutionTimeout = () => {
170-if (preExecutionTimeoutId || executionStarted) {
171-return;
172-}
173-preExecutionTimeoutId = setTimeout(() => {
174-if (!executionStarted) {
175-triggerTimeout(preExecutionTimeoutErrorMessage(activeExecution));
176-}
177-}, resolveCronAgentPreExecutionWatchdogMs(jobTimeoutMs));
178-};
179-const clearPreExecutionTimeout = () => {
180-if (!preExecutionTimeoutId) {
181-return;
182-}
183-clearTimeout(preExecutionTimeoutId);
184-preExecutionTimeoutId = undefined;
185-};
186-const noteExecutionProgress = (info?: CronAgentExecutionStarted) => {
187-if (info) {
188-activeExecution = { ...activeExecution, ...info };
189-if (isCronAgentExecutionStarted(info)) {
190-executionStarted = true;
191-clearPreExecutionTimeout();
192-}
193-}
194-};
195-const onExecutionStarted = (info?: CronAgentExecutionStarted) => {
196-runnerStarted = true;
197-noteExecutionProgress(info);
198-clearSetupTimeout();
199-startTimeout();
200-startPreExecutionTimeout();
201-};
202-const onExecutionPhase = (info: CronAgentExecutionPhaseUpdate) => {
203-noteExecutionProgress(info);
204-};
173+const watchdog = createCronAgentWatchdog({
174+deferUntilRunner: deferTimeoutUntilExecutionStart,
175+ jobTimeoutMs,
176+ triggerTimeout,
177+});
205178const corePromise = executeJobCore(state, job, runAbortController.signal, {
206-onExecutionStarted: deferTimeoutUntilExecutionStart ? onExecutionStarted : undefined,
207-onExecutionPhase: deferTimeoutUntilExecutionStart ? onExecutionPhase : undefined,
179+onExecutionStarted: deferTimeoutUntilExecutionStart ? watchdog.noteRunnerStarted : undefined,
180+onExecutionPhase: deferTimeoutUntilExecutionStart ? watchdog.notePhase : undefined,
208181});
209-if (!deferTimeoutUntilExecutionStart) {
210-startTimeout();
211-} else {
212-startSetupTimeout();
213-}
182+watchdog.start();
214183void corePromise.catch((err) => {
215184if (runAbortController.signal.aborted) {
216185state.deps.log.warn(
@@ -224,6 +193,7 @@ export async function executeJobCoreWithTimeout(
224193if (first !== timeoutMarker) {
225194return first;
226195}
196+const activeExecution = watchdog.activeExecution();
227197await cleanupTimedOutCronAgentRun(state, job, jobTimeoutMs, activeExecution);
228198const error = timeoutReason ?? timeoutErrorMessage(activeExecution);
229199return {
@@ -234,14 +204,113 @@ export async function executeJobCoreWithTimeout(
234204}),
235205};
236206} finally {
237-if (timeoutId) {
238-clearTimeout(timeoutId);
239-}
240-clearSetupTimeout();
241-clearPreExecutionTimeout();
207+watchdog.dispose();
242208}
243209}
244210211+function createCronAgentWatchdog(params: {
212+deferUntilRunner: boolean;
213+jobTimeoutMs: number;
214+triggerTimeout: (reason: string) => void;
215+}): CronAgentWatchdog {
216+let state: CronAgentWatchdogState = params.deferUntilRunner ? "waiting_for_runner" : "executing";
217+let timeoutId: NodeJS.Timeout | undefined;
218+let setupTimeoutId: NodeJS.Timeout | undefined;
219+let preExecutionTimeoutId: NodeJS.Timeout | undefined;
220+let activeExecution: CronAgentExecutionStarted | undefined;
221+222+const setTimedOut = (reason: string) => {
223+if (state === "timed_out" || state === "disposed") {
224+return;
225+}
226+state = "timed_out";
227+params.triggerTimeout(reason);
228+};
229+const startTimeout = () => {
230+if (timeoutId || state === "disposed") {
231+return;
232+}
233+timeoutId = setTimeout(() => {
234+setTimedOut(timeoutErrorMessage(activeExecution));
235+}, params.jobTimeoutMs);
236+};
237+const clearSetupTimeout = () => {
238+if (!setupTimeoutId) {
239+return;
240+}
241+clearTimeout(setupTimeoutId);
242+setupTimeoutId = undefined;
243+};
244+const clearPreExecutionTimeout = () => {
245+if (!preExecutionTimeoutId) {
246+return;
247+}
248+clearTimeout(preExecutionTimeoutId);
249+preExecutionTimeoutId = undefined;
250+};
251+const startPreExecutionTimeout = () => {
252+if (preExecutionTimeoutId || state !== "waiting_for_execution") {
253+return;
254+}
255+preExecutionTimeoutId = setTimeout(() => {
256+if (state === "waiting_for_execution") {
257+setTimedOut(preExecutionTimeoutErrorMessage(activeExecution));
258+}
259+}, resolveCronAgentPreExecutionWatchdogMs(params.jobTimeoutMs));
260+};
261+const noteExecutionProgress = (info?: CronAgentExecutionStarted) => {
262+if (!info) {
263+return;
264+}
265+activeExecution = { ...activeExecution, ...info };
266+if (isCronAgentExecutionStarted(info)) {
267+state = "executing";
268+clearPreExecutionTimeout();
269+}
270+};
271+272+return {
273+start: () => {
274+if (params.deferUntilRunner) {
275+setupTimeoutId = setTimeout(() => {
276+if (state === "waiting_for_runner") {
277+setTimedOut(setupTimeoutErrorMessage(activeExecution));
278+}
279+}, CRON_AGENT_SETUP_WATCHDOG_MS);
280+return;
281+}
282+startTimeout();
283+},
284+noteRunnerStarted: (info?: CronAgentExecutionStarted) => {
285+if (state === "disposed" || state === "timed_out") {
286+return;
287+}
288+clearSetupTimeout();
289+startTimeout();
290+if (state !== "executing") {
291+state = "waiting_for_execution";
292+}
293+noteExecutionProgress(info);
294+startPreExecutionTimeout();
295+},
296+notePhase: (info: CronAgentExecutionPhaseUpdate) => {
297+if (state === "disposed" || state === "timed_out") {
298+return;
299+}
300+noteExecutionProgress(info);
301+},
302+activeExecution: () => activeExecution,
303+dispose: () => {
304+state = "disposed";
305+if (timeoutId) {
306+clearTimeout(timeoutId);
307+}
308+clearSetupTimeout();
309+clearPreExecutionTimeout();
310+},
311+};
312+}
313+245314async function cleanupTimedOutCronAgentRun(
246315state: CronServiceState,
247316job: CronJob,
@@ -302,30 +371,14 @@ function preExecutionTimeoutErrorMessage(execution?: CronAgentExecutionStarted):
302371}
303372304373function formatCronAgentExecutionPhase(execution?: CronAgentExecutionStarted): string | undefined {
305-return execution?.phase?.replaceAll("_", "-");
374+return formatEmbeddedAgentExecutionPhase(execution?.phase);
306375}
307376308-const CRON_AGENT_EXECUTION_STARTED_BY_PHASE = {
309-runner_entered: false,
310-workspace: false,
311-runtime_plugins: false,
312-model_resolution: false,
313-auth: false,
314-context_engine: false,
315-attempt_dispatch: true,
316-context_assembled: true,
317-turn_accepted: true,
318-process_spawned: true,
319-tool_execution_started: true,
320-assistant_output_started: true,
321-model_call_started: true,
322-} as const satisfies Record<CronAgentExecutionPhase, boolean>;
323-324377function isCronAgentExecutionStarted(info: CronAgentExecutionStarted): boolean {
325378if (info.firstModelCallStarted) {
326379return true;
327380}
328-return info.phase ? CRON_AGENT_EXECUTION_STARTED_BY_PHASE[info.phase] : false;
381+return info.phase ? CRON_AGENT_PHASE_WATCHDOG_STAGE[info.phase] === "execution" : false;
329382}
330383331384function resolveCronAgentPreExecutionWatchdogMs(jobTimeoutMs: number): number {
此内容由惯性聚合(RSS阅读器)自动聚合整理，仅供阅读参考。 原文来自 — 版权归原作者所有。