@@ -45,6 +45,7 @@ type SlackChannelStatus = {
4545const SLACK_QA_READY_TIMEOUT_MS = 45_000;
4646const SLACK_QA_READY_STABILITY_MS = 3_000;
4747const SLACK_QA_GATEWAY_STOP_SETTLE_MS = 3_000;
48+const SLACK_QA_RETRYABLE_SCENARIO_ATTEMPTS = 2;
48494950type SlackQaScenarioId =
5051| "slack-allowlist-block"
@@ -802,6 +803,10 @@ async function waitForSlackChannelStable(
802803);
803804}
804805806+function isRetryableSlackQaScenarioError(error: unknown) {
807+return /timed out after \d+ms waiting for Slack message/iu.test(formatErrorMessage(error));
808+}
809+805810function toObservedSlackArtifacts(params: {
806811includeContent: boolean;
807812messages: SlackObservedMessage[];
@@ -930,138 +935,164 @@ export async function runSlackQaLive(params: {
930935timeout: SLACK_QA_WEB_API_TIMEOUT_MS,
931936});
932937for (const scenario of scenarios) {
933-let gatewayHarness: Awaited<ReturnType<typeof startQaLiveLaneGateway>> | undefined;
934-try {
935-assertLeaseHealthy();
936-gatewayHarness = await startQaLiveLaneGateway({
937- repoRoot,
938-transport: {
939-requiredPluginIds: [],
940-createGatewayConfig: () => ({}),
941-},
942-transportBaseUrl: "http://127.0.0.1:0",
943- providerMode,
944- primaryModel,
945- alternateModel,
946-fastMode: params.fastMode,
947-controlUiEnabled: false,
948-mutateConfig: (cfg) =>
949-buildSlackQaConfig(cfg, {
950-channelId: activeRuntimeEnv.channelId,
951-driverBotUserId: driverIdentity.userId,
952-overrides: scenario.configOverrides,
953- sutAccountId,
954-sutAppToken: activeRuntimeEnv.sutAppToken,
955-sutBotToken: activeRuntimeEnv.sutBotToken,
956-}),
957-});
958-const activeGatewayHarness = gatewayHarness;
959-await waitForSlackChannelStable(activeGatewayHarness.gateway, sutAccountId);
960-const scenarioRun = scenario.buildRun(sutIdentity.userId);
961-const baseScenarioContext = {
962-channelId: activeRuntimeEnv.channelId,
963- driverClient,
964-gateway: activeGatewayHarness.gateway,
965-postSlackMessage: async (message: { text: string; threadTs?: string }) =>
966-await sendSlackChannelMessage({
967-channelId: activeRuntimeEnv.channelId,
968-client: driverClient,
969-text: message.text,
970-threadTs: message.threadTs,
971-}),
972- sutIdentity,
973- sutReadClient,
974-waitForReady: async () =>
975-await waitForSlackChannelStable(activeGatewayHarness.gateway, sutAccountId),
976-};
977-const beforeRunResult = await scenarioRun.beforeRun?.(baseScenarioContext);
978-const beforeRunDetails =
979-typeof beforeRunResult === "string" ? beforeRunResult : beforeRunResult?.details;
980-const requestStartedAt = new Date();
981-const sent = await sendSlackChannelMessage({
982-channelId: activeRuntimeEnv.channelId,
983-client: driverClient,
984-text: scenarioRun.input,
985-threadTs:
986-typeof beforeRunResult === "object" ? beforeRunResult?.inputThreadTs : undefined,
987-});
988-const requestThreadTs =
989-(typeof beforeRunResult === "object" ? beforeRunResult?.inputThreadTs : undefined) ??
990-sent.ts;
991-if (scenarioRun.expectReply) {
992-const reply = await waitForSlackScenarioReply({
993-channelId: activeRuntimeEnv.channelId,
994-client: sutReadClient,
995-matchText: scenarioRun.matchText,
996- observedMessages,
997-observationScenarioId: scenario.id,
998-observationScenarioTitle: scenario.title,
999-sentTs: sent.ts,
1000-threadTs: requestThreadTs,
1001- sutIdentity,
1002-timeoutMs: scenario.timeoutMs,
1003-});
1004-scenarioRun.verify?.(reply.message, { requestThreadTs, sentTs: sent.ts });
1005-const responseObservedAt = new Date(reply.observedAt);
1006-const rttMs = responseObservedAt.getTime() - requestStartedAt.getTime();
1007-const afterReplyDetails = await scenarioRun.afterReply?.(reply.message, {
1008- ...baseScenarioContext,
1009-sentTs: sent.ts,
1010-});
1011-scenarioResults.push({
1012-id: scenario.id,
1013-title: scenario.title,
1014-status: "pass",
1015-details: [`reply matched in ${rttMs}ms`, beforeRunDetails, afterReplyDetails]
1016-.filter(Boolean)
1017-.join("; "),
1018- rttMs,
1019-requestStartedAt: requestStartedAt.toISOString(),
1020-responseObservedAt: responseObservedAt.toISOString(),
938+let scenarioAttempt = 1;
939+while (true) {
940+let gatewayHarness: Awaited<ReturnType<typeof startQaLiveLaneGateway>> | undefined;
941+try {
942+assertLeaseHealthy();
943+gatewayHarness = await startQaLiveLaneGateway({
944+ repoRoot,
945+transport: {
946+requiredPluginIds: [],
947+createGatewayConfig: () => ({}),
948+},
949+transportBaseUrl: "http://127.0.0.1:0",
950+ providerMode,
951+ primaryModel,
952+ alternateModel,
953+fastMode: params.fastMode,
954+controlUiEnabled: false,
955+mutateConfig: (cfg) =>
956+buildSlackQaConfig(cfg, {
957+channelId: activeRuntimeEnv.channelId,
958+driverBotUserId: driverIdentity.userId,
959+overrides: scenario.configOverrides,
960+ sutAccountId,
961+sutAppToken: activeRuntimeEnv.sutAppToken,
962+sutBotToken: activeRuntimeEnv.sutBotToken,
963+}),
1021964});
1022-} else {
1023-await waitForSlackNoReply({
965+const activeGatewayHarness = gatewayHarness;
966+await waitForSlackChannelStable(activeGatewayHarness.gateway, sutAccountId);
967+const scenarioRun = scenario.buildRun(sutIdentity.userId);
968+const baseScenarioContext = {
1024969channelId: activeRuntimeEnv.channelId,
1025-client: sutReadClient,
1026-matchText: scenarioRun.matchText,
1027- observedMessages,
1028-observationScenarioId: scenario.id,
1029-observationScenarioTitle: scenario.title,
1030-sentTs: sent.ts,
970+ driverClient,
971+gateway: activeGatewayHarness.gateway,
972+postSlackMessage: async (message: { text: string; threadTs?: string }) =>
973+await sendSlackChannelMessage({
974+channelId: activeRuntimeEnv.channelId,
975+client: driverClient,
976+text: message.text,
977+threadTs: message.threadTs,
978+}),
1031979 sutIdentity,
1032-timeoutMs: scenario.timeoutMs,
980+ sutReadClient,
981+waitForReady: async () =>
982+await waitForSlackChannelStable(activeGatewayHarness.gateway, sutAccountId),
983+};
984+const beforeRunResult = await scenarioRun.beforeRun?.(baseScenarioContext);
985+const beforeRunDetails =
986+typeof beforeRunResult === "string" ? beforeRunResult : beforeRunResult?.details;
987+const requestStartedAt = new Date();
988+const sent = await sendSlackChannelMessage({
989+channelId: activeRuntimeEnv.channelId,
990+client: driverClient,
991+text: scenarioRun.input,
992+threadTs:
993+typeof beforeRunResult === "object" ? beforeRunResult?.inputThreadTs : undefined,
1033994});
995+const requestThreadTs =
996+(typeof beforeRunResult === "object" ? beforeRunResult?.inputThreadTs : undefined) ??
997+sent.ts;
998+if (scenarioRun.expectReply) {
999+const reply = await waitForSlackScenarioReply({
1000+channelId: activeRuntimeEnv.channelId,
1001+client: sutReadClient,
1002+matchText: scenarioRun.matchText,
1003+ observedMessages,
1004+observationScenarioId: scenario.id,
1005+observationScenarioTitle: scenario.title,
1006+sentTs: sent.ts,
1007+threadTs: requestThreadTs,
1008+ sutIdentity,
1009+timeoutMs: scenario.timeoutMs,
1010+});
1011+scenarioRun.verify?.(reply.message, { requestThreadTs, sentTs: sent.ts });
1012+const responseObservedAt = new Date(reply.observedAt);
1013+const rttMs = responseObservedAt.getTime() - requestStartedAt.getTime();
1014+const afterReplyDetails = await scenarioRun.afterReply?.(reply.message, {
1015+ ...baseScenarioContext,
1016+sentTs: sent.ts,
1017+});
1018+scenarioResults.push({
1019+id: scenario.id,
1020+title: scenario.title,
1021+status: "pass",
1022+details: [
1023+`reply matched in ${rttMs}ms`,
1024+beforeRunDetails,
1025+afterReplyDetails,
1026+scenarioAttempt > 1 ? `retried ${scenarioAttempt - 1}x` : undefined,
1027+]
1028+.filter(Boolean)
1029+.join("; "),
1030+ rttMs,
1031+requestStartedAt: requestStartedAt.toISOString(),
1032+responseObservedAt: responseObservedAt.toISOString(),
1033+});
1034+} else {
1035+await waitForSlackNoReply({
1036+channelId: activeRuntimeEnv.channelId,
1037+client: sutReadClient,
1038+matchText: scenarioRun.matchText,
1039+ observedMessages,
1040+observationScenarioId: scenario.id,
1041+observationScenarioTitle: scenario.title,
1042+sentTs: sent.ts,
1043+ sutIdentity,
1044+timeoutMs: scenario.timeoutMs,
1045+});
1046+scenarioResults.push({
1047+id: scenario.id,
1048+title: scenario.title,
1049+status: "pass",
1050+details:
1051+scenarioAttempt > 1 ? `no reply; retried ${scenarioAttempt - 1}x` : "no reply",
1052+});
1053+}
1054+break;
1055+} catch (error) {
1056+if (
1057+scenarioAttempt < SLACK_QA_RETRYABLE_SCENARIO_ATTEMPTS &&
1058+isRetryableSlackQaScenarioError(error)
1059+) {
1060+scenarioAttempt += 1;
1061+continue;
1062+}
10341063scenarioResults.push({
10351064id: scenario.id,
10361065title: scenario.title,
1037-status: "pass",
1038-details: "no reply",
1066+status: "fail",
1067+details:
1068+scenarioAttempt > 1
1069+ ? `${formatErrorMessage(error)}; retried ${scenarioAttempt - 1}x`
1070+ : formatErrorMessage(error),
10391071});
1040-}
1041-} catch (error) {
1042-scenarioResults.push({
1043-id: scenario.id,
1044-title: scenario.title,
1045-status: "fail",
1046-details: formatErrorMessage(error),
1047-});
1048-preservedGatewayDebugArtifacts = true;
1049-if (gatewayHarness) {
1050-await gatewayHarness
1051-.stop({ keepTemp: true, preserveToDir: gatewayDebugDirPath })
1052-.catch((stopError) => {
1053-appendLiveLaneIssue(cleanupIssues, "gateway debug preservation failed", stopError);
1072+preservedGatewayDebugArtifacts = true;
1073+if (gatewayHarness) {
1074+await gatewayHarness
1075+.stop({ keepTemp: true, preserveToDir: gatewayDebugDirPath })
1076+.catch((stopError) => {
1077+appendLiveLaneIssue(cleanupIssues, "gateway debug preservation failed", stopError);
1078+});
1079+}
1080+break;
1081+} finally {
1082+if (!preservedGatewayDebugArtifacts && gatewayHarness) {
1083+await gatewayHarness.stop().catch((error) => {
1084+appendLiveLaneIssue(cleanupIssues, "gateway stop failed", error);
10541085});
1086+await new Promise((resolve) => setTimeout(resolve, SLACK_QA_GATEWAY_STOP_SETTLE_MS));
1087+}
10551088}
1056-break;
1057-} finally {
1058-if (!preservedGatewayDebugArtifacts && gatewayHarness) {
1059-await gatewayHarness.stop().catch((error) => {
1060-appendLiveLaneIssue(cleanupIssues, "gateway stop failed", error);
1061-});
1062-await new Promise((resolve) => setTimeout(resolve, SLACK_QA_GATEWAY_STOP_SETTLE_MS));
1089+if (scenarioResults.at(-1)?.id === scenario.id) {
1090+break;
10631091}
10641092}
1093+if (scenarioResults.at(-1)?.status === "fail") {
1094+break;
1095+}
10651096}
10661097} catch (error) {
10671098cleanupIssues.push(
此内容由惯性聚合(RSS阅读器)自动聚合整理，仅供阅读参考。 原文来自 — 版权归原作者所有。