@@ -83,9 +83,30 @@ import type { QaProviderModeInput } from "./run-config.js";
83838484describe("qa cli runtime", () => {
8585let stdoutWrite: ReturnType<typeof vi.spyOn>;
86-87-beforeEach(() => {
86+let stderrWrite: ReturnType<typeof vi.spyOn>;
87+let suiteArtifactsDir: string;
88+let suiteReportPath: string;
89+let suiteSummaryPath: string;
90+91+beforeEach(async () => {
92+suiteArtifactsDir = await fs.mkdtemp(path.join(os.tmpdir(), "qa-suite-runtime-"));
93+suiteReportPath = path.join(suiteArtifactsDir, "qa-suite-report.md");
94+suiteSummaryPath = path.join(suiteArtifactsDir, "qa-suite-summary.json");
95+await fs.writeFile(suiteReportPath, "# QA Suite Report\n", "utf8");
96+await fs.writeFile(
97+suiteSummaryPath,
98+JSON.stringify({
99+counts: {
100+total: 1,
101+passed: 1,
102+failed: 0,
103+},
104+scenarios: [],
105+}),
106+"utf8",
107+);
88108stdoutWrite = vi.spyOn(process.stdout, "write").mockReturnValue(true);
109+stderrWrite = vi.spyOn(process.stderr, "write").mockReturnValue(true);
89110runQaSuiteFromRuntime.mockReset();
90111runQaCharacterEval.mockReset();
91112runQaManualLane.mockReset();
@@ -101,8 +122,8 @@ describe("qa cli runtime", () => {
101122);
102123runQaSuiteFromRuntime.mockResolvedValue({
103124watchUrl: "http://127.0.0.1:43124",
104-reportPath: "/tmp/report.md",
105-summaryPath: "/tmp/summary.json",
125+reportPath: suiteReportPath,
126+summaryPath: suiteSummaryPath,
106127scenarios: [],
107128});
108129runQaCharacterEval.mockResolvedValue({
@@ -153,9 +174,11 @@ describe("qa cli runtime", () => {
153174});
154175});
155176156-afterEach(() => {
177+afterEach(async () => {
157178stdoutWrite.mockRestore();
179+stderrWrite.mockRestore();
158180vi.clearAllMocks();
181+await fs.rm(suiteArtifactsDir, { recursive: true, force: true });
159182});
160183161184it("resolves suite repo-root-relative paths before dispatching", async () => {
@@ -307,10 +330,22 @@ describe("qa cli runtime", () => {
307330it("sets a failing exit code when host suite scenarios fail", async () => {
308331const priorExitCode = process.exitCode;
309332process.exitCode = undefined;
333+await fs.writeFile(
334+suiteSummaryPath,
335+JSON.stringify({
336+counts: {
337+total: 1,
338+passed: 0,
339+failed: 1,
340+},
341+scenarios: [{ name: "channel chat baseline", status: "fail" }],
342+}),
343+"utf8",
344+);
310345runQaSuiteFromRuntime.mockResolvedValueOnce({
311346watchUrl: "http://127.0.0.1:43124",
312-reportPath: "/tmp/report.md",
313-summaryPath: "/tmp/summary.json",
347+reportPath: suiteReportPath,
348+summaryPath: suiteSummaryPath,
314349scenarios: [
315350{
316351name: "channel chat baseline",
@@ -333,10 +368,22 @@ describe("qa cli runtime", () => {
333368it("keeps host suite exit code clear when --allow-failures is set", async () => {
334369const priorExitCode = process.exitCode;
335370process.exitCode = undefined;
371+await fs.writeFile(
372+suiteSummaryPath,
373+JSON.stringify({
374+counts: {
375+total: 1,
376+passed: 0,
377+failed: 1,
378+},
379+scenarios: [{ name: "channel chat baseline", status: "fail" }],
380+}),
381+"utf8",
382+);
336383runQaSuiteFromRuntime.mockResolvedValueOnce({
337384watchUrl: "http://127.0.0.1:43124",
338-reportPath: "/tmp/report.md",
339-summaryPath: "/tmp/summary.json",
385+reportPath: suiteReportPath,
386+summaryPath: suiteSummaryPath,
340387scenarios: [
341388{
342389name: "channel chat baseline",
@@ -357,6 +404,203 @@ describe("qa cli runtime", () => {
357404}
358405});
359406407+it("retries host suite runs once for retryable infra failures", async () => {
408+runQaSuiteFromRuntime
409+.mockRejectedValueOnce(new Error("agent.wait timeout while waiting for transport ready"))
410+.mockResolvedValueOnce({
411+watchUrl: "http://127.0.0.1:43124",
412+reportPath: suiteReportPath,
413+summaryPath: suiteSummaryPath,
414+scenarios: [],
415+});
416+417+await runQaSuiteCommand({
418+repoRoot: "/tmp/openclaw-repo",
419+});
420+421+expect(runQaSuiteFromRuntime).toHaveBeenCalledTimes(2);
422+expect(stderrWrite).toHaveBeenCalledWith(
423+expect.stringContaining("[qa-suite] infra retry 1/1: agent.wait timeout"),
424+);
425+});
426+427+it("retries host suite runs once for qa-channel readiness timeouts", async () => {
428+runQaSuiteFromRuntime
429+.mockRejectedValueOnce(
430+new Error(
431+"timed out after 180000ms waiting for qa-channel ready; last status: no qa-channel accounts reported",
432+),
433+)
434+.mockResolvedValueOnce({
435+watchUrl: "http://127.0.0.1:43124",
436+reportPath: suiteReportPath,
437+summaryPath: suiteSummaryPath,
438+scenarios: [],
439+});
440+441+await runQaSuiteCommand({
442+repoRoot: "/tmp/openclaw-repo",
443+});
444+445+expect(runQaSuiteFromRuntime).toHaveBeenCalledTimes(2);
446+expect(stderrWrite).toHaveBeenCalledWith(
447+expect.stringContaining(
448+"[qa-suite] infra retry 1/1: timed out after 180000ms waiting for qa-channel ready",
449+),
450+);
451+});
452+453+it("does not retry host suite runs for generic timeout wording", async () => {
454+runQaSuiteFromRuntime.mockRejectedValueOnce(
455+new Error("approval-turn timed out waiting for post-approval read"),
456+);
457+458+await expect(
459+runQaSuiteCommand({
460+repoRoot: "/tmp/openclaw-repo",
461+}),
462+).rejects.toThrow("approval-turn timed out waiting for post-approval read");
463+464+expect(runQaSuiteFromRuntime).toHaveBeenCalledTimes(1);
465+});
466+467+it("does not retry host suite runs for semantic failures", async () => {
468+const priorExitCode = process.exitCode;
469+process.exitCode = undefined;
470+await fs.writeFile(
471+suiteSummaryPath,
472+JSON.stringify({
473+counts: {
474+total: 1,
475+passed: 0,
476+failed: 1,
477+},
478+scenarios: [{ name: "channel chat baseline", status: "fail" }],
479+}),
480+"utf8",
481+);
482+runQaSuiteFromRuntime.mockResolvedValueOnce({
483+watchUrl: "http://127.0.0.1:43124",
484+reportPath: suiteReportPath,
485+summaryPath: suiteSummaryPath,
486+scenarios: [
487+{
488+name: "channel chat baseline",
489+status: "fail",
490+steps: [],
491+},
492+],
493+});
494+495+try {
496+await runQaSuiteCommand({
497+repoRoot: "/tmp/openclaw-repo",
498+});
499+expect(runQaSuiteFromRuntime).toHaveBeenCalledTimes(1);
500+expect(process.exitCode).toBe(1);
501+} finally {
502+process.exitCode = priorExitCode;
503+}
504+});
505+506+it("runs a host-only parity preflight against the sentinel scenario", async () => {
507+await runQaSuiteCommand({
508+repoRoot: "/tmp/openclaw-repo",
509+providerMode: "mock-openai",
510+primaryModel: "openai/gpt-5.4",
511+alternateModel: "anthropic/claude-opus-4-6",
512+preflight: true,
513+});
514+515+expect(runQaSuiteFromRuntime).toHaveBeenCalledWith({
516+repoRoot: path.resolve("/tmp/openclaw-repo"),
517+outputDir: expect.stringMatching(
518+/^\/tmp\/openclaw-repo\/\.artifacts\/qa-e2e\/preflight\/suite-/,
519+),
520+transportId: "qa-channel",
521+providerMode: "mock-openai",
522+primaryModel: "openai/gpt-5.4",
523+alternateModel: "anthropic/claude-opus-4-6",
524+scenarioIds: ["approval-turn-tool-followthrough"],
525+concurrency: 1,
526+});
527+expect(stdoutWrite).toHaveBeenCalledWith(
528+expect.stringContaining("QA parity preflight summary:"),
529+);
530+});
531+532+it("throws when parity preflight finds a failing sentinel scenario", async () => {
533+await fs.writeFile(
534+suiteSummaryPath,
535+JSON.stringify({
536+counts: {
537+total: 1,
538+passed: 0,
539+failed: 1,
540+},
541+scenarios: [{ name: "approval turn tool followthrough", status: "fail" }],
542+}),
543+"utf8",
544+);
545+runQaSuiteFromRuntime.mockResolvedValueOnce({
546+watchUrl: "http://127.0.0.1:43124",
547+reportPath: suiteReportPath,
548+summaryPath: suiteSummaryPath,
549+scenarios: [{ name: "approval turn tool followthrough", status: "fail", steps: [] }],
550+});
551+552+await expect(
553+runQaSuiteCommand({
554+repoRoot: "/tmp/openclaw-repo",
555+preflight: true,
556+}),
557+).rejects.toThrow("QA parity preflight failed with 1 failing scenario.");
558+});
559+560+it("keeps parity preflight exit code clear when --allow-failures is set", async () => {
561+const priorExitCode = process.exitCode;
562+process.exitCode = undefined;
563+await fs.writeFile(
564+suiteSummaryPath,
565+JSON.stringify({
566+counts: {
567+total: 1,
568+passed: 0,
569+failed: 1,
570+},
571+scenarios: [{ name: "approval turn tool followthrough", status: "fail" }],
572+}),
573+"utf8",
574+);
575+runQaSuiteFromRuntime.mockResolvedValueOnce({
576+watchUrl: "http://127.0.0.1:43124",
577+reportPath: suiteReportPath,
578+summaryPath: suiteSummaryPath,
579+scenarios: [{ name: "approval turn tool followthrough", status: "fail", steps: [] }],
580+});
581+582+try {
583+await runQaSuiteCommand({
584+repoRoot: "/tmp/openclaw-repo",
585+preflight: true,
586+allowFailures: true,
587+});
588+expect(process.exitCode).toBeUndefined();
589+} finally {
590+process.exitCode = priorExitCode;
591+}
592+});
593+594+it("rejects preflight on the multipass runner", async () => {
595+await expect(
596+runQaSuiteCommand({
597+repoRoot: "/tmp/openclaw-repo",
598+runner: "multipass",
599+preflight: true,
600+}),
601+).rejects.toThrow("--preflight requires --runner host.");
602+});
603+360604it("passes host suite CLI auth mode through", async () => {
361605await runQaSuiteCommand({
362606repoRoot: "/tmp/openclaw-repo",
此内容由惯性聚合(RSS阅读器)自动聚合整理，仅供阅读参考。 原文来自 — 版权归原作者所有。