@@ -719,6 +719,54 @@ describe("qa confidence report", () => {
|
719 | 719 | expect(report.lanes[0]?.details).toContain("count/scenario mismatch"); |
720 | 720 | }); |
721 | 721 | |
| 722 | +it("treats impossible suite counts as unknown", async () => { |
| 723 | +for (const [artifact, expectedDetail] of [ |
| 724 | +[ |
| 725 | +{ counts: { total: 1, passed: -1, skipped: 0, failed: 0 } }, |
| 726 | +"counts.passed must be a non-negative integer", |
| 727 | +], |
| 728 | +[ |
| 729 | +{ counts: { total: 1, passed: 2, failed: 0 } }, |
| 730 | +"counts.total=1 is less than provided count sum=2", |
| 731 | +], |
| 732 | +[ |
| 733 | +{ counts: { total: 1, skipped: 2, failed: 0 } }, |
| 734 | +"counts.total=1 is less than provided count sum=2", |
| 735 | +], |
| 736 | +[ |
| 737 | +{ counts: { total: 5, passed: 2, skipped: 2, failed: 0 } }, |
| 738 | +"counts.total=5 does not match counts.passed+counts.failed+counts.skipped=4", |
| 739 | +], |
| 740 | +] as const) { |
| 741 | +await writeJson("live/qa-suite-summary.json", artifact); |
| 742 | + |
| 743 | +const report = await buildQaConfidenceReport({ |
| 744 | +manifest: { |
| 745 | +version: 1, |
| 746 | +profile: "codex-100", |
| 747 | +lanes: [ |
| 748 | +{ |
| 749 | +id: "first-hour-live", |
| 750 | +title: "First hour live", |
| 751 | +kind: "qa-suite-summary", |
| 752 | +artifact: "live/qa-suite-summary.json", |
| 753 | +required: true, |
| 754 | +failureVerdict: "qa-harness-bug", |
| 755 | +}, |
| 756 | +], |
| 757 | +}, |
| 758 | +artifactRoot: tempRoot, |
| 759 | +strictZeroUnknowns: true, |
| 760 | +generatedAt: "2026-05-13T00:00:00.000Z", |
| 761 | +}); |
| 762 | + |
| 763 | +expect(report.pass).toBe(false); |
| 764 | +expect(report.counts).toMatchObject({ failed: 0, unknown: 1 }); |
| 765 | +expect(report.lanes[0]).toMatchObject({ status: "unknown" }); |
| 766 | +expect(report.lanes[0]?.details).toContain(expectedDetail); |
| 767 | +} |
| 768 | +}); |
| 769 | + |
722 | 770 | it("requires generic summary lanes to expose an explicit pass signal", async () => { |
723 | 771 | await writeJson("runtime/qa-runtime-parity-summary.json", {}); |
724 | 772 | |
|
