@@ -0,0 +1,191 @@
1+import { describe, expect, it } from "vitest";
2+import type {
3+RuntimeId,
4+RuntimeParityCell,
5+RuntimeParityResult,
6+RuntimeParityToolCall,
7+} from "./runtime-parity.js";
8+import {
9+buildTokenEfficiencyReport,
10+renderTokenEfficiencyMarkdownReport,
11+type TokenEfficiencySuiteSummary,
12+} from "./token-efficiency-report.js";
13+14+function makeToolCall(tool: string): RuntimeParityToolCall {
15+return {
16+ tool,
17+argsHash: `${tool}-args`,
18+resultHash: `${tool}-result`,
19+};
20+}
21+22+function makeCell(
23+runtime: RuntimeId,
24+usage: RuntimeParityCell["usage"],
25+toolCalls: RuntimeParityToolCall[] = [],
26+): RuntimeParityCell {
27+return {
28+ runtime,
29+transcriptBytes: '{"role":"assistant"}\n',
30+ toolCalls,
31+finalText: "done",
32+ usage,
33+wallClockMs: 10,
34+bootStateLines: [],
35+};
36+}
37+38+function makeRuntimeParity(
39+scenarioId: string,
40+pi: RuntimeParityCell,
41+codex: RuntimeParityCell,
42+): RuntimeParityResult {
43+return {
44+ scenarioId,
45+drift: "none",
46+cells: { pi, codex },
47+};
48+}
49+50+function makeLiveSummary(runtimeParity: RuntimeParityResult[]): TokenEfficiencySuiteSummary {
51+return {
52+scenarios: runtimeParity.map((result) => ({
53+name: result.scenarioId,
54+status: "pass" as const,
55+runtimeParity: result,
56+})),
57+run: {
58+providerMode: "live-frontier",
59+runtimePair: ["pi", "codex"],
60+},
61+};
62+}
63+64+describe("token efficiency report", () => {
65+it("does not fail live reports solely because Codex uses fewer tokens", () => {
66+const report = buildTokenEfficiencyReport({
67+generatedAt: "2026-05-10T00:00:00.000Z",
68+summary: makeLiveSummary([
69+makeRuntimeParity(
70+"codex-savings",
71+makeCell("pi", { inputTokens: 120, outputTokens: 80, totalTokens: 200 }),
72+makeCell("codex", { inputTokens: 60, outputTokens: 40, totalTokens: 100 }),
73+),
74+]),
75+});
76+77+expect(report.pass).toBe(true);
78+expect(report.aggregate.flaggedScenarios).toEqual([]);
79+expect(report.aggregate.savingsScenarios).toEqual(["codex-savings"]);
80+expect(report.rows[0]).toMatchObject({
81+deltaPercent: -50,
82+classification: "savings",
83+flagged: false,
84+});
85+});
86+87+it("fails live reports on positive Codex token increases over the threshold", () => {
88+const report = buildTokenEfficiencyReport({
89+generatedAt: "2026-05-10T00:00:00.000Z",
90+summary: makeLiveSummary([
91+makeRuntimeParity(
92+"runtime-tool-fs-read",
93+makeCell("pi", { inputTokens: 72_000, outputTokens: 381, totalTokens: 72_381 }, [
94+makeToolCall("fs.read"),
95+makeToolCall("fs.read"),
96+]),
97+makeCell(
98+"codex",
99+{ inputTokens: 118_000, outputTokens: 1_489, totalTokens: 119_489 },
100+Array.from({ length: 40 }, () => makeToolCall("fs.read")),
101+),
102+),
103+]),
104+});
105+106+expect(report.pass).toBe(false);
107+expect(report.aggregate.flaggedScenarios).toEqual(["runtime-tool-fs-read"]);
108+expect(report.rows[0]).toMatchObject({
109+classification: "regression",
110+flagged: true,
111+toolsUsed: ["fs.read"],
112+});
113+expect(report.failures).toEqual([
114+"runtime-tool-fs-read token delta=+65.1% exceeds 15.0% Codex increase threshold",
115+]);
116+});
117+118+it("keeps live zero-usage rows failing instead of passing as neutral", () => {
119+const report = buildTokenEfficiencyReport({
120+summary: makeLiveSummary([
121+makeRuntimeParity(
122+"missing-live-usage",
123+makeCell("pi", { inputTokens: 0, outputTokens: 0, totalTokens: 0 }),
124+makeCell("codex", { inputTokens: 0, outputTokens: 0, totalTokens: 0 }),
125+),
126+]),
127+});
128+129+expect(report.pass).toBe(false);
130+expect(report.failures).toEqual([
131+"missing-live-usage pi live usage totalTokens=0",
132+"missing-live-usage codex live usage totalTokens=0",
133+]);
134+});
135+136+it("labels mock-estimated Codex increases as regressions without failing the live gate", () => {
137+const report = buildTokenEfficiencyReport({
138+summary: {
139+scenarios: [
140+{
141+name: "mock-regression",
142+status: "pass",
143+runtimeParity: makeRuntimeParity(
144+"mock-regression",
145+makeCell("pi", { inputTokens: 100, outputTokens: 0, totalTokens: 100 }),
146+makeCell("codex", { inputTokens: 130, outputTokens: 0, totalTokens: 130 }),
147+),
148+},
149+],
150+run: {
151+providerMode: "mock-openai",
152+runtimePair: ["pi", "codex"],
153+},
154+},
155+});
156+157+expect(report.status).toBe("estimated");
158+expect(report.pass).toBe(true);
159+expect(report.aggregate.flaggedScenarios).toEqual([]);
160+expect(report.rows[0]).toMatchObject({
161+usageSource: "mock-estimate",
162+classification: "regression",
163+flagged: false,
164+});
165+});
166+167+it("renders savings and regression classifications in the markdown report", () => {
168+const report = buildTokenEfficiencyReport({
169+generatedAt: "2026-05-10T00:00:00.000Z",
170+summary: makeLiveSummary([
171+makeRuntimeParity(
172+"codex-savings",
173+makeCell("pi", { inputTokens: 100, outputTokens: 100, totalTokens: 200 }),
174+makeCell("codex", { inputTokens: 50, outputTokens: 50, totalTokens: 100 }),
175+),
176+makeRuntimeParity(
177+"codex-regression",
178+makeCell("pi", { inputTokens: 100, outputTokens: 0, totalTokens: 100 }),
179+makeCell("codex", { inputTokens: 130, outputTokens: 0, totalTokens: 130 }),
180+),
181+]),
182+});
183+184+const markdown = renderTokenEfficiencyMarkdownReport(report);
185+expect(markdown).toContain("p50 per scenario");
186+expect(markdown).toContain("| codex-savings | live-usage |");
187+expect(markdown).toContain("| -50.0% | savings | no |");
188+expect(markdown).toContain("| codex-regression | live-usage |");
189+expect(markdown).toContain("| +30.0% | regression | yes |");
190+});
191+});
此内容由惯性聚合(RSS阅读器)自动聚合整理，仅供阅读参考。 原文来自 — 版权归原作者所有。