@@ -112,6 +112,32 @@ function makeSuiteResult(params: { outputDir: string; model: string; transcript:
112112} satisfies QaSuiteResult;
113113}
114114115+function requireRunSuiteParams(runSuite: ReturnType<typeof vi.fn>, index = 0) {
116+const params = runSuite.mock.calls[index]?.[0] as CharacterRunSuiteParams | undefined;
117+if (!params) {
118+throw new Error(`runSuite call ${index} missing`);
119+}
120+return params;
121+}
122+123+function requireRunJudgeParams(runJudge: ReturnType<typeof vi.fn>, index = 0) {
124+const params = runJudge.mock.calls[index]?.[0] as CharacterRunJudgeParams | undefined;
125+if (!params) {
126+throw new Error(`runJudge call ${index} missing`);
127+}
128+return params;
129+}
130+131+function expectFirstRunFailure(
132+result: Awaited<ReturnType<typeof runQaCharacterEval>>,
133+expected: { model: string; error: string },
134+) {
135+const run = result.runs[0];
136+expect(run?.model).toBe(expected.model);
137+expect(run?.status).toBe("fail");
138+expect(run?.error).toBe(expected.error);
139+}
140+115141describe("runQaCharacterEval", () => {
116142let tempRoot: string;
117143@@ -160,24 +186,17 @@ describe("runQaCharacterEval", () => {
160186});
161187162188expect(runSuite).toHaveBeenCalledTimes(2);
163-expect(runSuite).toHaveBeenNthCalledWith(
164-1,
165-expect.objectContaining({
166-providerMode: "live-frontier",
167-primaryModel: "openai/gpt-5.5",
168-alternateModel: "openai/gpt-5.5",
169-fastMode: true,
170-scenarioIds: ["character-vibes-gollum"],
171-}),
172-);
173-expect(runJudge).toHaveBeenCalledWith(
174-expect.objectContaining({
175-judgeModel: "openai/gpt-5.5",
176-judgeThinkingDefault: "xhigh",
177-judgeFastMode: true,
178-timeoutMs: 300_000,
179-}),
180-);
189+const firstRunParams = requireRunSuiteParams(runSuite);
190+expect(firstRunParams.providerMode).toBe("live-frontier");
191+expect(firstRunParams.primaryModel).toBe("openai/gpt-5.5");
192+expect(firstRunParams.alternateModel).toBe("openai/gpt-5.5");
193+expect(firstRunParams.fastMode).toBe(true);
194+expect(firstRunParams.scenarioIds).toEqual(["character-vibes-gollum"]);
195+const judgeParams = requireRunJudgeParams(runJudge);
196+expect(judgeParams.judgeModel).toBe("openai/gpt-5.5");
197+expect(judgeParams.judgeThinkingDefault).toBe("xhigh");
198+expect(judgeParams.judgeFastMode).toBe(true);
199+expect(judgeParams.timeoutMs).toBe(300_000);
181200expect(result.judgments).toHaveLength(1);
182201expect(result.judgments[0]?.rankings.map((ranking) => ranking.model)).toEqual([
183202"openai/gpt-5.5",
@@ -404,9 +423,8 @@ describe("runQaCharacterEval", () => {
404423 runJudge,
405424});
406425407-expect(result.runs[0]).toMatchObject({
426+expectFirstRunFailure(result, {
408427model: "qwen/qwen3.6-plus",
409-status: "fail",
410428error: "model unsupported error leaked into transcript",
411429});
412430});
@@ -432,9 +450,8 @@ describe("runQaCharacterEval", () => {
432450 runJudge,
433451});
434452435-expect(result.runs[0]).toMatchObject({
453+expectFirstRunFailure(result, {
436454model: "qwen/qwen3.5-plus",
437-status: "fail",
438455error: "tool failure leaked into transcript",
439456});
440457});
@@ -461,9 +478,8 @@ describe("runQaCharacterEval", () => {
461478 runJudge,
462479});
463480464-expect(result.runs[0]).toMatchObject({
481+expectFirstRunFailure(result, {
465482model: "qa/generic-fallback-model",
466-status: "fail",
467483error: "generic request failure leaked into transcript",
468484});
469485});
@@ -490,9 +506,8 @@ describe("runQaCharacterEval", () => {
490506 runJudge,
491507});
492508493-expect(result.runs[0]).toMatchObject({
509+expectFirstRunFailure(result, {
494510model: "google/gemini-test",
495-status: "fail",
496511error: "LLM timeout leaked into transcript",
497512});
498513});
@@ -519,9 +534,8 @@ describe("runQaCharacterEval", () => {
519534 runJudge,
520535});
521536522-expect(result.runs[0]).toMatchObject({
537+expectFirstRunFailure(result, {
523538model: "codex/gpt-5.5",
524-status: "fail",
525539error: "internal harness/meta text leaked into transcript",
526540});
527541});
此内容由惯性聚合(RSS阅读器)自动聚合整理，仅供阅读参考。 原文来自 — 版权归原作者所有。