@@ -2,11 +2,7 @@ import {
22QA_AGENTIC_PARITY_SCENARIO_TITLES,
33QA_AGENTIC_PARITY_TOOL_BACKED_SCENARIO_TITLES,
44} from "./agentic-parity.js";
5-import type {
6-RuntimeId,
7-RuntimeParityDrift,
8-RuntimeParityResult,
9-} from "./runtime-parity.js";
5+import type { RuntimeId, RuntimeParityDrift, RuntimeParityResult } from "./runtime-parity.js";
106import { isRuntimeParityResultPass, runtimeParityCellStatus } from "./runtime-parity.js";
117128type QaParityReportStep = {
@@ -251,6 +247,23 @@ function buildRuntimeParityDriftCounts(): Record<RuntimeParityDrift, number> {
251247};
252248}
253249250+function isLiveProviderMode(providerMode: string | undefined) {
251+return providerMode?.startsWith("live-") === true;
252+}
253+254+function describeLiveUsageFailure(scenarioName: string, scenario: QaRuntimeParityScenarioReport) {
255+const missing = [
256+scenario.piTokens > 0 ? undefined : `${scenario.piStatus === "pass" ? "pi" : "pi failed"}=0`,
257+scenario.codexTokens > 0
258+ ? undefined
259+ : `${scenario.codexStatus === "pass" ? "codex" : "codex failed"}=0`,
260+].filter((entry): entry is string => !!entry);
261+if (missing.length === 0) {
262+return undefined;
263+}
264+return `${scenarioName} missing live assistant-message usage (${missing.join(", ")}).`;
265+}
266+254267function normalizeRuntimePair(
255268pair: [RuntimeId, RuntimeId] | null | undefined,
256269): [RuntimeId, RuntimeId] {
@@ -608,9 +621,11 @@ export function buildQaRuntimeParityReport(params: {
608621comparedAt?: string;
609622}): QaRuntimeParityReport {
610623const runtimePair = normalizeRuntimePair(params.summary.run?.runtimePair);
624+const providerMode = params.summary.run?.providerMode;
625+const requiresLiveUsage = isLiveProviderMode(providerMode);
611626const driftCounts = buildRuntimeParityDriftCounts();
612627const failures: string[] = [];
613-const scenarios = params.summary.scenarios.map((scenario) => {
628+const scenarios: QaRuntimeParityScenarioReport[] = params.summary.scenarios.map((scenario) => {
614629const parity = scenario.runtimeParity;
615630if (!parity) {
616631failures.push(`Missing runtime parity capture for ${scenario.name}.`);
@@ -632,15 +647,10 @@ export function buildQaRuntimeParityReport(params: {
632647const codexCell = parity.cells.codex;
633648const piStatus = runtimeParityCellStatus(piCell);
634649const codexStatus = runtimeParityCellStatus(codexCell);
635-const status = isRuntimeParityResultPass(parity) ? "pass" : "fail";
636-if (status === "fail") {
637-failures.push(
638-`${scenario.name} drift=${parity.drift}${parity.driftDetails ? ` (${parity.driftDetails})` : ""}.`,
639-);
640-}
641-return {
650+const parityStatus = isRuntimeParityResultPass(parity) ? "pass" : "fail";
651+const reportScenario = {
642652name: scenario.name,
643- status,
653+status: parityStatus,
644654drift: parity.drift,
645655driftDetails: parity.driftDetails,
646656 piStatus,
@@ -650,6 +660,19 @@ export function buildQaRuntimeParityReport(params: {
650660piToolCalls: piCell.toolCalls.length,
651661codexToolCalls: codexCell.toolCalls.length,
652662} satisfies QaRuntimeParityScenarioReport;
663+if (parityStatus === "fail") {
664+failures.push(
665+`${scenario.name} drift=${parity.drift}${parity.driftDetails ? ` (${parity.driftDetails})` : ""}.`,
666+);
667+}
668+const usageFailure = requiresLiveUsage
669+ ? describeLiveUsageFailure(scenario.name, reportScenario)
670+ : undefined;
671+if (usageFailure) {
672+failures.push(usageFailure);
673+return { ...reportScenario, status: "fail" };
674+}
675+return reportScenario;
653676});
654677655678const totalScenarios = params.summary.counts?.total ?? scenarios.length;
@@ -659,7 +682,7 @@ export function buildQaRuntimeParityReport(params: {
659682return {
660683 runtimePair,
661684comparedAt: params.comparedAt ?? new Date().toISOString(),
662-providerMode: params.summary.run?.providerMode,
685+ providerMode,
663686primaryModel: params.summary.run?.primaryModel,
664687 totalScenarios,
665688 passedScenarios,
此内容由惯性聚合(RSS阅读器)自动聚合整理，仅供阅读参考。 原文来自 — 版权归原作者所有。