@@ -31,6 +31,7 @@ export type QaToolCoverageBucket = QaRuntimeToolBucket;
31313232export type QaToolCoverageRow = {
3333tool: string;
34+runtimeToolName?: string;
3435bucket: QaToolCoverageBucket;
3536expectedLayer: QaRuntimeToolExpectedLayer;
3637capabilityLayer: QaRuntimeCapabilityLayer;
@@ -41,6 +42,8 @@ export type QaToolCoverageRow = {
4142pi: QaToolCoverageStatus;
4243codex: QaToolCoverageStatus;
4344drift: QaToolCoverageDrift;
45+piToolCalls: number;
46+codexToolCalls: number;
4447tracking?: string;
4548codexDefaultImpact?: string;
4649qaImpact?: string;
@@ -146,6 +149,12 @@ function readScenarioTracking(scenario: QaSeedScenarioWithSource): string | unde
146149return issue;
147150}
148151152+function readScenarioRuntimeToolName(scenario: QaSeedScenarioWithSource): string | undefined {
153+const config = scenario.execution.config;
154+const toolCoverage = isRecord(config?.toolCoverage) ? config.toolCoverage : undefined;
155+return readString(toolCoverage?.actualTool) ?? readString(config?.toolName);
156+}
157+149158function summaryByScenarioId(
150159summary: QaToolCoverageSuiteSummary | undefined,
151160): Map<string, RuntimeParityResult> {
@@ -177,6 +186,17 @@ function isPassingToolCoverageDrift(drift: QaToolCoverageDrift, evaluated: boole
177186return PASSING_DRIFTS.has(drift) || (!evaluated && drift === "not-run");
178187}
179188189+function countRuntimeToolCalls(
190+result: RuntimeParityResult | undefined,
191+runtime: RuntimeId,
192+toolName: string | undefined,
193+) {
194+if (!result || !toolName) {
195+return 0;
196+}
197+return result.cells[runtime].toolCalls.filter((call) => call.tool === toolName).length;
198+}
199+180200function buildRow(params: {
181201group: ToolFixtureGroup;
182202results: ReadonlyMap<string, RuntimeParityResult>;
@@ -188,8 +208,12 @@ function buildRow(params: {
188208.find((entry) => entry.required);
189209const fallbackMetadata = readScenarioRuntimeToolCoverageMetadata(params.group.scenarios[0]);
190210const rowMetadata = metadata ?? fallbackMetadata;
211+const runtimeToolName = params.group.scenarios
212+.map(readScenarioRuntimeToolName)
213+.find(Boolean);
191214return {
192215tool: params.group.tool,
216+ ...(runtimeToolName ? { runtimeToolName } : {}),
193217bucket: rowMetadata.bucket,
194218expectedLayer: rowMetadata.expectedLayer,
195219capabilityLayer: rowMetadata.capabilityLayer,
@@ -200,6 +224,8 @@ function buildRow(params: {
200224pi: result ? cellStatus(result.cells.pi) : "not-run",
201225codex: result ? cellStatus(result.cells.codex) : "not-run",
202226drift: result?.drift ?? "not-run",
227+piToolCalls: countRuntimeToolCalls(result, "pi", runtimeToolName),
228+codexToolCalls: countRuntimeToolCalls(result, "codex", runtimeToolName),
203229 ...(tracking ? { tracking } : {}),
204230 ...(rowMetadata.codexDefaultImpact
205231 ? { codexDefaultImpact: rowMetadata.codexDefaultImpact }
@@ -210,6 +236,28 @@ function buildRow(params: {
210236};
211237}
212238239+function coverageFailureForRow(row: QaToolCoverageRow): string | undefined {
240+if (!row.required || row.tracking) {
241+return undefined;
242+}
243+if (row.drift === "not-run") {
244+return `${row.tool} drift=not-run`;
245+}
246+if (row.pi !== "pass" || row.codex !== "pass") {
247+return `${row.tool} status pi=${row.pi} codex=${row.codex}`;
248+}
249+if (row.drift === "failure-mode") {
250+return `${row.tool} drift=failure-mode${row.details ? ` (${row.details})` : ""}`;
251+}
252+if (row.runtimeToolName && row.piToolCalls === 0) {
253+return `${row.tool} missing pi tool call ${row.runtimeToolName}`;
254+}
255+if (row.runtimeToolName && row.codexToolCalls === 0) {
256+return `${row.tool} missing codex tool call ${row.runtimeToolName}`;
257+}
258+return undefined;
259+}
260+213261export function buildQaToolCoverageReport(params: {
214262scenarios: readonly QaSeedScenarioWithSource[];
215263summary?: QaToolCoverageSuiteSummary;
@@ -225,11 +273,7 @@ export function buildQaToolCoverageReport(params: {
225273);
226274const evaluated = Boolean(params.summary);
227275const failures = evaluated
228- ? rows
229-.filter(
230-(row) => row.required && !row.tracking && !isPassingToolCoverageDrift(row.drift, true),
231-)
232-.map((row) => `${row.tool} drift=${row.drift}${row.details ? ` (${row.details})` : ""}`)
276+ ? rows.map(coverageFailureForRow).filter((failure): failure is string => Boolean(failure))
233277 : [];
234278return {
235279runtimePair: normalizeRuntimePair(params.runtimePair ?? params.summary?.run?.runtimePair),
@@ -244,7 +288,13 @@ export function buildQaToolCoverageReport(params: {
244288.length,
245289optionalTools: rows.filter((row) => row.bucket === "optional-profile-or-plugin").length,
246290passingTools: evaluated
247- ? rows.filter((row) => isPassingToolCoverageDrift(row.drift, true)).length
291+ ? rows.filter(
292+(row) =>
293+!row.tracking &&
294+row.pi === "pass" &&
295+row.codex === "pass" &&
296+(isPassingToolCoverageDrift(row.drift, true) || !coverageFailureForRow(row)),
297+).length
248298 : 0,
249299failingTools: failures.length,
250300 rows,
此内容由惯性聚合(RSS阅读器)自动聚合整理，仅供阅读参考。 原文来自 — 版权归原作者所有。