@@ -182,9 +182,9 @@ describe("qa coverage report", () => {
182182expect(inventory.scorecardTaxonomy.requiredCategoryCount).toBeLessThanOrEqual(
183183inventory.scorecardTaxonomy.categoryCount,
184184);
185-expect(inventory.scorecardTaxonomy.requiredFeatureCount).toBeGreaterThan(0);
186-expect(inventory.scorecardTaxonomy.fulfilledFeatureCount).toBeGreaterThan(0);
187-expect(inventory.scorecardTaxonomy.taxonomyFulfillmentPercent).toBeGreaterThan(0);
185+expect(inventory.scorecardTaxonomy.requiredCoverageIdCount).toBeGreaterThan(0);
186+expect(inventory.scorecardTaxonomy.fulfilledCoverageIdCount).toBeGreaterThan(0);
187+expect(inventory.scorecardTaxonomy.coverageIdFulfillmentPercent).toBeGreaterThan(0);
188188expect(inventory.scorecardTaxonomy.evidenceRefCount).toBeGreaterThan(0);
189189expect(inventory.scorecardTaxonomy.scenarioCoverageIdCount).toBeGreaterThan(0);
190190expect(inventory.scorecardTaxonomy.unknownCoverageIdCount).toBe(0);
@@ -259,7 +259,7 @@ describe("qa coverage report", () => {
259259expect(report).toContain("## Scorecard Taxonomy");
260260expect(report).toContain("- Taxonomy: taxonomy.yaml");
261261expect(report).toContain("- Fulfilled taxonomy categories:");
262-expect(report).toContain("- Fulfilled taxonomy features:");
262+expect(report).toContain("- Fulfilled taxonomy coverage IDs:");
263263expect(report).toContain("- Evidence refs:");
264264expect(report).toContain("- Scenario coverage IDs:");
265265expect(report).toContain(
@@ -347,7 +347,7 @@ describe("qa coverage report", () => {
347347],
348348});
349349350-expect(report.fulfilledFeatureCount).toBe(0);
350+expect(report.fulfilledCoverageIdCount).toBe(0);
351351expect(report.categories[0]?.coverageStatus).toBe("missing");
352352expect(report.validationIssues.map((issue) => issue.code)).toEqual([
353353"coverage-id-not-found",
@@ -375,7 +375,7 @@ describe("qa coverage report", () => {
375375376376expect(report.validationIssues).toStrictEqual([]);
377377expect(report.fulfilledCategoryCount).toBe(1);
378-expect(report.fulfilledFeatureCount).toBe(1);
378+expect(report.fulfilledCoverageIdCount).toBe(1);
379379expect(report.categories[0]?.coverageStatus).toBe("covered");
380380expect(report.categories[0]?.scenarioRefs).toStrictEqual([
381381"qa/scenarios/ui/control-ui-chat-flow-playwright.yaml",
@@ -391,7 +391,7 @@ describe("qa coverage report", () => {
391391]);
392392});
393393394-it("requires every coverage ID on a taxonomy feature to have primary evidence", () => {
394+it("counts partial coverage IDs proportionately for taxonomy fulfillment", () => {
395395const report = buildQaScorecardTaxonomyReport({
396396taxonomy: testMaturityTaxonomy({
397397featureCoverageIds: [[TEST_EXECUTABLE_COVERAGE_ID, TEST_WEBCHAT_COVERAGE_ID]],
@@ -407,7 +407,9 @@ describe("qa coverage report", () => {
407407});
408408409409expect(report.fulfilledCategoryCount).toBe(0);
410-expect(report.fulfilledFeatureCount).toBe(0);
410+expect(report.requiredCoverageIdCount).toBe(2);
411+expect(report.fulfilledCoverageIdCount).toBe(1);
412+expect(report.coverageIdFulfillmentPercent).toBe(50);
411413expect(report.categories[0]?.coverageStatus).toBe("partial");
412414expect(report.categories[0]?.fulfilledCoverageIds).toStrictEqual([TEST_EXECUTABLE_COVERAGE_ID]);
413415expect(report.validationIssues).toContainEqual(
@@ -418,6 +420,75 @@ describe("qa coverage report", () => {
418420);
419421});
420422423+it("counts each required taxonomy coverage ID once across categories", () => {
424+const taxonomy: QaMaturityTaxonomy = {
425+ ...testMaturityTaxonomy(),
426+profiles: [
427+{
428+id: "release",
429+description: "Test release profile.",
430+includeAllCategories: false,
431+channelDriver: "qa-channel",
432+categoryIds: [
433+"agent-runtime-and-provider-execution.agent-turn-execution",
434+"agent-runtime-and-provider-execution.tool-execution-controls",
435+],
436+},
437+],
438+surfaces: [
439+{
440+id: "agent-runtime-and-provider-execution",
441+name: "Agent Runtime",
442+family: "test",
443+level: "experimental",
444+categories: [
445+{
446+id: "agent-turn-execution",
447+name: "Agent Turn Execution",
448+category_note: "agent-turn-execution.md",
449+docs: [],
450+search_anchors: [],
451+features: [
452+{
453+name: "shared plus unique",
454+coverageIds: [TEST_EXECUTABLE_COVERAGE_ID, TEST_WEBCHAT_COVERAGE_ID],
455+},
456+],
457+},
458+{
459+id: "tool-execution-controls",
460+name: "Tool Execution Controls",
461+category_note: "tool-execution-controls.md",
462+docs: [],
463+search_anchors: [],
464+features: [
465+{
466+name: "shared",
467+coverageIds: [TEST_EXECUTABLE_COVERAGE_ID],
468+},
469+],
470+},
471+],
472+},
473+],
474+};
475+const report = buildQaScorecardTaxonomyReport({
476+ taxonomy,
477+repoRoot: process.cwd(),
478+scenarios: [
479+scenarioWithCoverage({
480+primary: [TEST_EXECUTABLE_COVERAGE_ID],
481+secondary: [TEST_WEBCHAT_COVERAGE_ID],
482+sourcePath: "qa/scenarios/channels/dm-chat-baseline.yaml",
483+}),
484+],
485+});
486+487+expect(report.requiredCoverageIdCount).toBe(2);
488+expect(report.fulfilledCoverageIdCount).toBe(1);
489+expect(report.coverageIdFulfillmentPercent).toBe(50);
490+});
491+421492it("uses script producer evidence as coverage fulfillment", () => {
422493const report = buildQaScorecardTaxonomyReport({
423494taxonomy: testMaturityTaxonomy({
@@ -437,7 +508,7 @@ describe("qa coverage report", () => {
437508438509expect(report.validationIssues).toStrictEqual([]);
439510expect(report.fulfilledCategoryCount).toBe(1);
440-expect(report.fulfilledFeatureCount).toBe(1);
511+expect(report.fulfilledCoverageIdCount).toBe(1);
441512expect(report.categories[0]?.evidence).toStrictEqual([
442513{
443514coverageId: TEST_BROWSER_COVERAGE_ID,
@@ -555,7 +626,7 @@ describe("qa coverage report", () => {
555626],
556627});
557628558-expect(report.fulfilledFeatureCount).toBe(0);
629+expect(report.fulfilledCoverageIdCount).toBe(0);
559630expect(report.categories[0]?.coverageStatus).toBe("partial");
560631expect(report.validationIssues.map((issue) => issue.code)).toEqual([
561632"coverage-id-not-found",
此内容由惯性聚合(RSS阅读器)自动聚合整理，仅供阅读参考。 原文来自 — 版权归原作者所有。