|
| 1 | +# QA bus tool trace visibility |
| 2 | + |
| 3 | +```yaml qa-scenario |
| 4 | +id: qa-bus-tool-trace-visibility |
| 5 | +title: QA bus tool trace visibility |
| 6 | +surface: harness |
| 7 | +coverage: |
| 8 | +primary: |
| 9 | + - harness.tool-trace-visibility |
| 10 | +secondary: |
| 11 | + - runtime.qa-bus |
| 12 | + - tools.trace |
| 13 | +objective: Verify QA-Lab can assert sanitized tool-call traces directly on bus messages. |
| 14 | +successCriteria: |
| 15 | + - QA bus messages can carry a toolCalls array. |
| 16 | + - Readback preserves the tool name while redacting sensitive argument values. |
| 17 | + - QA bus search can locate the message by tool name. |
| 18 | +docsRefs: |
| 19 | + - docs/help/testing.md |
| 20 | + - qa/scenarios/index.md |
| 21 | +codeRefs: |
| 22 | + - extensions/qa-lab/src/bus-state.ts |
| 23 | + - extensions/qa-lab/src/bus-queries.ts |
| 24 | + - extensions/qa-lab/src/runtime-api.ts |
| 25 | +execution: |
| 26 | +kind: flow |
| 27 | +summary: Add a synthetic tool-backed bus message and verify sanitized trace assertions. |
| 28 | +config: |
| 29 | +expectedToolName: exec |
| 30 | +expectedRedaction: "[redacted]" |
| 31 | +searchQuery: exec |
| 32 | +``` |
| 33 | + |
| 34 | +```yaml qa-flow |
| 35 | +steps: |
| 36 | + - name: preserves searchable sanitized tool-call traces |
| 37 | +actions: |
| 38 | + - call: reset |
| 39 | + - call: state.addOutboundMessage |
| 40 | +saveAs: outbound |
| 41 | +args: |
| 42 | + - to: dm:qa-operator |
| 43 | +text: qa bus tool trace check |
| 44 | +toolCalls: |
| 45 | + - name: |
| 46 | +expr: config.expectedToolName |
| 47 | +arguments: |
| 48 | +command: pwd |
| 49 | +apiToken: qa-secret-token |
| 50 | + - set: readback |
| 51 | +value: |
| 52 | +expr: "state.readMessage({ messageId: outbound.id })" |
| 53 | + - assert: |
| 54 | +expr: "readback.toolCalls?.[0]?.name === config.expectedToolName" |
| 55 | +message: |
| 56 | +expr: "`expected tool name ${config.expectedToolName}, got ${String(readback.toolCalls?.[0]?.name ?? '')}`" |
| 57 | + - assert: |
| 58 | +expr: "readback.toolCalls?.[0]?.arguments?.command === config.expectedRedaction && readback.toolCalls?.[0]?.arguments?.apiToken === config.expectedRedaction" |
| 59 | +message: |
| 60 | +expr: "`expected redacted tool arguments, got ${JSON.stringify(readback.toolCalls?.[0]?.arguments ?? null)}`" |
| 61 | + - set: searchMatches |
| 62 | +value: |
| 63 | +expr: "state.searchMessages({ query: config.searchQuery })" |
| 64 | + - assert: |
| 65 | +expr: "searchMatches.some((message) => message.id === outbound.id)" |
| 66 | +message: |
| 67 | +expr: "`expected search query ${config.searchQuery} to find ${outbound.id}, got ${JSON.stringify(searchMatches.map((message) => message.id))}`" |
| 68 | +detailsExpr: "`${readback.toolCalls?.[0]?.name}:${String(readback.toolCalls?.[0]?.arguments?.command ?? '')}`" |
| 69 | +``` |