@@ -13,7 +13,7 @@ objective: Verify GPT-5.5 can switch from disabled thinking to medium thinking w
1313successCriteria:
1414 - Live runs target openai/gpt-5.5, not a mini or pro variant.
1515 - The session enables reasoning display before the comparison turns.
16- - The disabled-thinking turn returns its visible marker without a Reasoning-prefixed message.
16+ - The disabled-thinking turn returns its visible marker without a non-empty Reasoning summary.
1717 - The medium-thinking turn returns its visible marker and a separate Reasoning-prefixed message.
1818docsRefs:
1919 - docs/tools/thinking.md
@@ -77,22 +77,22 @@ steps:
7777 - lambda:
7878expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && /Reasoning visibility enabled/i.test(candidate.text)).at(-1)"
7979 - expr: liveTurnTimeoutMs(env, 20000)
80- - call: state.addInboundMessage
80+ - call: patchConfig
8181args:
82- - conversation:
83-id:
84-expr: config.conversationId
85-kind: direct
86-senderId: qa-operator
87-senderName: QA Operator
88-text:
89-expr: config.offDirective
90- - call: waitForCondition
91-saveAs: offAck
82+ - env:
83+ref: env
84+patch:
85+agents:
86+defaults:
87+thinkingDefault: "off"
88+ - call: waitForGatewayHealthy
9289args:
93- - lambda:
94-expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && /Thinking disabled/i.test(candidate.text)).at(-1)"
95- - expr: liveTurnTimeoutMs(env, 20000)
90+ - ref: env
91+ - 60000
92+ - call: waitForQaChannelReady
93+args:
94+ - ref: env
95+ - 60000
9696 - set: offCursor
9797value:
9898expr: state.getSnapshot().messages.length
@@ -105,7 +105,7 @@ steps:
105105senderId: qa-operator
106106senderName: QA Operator
107107text:
108-expr: "`${config.offDirective} ${config.offPrompt}`"
108+expr: config.offPrompt
109109 - call: waitForCondition
110110saveAs: offAnswer
111111args:
@@ -120,7 +120,7 @@ steps:
120120message:
121121expr: "`missing off marker; saw ${offMessages.map((message) => message.text).join(' | ')}`"
122122 - assert:
123-expr: "!offMessages.some((candidate) => candidate.text.trimStart().startsWith('Reasoning:'))"
123+expr: "!offMessages.some((candidate) => candidate.text.trimStart().startsWith('Reasoning:') && !candidate.text.includes('Native reasoning was produced; no summary text was returned.'))"
124124message:
125125expr: "`disabled thinking unexpectedly emitted reasoning: ${offMessages.map((message) => message.text).join(' | ')}`"
126126 - if:
@@ -136,26 +136,26 @@ steps:
136136expr: "String(offRequest?.model ?? '').includes('gpt-5.5')"
137137message:
138138expr: "`expected GPT-5.5 off mock request, got ${String(offRequest?.model ?? '')}`"
139-detailsExpr: "`off ack=${offAck.text}; off answer=${offAnswer.text}`"
139+detailsExpr: "`reasoning ack=${reasoningAck.text}; off answer=${offAnswer.text}`"
140140 - name: switches to medium thinking
141141actions:
142- - call: state.addInboundMessage
142+ - call: patchConfig
143143args:
144- - conversation:
145-id:
146-expr: config.conversationId
147-kind: direct
148-senderId: qa-operator
149-senderName: QA Operator
150-text:
151-expr: config.maxDirective
152- - call: waitForCondition
153-saveAs: maxAck
144+ - env:
145+ref: env
146+patch:
147+agents:
148+defaults:
149+thinkingDefault: "medium"
150+ - call: waitForGatewayHealthy
154151args:
155- - lambda:
156-expr: "state.getSnapshot().messages.filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && /Thinking level set to medium/i.test(candidate.text)).at(-1)"
157- - expr: liveTurnTimeoutMs(env, 20000)
158-detailsExpr: "`max ack=${maxAck.text}`"
152+ - ref: env
153+ - 60000
154+ - call: waitForQaChannelReady
155+args:
156+ - ref: env
157+ - 60000
158+detailsExpr: "`thinking default patched to medium`"
159159 - name: verifies medium thinking emits visible reasoning
160160actions:
161161 - set: maxCursor
@@ -170,7 +170,7 @@ steps:
170170senderId: qa-operator
171171senderName: QA Operator
172172text:
173-expr: "`${config.maxDirective} ${config.maxPrompt}`"
173+expr: config.maxPrompt
174174 - call: waitForCondition
175175saveAs: maxReasoning
176176args:
此内容由惯性聚合(RSS阅读器)自动聚合整理，仅供阅读参考。 原文来自 — 版权归原作者所有。