@@ -19,7 +19,7 @@ scenario:
|
19 | 19 | anthropic/claude-sonnet-4-6: |
20 | 20 | params: {} |
21 | 21 | successCriteria: |
22 | | - - Anthropic Claude Sonnet 4.6 advertises adaptive but not OpenAI-only xhigh or Opus max. |
| 22 | + - Anthropic Claude Sonnet 4.6 advertises adaptive and native max, but not OpenAI-only xhigh. |
23 | 23 | - A stored adaptive level remaps to medium when switching to OpenAI GPT-5.5. |
24 | 24 | - OpenAI GPT-5.5 advertises xhigh but not adaptive or max. |
25 | 25 | - A stored xhigh level remaps to high when switching to an Anthropic model without xhigh support. |
@@ -36,7 +36,7 @@ scenario:
|
36 | 36 | - extensions/openai/openai-provider.ts |
37 | 37 | execution: |
38 | 38 | kind: flow |
39 | | -summary: Select Anthropic, set adaptive, switch to OpenAI and verify medium fallback, then set xhigh and verify high fallback on a non-xhigh model. |
| 39 | +summary: Select Anthropic, set adaptive, switch to OpenAI and verify medium fallback, then set xhigh and verify high fallback on a model without xhigh support. |
40 | 40 | config: |
41 | 41 | requiredProviderMode: live-frontier |
42 | 42 | requiredProvider: openai |
|
90 | 90 | message: |
91 | 91 | expr: "`expected Anthropic /think options to include adaptive, got ${anthropicThinkStatus.text}`" |
92 | 92 | - assert: |
93 | | -expr: "!/Options: .*\\bxhigh\\b/i.test(anthropicThinkStatus.text) && !/Options: .*\\bmax\\b/i.test(anthropicThinkStatus.text)" |
| 93 | +expr: "!/Options: .*\\bxhigh\\b/i.test(anthropicThinkStatus.text) && /Options: .*\\bmax\\b/i.test(anthropicThinkStatus.text)" |
94 | 94 | message: |
95 | | -expr: "`expected Sonnet /think options to omit xhigh/max, got ${anthropicThinkStatus.text}`" |
| 95 | +expr: "`expected Sonnet /think options to include max and omit xhigh, got ${anthropicThinkStatus.text}`" |
96 | 96 | detailsExpr: "`model=${JSON.stringify(anthropicModelAck.resolved)}; think=${anthropicThinkStatus.text}`" |
97 | 97 | - name: maps adaptive to medium when switching to OpenAI |
98 | 98 | actions: |
@@ -190,7 +190,7 @@ flow:
|
190 | 190 | expr: "state.getSnapshot().messages.slice(cursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && /Current thinking level: high/i.test(candidate.text)).at(-1)" |
191 | 191 | - expr: liveTurnTimeoutMs(env, 20000) |
192 | 192 | - assert: |
193 | | -expr: "/Options: .*\\badaptive\\b/i.test(noXhighThinkStatus.text) && !/Options: .*\\bxhigh\\b/i.test(noXhighThinkStatus.text) && !/Options: .*\\bmax\\b/i.test(noXhighThinkStatus.text)" |
| 193 | +expr: "/Options: .*\\badaptive\\b/i.test(noXhighThinkStatus.text) && /Options: .*\\bmax\\b/i.test(noXhighThinkStatus.text) && !/Options: .*\\bxhigh\\b/i.test(noXhighThinkStatus.text)" |
194 | 194 | message: |
195 | | -expr: "`expected non-xhigh model /think options to include adaptive and omit xhigh/max, got ${noXhighThinkStatus.text}`" |
| 195 | +expr: "`expected non-xhigh model /think options to include adaptive/max and omit xhigh, got ${noXhighThinkStatus.text}`" |
196 | 196 | detailsExpr: "`xhigh=${xhighAck.text}; switch=${JSON.stringify(noXhighModelAck.resolved)}; think=${noXhighThinkStatus.text}`" |