@@ -57,6 +57,7 @@ let mockedResolveAuthProfileOrder: ReturnType<
5757typeof vi.mocked<AuthProfilesOrderModule["resolveAuthProfileOrder"]>
5858>;
5959let runWithModelFallback: ModelFallbackModule["runWithModelFallback"];
60+let modelFallbackTesting: ModelFallbackModule["__testing"];
6061let _probeThrottleInternals: ModelFallbackModule["_probeThrottleInternals"];
6162let resetLogger: LoggerModule["resetLogger"];
6263let setLoggerOverride: LoggerModule["setLoggerOverride"];
@@ -82,6 +83,7 @@ async function loadModelFallbackProbeModules() {
8283);
8384mockedResolveAuthProfileOrder = vi.mocked(authProfilesOrderModule.resolveAuthProfileOrder);
8485runWithModelFallback = modelFallbackModule.runWithModelFallback;
86+modelFallbackTesting = modelFallbackModule.__testing;
8587_probeThrottleInternals = modelFallbackModule._probeThrottleInternals;
8688resetLogger = loggerModule.resetLogger;
8789setLoggerOverride = loggerModule.setLoggerOverride;
@@ -187,6 +189,34 @@ describe("runWithModelFallback – probe logic", () => {
187189 run,
188190});
189191192+function resolveOpenAiCooldownDecision(params: {
193+reason: "rate_limit" | "overloaded" | "timeout" | "auth" | "billing";
194+soonest: number | null;
195+isPrimary?: boolean;
196+hasFallbackCandidates?: boolean;
197+requestedModel?: boolean;
198+throttleKey?: string;
199+}) {
200+mockedGetSoonestCooldownExpiry.mockReturnValue(params.soonest);
201+mockedResolveProfilesUnavailableReason.mockReturnValue(params.reason);
202+return modelFallbackTesting.resolveCooldownDecision({
203+candidate: { provider: "openai", model: "gpt-4.1-mini" },
204+isPrimary: params.isPrimary ?? true,
205+requestedModel: params.requestedModel ?? true,
206+hasFallbackCandidates: params.hasFallbackCandidates ?? true,
207+now: NOW,
208+probeThrottleKey: params.throttleKey ?? "openai",
209+authRuntime: {
210+getSoonestCooldownExpiry: mockedGetSoonestCooldownExpiry,
211+resolveProfilesUnavailableReason: mockedResolveProfilesUnavailableReason,
212+} as unknown as Parameters<
213+typeof modelFallbackTesting.resolveCooldownDecision
214+>[0]["authRuntime"],
215+authStore: { version: 1, profiles: {} },
216+profileIds: ["openai-profile-1"],
217+});
218+}
219+190220async function expectPrimarySkippedAfterLongCooldown(reason: "billing" | "rate_limit") {
191221const cfg = makeCfg();
192222const expiresIn30Min = NOW + 30 * 60 * 1000;
@@ -202,6 +232,7 @@ describe("runWithModelFallback – probe logic", () => {
202232beforeEach(() => {
203233realDateNow = Date.now;
204234Date.now = vi.fn(() => NOW);
235+setLoggerOverride({ level: "silent", consoleLevel: "silent" });
205236206237// Clear throttle state between tests
207238_probeThrottleInternals.lastProbeAttempt.clear();
@@ -261,16 +292,35 @@ describe("runWithModelFallback – probe logic", () => {
261292await expectPrimarySkippedAfterLongCooldown("billing");
262293});
263294264-it("probes primary model when within 2-min margin of cooldown expiry", async () => {
265-const cfg = makeCfg();
266-// Cooldown expires in 1 minute — within 2-min probe margin
267-const expiresIn1Min = NOW + 60 * 1000;
268-mockedGetSoonestCooldownExpiry.mockReturnValue(expiresIn1Min);
269-270-const run = vi.fn().mockResolvedValue("probed-ok");
271-272-const result = await runPrimaryCandidate(cfg, run);
273-expectPrimaryProbeSuccess(result, run, "probed-ok");
295+it("decides when cooldowned primary probes are allowed", () => {
296+expect(
297+resolveOpenAiCooldownDecision({
298+reason: "rate_limit",
299+soonest: NOW + 60 * 1000,
300+}),
301+).toEqual({ type: "attempt", reason: "rate_limit", markProbe: true });
302+expect(
303+resolveOpenAiCooldownDecision({
304+reason: "rate_limit",
305+soonest: NOW - 5 * 60 * 1000,
306+}),
307+).toEqual({ type: "attempt", reason: "rate_limit", markProbe: true });
308+expect(
309+resolveOpenAiCooldownDecision({
310+reason: "rate_limit",
311+soonest: NOW + 30 * 1000,
312+throttleKey: "recent-openai",
313+}),
314+).toEqual({ type: "attempt", reason: "rate_limit", markProbe: true });
315+316+_probeThrottleInternals.lastProbeAttempt.set("recent-openai", NOW - 10_000);
317+expect(
318+resolveOpenAiCooldownDecision({
319+reason: "rate_limit",
320+soonest: NOW + 30 * 1000,
321+throttleKey: "recent-openai",
322+}),
323+).toMatchObject({ type: "skip", reason: "rate_limit" });
274324});
275325276326it("logs primary metadata on probe success and failure fallback decisions", async () => {
@@ -399,18 +449,6 @@ describe("runWithModelFallback – probe logic", () => {
399449);
400450});
401451402-it("probes primary model when cooldown already expired", async () => {
403-const cfg = makeCfg();
404-// Cooldown expired 5 min ago
405-const expiredAlready = NOW - 5 * 60 * 1000;
406-mockedGetSoonestCooldownExpiry.mockReturnValue(expiredAlready);
407-408-const run = vi.fn().mockResolvedValue("recovered");
409-410-const result = await runPrimaryCandidate(cfg, run);
411-expectPrimaryProbeSuccess(result, run, "recovered");
412-});
413-414452it.each([
415453{
416454label: "rate-limit",
@@ -503,37 +541,6 @@ describe("runWithModelFallback – probe logic", () => {
503541expect(run).toHaveBeenNthCalledWith(3, "deepseek", "deepseek-chat");
504542});
505543506-it("throttles probe when called within 30s interval", async () => {
507-const cfg = makeCfg();
508-// Cooldown just about to expire (within probe margin)
509-const almostExpired = NOW + 30 * 1000;
510-mockedGetSoonestCooldownExpiry.mockReturnValue(almostExpired);
511-512-// Simulate a recent probe 10s ago
513-_probeThrottleInternals.lastProbeAttempt.set("openai", NOW - 10_000);
514-515-const run = vi.fn().mockResolvedValue("ok");
516-517-const result = await runPrimaryCandidate(cfg, run);
518-519-// Should be throttled → skip primary, use fallback
520-expectFallbackUsed(result, run);
521-});
522-523-it("allows probe when 30s have passed since last probe", async () => {
524-const cfg = makeCfg();
525-const almostExpired = NOW + 30 * 1000;
526-mockedGetSoonestCooldownExpiry.mockReturnValue(almostExpired);
527-528-// Last probe was 31s ago — should NOT be throttled
529-_probeThrottleInternals.lastProbeAttempt.set("openai", NOW - 31_000);
530-531-const run = vi.fn().mockResolvedValue("probed-ok");
532-533-const result = await runPrimaryCandidate(cfg, run);
534-expectPrimaryProbeSuccess(result, run, "probed-ok");
535-});
536-537544it("prunes stale probe throttle entries before checking eligibility", () => {
538545_probeThrottleInternals.lastProbeAttempt.set(
539546"stale",
@@ -564,21 +571,21 @@ describe("runWithModelFallback – probe logic", () => {
564571expect(_probeThrottleInternals.lastProbeAttempt.has("key-0")).toBe(true);
565572});
566573567-it("handles missing or non-finite soonest safely (treats as probe-worthy)", async () => {
568-const cfg = makeCfg();
569-574+it("handles missing or non-finite soonest safely (treats as probe-worthy)", () => {
570575for (const [label, soonest] of [
571576["infinity", Infinity],
572577["nan", Number.NaN],
573578["null", null],
574579] as const) {
575580_probeThrottleInternals.lastProbeAttempt.clear();
576-mockedGetSoonestCooldownExpiry.mockReturnValue(soonest);
577-578-const run = vi.fn().mockResolvedValue(`ok-${label}`);
579581580-const result = await runPrimaryCandidate(cfg, run);
581-expectPrimaryProbeSuccess(result, run, `ok-${label}`);
582+expect(
583+resolveOpenAiCooldownDecision({
584+reason: "rate_limit",
585+ soonest,
586+}),
587+label,
588+).toEqual({ type: "attempt", reason: "rate_limit", markProbe: true });
582589}
583590});
584591@@ -612,84 +619,49 @@ describe("runWithModelFallback – probe logic", () => {
612619expect(run).not.toHaveBeenCalled();
613620});
614621615-it("scopes probe throttling by agentDir to avoid cross-agent suppression", async () => {
616-const cfg = makeCfg();
617-const almostExpired = NOW + 30 * 1000;
618-mockedGetSoonestCooldownExpiry.mockReturnValue(almostExpired);
619-620-const run = vi.fn().mockResolvedValue("probed-ok");
621-622-await runWithModelFallback({
623- cfg,
624-provider: "openai",
625-model: "gpt-4.1-mini",
626-agentDir: "/tmp/agent-a",
627- run,
628-});
622+it("scopes probe throttling by agentDir to avoid cross-agent suppression", () => {
623+const agentAKey = _probeThrottleInternals.resolveProbeThrottleKey("openai", "/tmp/agent-a");
624+const agentBKey = _probeThrottleInternals.resolveProbeThrottleKey("openai", "/tmp/agent-b");
625+_probeThrottleInternals.lastProbeAttempt.set(agentAKey, NOW - 10_000);
629626630-await runWithModelFallback({
631-cfg,
632-provider: "openai",
633-model: "gpt-4.1-mini",
634-agentDir: "/tmp/agent-b",
635-run,
636-});
637-638-expect(run).toHaveBeenNthCalledWith(1, "openai", "gpt-4.1-mini", {
639-allowTransientCooldownProbe: true,
640-});
641-expect(run).toHaveBeenNthCalledWith(2, "openai", "gpt-4.1-mini", {
642-allowTransientCooldownProbe: true,
643-});
627+expect(
628+resolveOpenAiCooldownDecision({
629+ reason: "rate_limit",
630+ soonest: NOW + 30 * 1000,
631+ throttleKey: agentAKey,
632+}),
633+).toMatchObject({ type: "skip", reason: "rate_limit" });
634+ expect(
635+ resolveOpenAiCooldownDecision({
636+ reason: "rate_limit",
637+ soonest: NOW + 30 * 1000,
638+ throttleKey: agentBKey,
639+}),
640+).toEqual({ type: "attempt", reason: "rate_limit", markProbe: true });
644641});
645642646-it("probes billing-cooldowned primary when no fallback candidates exist", async () => {
647-const cfg = makeCfg({
648-agents: {
649-defaults: {
650-model: {
651-primary: "openai/gpt-4.1-mini",
652-fallbacks: [],
653-},
654-},
655-},
656-} as Partial<OpenClawConfig>);
657-643+it("decides when billing cooldowns should probe", () => {
658644// Single-provider setups need periodic probes even when the billing
659645// cooldown is far from expiry, otherwise topping up credits never recovers
660646// without a restart.
661-const expiresIn30Min = NOW + 30 * 60 * 1000;
662-mockedGetSoonestCooldownExpiry.mockReturnValue(expiresIn30Min);
663-mockedResolveProfilesUnavailableReason.mockReturnValue("billing");
664-665-const run = vi.fn().mockResolvedValue("billing-recovered");
666-667-const result = await runWithModelFallback({
668- cfg,
669-provider: "openai",
670-model: "gpt-4.1-mini",
671-fallbacksOverride: [],
672- run,
673-});
674-675-expect(result.result).toBe("billing-recovered");
676-expect(run).toHaveBeenCalledTimes(1);
677-expect(run).toHaveBeenCalledWith("openai", "gpt-4.1-mini", {
678-allowTransientCooldownProbe: true,
679-});
680-});
681-682-it("probes billing-cooldowned primary with fallbacks when near cooldown expiry", async () => {
683-const cfg = makeCfg();
684-// Cooldown expires in 1 minute — within 2-min probe margin
685-const expiresIn1Min = NOW + 60 * 1000;
686-mockedGetSoonestCooldownExpiry.mockReturnValue(expiresIn1Min);
687-mockedResolveProfilesUnavailableReason.mockReturnValue("billing");
688-689-const run = vi.fn().mockResolvedValue("billing-probe-ok");
690-691-const result = await runPrimaryCandidate(cfg, run);
692-693-expectPrimaryProbeSuccess(result, run, "billing-probe-ok");
647+expect(
648+resolveOpenAiCooldownDecision({
649+reason: "billing",
650+soonest: NOW + 30 * 60 * 1000,
651+hasFallbackCandidates: false,
652+}),
653+).toEqual({ type: "attempt", reason: "billing", markProbe: true });
654+expect(
655+resolveOpenAiCooldownDecision({
656+reason: "billing",
657+soonest: NOW + 60 * 1000,
658+}),
659+).toEqual({ type: "attempt", reason: "billing", markProbe: true });
660+expect(
661+resolveOpenAiCooldownDecision({
662+reason: "billing",
663+soonest: NOW + 30 * 60 * 1000,
664+}),
665+).toMatchObject({ type: "skip", reason: "billing" });
694666});
695667});
此内容由惯性聚合(RSS阅读器)自动聚合整理，仅供阅读参考。 原文来自 — 版权归原作者所有。