@@ -15,6 +15,68 @@ type StreamModel = Parameters<StreamFn>[0];
15151616const preloadInFlight = new Map<string, Promise<void>>();
171718+/**
19+ * Cooldown state for the LM Studio preload endpoint.
20+ *
21+ * Without this, every chat request would retry preload ~every 2s even when
22+ * LM Studio has rejected the load (for example the memory guardrail will keep
23+ * rejecting until the user adjusts the setting or frees RAM). That produced
24+ * hundreds of `LM Studio inference preload failed` WARN lines per hour without
25+ * actually helping the user. The cooldown applies an exponential backoff per
26+ * preloadKey and, while the cooldown is active, the wrapper skips the preload
27+ * step entirely and proceeds directly to streaming — the model is often
28+ * already loaded from the user's LM Studio UI, so inference can succeed even
29+ * when preload keeps being rejected.
30+ */
31+type PreloadCooldownEntry = {
32+untilMs: number;
33+consecutiveFailures: number;
34+};
35+36+const preloadCooldown = new Map<string, PreloadCooldownEntry>();
37+38+const PRELOAD_BACKOFF_BASE_MS = 5_000;
39+const PRELOAD_BACKOFF_MAX_MS = 300_000;
40+41+function computePreloadBackoffMs(consecutiveFailures: number): number {
42+const exponent = Math.max(0, consecutiveFailures - 1);
43+const raw = PRELOAD_BACKOFF_BASE_MS * 2 ** exponent;
44+return Math.min(PRELOAD_BACKOFF_MAX_MS, raw);
45+}
46+47+function recordPreloadSuccess(preloadKey: string): void {
48+preloadCooldown.delete(preloadKey);
49+}
50+51+function recordPreloadFailure(preloadKey: string, now: number): PreloadCooldownEntry {
52+const existing = preloadCooldown.get(preloadKey);
53+const consecutiveFailures = (existing?.consecutiveFailures ?? 0) + 1;
54+const entry: PreloadCooldownEntry = {
55+ consecutiveFailures,
56+untilMs: now + computePreloadBackoffMs(consecutiveFailures),
57+};
58+preloadCooldown.set(preloadKey, entry);
59+return entry;
60+}
61+62+function isPreloadCoolingDown(preloadKey: string, now: number): PreloadCooldownEntry | undefined {
63+const entry = preloadCooldown.get(preloadKey);
64+if (!entry) {
65+return undefined;
66+}
67+if (entry.untilMs <= now) {
68+preloadCooldown.delete(preloadKey);
69+return undefined;
70+}
71+return entry;
72+}
73+74+/** Test-only hook for clearing preload cooldown state between cases. */
75+export function __resetLmstudioPreloadCooldownForTest(): void {
76+preloadCooldown.clear();
77+preloadInFlight.clear();
78+}
79+1880function normalizeLmstudioModelKey(modelId: string): string {
1981const trimmed = modelId.trim();
2082if (trimmed.toLowerCase().startsWith("lmstudio/")) {
@@ -131,29 +193,67 @@ export function wrapLmstudioInferencePreload(ctx: ProviderWrapStreamFnContext):
131193 modelKey,
132194 requestedContextLength,
133195});
196+197+const cooldownEntry = isPreloadCoolingDown(preloadKey, Date.now());
134198const existing = preloadInFlight.get(preloadKey);
135-const preloadPromise =
199+const preloadPromise: Promise<void> | undefined =
136200existing ??
137-ensureLmstudioModelLoadedBestEffort({
138-baseUrl: resolvedBaseUrl,
139- modelKey,
140- requestedContextLength,
141- options,
142- ctx,
143-modelHeaders: resolveModelHeaders(model),
144-}).finally(() => {
145-preloadInFlight.delete(preloadKey);
146-});
147-if (!existing) {
148-preloadInFlight.set(preloadKey, preloadPromise);
149-}
201+(cooldownEntry
202+ ? undefined
203+ : (() => {
204+const created = ensureLmstudioModelLoadedBestEffort({
205+baseUrl: resolvedBaseUrl,
206+ modelKey,
207+ requestedContextLength,
208+ options,
209+ ctx,
210+modelHeaders: resolveModelHeaders(model),
211+})
212+.then(
213+() => {
214+recordPreloadSuccess(preloadKey);
215+},
216+(error) => {
217+const entry = recordPreloadFailure(preloadKey, Date.now());
218+throw Object.assign(new Error("preload-failed"), {
219+cause: error,
220+consecutiveFailures: entry.consecutiveFailures,
221+cooldownMs: entry.untilMs - Date.now(),
222+});
223+},
224+)
225+.finally(() => {
226+preloadInFlight.delete(preloadKey);
227+});
228+preloadInFlight.set(preloadKey, created);
229+return created;
230+})());
150231151232return (async () => {
152-try {
153-await preloadPromise;
154-} catch (error) {
155-log.warn(
156-`LM Studio inference preload failed for "${modelKey}"; continuing without preload: ${String(error)}`,
233+if (preloadPromise) {
234+try {
235+await preloadPromise;
236+} catch (error) {
237+const annotated = error as {
238+cause?: unknown;
239+consecutiveFailures?: number;
240+cooldownMs?: number;
241+};
242+const cause = annotated.cause ?? error;
243+const failures = annotated.consecutiveFailures ?? 1;
244+const cooldownSec = Math.max(
245+0,
246+Math.round((annotated.cooldownMs ?? 0) / 1000),
247+);
248+log.warn(
249+`LM Studio inference preload failed for "${modelKey}" (${failures} consecutive failure${
250+ failures === 1 ? "" : "s"
251+ }, next preload attempt skipped for ~${cooldownSec}s); continuing without preload: ${String(cause)}`,
252+);
253+}
254+} else if (cooldownEntry) {
255+log.debug(
256+`LM Studio inference preload for "${modelKey}" skipped while backoff active (${cooldownEntry.consecutiveFailures} prior failures)`,
157257);
158258}
159259// LM Studio uses OpenAI-compatible streaming usage payloads when requested via
此内容由惯性聚合(RSS阅读器)自动聚合整理，仅供阅读参考。 原文来自 — 版权归原作者所有。