@@ -40,6 +40,55 @@ function makeRunSuite(transcriptForModel: (model: string) => string = defaultMod
4040);
4141}
424243+function createConcurrencyGate(expectedActive: number) {
44+let active = 0;
45+let maxActive = 0;
46+let releaseStartedTasks = false;
47+let resolveExpectedActive: () => void = () => {};
48+const expectedActiveReached = new Promise<void>((resolve) => {
49+resolveExpectedActive = resolve;
50+});
51+const taskReleases: Array<() => void> = [];
52+const releaseQueuedTasks = () => {
53+if (!releaseStartedTasks) {
54+return;
55+}
56+let releaseTask: (() => void) | undefined;
57+while ((releaseTask = taskReleases.shift())) {
58+releaseTask();
59+}
60+};
61+62+return {
63+get maxActive() {
64+return maxActive;
65+},
66+async run<T>(work: () => T | Promise<T>): Promise<T> {
67+active += 1;
68+maxActive = Math.max(maxActive, active);
69+if (active >= expectedActive) {
70+resolveExpectedActive();
71+}
72+await new Promise<void>((resolve) => {
73+taskReleases.push(resolve);
74+releaseQueuedTasks();
75+});
76+try {
77+return await work();
78+} finally {
79+active -= 1;
80+}
81+},
82+async waitForExpectedActive(): Promise<void> {
83+await expectedActiveReached;
84+},
85+releaseStartedTasks(): void {
86+releaseStartedTasks = true;
87+releaseQueuedTasks();
88+},
89+};
90+}
91+4392function makeSuiteResult(params: { outputDir: string; model: string; transcript: string }) {
4493return {
4594outputDir: params.outputDir,
@@ -265,22 +314,17 @@ describe("runQaCharacterEval", () => {
265314});
266315267316it("runs candidate models with bounded concurrency while preserving result order", async () => {
268-let activeRuns = 0;
269-let maxActiveRuns = 0;
270-const runSuite = vi.fn(async (params: CharacterRunSuiteParams) => {
271-activeRuns += 1;
272-maxActiveRuns = Math.max(maxActiveRuns, activeRuns);
273-await new Promise((resolve) => setTimeout(resolve, 10));
274-activeRuns -= 1;
275-return makeReplySuiteResult(params);
276-});
317+const runGate = createConcurrencyGate(2);
318+const runSuite = vi.fn(async (params: CharacterRunSuiteParams) =>
319+runGate.run(() => makeReplySuiteResult(params)),
320+);
277321const runJudge = makeRunJudge([
278322{ model: "openai/gpt-5.5", rank: 1, score: 8, summary: "ok" },
279323{ model: "anthropic/claude-sonnet-4-6", rank: 2, score: 7, summary: "ok" },
280324{ model: "moonshot/kimi-k2.5", rank: 3, score: 6, summary: "ok" },
281325]);
282326283-const result = await runQaCharacterEval({
327+const resultPromise = runQaCharacterEval({
284328repoRoot: tempRoot,
285329outputDir: path.join(tempRoot, "character"),
286330models: ["openai/gpt-5.5", "anthropic/claude-sonnet-4-6", "moonshot/kimi-k2.5"],
@@ -290,7 +334,10 @@ describe("runQaCharacterEval", () => {
290334 runJudge,
291335});
292336293-expect(maxActiveRuns).toBe(2);
337+await runGate.waitForExpectedActive();
338+expect(runGate.maxActive).toBe(2);
339+runGate.releaseStartedTasks();
340+const result = await resultPromise;
294341expect(result.runs.map((run) => run.model)).toEqual([
295342"openai/gpt-5.5",
296343"anthropic/claude-sonnet-4-6",
@@ -299,33 +346,25 @@ describe("runQaCharacterEval", () => {
299346});
300347301348it("defaults candidate and judge concurrency to sixteen", async () => {
302-let activeRuns = 0;
303-let maxActiveRuns = 0;
304-const runSuite = vi.fn(async (params: CharacterRunSuiteParams) => {
305-activeRuns += 1;
306-maxActiveRuns = Math.max(maxActiveRuns, activeRuns);
307-await new Promise((resolve) => setTimeout(resolve, 10));
308-activeRuns -= 1;
309-return makeReplySuiteResult(params);
310-});
311-let activeJudges = 0;
312-let maxActiveJudges = 0;
349+const runGate = createConcurrencyGate(16);
350+const judgeGate = createConcurrencyGate(16);
351+const runSuite = vi.fn(async (params: CharacterRunSuiteParams) =>
352+runGate.run(() => makeReplySuiteResult(params)),
353+);
313354const runJudge = vi.fn(async (_params: CharacterRunJudgeParams) => {
314-activeJudges += 1;
315-maxActiveJudges = Math.max(maxActiveJudges, activeJudges);
316-await new Promise((resolve) => setTimeout(resolve, 10));
317-activeJudges -= 1;
318-return makeJudgeReply(
319-Array.from({ length: 20 }, (_, index) => ({
320-model: `provider/model-${index + 1}`,
321-rank: index + 1,
322-score: 10 - index,
323-summary: "ok",
324-})),
355+return await judgeGate.run(() =>
356+makeJudgeReply(
357+Array.from({ length: 20 }, (_, index) => ({
358+model: `provider/model-${index + 1}`,
359+rank: index + 1,
360+score: 10 - index,
361+summary: "ok",
362+})),
363+),
325364);
326365});
327366328-await runQaCharacterEval({
367+const resultPromise = runQaCharacterEval({
329368repoRoot: tempRoot,
330369outputDir: path.join(tempRoot, "character"),
331370models: Array.from({ length: 20 }, (_, index) => `provider/model-${index + 1}`),
@@ -334,8 +373,13 @@ describe("runQaCharacterEval", () => {
334373 runJudge,
335374});
336375337-expect(maxActiveRuns).toBe(16);
338-expect(maxActiveJudges).toBe(16);
376+await runGate.waitForExpectedActive();
377+expect(runGate.maxActive).toBe(16);
378+runGate.releaseStartedTasks();
379+await judgeGate.waitForExpectedActive();
380+expect(judgeGate.maxActive).toBe(16);
381+judgeGate.releaseStartedTasks();
382+await resultPromise;
339383});
340384341385it("marks raw provider error transcripts as failed output", async () => {
此内容由惯性聚合(RSS阅读器)自动聚合整理，仅供阅读参考。 原文来自 — 版权归原作者所有。