expect(report).not.toContain("Judge Raw Reply");

});

it("creates a unique default output directory under repo artifacts", async () => {

const runSuite = vi.fn(async (params: CharacterRunSuiteParams) =>

makeSuiteResult({

outputDir: params.outputDir,

model: params.primaryModel,

transcript: "USER Alice: hi\n\nASSISTANT openclaw: default dir reply",

}),

);

const runJudge = makeRunJudge([

{

model: "openai/gpt-5.5",

rank: 1,

score: 8,

summary: "solid",

strengths: ["clear"],

weaknesses: [],

},

]);

const result = await runQaCharacterEval({

repoRoot: tempRoot,

models: ["openai/gpt-5.5"],

runSuite,

runJudge,

});

expect(path.dirname(result.outputDir)).toBe(path.join(tempRoot, ".artifacts", "qa-e2e"));

expect(path.basename(result.outputDir)).toMatch(

/^character-eval-[a-z0-9]+-[a-f0-9]{8}$/u,

);

await expect(fs.stat(result.reportPath).then((stats) => stats.isFile())).resolves.toBe(true);

});

it("can hide candidate model refs from judge prompts and map rankings back", async () => {

const runSuite = vi.fn(async (params: CharacterRunSuiteParams) =>

makeSuiteResult({

import path from "node:path";

import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";

import { normalizeStringEntries, uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";

import { createQaArtifactRunId } from "./artifact-run-id.js";

import { isQaFastModeModelRef, type QaProviderMode } from "./model-selection.js";

import {

QA_FRONTIER_CHARACTER_EVAL_MODELS,

const outputDir =

params.outputDir ??

path.join(repoRoot, ".artifacts", "qa-e2e", `character-eval-${Date.now().toString(36)}`);

path.join(repoRoot, ".artifacts", "qa-e2e", `character-eval-${createQaArtifactRunId()}`);

const runsDir = path.join(outputDir, "runs");

await fs.mkdir(runsDir, { recursive: true });

import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";

import { chromium } from "playwright-core";

import { z } from "zod";

import { createQaArtifactRunId } from "../../artifact-run-id.js";

import { QA_EVIDENCE_FILENAME, buildLiveTransportEvidenceSummary } from "../../evidence-summary.js";

import { startQaGatewayChild } from "../../gateway-child.js";

import { isTruthyOptIn } from "../../mantis-options.runtime.js";

const repoRoot = path.resolve(params.repoRoot ?? process.cwd());

const outputDir =

params.outputDir ??

path.join(repoRoot, ".artifacts", "qa-e2e", `discord-${Date.now().toString(36)}`);

path.join(repoRoot, ".artifacts", "qa-e2e", `discord-${createQaArtifactRunId()}`);

await fs.mkdir(outputDir, { recursive: true });

const providerMode = normalizeQaProviderMode(

import { parseStrictPositiveInteger } from "openclaw/plugin-sdk/number-runtime";

import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";

import { z } from "zod";

import { createQaArtifactRunId } from "../../artifact-run-id.js";

import { QA_EVIDENCE_FILENAME, buildLiveTransportEvidenceSummary } from "../../evidence-summary.js";

import { startQaGatewayChild } from "../../gateway-child.js";

import { isTruthyOptIn } from "../../mantis-options.runtime.js";

const repoRoot = path.resolve(params.repoRoot ?? process.cwd());

const outputDir =

params.outputDir ??

path.join(repoRoot, ".artifacts", "qa-e2e", `slack-${Date.now().toString(36)}`);

path.join(repoRoot, ".artifacts", "qa-e2e", `slack-${createQaArtifactRunId()}`);

await fs.mkdir(outputDir, { recursive: true });

const providerMode = normalizeQaProviderMode(

import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";

import { isRecord, uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";

import { z } from "zod";

import { createQaArtifactRunId } from "../../artifact-run-id.js";

import {

QA_EVIDENCE_FILENAME,

buildLiveTransportEvidenceSummary,

const repoRoot = path.resolve(params.repoRoot ?? process.cwd());

const outputDir =

params.outputDir ??

path.join(repoRoot, ".artifacts", "qa-e2e", `telegram-${Date.now().toString(36)}`);

path.join(repoRoot, ".artifacts", "qa-e2e", `telegram-${createQaArtifactRunId()}`);

await fs.mkdir(outputDir, { recursive: true });

const providerMode = normalizeQaProviderMode(

import { normalizeStringEntries, uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";

import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path";

import { z } from "zod";

import { createQaArtifactRunId } from "../../artifact-run-id.js";

import { QA_EVIDENCE_FILENAME, buildLiveTransportEvidenceSummary } from "../../evidence-summary.js";

import { startQaGatewayChild } from "../../gateway-child.js";

import { isTruthyOptIn } from "../../mantis-options.runtime.js";

const repoRoot = path.resolve(params.repoRoot ?? process.cwd());

const outputDir =

params.outputDir ??

path.join(repoRoot, ".artifacts", "qa-e2e", `whatsapp-${Date.now().toString(36)}`);

path.join(repoRoot, ".artifacts", "qa-e2e", `whatsapp-${createQaArtifactRunId()}`);

await fs.mkdir(outputDir, { recursive: true });

const providerMode = normalizeQaProviderMode(