11// Bench Cli Startup script supports OpenClaw repository automation.
22import { spawn } from "node:child_process";
3-import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
3+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
44import os from "node:os";
55import path from "node:path";
66import { pathToFileURL } from "node:url";
@@ -61,8 +61,36 @@ type SuiteResult = {
6161}>;
6262};
636364+type BenchmarkReport = {
65+primary: SuiteResult;
66+secondary?: SuiteResult | null;
67+};
68+69+type CaseDelta = {
70+id: string;
71+name: string;
72+durationAvgDeltaMs: number;
73+durationAvgDeltaPct: number;
74+maxRssAvgDeltaMb: number | null;
75+maxRssAvgDeltaPct: number | null;
76+};
77+78+type BenchmarkComparison = {
79+baseline: string;
80+candidate: string;
81+deltas: CaseDelta[];
82+};
83+84+type BenchmarkComparisonResult = {
85+baseline: SuiteResult;
86+candidate: SuiteResult;
87+comparison: BenchmarkComparison;
88+};
89+6490type CliOptions = {
6591cases: CommandCase[];
92+compareBaseline?: string;
93+compareCandidate?: string;
6694entryPrimary: string;
6795entrySecondary?: string;
6896runs: number;
@@ -797,8 +825,26 @@ function printSuite(result: SuiteResult): void {
797825}
798826799827function printDelta(primary: SuiteResult, secondary: SuiteResult): void {
800-const primaryById = new Map(primary.cases.map((commandCase) => [commandCase.id, commandCase]));
828+const deltas = buildCaseDeltas(primary, secondary);
801829console.log("Delta (secondary - primary, avg)");
830+for (const delta of deltas) {
831+const durationDelta = delta.durationAvgDeltaMs;
832+const durationPct = delta.durationAvgDeltaPct;
833+const durationSign = durationDelta > 0 ? "+" : "";
834+let line = `${delta.name.padEnd(24)} ${durationSign}${formatMs(durationDelta)} (${durationSign}${durationPct.toFixed(1)}%)`;
835+if (delta.maxRssAvgDeltaMb != null && delta.maxRssAvgDeltaPct != null) {
836+const rssDelta = delta.maxRssAvgDeltaMb;
837+const rssPct = delta.maxRssAvgDeltaPct;
838+const rssSign = rssDelta > 0 ? "+" : "";
839+line += ` rss ${rssSign}${formatMb(rssDelta)} (${rssSign}${rssPct.toFixed(1)}%)`;
840+}
841+console.log(line);
842+}
843+}
844+845+function buildCaseDeltas(primary: SuiteResult, secondary: SuiteResult): CaseDelta[] {
846+const primaryById = new Map(primary.cases.map((commandCase) => [commandCase.id, commandCase]));
847+const deltas: CaseDelta[] = [];
802848for (const commandCase of secondary.cases) {
803849const baseline = primaryById.get(commandCase.id);
804850if (!baseline) {
@@ -809,17 +855,24 @@ function printDelta(primary: SuiteResult, secondary: SuiteResult): void {
809855baseline.summary.durationMs.avg > 0
810856 ? (durationDelta / baseline.summary.durationMs.avg) * 100
811857 : 0;
812-const durationSign = durationDelta > 0 ? "+" : "";
813-let line = `${commandCase.name.padEnd(24)} ${durationSign}${formatMs(durationDelta)} (${durationSign}${durationPct.toFixed(1)}%)`;
814-if (baseline.summary.maxRssMb && commandCase.summary.maxRssMb) {
815-const rssDelta = commandCase.summary.maxRssMb.avg - baseline.summary.maxRssMb.avg;
816-const rssPct =
817-baseline.summary.maxRssMb.avg > 0 ? (rssDelta / baseline.summary.maxRssMb.avg) * 100 : 0;
818-const rssSign = rssDelta > 0 ? "+" : "";
819-line += ` rss ${rssSign}${formatMb(rssDelta)} (${rssSign}${rssPct.toFixed(1)}%)`;
820-}
821-console.log(line);
858+const rssDelta =
859+baseline.summary.maxRssMb && commandCase.summary.maxRssMb
860+ ? commandCase.summary.maxRssMb.avg - baseline.summary.maxRssMb.avg
861+ : null;
862+const rssPct =
863+rssDelta != null && baseline.summary.maxRssMb && baseline.summary.maxRssMb.avg > 0
864+ ? (rssDelta / baseline.summary.maxRssMb.avg) * 100
865+ : null;
866+deltas.push({
867+id: commandCase.id,
868+name: commandCase.name,
869+durationAvgDeltaMs: durationDelta,
870+durationAvgDeltaPct: durationPct,
871+maxRssAvgDeltaMb: rssDelta,
872+maxRssAvgDeltaPct: rssPct,
873+});
822874}
875+return deltas;
823876}
824877825878export function collectFailedSamples(result: SuiteResult): string[] {
@@ -910,6 +963,8 @@ function parseOptions(): CliOptions {
910963});
911964return {
912965 cases,
966+compareBaseline: parseFlagValue("--compare-baseline"),
967+compareCandidate: parseFlagValue("--compare-candidate"),
913968entryPrimary: parseFlagValue("--entry-primary") ?? parseFlagValue("--entry") ?? DEFAULT_ENTRY,
914969entrySecondary: parseFlagValue("--entry-secondary"),
915970runs: parsePositiveInt(parseFlagValue("--runs"), DEFAULT_RUNS, "--runs"),
@@ -938,6 +993,8 @@ Options:
938993 --warmup <n> Warmup runs per case (default: ${DEFAULT_WARMUP})
939994 --timeout-ms <ms> Per-run timeout (default: ${DEFAULT_TIMEOUT_MS})
940995 --output <path> Write machine-readable JSON to a file
996+ --compare-baseline <path> Read a saved JSON report as the baseline
997+ --compare-candidate <path> Read a saved JSON report as the candidate and print deltas
941998 --cpu-prof-dir <dir> Write V8 CPU profiles for each run
942999 --heap-prof-dir <dir> Write V8 heap profiles for each run
9431000 --json Emit machine-readable JSON
@@ -948,13 +1005,64 @@ Case ids:
9481005`);
9491006}
95010071008+function readBenchmarkReport(filePath: string): BenchmarkReport {
1009+return JSON.parse(readFileSync(filePath, "utf8")) as BenchmarkReport;
1010+}
1011+1012+function writeJsonOutput(filePath: string, value: unknown): void {
1013+mkdirSync(path.dirname(filePath), { recursive: true });
1014+writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
1015+}
1016+1017+function readBenchmarkComparison(
1018+baselinePath: string,
1019+candidatePath: string,
1020+): BenchmarkComparisonResult {
1021+const baseline = readBenchmarkReport(baselinePath);
1022+const candidate = readBenchmarkReport(candidatePath);
1023+return {
1024+baseline: baseline.primary,
1025+candidate: candidate.primary,
1026+comparison: {
1027+baseline: baselinePath,
1028+candidate: candidatePath,
1029+deltas: buildCaseDeltas(baseline.primary, candidate.primary),
1030+},
1031+};
1032+}
1033+1034+function readBenchmarkComparisonForTesting(
1035+baselinePath: string,
1036+candidatePath: string,
1037+): { comparison: unknown } {
1038+return readBenchmarkComparison(baselinePath, candidatePath);
1039+}
1040+9511041async function main(): Promise<void> {
9521042if (hasFlag("--help")) {
9531043printUsage();
9541044return;
9551045}
95610469571047const options = parseOptions();
1048+if (options.compareBaseline || options.compareCandidate) {
1049+if (!options.compareBaseline || !options.compareCandidate) {
1050+throw new Error("--compare-baseline and --compare-candidate must be provided together");
1051+}
1052+const { baseline, candidate, comparison } = readBenchmarkComparison(
1053+options.compareBaseline,
1054+options.compareCandidate,
1055+);
1056+if (options.output) {
1057+writeJsonOutput(options.output, comparison);
1058+}
1059+if (options.json) {
1060+console.log(JSON.stringify(comparison, null, 2));
1061+return;
1062+}
1063+printDelta(baseline, candidate);
1064+return;
1065+}
9581066const tmpDir = mkdtempSync(path.join(os.tmpdir(), "openclaw-cli-bench-"));
9591067const rssHookPath = buildRssHook(tmpDir);
9601068try {
@@ -987,8 +1095,7 @@ async function main(): Promise<void> {
9871095];
98810969891097if (options.output) {
990-mkdirSync(path.dirname(options.output), { recursive: true });
991-writeFileSync(options.output, `${JSON.stringify(report, null, 2)}\n`, "utf8");
1098+writeJsonOutput(options.output, report);
9921099}
99311009941101if (options.json) {
@@ -1040,6 +1147,8 @@ export const testing = {
10401147 parseGatewayPortEnv,
10411148 parseNonNegativeInt,
10421149 parsePositiveInt,
1150+readBenchmarkComparison: readBenchmarkComparisonForTesting,
1151+ writeJsonOutput,
10431152};
1044115310451154if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
此内容由惯性聚合(RSS阅读器)自动聚合整理，仅供阅读参考。 原文来自 — 版权归原作者所有。