1+import { beforeEach, describe, expect, it, vi } from "vitest";
2+import {
3+installPwToolsCoreTestHooks,
4+setPwToolsCoreCurrentPage,
5+setPwToolsCoreCurrentRefLocator,
6+} from "./pw-tools-core.test-harness.js";
7+8+installPwToolsCoreTestHooks();
9+const mod = await import("./pw-tools-core.js");
10+11+type EvaluateArg = unknown;
12+13+function evaluateMockReturning(view: { x: number; y: number; width?: number; height?: number }) {
14+// Caller reads { x, y, width, height } in one evaluate; default to a normal
15+// desktop viewport so refs near the top stay in-viewport unless a test puts
16+// them out of range explicitly.
17+const result = { width: 1280, height: 720, ...view };
18+return vi.fn(async (arg: EvaluateArg) => {
19+if (typeof arg === "function") {
20+return result;
21+}
22+return true;
23+});
24+}
25+26+describe("screenshotWithLabelsViaPlaywright (viewport)", () => {
27+beforeEach(() => {
28+vi.clearAllMocks();
29+});
30+31+it("calls page.screenshot without fullPage and returns annotations", async () => {
32+const evaluate = evaluateMockReturning({ x: 0, y: 100 });
33+const screenshot = vi.fn(async () => Buffer.from("PNG"));
34+setPwToolsCoreCurrentPage({ evaluate, screenshot, url: () => "https://example.com" });
35+setPwToolsCoreCurrentRefLocator({
36+boundingBox: async () => ({ x: 10, y: 200, width: 50, height: 20 }),
37+});
38+39+const result = await mod.screenshotWithLabelsViaPlaywright({
40+cdpUrl: "http://127.0.0.1:18792",
41+targetId: "T1",
42+refs: { e1: { role: "button", name: "Submit" } },
43+type: "png",
44+});
45+46+expect(screenshot).toHaveBeenCalledWith(expect.objectContaining({ type: "png" }));
47+expect(screenshot).not.toHaveBeenCalledWith(expect.objectContaining({ fullPage: true }));
48+49+expect(result.annotations).toHaveLength(1);
50+expect(result.annotations[0]).toMatchObject({
51+ref: "e1",
52+number: 1,
53+role: "button",
54+name: "Submit",
55+});
56+// viewport-mode box = doc(box.x + scroll.x, box.y + scroll.y) - scroll = bbox
57+expect(result.annotations[0]?.box).toEqual({ x: 10, y: 200, width: 50, height: 20 });
58+expect(result.skipped).toBe(0);
59+});
60+61+it("runs the clear script even when screenshot throws", async () => {
62+const evaluate = evaluateMockReturning({ x: 0, y: 0 });
63+const screenshot = vi.fn(async () => {
64+throw new Error("boom");
65+});
66+setPwToolsCoreCurrentPage({ evaluate, screenshot });
67+setPwToolsCoreCurrentRefLocator({
68+boundingBox: async () => ({ x: 0, y: 0, width: 1, height: 1 }),
69+});
70+71+await expect(
72+mod.screenshotWithLabelsViaPlaywright({
73+cdpUrl: "http://127.0.0.1:18792",
74+targetId: "T1",
75+refs: { e1: { role: "button" } },
76+}),
77+).rejects.toThrow(/boom/);
78+79+// The clear script must have run (string evaluate calls include the overlay attr)
80+const clearCalls = evaluate.mock.calls.filter(
81+([arg]) => typeof arg === "string" && arg.includes("data-openclaw-labels"),
82+);
83+// inject + clear = at least 2 string evaluations
84+expect(clearCalls.length).toBeGreaterThanOrEqual(2);
85+});
86+87+it("counts off-viewport refs as skipped but still surfaces them in annotations", async () => {
88+const evaluate = evaluateMockReturning({ x: 0, y: 0, width: 1280, height: 720 });
89+const screenshot = vi.fn(async () => Buffer.from("PNG"));
90+setPwToolsCoreCurrentPage({ evaluate, screenshot });
91+// bbox is far below the viewport (y: 5000): not drawn, but still reported
92+// so callers keep the position and a non-zero skipped count.
93+setPwToolsCoreCurrentRefLocator({
94+boundingBox: async () => ({ x: 0, y: 5000, width: 50, height: 20 }),
95+});
96+97+const result = await mod.screenshotWithLabelsViaPlaywright({
98+cdpUrl: "http://127.0.0.1:18792",
99+targetId: "T1",
100+refs: { e1: { role: "button" } },
101+});
102+103+expect(result.skipped).toBe(1);
104+expect(result.labels).toBe(0);
105+expect(result.annotations).toHaveLength(1);
106+expect(result.annotations[0]?.ref).toBe("e1");
107+});
108+});
109+110+describe("screenshotWithLabelsViaPlaywright (fullpage)", () => {
111+beforeEach(() => vi.clearAllMocks());
112+113+it("forwards fullPage:true to page.screenshot and uses doc-space annotations", async () => {
114+const evaluate = evaluateMockReturning({ x: 0, y: 1000 });
115+const screenshot = vi.fn(async () => Buffer.from("FULL"));
116+setPwToolsCoreCurrentPage({ evaluate, screenshot });
117+setPwToolsCoreCurrentRefLocator({
118+boundingBox: async () => ({ x: 10, y: 200, width: 50, height: 20 }),
119+});
120+121+const result = await mod.screenshotWithLabelsViaPlaywright({
122+cdpUrl: "http://127.0.0.1:18792",
123+targetId: "T1",
124+refs: { e1: { role: "button" } },
125+fullPage: true,
126+});
127+128+expect(screenshot).toHaveBeenCalledWith(expect.objectContaining({ fullPage: true }));
129+// doc-space: scroll y=1000 + bbox y=200 = 1200
130+expect(result.annotations[0]?.box.y).toBe(1200);
131+expect(result.annotations[0]?.box.x).toBe(10);
132+});
133+});
134+135+describe("screenshotWithLabelsViaPlaywright (element/ref)", () => {
136+beforeEach(() => vi.clearAllMocks());
137+138+it("uses refLocator.screenshot for ref mode and projects relative to element", async () => {
139+const evaluate = evaluateMockReturning({ x: 0, y: 0 });
140+// First call resolves the element rect (container), second resolves e1 annotation bbox.
141+const boundingBox = vi
142+.fn<() => Promise<{ x: number; y: number; width: number; height: number } | null>>()
143+.mockResolvedValueOnce({ x: 50, y: 100, width: 200, height: 300 })
144+.mockResolvedValueOnce({ x: 60, y: 110, width: 30, height: 20 });
145+const elementScreenshot = vi.fn(async () => Buffer.from("ELEM"));
146+setPwToolsCoreCurrentPage({ evaluate, screenshot: vi.fn() });
147+setPwToolsCoreCurrentRefLocator({ boundingBox, screenshot: elementScreenshot });
148+149+const result = await mod.screenshotWithLabelsViaPlaywright({
150+cdpUrl: "http://127.0.0.1:18792",
151+targetId: "T1",
152+refs: { e1: { role: "button" } },
153+ref: "container",
154+});
155+156+expect(elementScreenshot).toHaveBeenCalledTimes(1);
157+// Element-relative: doc(60,110) - elementRect(50,100) = (10,10)
158+expect(result.annotations).toHaveLength(1);
159+expect(result.annotations[0]?.box).toEqual({ x: 10, y: 10, width: 30, height: 20 });
160+});
161+162+it("throws when ref/element cannot be resolved", async () => {
163+const evaluate = evaluateMockReturning({ x: 0, y: 0 });
164+setPwToolsCoreCurrentPage({ evaluate, screenshot: vi.fn() });
165+setPwToolsCoreCurrentRefLocator({
166+boundingBox: async () => null,
167+screenshot: vi.fn(),
168+});
169+170+await expect(
171+mod.screenshotWithLabelsViaPlaywright({
172+cdpUrl: "http://127.0.0.1:18792",
173+targetId: "T1",
174+refs: { e1: { role: "button" } },
175+ref: "missing",
176+}),
177+).rejects.toThrow(/element not found/i);
178+});
179+});
180+181+describe("screenshotWithLabelsViaPlaywright (skipped accounting)", () => {
182+beforeEach(() => vi.clearAllMocks());
183+184+it("counts refs whose boundingBox is null toward skipped", async () => {
185+const evaluate = evaluateMockReturning({ x: 0, y: 0 });
186+const screenshot = vi.fn(async () => Buffer.from("PNG"));
187+setPwToolsCoreCurrentPage({ evaluate, screenshot });
188+// Two refs: first returns a box, second returns null (e.g. element detached).
189+const boundingBox = vi
190+.fn<() => Promise<{ x: number; y: number; width: number; height: number } | null>>()
191+.mockResolvedValueOnce({ x: 10, y: 20, width: 30, height: 40 })
192+.mockResolvedValueOnce(null);
193+setPwToolsCoreCurrentRefLocator({ boundingBox });
194+195+const result = await mod.screenshotWithLabelsViaPlaywright({
196+cdpUrl: "http://127.0.0.1:18792",
197+targetId: "T1",
198+refs: { e1: { role: "button" }, e2: { role: "link" } },
199+});
200+201+expect(result.annotations).toHaveLength(1);
202+expect(result.annotations[0]?.ref).toBe("e1");
203+expect(result.skipped).toBe(1);
204+});
205+});
此内容由惯性聚合(RSS阅读器)自动聚合整理，仅供阅读参考。 原文来自 — 版权归原作者所有。