1+import { pathToFileURL } from "node:url";
12import { fetchFirecrawlContent } from "../extensions/firecrawl/api.ts";
23import { extractReadableContent } from "../src/agents/tools/web-tools.js";
34import { formatErrorMessage } from "../src/infra/errors.ts";
@@ -18,6 +19,12 @@ const baseUrl = process.env.FIRECRAWL_BASE_URL ?? "https://api.firecrawl.dev";
1819const userAgent =
1920"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
2021const timeoutMs = 30_000;
22+const FETCH_HTML_MAX_BYTES = 5 * 1024 * 1024;
23+24+type FetchHtmlOptions = {
25+fetchImpl?: typeof fetch;
26+maxBytes?: number;
27+};
21282229function truncate(value: string, max = 180): string {
2330if (!value) {
@@ -26,7 +33,97 @@ function truncate(value: string, max = 180): string {
2633return value.length > max ? `${value.slice(0, max)}…` : value;
2734}
283529-async function fetchHtml(url: string): Promise<{
36+function responseBodyTooLargeError(label: string, maxBytes: number): Error {
37+return new Error(`${label} response body exceeded ${maxBytes} bytes`);
38+}
39+40+async function readBoundedResponseText(
41+response: Response,
42+label: string,
43+signal: AbortSignal,
44+maxBytes = FETCH_HTML_MAX_BYTES,
45+): Promise<string> {
46+const contentLength = Number(response.headers.get("content-length") ?? "");
47+if (Number.isSafeInteger(contentLength) && contentLength > maxBytes) {
48+await response.body?.cancel().catch(() => undefined);
49+throw responseBodyTooLargeError(label, maxBytes);
50+}
51+if (!response.body) {
52+return "";
53+}
54+55+const reader = response.body.getReader();
56+const decoder = new TextDecoder();
57+const chunks: string[] = [];
58+let totalBytes = 0;
59+let canceled = false;
60+61+try {
62+for (;;) {
63+const { done, value } = await readResponseChunk(reader, label, signal, () => {
64+canceled = true;
65+});
66+if (done) {
67+const tail = decoder.decode();
68+if (tail) {
69+chunks.push(tail);
70+}
71+break;
72+}
73+74+totalBytes += value.byteLength;
75+if (totalBytes > maxBytes) {
76+canceled = true;
77+await reader.cancel().catch(() => undefined);
78+throw responseBodyTooLargeError(label, maxBytes);
79+}
80+chunks.push(decoder.decode(value, { stream: true }));
81+}
82+} finally {
83+if (!canceled) {
84+reader.releaseLock();
85+}
86+}
87+88+return chunks.join("");
89+}
90+91+async function readResponseChunk(
92+reader: ReadableStreamDefaultReader<Uint8Array>,
93+label: string,
94+signal: AbortSignal,
95+markCanceled: () => void,
96+): Promise<ReadableStreamReadResult<Uint8Array>> {
97+if (signal.aborted) {
98+markCanceled();
99+await reader.cancel().catch(() => undefined);
100+throw signal.reason instanceof Error ? signal.reason : new Error(`${label} request aborted`);
101+}
102+103+let removeAbortListener: (() => void) | undefined;
104+const abortPromise = new Promise<ReadableStreamReadResult<Uint8Array>>((_resolve, reject) => {
105+const onAbort = () => {
106+markCanceled();
107+void reader.cancel().catch(() => undefined);
108+reject(
109+signal.reason instanceof Error ? signal.reason : new Error(`${label} request aborted`),
110+);
111+};
112+signal.addEventListener("abort", onAbort, { once: true });
113+removeAbortListener = () => signal.removeEventListener("abort", onAbort);
114+});
115+116+try {
117+return await Promise.race([reader.read(), abortPromise]);
118+} finally {
119+removeAbortListener?.();
120+}
121+}
122+123+async function fetchHtml(
124+url: string,
125+options: FetchHtmlOptions = {},
126+): Promise<{
30127ok: boolean;
31128status: number;
32129contentType: string;
@@ -35,14 +132,20 @@ async function fetchHtml(url: string): Promise<{
35132}> {
36133const controller = new AbortController();
37134const timer = setTimeout(() => controller.abort(), timeoutMs);
135+const fetchImpl = options.fetchImpl ?? fetch;
38136try {
39-const res = await fetch(url, {
137+const res = await fetchImpl(url, {
40138method: "GET",
41139headers: { Accept: "*/*", "User-Agent": userAgent },
42140signal: controller.signal,
43141});
44142const contentType = res.headers.get("content-type") ?? "application/octet-stream";
45-const body = await res.text();
143+const body = await readBoundedResponseText(
144+res,
145+"local HTML fetch",
146+controller.signal,
147+options.maxBytes ?? FETCH_HTML_MAX_BYTES,
148+);
46149return {
47150ok: res.ok,
48151status: res.status,
@@ -135,7 +238,15 @@ async function run() {
135238process.exit(0);
136239}
137240138-run().catch((error) => {
139-console.error(error);
140-process.exit(1);
141-});
241+if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
242+run().catch((error) => {
243+console.error(error);
244+process.exit(1);
245+});
246+}
247+248+export const testing = {
249+FETCH_HTML_MAX_BYTES,
250+ fetchHtml,
251+ readBoundedResponseText,
252+};
此内容由惯性聚合(RSS阅读器)自动聚合整理，仅供阅读参考。 原文来自 — 版权归原作者所有。