@@ -0,0 +1,211 @@
1+import type {
2+WebContentExtractionRequest,
3+WebContentExtractionResult,
4+WebContentExtractorPlugin,
5+} from "openclaw/plugin-sdk/web-content-extractor";
6+import {
7+htmlToMarkdown,
8+normalizeWhitespace,
9+sanitizeHtml,
10+stripInvisibleUnicode,
11+} from "openclaw/plugin-sdk/web-content-extractor";
12+13+const READABILITY_MAX_HTML_CHARS = 1_000_000;
14+const READABILITY_MAX_ESTIMATED_NESTING_DEPTH = 3_000;
15+16+type ParsedHtml = {
17+document: Document;
18+};
19+20+type ParseHtml = (html: string) => ParsedHtml;
21+22+type ReadabilityResult = {
23+content?: string;
24+textContent?: string | null;
25+title?: string | null;
26+};
27+28+type ReadabilityInstance = {
29+parse(): ReadabilityResult | null;
30+};
31+32+type ReadabilityConstructor = new (
33+document: Document,
34+options: { charThreshold: number },
35+) => ReadabilityInstance;
36+37+type ReadabilityModule = {
38+Readability: ReadabilityConstructor;
39+};
40+41+type LinkedomModule = {
42+parseHTML: ParseHtml;
43+};
44+45+const READABILITY_MODULE = "@mozilla/readability";
46+const LINKEDOM_MODULE = "linkedom";
47+48+let readabilityDepsPromise:
49+| Promise<{
50+Readability: ReadabilityConstructor;
51+parseHTML: ParseHtml;
52+}>
53+| undefined;
54+55+async function loadReadabilityDeps(): Promise<{
56+Readability: ReadabilityConstructor;
57+parseHTML: ParseHtml;
58+}> {
59+if (!readabilityDepsPromise) {
60+readabilityDepsPromise = Promise.all([
61+import(READABILITY_MODULE) as Promise<ReadabilityModule>,
62+import(LINKEDOM_MODULE) as Promise<LinkedomModule>,
63+]).then(([readability, linkedom]) => ({
64+Readability: readability.Readability,
65+parseHTML: linkedom.parseHTML,
66+}));
67+}
68+try {
69+return await readabilityDepsPromise;
70+} catch (error) {
71+readabilityDepsPromise = undefined;
72+throw error;
73+}
74+}
75+76+function normalizeLowercaseStringOrEmpty(value: string): string {
77+return value.trim().toLowerCase();
78+}
79+80+function exceedsEstimatedHtmlNestingDepth(html: string, maxDepth: number): boolean {
81+const voidTags = new Set([
82+"area",
83+"base",
84+"br",
85+"col",
86+"embed",
87+"hr",
88+"img",
89+"input",
90+"link",
91+"meta",
92+"param",
93+"source",
94+"track",
95+"wbr",
96+]);
97+98+let depth = 0;
99+const len = html.length;
100+for (let i = 0; i < len; i++) {
101+if (html.charCodeAt(i) !== 60) {
102+continue;
103+}
104+const next = html.charCodeAt(i + 1);
105+if (next === 33 || next === 63) {
106+continue;
107+}
108+109+let j = i + 1;
110+let closing = false;
111+if (html.charCodeAt(j) === 47) {
112+closing = true;
113+j += 1;
114+}
115+116+while (j < len && html.charCodeAt(j) <= 32) {
117+j += 1;
118+}
119+120+const nameStart = j;
121+while (j < len) {
122+const c = html.charCodeAt(j);
123+const isNameChar =
124+(c >= 65 && c <= 90) ||
125+(c >= 97 && c <= 122) ||
126+(c >= 48 && c <= 57) ||
127+c === 58 ||
128+c === 45;
129+if (!isNameChar) {
130+break;
131+}
132+j += 1;
133+}
134+135+const tagName = normalizeLowercaseStringOrEmpty(html.slice(nameStart, j));
136+if (!tagName) {
137+continue;
138+}
139+140+if (closing) {
141+depth = Math.max(0, depth - 1);
142+continue;
143+}
144+if (voidTags.has(tagName)) {
145+continue;
146+}
147+148+let selfClosing = false;
149+for (let k = j; k < len && k < j + 200; k++) {
150+const c = html.charCodeAt(k);
151+if (c === 62) {
152+selfClosing = html.charCodeAt(k - 1) === 47;
153+break;
154+}
155+}
156+if (selfClosing) {
157+continue;
158+}
159+160+depth += 1;
161+if (depth > maxDepth) {
162+return true;
163+}
164+}
165+return false;
166+}
167+168+async function extractWithReadability(
169+request: WebContentExtractionRequest,
170+): Promise<WebContentExtractionResult | null> {
171+const cleanHtml = await sanitizeHtml(request.html);
172+if (
173+cleanHtml.length > READABILITY_MAX_HTML_CHARS ||
174+exceedsEstimatedHtmlNestingDepth(cleanHtml, READABILITY_MAX_ESTIMATED_NESTING_DEPTH)
175+) {
176+return null;
177+}
178+try {
179+const { Readability, parseHTML } = await loadReadabilityDeps();
180+const { document } = parseHTML(cleanHtml);
181+try {
182+(document as { baseURI?: string }).baseURI = request.url;
183+} catch {
184+// Best-effort base URI for relative links.
185+}
186+const reader = new Readability(document, { charThreshold: 0 });
187+const parsed = reader.parse();
188+if (!parsed?.content) {
189+return null;
190+}
191+const title = parsed.title || undefined;
192+if (request.extractMode === "text") {
193+const text = stripInvisibleUnicode(normalizeWhitespace(parsed.textContent ?? ""));
194+return text ? { text, title } : null;
195+}
196+const rendered = htmlToMarkdown(parsed.content);
197+const text = stripInvisibleUnicode(rendered.text);
198+return text ? { text, title: title ?? rendered.title } : null;
199+} catch {
200+return null;
201+}
202+}
203+204+export function createReadabilityWebContentExtractor(): WebContentExtractorPlugin {
205+return {
206+id: "readability",
207+label: "Readability",
208+autoDetectOrder: 10,
209+extract: extractWithReadability,
210+};
211+}
此内容由惯性聚合(RSS阅读器)自动聚合整理，仅供阅读参考。 原文来自 — 版权归原作者所有。