@@ -5,6 +5,10 @@ import type { OpenClawConfig } from "../../config/types.openclaw.js";
55import { logVerbose } from "../../globals.js";
66import { formatErrorMessage } from "../../infra/errors.js";
77import type { ImageContent } from "../../llm/types.js";
8+import {
9+stripExtractedFileImageMetadata,
10+type ExtractedFileImage,
11+} from "../../media-understanding/extracted-file-images.js";
812import type { PromptImageOrderEntry } from "../../media/prompt-image-order.js";
913import type { MsgContext } from "../templating.js";
1014import { resolveAgentTurnAttachments } from "./agent-turn-attachments.js";
@@ -15,6 +19,13 @@ type CurrentImageAttachment = {
1519mediaType: string;
1620};
172122+type OrderedTurnImage = {
23+image: ImageContent;
24+imageOrder: PromptImageOrderEntry;
25+sourceIndex?: number;
26+sequence: number;
27+};
28+1829function isGenericMediaType(mediaType: string | undefined): boolean {
1930if (!mediaType) {
2031return true;
@@ -88,30 +99,82 @@ function createUndescribedImageContext(
8899};
89100}
90101102+function appendOrderedImages(params: {
103+entries: OrderedTurnImage[];
104+images: ImageContent[] | undefined;
105+imageOrder?: PromptImageOrderEntry[];
106+sourceIndex?: number;
107+}) {
108+if (!params.images || params.images.length === 0) {
109+return;
110+}
111+for (const [index, image] of params.images.entries()) {
112+params.entries.push({
113+ image,
114+imageOrder: params.imageOrder?.[index] ?? "inline",
115+sourceIndex: params.sourceIndex,
116+sequence: params.entries.length,
117+});
118+}
119+}
120+121+function resolveMergedTurnImages(entries: OrderedTurnImage[]): {
122+images?: ImageContent[];
123+imageOrder?: PromptImageOrderEntry[];
124+} {
125+if (entries.length === 0) {
126+return {};
127+}
128+const merged = entries.toSorted((left, right) => {
129+if (left.sourceIndex !== undefined && right.sourceIndex !== undefined) {
130+return left.sourceIndex - right.sourceIndex || left.sequence - right.sequence;
131+}
132+if (left.sourceIndex !== undefined || right.sourceIndex !== undefined) {
133+return left.sequence - right.sequence;
134+}
135+return left.sequence - right.sequence;
136+});
137+return {
138+images: merged.map((entry) => entry.image),
139+imageOrder: merged.map((entry) => entry.imageOrder),
140+};
141+}
142+91143/** Resolves current-turn image attachments that were not already described by media understanding. */
92144export async function resolveCurrentTurnImages(params: {
93145ctx: MsgContext;
94146cfg: OpenClawConfig;
95147images?: ImageContent[];
96148imageOrder?: PromptImageOrderEntry[];
149+extractedFileImages?: ExtractedFileImage[];
97150}): Promise<{
98151images?: ImageContent[];
99152imageOrder?: PromptImageOrderEntry[];
100153}> {
101-if (Array.isArray(params.images) && params.images.length > 0) {
102-return { images: params.images, imageOrder: params.imageOrder };
154+const entries: OrderedTurnImage[] = [];
155+appendOrderedImages({
156+ entries,
157+images: params.images,
158+imageOrder: params.imageOrder,
159+});
160+for (const image of params.extractedFileImages ?? []) {
161+appendOrderedImages({
162+ entries,
163+images: [stripExtractedFileImageMetadata(image)],
164+sourceIndex: image.attachmentIndex,
165+});
103166}
104167105168const currentImageAttachments = collectCurrentImageAttachments(params.ctx);
106169if (currentImageAttachments.length === 0) {
107-return { images: params.images, imageOrder: params.imageOrder };
170+return resolveMergedTurnImages(entries);
108171}
109172const describedImageIndexes = collectDescribedImageAttachmentIndexes(params.ctx);
110173const undescribedImageAttachments = currentImageAttachments.filter(
111174(attachment) => !describedImageIndexes.has(attachment.index),
112175);
113176if (undescribedImageAttachments.length === 0) {
114-return { images: params.images, imageOrder: params.imageOrder };
177+return resolveMergedTurnImages(entries);
115178}
116179117180try {
@@ -132,15 +195,20 @@ export async function resolveCurrentTurnImages(params: {
132195logVerbose(
133196`agent-runner: native OpenClaw media resolution produced ${images.length}/${undescribedImageAttachments.length} current image attachment(s); falling back to prompt image refs`,
134197);
135-return { images: params.images, imageOrder: params.imageOrder };
198+return resolveMergedTurnImages(entries);
199+}
200+for (const [index, image] of images.entries()) {
201+appendOrderedImages({
202+ entries,
203+images: [image],
204+sourceIndex: undescribedImageAttachments[index]?.index,
205+});
136206}
137-return images.length > 0
138- ? { images, imageOrder: images.map(() => "inline" as const) }
139- : { images: params.images, imageOrder: params.imageOrder };
207+return resolveMergedTurnImages(entries);
140208} catch (error) {
141209logVerbose(
142210`agent-runner: media attachment image resolution failed, proceeding without native images: ${formatErrorMessage(error)}`,
143211);
144-return { images: params.images, imageOrder: params.imageOrder };
212+return resolveMergedTurnImages(entries);
145213}
146214}
此内容由惯性聚合(RSS阅读器)自动聚合整理，仅供阅读参考。 原文来自 — 版权归原作者所有。