@@ -140,6 +140,56 @@ async function createTempAgentDir(): Promise<string> {
140140return await fs.mkdtemp(path.join(os.tmpdir(), "openai-plugin-live-"));
141141}
142142143+async function waitForLiveExpectation(expectation: () => void, timeoutMs = 30_000) {
144+const started = Date.now();
145+let lastError: unknown;
146+while (Date.now() - started < timeoutMs) {
147+try {
148+expectation();
149+return;
150+} catch (error) {
151+lastError = error;
152+await new Promise((resolve) => setTimeout(resolve, 100));
153+}
154+}
155+throw lastError;
156+}
157+158+function normalizeTranscriptForMatch(value: string): string {
159+return value.toLowerCase().replace(/[^a-z0-9]+/g, "");
160+}
161+162+function linearToMulaw(sample: number): number {
163+const bias = 132;
164+const clip = 32635;
165+let next = Math.max(-clip, Math.min(clip, sample));
166+const sign = next < 0 ? 0x80 : 0;
167+if (next < 0) {
168+next = -next;
169+}
170+171+next += bias;
172+let exponent = 7;
173+for (let expMask = 0x4000; (next & expMask) === 0 && exponent > 0; exponent -= 1) {
174+expMask >>= 1;
175+}
176+177+const mantissa = (next >> (exponent + 3)) & 0x0f;
178+return ~(sign | (exponent << 4) | mantissa) & 0xff;
179+}
180+181+function convertPcm24kToMulaw8k(pcm: Buffer): Buffer {
182+const inputSamples = Math.floor(pcm.length / 2);
183+const outputSamples = Math.floor(inputSamples / 3);
184+const mulaw = Buffer.alloc(outputSamples);
185+186+for (let i = 0; i < outputSamples; i += 1) {
187+mulaw[i] = linearToMulaw(pcm.readInt16LE(i * 3 * 2));
188+}
189+190+return mulaw;
191+}
192+143193describeLive("openai plugin live", () => {
144194it("registers an OpenAI provider that can complete a live request", async () => {
145195const { providers } = await registerOpenAIPlugin();
@@ -247,6 +297,89 @@ describeLive("openai plugin live", () => {
247297expect(text).toMatch(/\bok\b/);
248298}, 45_000);
249299300+it("opens OpenAI realtime STT before sending audio", async () => {
301+const { realtimeTranscriptionProviders } = await registerOpenAIPlugin();
302+const realtimeProvider = requireRegisteredProvider(realtimeTranscriptionProviders, "openai");
303+const errors: Error[] = [];
304+const session = realtimeProvider.createSession({
305+providerConfig: {
306+apiKey: OPENAI_API_KEY,
307+language: "en",
308+},
309+onError: (error) => errors.push(error),
310+});
311+312+try {
313+await session.connect();
314+await new Promise((resolve) => setTimeout(resolve, 1_000));
315+expect(errors).toEqual([]);
316+expect(session.isConnected()).toBe(true);
317+} finally {
318+session.close();
319+}
320+}, 30_000);
321+322+it("streams realtime STT through the registered transcription provider", async () => {
323+const { realtimeTranscriptionProviders, speechProviders } = await registerOpenAIPlugin();
324+const realtimeProvider = requireRegisteredProvider(realtimeTranscriptionProviders, "openai");
325+const speechProvider = requireRegisteredProvider(speechProviders, "openai");
326+const cfg = createLiveConfig();
327+const ttsConfig = createLiveTtsConfig();
328+const phrase = "Testing OpenClaw OpenAI realtime transcription integration test OK.";
329+330+const telephony = await speechProvider.synthesizeTelephony?.({
331+text: phrase,
332+ cfg,
333+providerConfig: ttsConfig.providerConfigs.openai ?? {},
334+timeoutMs: ttsConfig.timeoutMs,
335+});
336+if (!telephony) {
337+throw new Error("OpenAI telephony synthesis did not return audio");
338+}
339+expect(telephony.outputFormat).toBe("pcm");
340+expect(telephony.sampleRate).toBe(24_000);
341+342+const transcripts: string[] = [];
343+const partials: string[] = [];
344+const errors: Error[] = [];
345+const session = realtimeProvider.createSession({
346+providerConfig: {
347+apiKey: OPENAI_API_KEY,
348+language: "en",
349+silenceDurationMs: 500,
350+},
351+onPartial: (partial) => partials.push(partial),
352+onTranscript: (transcript) => transcripts.push(transcript),
353+onError: (error) => errors.push(error),
354+});
355+356+try {
357+await session.connect();
358+const speech = convertPcm24kToMulaw8k(telephony.audioBuffer);
359+const silence = Buffer.alloc(8_000, 0xff);
360+const audio = Buffer.concat([silence.subarray(0, 4_000), speech, silence]);
361+for (let offset = 0; offset < audio.byteLength; offset += 160) {
362+session.sendAudio(audio.subarray(offset, offset + 160));
363+await new Promise((resolve) => setTimeout(resolve, 5));
364+}
365+366+await waitForLiveExpectation(() => {
367+if (errors[0]) {
368+throw errors[0];
369+}
370+expect(normalizeTranscriptForMatch(transcripts.join(" "))).toContain("openclaw");
371+}, 60_000);
372+} finally {
373+session.close();
374+}
375+376+const normalized = transcripts.join(" ").toLowerCase();
377+const compact = normalizeTranscriptForMatch(normalized);
378+expect(compact).toContain("openclaw");
379+expect(normalized).toContain("transcription");
380+expect(partials.length + transcripts.length).toBeGreaterThan(0);
381+}, 180_000);
382+250383it("generates an image through the registered image provider", async () => {
251384const { imageProviders } = await registerOpenAIPlugin();
252385const imageProvider = requireRegisteredProvider(imageProviders, "openai");
此内容由惯性聚合(RSS阅读器)自动聚合整理，仅供阅读参考。 原文来自 — 版权归原作者所有。