@@ -8,6 +8,7 @@ import {
88createRealtimeVoiceAgentTalkbackQueue,
99createRealtimeVoiceBridgeSession,
1010createRealtimeVoiceForcedConsultCoordinator,
11+createRealtimeVoiceOutputActivityTracker,
1112createRealtimeVoiceTurnContextTracker,
1213matchRealtimeVoiceActivationName,
1314matchRealtimeVoiceConsultQuestions,
@@ -30,6 +31,7 @@ import {
3031type RealtimeVoiceToolCallEvent,
3132type RealtimeVoiceForcedConsultCoordinator,
3233type RealtimeVoiceForcedConsultHandle,
34+type RealtimeVoiceOutputActivityTracker,
3335type RealtimeVoiceTurnContextHandle,
3436type RealtimeVoiceTurnContextTracker,
3537sortRealtimeVoiceActivationNames,
@@ -376,15 +378,10 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
376378ignoredContextTtlMs: DISCORD_REALTIME_IGNORED_WAKE_NAME_CONTEXT_TTL_MS,
377379},
378380);
379-private outputAudioTimestampMs = 0;
380-private outputAudioDiscordBytes = 0;
381-private outputAudioRealtimeBytes = 0;
382-private outputAudioChunks = 0;
383-private outputAudioStartedAt: number | undefined;
381+private readonly outputActivity: RealtimeVoiceOutputActivityTracker =
382+createRealtimeVoiceOutputActivityTracker();
384383private outputPlaybackWatchdog: ReturnType<typeof setTimeout> | undefined;
385-private outputStreamEnding = false;
386384private outputPacedBuffer: Buffer = Buffer.alloc(0);
387-private outputPlaybackStarted = false;
388385private realtimeProviderId: string | undefined;
389386private queuedExactSpeechMessages: string[] = [];
390387private exactSpeechResponseActive = false;
@@ -653,7 +650,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
653650turn.inputChunks += 1;
654651if (turn.inputChunks === 1) {
655652logger.info(
656-`discord voice: realtime input audio started guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} user=${turn.context.userId} speaker=${turn.context.speakerLabel} discordBytes=${discordPcm48kStereo.length} realtimeBytes=${realtimePcm.length} outputAudioMs=${Math.floor(this.outputAudioTimestampMs)} outputActive=${this.isOutputAudioActive()}`,
653+`discord voice: realtime input audio started guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} user=${turn.context.userId} speaker=${turn.context.speakerLabel} discordBytes=${discordPcm48kStereo.length} realtimeBytes=${realtimePcm.length} outputAudioMs=${this.outputAudioMs()} outputActive=${this.isOutputAudioActive()}`,
657654);
658655}
659656const outputActive = this.hasInterruptibleOutputAudio();
@@ -663,7 +660,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
663660`realtime barge-in from active speaker audio: guild ${this.params.entry.guildId} channel ${this.params.entry.channelId} user ${turn.context.userId}`,
664661);
665662logger.info(
666-`discord voice: realtime barge-in detected source=active-speaker-audio guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} user=${turn.context.userId} speaker=${turn.context.speakerLabel} outputAudioMs=${Math.floor(this.outputAudioTimestampMs)} outputActive=${this.isOutputAudioActive()} discordBytes=${discordPcm48kStereo.length} realtimeBytes=${realtimePcm.length}`,
663+`discord voice: realtime barge-in detected source=active-speaker-audio guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} user=${turn.context.userId} speaker=${turn.context.speakerLabel} outputAudioMs=${this.outputAudioMs()} outputActive=${this.isOutputAudioActive()} discordBytes=${discordPcm48kStereo.length} realtimeBytes=${realtimePcm.length}`,
667664);
668665this.handleBargeIn("active-speaker-audio");
669666}
@@ -681,12 +678,12 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
681678const outputActive = this.hasInterruptibleOutputAudio();
682679if (!outputActive) {
683680logger.info(
684-`discord voice: realtime barge-in ignored reason=${reason} outputActive=false guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} playbackChunks=${this.outputAudioChunks}`,
681+`discord voice: realtime barge-in ignored reason=${reason} outputActive=false guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} playbackChunks=${this.outputAudioChunks()}`,
685682);
686683return;
687684}
688685logger.info(
689-`discord voice: realtime barge-in requested reason=${reason} guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} outputAudioMs=${Math.floor(this.outputAudioTimestampMs)} outputActive=${this.isOutputAudioActive()} playbackChunks=${this.outputAudioChunks}`,
686+`discord voice: realtime barge-in requested reason=${reason} guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} outputAudioMs=${this.outputAudioMs()} outputActive=${this.isOutputAudioActive()} playbackChunks=${this.outputAudioChunks()}`,
690687);
691688this.bridge?.handleBargeIn({ audioPlaybackActive: true });
692689}
@@ -704,9 +701,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
704701705702private hasInterruptibleOutputAudio(): boolean {
706703this.syncOutputAudioTimestamp();
707-return (
708-this.isOutputAudioActive() || this.outputAudioChunks > 0 || this.outputAudioTimestampMs > 0
709-);
704+return this.outputActivity.isInterruptible(this.isOutputStreamActive());
710705}
711706712707private get realtimeConfig(): DiscordRealtimeVoiceConfig {
@@ -719,7 +714,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
719714return;
720715}
721716this.syncOutputAudioTimestamp();
722-if (this.outputStreamEnding) {
717+if (this.outputActivity.snapshot().streamEnding) {
723718logVoiceVerbose(
724719`realtime output audio ignored after stream ending: guild ${this.params.entry.guildId} channel ${this.params.entry.channelId}`,
725720);
@@ -729,13 +724,14 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
729724if (this.exactSpeechResponseActive) {
730725this.exactSpeechAudioStarted = true;
731726}
732-this.outputAudioDiscordBytes += discordPcm.length;
733-this.outputAudioRealtimeBytes += realtimePcm24kMono.length;
734-this.outputAudioChunks += 1;
735-this.outputAudioTimestampMs += pcm16MonoDurationMs(
736-realtimePcm24kMono,
737-REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ.sampleRateHz,
738-);
727+this.outputActivity.markAudio({
728+audioMs: pcm16MonoDurationMs(
729+realtimePcm24kMono,
730+REALTIME_VOICE_AUDIO_FORMAT_PCM16_24KHZ.sampleRateHz,
731+),
732+sourceAudioBytes: realtimePcm24kMono.length,
733+sinkAudioBytes: discordPcm.length,
734+});
739735this.queueOutputAudio(stream, discordPcm);
740736}
741737@@ -746,7 +742,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
746742const stream = new PassThrough({ highWaterMark: DISCORD_RAW_PCM_FRAME_BYTES * 128 });
747743this.outputStream = stream;
748744this.outputPacedBuffer = Buffer.alloc(0);
749-this.outputPlaybackStarted = false;
745+this.outputActivity.markStreamOpened();
750746stream.once("close", () => {
751747if (this.outputStream === stream) {
752748this.logOutputAudioStopped("stream-close");
@@ -759,7 +755,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
759755}
760756761757private queueOutputAudio(stream: PassThrough, discordPcm: Buffer): void {
762-if (this.outputPlaybackStarted) {
758+if (this.outputActivity.snapshot().playbackStarted) {
763759stream.write(discordPcm);
764760return;
765761}
@@ -776,7 +772,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
776772}
777773778774private startOutputPlayback(stream: PassThrough): void {
779-if (this.outputPlaybackStarted || stream.destroyed) {
775+if (this.outputActivity.snapshot().playbackStarted || stream.destroyed) {
780776return;
781777}
782778const voiceSdk = loadDiscordVoiceSdk();
@@ -788,8 +784,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
788784inputType: voiceSdk.StreamType.Raw,
789785});
790786this.params.entry.player.play(resource);
791-this.outputPlaybackStarted = true;
792-this.outputAudioStartedAt = Date.now();
787+this.outputActivity.markPlaybackStarted();
793788const realtimeConfig = this.realtimeConfig;
794789logger.info(
795790`discord voice: realtime audio playback started guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} mode=${this.params.mode} model=${realtimeConfig?.model ?? "provider-default"} voice=${realtimeConfig?.voice ?? "provider-default"}`,
@@ -807,7 +802,6 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
807802this.logOutputAudioStopped(reason);
808803this.outputStream = null;
809804this.outputPacedBuffer = Buffer.alloc(0);
810-this.outputPlaybackStarted = false;
811805this.resetOutputAudioStats();
812806stream?.end();
813807stream?.destroy();
@@ -818,12 +812,12 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
818812{ playBuffered = true }: { playBuffered?: boolean } = {},
819813): void {
820814const stream = this.outputStream;
821-if (!stream || stream.destroyed || this.outputStreamEnding) {
815+if (!stream || stream.destroyed || this.outputActivity.snapshot().streamEnding) {
822816return;
823817}
824-this.outputStreamEnding = true;
818+this.outputActivity.markStreamEnding();
825819logger.info(
826-`discord voice: realtime audio playback finishing reason=${reason} guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} audioMs=${Math.floor(this.outputAudioTimestampMs)} chunks=${this.outputAudioChunks}`,
820+`discord voice: realtime audio playback finishing reason=${reason} guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} audioMs=${this.outputAudioMs()} chunks=${this.outputAudioChunks()}`,
827821);
828822if (playBuffered) {
829823this.startOutputPlayback(stream);
@@ -839,14 +833,12 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
839833840834private scheduleOutputPlaybackWatchdog(reason: string, stream: PassThrough): void {
841835this.clearOutputPlaybackWatchdog();
842-if (!this.outputAudioStartedAt || this.outputAudioTimestampMs <= 0) {
836+const timeoutMs = this.outputActivity.playbackWatchdogDelayMs({
837+marginMs: DISCORD_REALTIME_OUTPUT_PLAYBACK_WATCHDOG_MARGIN_MS,
838+});
839+if (timeoutMs === undefined) {
843840return;
844841}
845-const elapsedMs = Date.now() - this.outputAudioStartedAt;
846-const timeoutMs = Math.max(
847-1_000,
848-this.outputAudioTimestampMs - elapsedMs + DISCORD_REALTIME_OUTPUT_PLAYBACK_WATCHDOG_MARGIN_MS,
849-);
850842this.outputPlaybackWatchdog = setTimeout(() => {
851843this.outputPlaybackWatchdog = undefined;
852844if (this.outputStream && this.outputStream !== stream) {
@@ -857,7 +849,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
857849return;
858850}
859851logger.warn(
860-`discord voice: realtime audio playback watchdog fired reason=${reason} guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} audioMs=${Math.floor(this.outputAudioTimestampMs)} elapsedMs=${this.outputAudioStartedAt ? Date.now() - this.outputAudioStartedAt : 0}`,
852+`discord voice: realtime audio playback watchdog fired reason=${reason} guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} audioMs=${this.outputAudioMs()} elapsedMs=${this.outputActivity.elapsedPlaybackMs()}`,
861853);
862854this.clearOutputAudio("playback-watchdog");
863855this.completeExactSpeechResponse("playback-watchdog");
@@ -879,7 +871,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
879871if (this.exactSpeechResponseActive || this.hasInterruptibleOutputAudio()) {
880872this.queuedExactSpeechMessages.push(text);
881873logger.info(
882-`discord voice: realtime exact speech queued guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} queued=${this.queuedExactSpeechMessages.length} outputAudioMs=${Math.floor(this.outputAudioTimestampMs)} outputActive=${this.isOutputAudioActive()}`,
874+`discord voice: realtime exact speech queued guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} queued=${this.queuedExactSpeechMessages.length} outputAudioMs=${this.outputAudioMs()} outputActive=${this.isOutputAudioActive()}`,
883875);
884876return;
885877}
@@ -986,11 +978,12 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
986978}
987979988980private logOutputAudioStopped(reason: string): void {
989-const audioMs = Math.floor(this.outputAudioTimestampMs);
990-const chunks = this.outputAudioChunks;
991-const discordBytes = this.outputAudioDiscordBytes;
992-const realtimeBytes = this.outputAudioRealtimeBytes;
993-const elapsedMs = this.outputAudioStartedAt ? Date.now() - this.outputAudioStartedAt : 0;
981+const activity = this.outputActivity.snapshot();
982+const audioMs = Math.floor(activity.audioMs);
983+const chunks = activity.chunks;
984+const discordBytes = activity.sinkAudioBytes;
985+const realtimeBytes = activity.sourceAudioBytes;
986+const elapsedMs = this.outputActivity.elapsedPlaybackMs();
994987if (this.outputStream || chunks > 0 || audioMs > 0) {
995988logger.info(
996989`discord voice: realtime audio playback stopped reason=${reason} guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} audioMs=${audioMs} elapsedMs=${elapsedMs} chunks=${chunks} discordBytes=${discordBytes} realtimeBytes=${realtimeBytes}`,
@@ -999,22 +992,28 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
999992}
10009931001994private resetOutputAudioStats(): void {
1002-this.outputAudioTimestampMs = 0;
1003-this.outputAudioDiscordBytes = 0;
1004-this.outputAudioRealtimeBytes = 0;
1005-this.outputAudioChunks = 0;
1006-this.outputAudioStartedAt = undefined;
1007-this.outputStreamEnding = false;
1008995this.outputPacedBuffer = Buffer.alloc(0);
1009-this.outputPlaybackStarted = false;
996+this.outputActivity.reset();
1010997}
10119981012999private syncOutputAudioTimestamp(): void {
1013-this.bridge?.setMediaTimestamp(Math.floor(this.outputAudioTimestampMs));
1000+this.bridge?.setMediaTimestamp(this.outputAudioMs());
1001+}
1002+1003+private outputAudioMs(): number {
1004+return Math.floor(this.outputActivity.snapshot().audioMs);
1005+}
1006+1007+private outputAudioChunks(): number {
1008+return this.outputActivity.snapshot().chunks;
1009+}
1010+1011+private isOutputStreamActive(): boolean {
1012+return Boolean(this.outputStream && !this.outputStream.destroyed);
10141013}
1015101410161015private isOutputAudioActive(): boolean {
1017-return Boolean(this.outputStream && !this.outputStream.destroyed) || this.outputAudioChunks > 0;
1016+return this.outputActivity.isActive(this.isOutputStreamActive());
10181017}
1019101810201019private logSpeakerTurnClosed(turn: PendingSpeakerTurn): void {
@@ -1385,7 +1384,7 @@ export class DiscordRealtimeVoiceSession implements VoiceRealtimeSession {
13851384);
13861385if (this.hasInterruptibleOutputAudio()) {
13871386logger.info(
1388-`discord voice: realtime forced agent consult preserving active playback guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} outputAudioMs=${Math.floor(this.outputAudioTimestampMs)} outputActive=${this.isOutputAudioActive()} playbackChunks=${this.outputAudioChunks}`,
1387+`discord voice: realtime forced agent consult preserving active playback guild=${this.params.entry.guildId} channel=${this.params.entry.channelId} outputAudioMs=${this.outputAudioMs()} outputActive=${this.isOutputAudioActive()} playbackChunks=${this.outputAudioChunks()}`,
13891388);
13901389}
13911390state.handledByForcedPlayback = true;
此内容由惯性聚合(RSS阅读器)自动聚合整理，仅供阅读参考。 原文来自 — 版权归原作者所有。