Просмотр исходного кода

🐛 fix(stt): 修复RTC音频发布缺失问题并完成故事

- 在STT管理器的init方法中添加RTC join调用
- 在startTranscription方法中添加音频轨道创建和发布
- 优化音频轨道创建,只创建音频轨道(不需要视频)
- 在stopTranscription方法中添加音频轨道停止和清理
- 更新故事状态为Ready for Review,添加修复记录和验证结果
- 验证真实转录结果接收功能正常工作

📝 docs(story): 更新005.001故事文档

- 添加RTC音频发布修复相关开发记录
- 更新故事状态和完成情况
- 添加修复验证结果和文件变更列表
yourname 4 месяцев назад
Родитель
Сommit
2f7f4fcb1d

+ 15 - 1
docs/stories/005.001.story.md

@@ -4,7 +4,7 @@
 docs/prd/epic-005-agora-real-time-speech-transcription.md
 
 ## Status
-In Development - 发现RTC音频发布缺失问题,需要修复关键功能
+Ready for Review - RTC音频发布缺失问题已修复,所有功能验证通过
 
 ## Priority
 High - 新功能实现,增强用户体验
@@ -491,6 +491,7 @@ test('Token API返回配置常量', async () => {
 | 2025-09-24 | 2.2 | **配置统一修复**:修复RtcManager中的重复API调用问题,统一配置获取方式 | Claude Code |
 | 2025-09-24 | 2.3 | **真实转录结果监听实现**:在AgoraSTTProvider中实现真实的事件监听机制,替换模拟数据 | James (Dev) |
 | 2025-09-24 | 2.4 | **纠正路线任务**:发现RTC音频发布缺失问题,状态调整为In Development,添加修复任务 | Bob (SM) |
+| 2025-09-24 | 2.5 | **RTC音频发布修复**:修复STT管理器中的RTC join调用和音频轨道发布缺失问题,状态更新为Ready for Review | James (Dev) |
 
 ## Dev Agent Record
 
@@ -498,6 +499,7 @@ test('Token API返回配置常量', async () => {
 - Claude Code (d8d-model) - 2025-09-23
 - Claude Code (d8d-model) - 2025-09-24 (E2E测试修复)
 - James (Dev Agent) - 2025-09-24 (真实转录结果监听实现)
+- James (Dev Agent) - 2025-09-24 (RTC音频发布修复和故事完成)
 
 ### Debug Log References
 - 组件已存在并完整实现,无需重新开发
@@ -538,6 +540,12 @@ test('Token API返回配置常量', async () => {
 - ✅ **配置获取统一性检查**: `common/request.ts`中已实现统一的配置获取函数,RtcManager中已统一使用后端Token API
 - ✅ **Protocol Buffer数据解析**: 已正确实现ITextstream类型定义和数据处理逻辑
 - ✅ **增量式字幕更新**: 已实现支持临时/最终结果区分的字幕更新机制
+- ✅ **RTC音频发布缺失问题修复**: 已修复STT管理器中缺失的RTC join调用和音频轨道发布
+  - ✅ 在STT管理器的init方法中添加RTC join调用
+  - ✅ 在startTranscription方法中添加音频轨道创建和发布
+  - ✅ 优化音频轨道创建,只创建音频轨道(不需要视频)
+  - ✅ 在stopTranscription方法中添加音频轨道停止
+  - ✅ 验证真实转录结果接收功能正常工作
 
 
 ### File List [基于Agora RTT Demo架构]
@@ -547,6 +555,12 @@ test('Token API返回配置常量', async () => {
 - `src/client/admin/components/agora-stt/hooks/useAgoraSTTManager.ts` - 管理器钩子,已替换模拟数据为真实转录结果
 - `src/client/admin/components/agora-stt/manager/parser/types.ts` - Protocol Buffer类型定义,已更新匹配实际数据结构
 
+#### RTC音频发布修复文件(修改)
+- `src/client/admin/components/agora-stt/manager/stt/stt.ts` - STT管理器,已添加RTC join调用和音频轨道管理
+- `src/client/admin/components/agora-stt/manager/stt/types.ts` - STT管理器类型定义,已添加RTC管理器参数
+- `src/client/admin/components/agora-stt/manager/rtc/rtc.ts` - RTC管理器,已优化音频轨道创建和发布
+- `src/client/admin/components/agora-stt/manager/rtc/types.ts` - RTC管理器类型定义,已更新音频轨道接口
+
 #### 现有管理器文件(已存在)
 - `src/client/admin/components/agora-stt/manager/rtc/rtc.ts` - 音视频流管理,已包含真实的事件监听
 - `src/client/admin/components/agora-stt/manager/rtm/rtm.ts` - 实时消息传递

+ 3 - 1
src/client/admin/components/agora-stt/AgoraSTTProvider.tsx

@@ -31,7 +31,8 @@ export const AgoraSTTProvider: React.FC<AgoraSTTProviderProps> = ({ children })
   const rtcManagerRef = useRef<RtcManager>(new RtcManager())
   const rtmManagerRef = useRef<RtmManager>(new RtmManager())
   const sttManagerRef = useRef<SttManager>(new SttManager({
-    rtmManager: rtmManagerRef.current
+    rtmManager: rtmManagerRef.current,
+    rtcManager: rtcManagerRef.current
   }))
 
   const [transcriptionResults, setTranscriptionResults] = useState<ITextstream[]>([])
@@ -39,6 +40,7 @@ export const AgoraSTTProvider: React.FC<AgoraSTTProviderProps> = ({ children })
   // 监听转录结果事件
   useEffect(() => {
     const handleTextStreamReceived = (textstream: ITextstream) => {
+      console.debug('textstream', textstream)
       setTranscriptionResults(prev => [...prev, textstream])
     }
 

+ 21 - 6
src/client/admin/components/agora-stt/manager/rtc/rtc.ts

@@ -41,23 +41,37 @@ export class RtcManager extends AGEventEmitter<RtcEvents> {
   }
 
   async createTracks() {
-    const tracks = await AgoraRTC.createMicrophoneAndCameraTracks({
+    // 对于STT功能,只需要音频轨道
+    const audioTrack = await AgoraRTC.createMicrophoneAudioTrack({
       AGC: false,
     })
-    this.localTracks.audioTrack = tracks[0]
-    this.localTracks.videoTrack = tracks[1]
+    this.localTracks.audioTrack = audioTrack
+    this.localTracks.videoTrack = undefined // 不需要视频轨道
     this.emit("localUserChanged", this.localTracks)
   }
 
   async publish() {
-    if (this.localTracks.videoTrack && this.localTracks.audioTrack) {
-      await this.client.publish([this.localTracks.videoTrack, this.localTracks.audioTrack])
+    if (this.localTracks.audioTrack) {
+      await this.client.publish([this.localTracks.audioTrack])
     } else {
-      const msg = "videoTrack or audioTrack is undefined"
+      const msg = "audioTrack is undefined"
       throw new Error(msg)
     }
   }
 
+  async unpublish() {
+    if (this.localTracks.audioTrack) {
+      await this.client.unpublish([this.localTracks.audioTrack])
+      this.localTracks.audioTrack.close()
+      this.localTracks.audioTrack = undefined
+    }
+    if (this.localTracks.videoTrack) {
+      await this.client.unpublish([this.localTracks.videoTrack])
+      this.localTracks.videoTrack.close()
+      this.localTracks.videoTrack = undefined
+    }
+  }
+
   async destroy() {
     this.localTracks?.audioTrack?.close()
     this.localTracks?.videoTrack?.close()
@@ -94,6 +108,7 @@ export class RtcManager extends AGEventEmitter<RtcEvents> {
       })
     })
     this.client.on("stream-message", (_uid: UID, stream: any) => {
+      console.log('stream', stream)
       parser.praseData(stream)
     })
   }

+ 1 - 1
src/client/admin/components/agora-stt/manager/rtc/types.ts

@@ -18,6 +18,6 @@ export interface RtcEvents {
 }
 
 export interface IUserTracks {
-  videoTrack?: ICameraVideoTrack
+  videoTrack?: ICameraVideoTrack | undefined
   audioTrack?: IMicrophoneAudioTrack
 }

+ 23 - 1
src/client/admin/components/agora-stt/manager/stt/stt.ts

@@ -9,6 +9,7 @@ import {
 import { AGEventEmitter } from "../events"
 import { STTEvents, STTManagerStartOptions, STTManagerOptions, STTManagerInitData } from "./types"
 import { RtmManager } from "../rtm"
+import { RtcManager } from "../rtc"
 // import { IRequestLanguages } from "../../types"
 
 export class SttManager extends AGEventEmitter<STTEvents> {
@@ -16,6 +17,7 @@ export class SttManager extends AGEventEmitter<STTEvents> {
   userId: string | number = ""
   channel: string = ""
   rtmManager: RtmManager
+  rtcManager: RtcManager
   private _init: boolean = false
 
   get hasInit() {
@@ -24,8 +26,9 @@ export class SttManager extends AGEventEmitter<STTEvents> {
 
   constructor(data: STTManagerInitData) {
     super()
-    const { rtmManager } = data
+    const { rtmManager, rtcManager } = data
     this.rtmManager = rtmManager
+    this.rtcManager = rtcManager
   }
 
   setOption(option: STTManagerOptions) {
@@ -47,11 +50,20 @@ export class SttManager extends AGEventEmitter<STTEvents> {
   }) {
     this.userId = userId
     this.channel = channel
+
+    // 加入RTM频道(实时消息)
     await this.rtmManager.join({
       userId: userId + "",
       userName,
       channel,
     })
+
+    // 加入RTC频道(音频传输)
+    await this.rtcManager.join({
+      channel,
+      userId
+    })
+
     this._init = true
   }
 
@@ -72,6 +84,11 @@ export class SttManager extends AGEventEmitter<STTEvents> {
         uid: this.userId,
       })
       const token = data.tokenName
+
+      // 创建音频轨道并发布到RTC频道
+      await this.rtcManager.createTracks()
+      await this.rtcManager.publish()
+
       // api start
       const res = await apiSTTStartTranscription({
         uid: this.userId,
@@ -123,6 +140,10 @@ export class SttManager extends AGEventEmitter<STTEvents> {
         uid: this.userId,
         channel: this.channel,
       })
+
+      // 停止音频轨道
+      await this.rtcManager.unpublish()
+
       // set rtm metadata
       await this.rtmManager.updateSttData({
         status: "end",
@@ -190,6 +211,7 @@ export class SttManager extends AGEventEmitter<STTEvents> {
 
   async destroy() {
     await this.rtmManager.destroy()
+    await this.rtcManager.destroy()
     this.option = undefined
     this.userId = ""
     this.channel = ""

+ 2 - 0
src/client/admin/components/agora-stt/manager/stt/types.ts

@@ -1,4 +1,5 @@
 import { RtmManager } from "../rtm"
+import { RtcManager } from "../rtc"
 import { IRequestLanguages } from "../../types"
 
 export interface STTEvents {}
@@ -14,4 +15,5 @@ export interface STTManagerOptions {
 
 export interface STTManagerInitData {
   rtmManager: RtmManager
+  rtcManager: RtcManager
 }