modify audio render

human/human_render.py

@@ -11,6 +11,8 @@ import numpy as np
 
 from human_handler import AudioHandler
 
+logger = logging.getLogger(__name__)
+
 
 class HumanRender(AudioHandler):
     def __init__(self, context, handler):
@@ -29,7 +31,7 @@ class HumanRender(AudioHandler):
         logging.info('human render run')
         while self._exit_event.is_set():
             self._run_step()
-            time.sleep(0.002)
+            time.sleep(0.02)
 
         logging.info('human render exit')
 

render/__init__.py

@@ -1,2 +1,4 @@
 #encoding = utf8
 
+from .audio_render import AudioRenderImpl
+from .video_render import VideoRenderImpl

render/audio_render.py

@@ -1,8 +1,69 @@
 #encoding = utf8
-from .base_render import BaseRender
+import logging
+import time
+from queue import Queue, Empty
+from threading import Event, Thread
+
+import numpy as np
+
+from audio_render import AudioRender
+from base_render import BaseRender
+
+logger = logging.getLogger(__name__)
+
+
+class AudioRenderImpl(BaseRender):
+    def __init__(self, start):
+        super().__init__(start)
+
+        self._queue = Queue()
+        self._exit_event = Event()
+        self._thread = Thread(target=self._on_run)
+        self._exit_event.set()
+        self._thread.start()
+        self._audio_render = AudioRender()
+        self._current_time = 0
+        self._display_time = 0
+
+    def _on_run(self):
+        logging.info('Audio render run')
+        while self._exit_event.is_set():
+            self._run_step()
+            time.sleep(0.02)
+
+        logging.info('Audio render exit')
+
+    def _run_step(self):
+        try:
+            audio_frames, ps = self._queue.get(block=True, timeout=0.01)
+        except Empty:
+            return
+
+        self._display_time = time.time()
+        self._current_time = ps
+
+        for audio_frame in audio_frames:
+            frame, type_ = audio_frame
+            frame = (frame * 32767).astype(np.int16)
+
+            if self._audio_render is not None:
+                try:
+                    self._audio_render.write(frame.tobytes(), int(frame.shape[0] * 2))
+                except Exception as e:
+                    logging.error(f'Error writing audio frame: {e}')
+
+    def put(self, frame):
+        ps = time.time() - self._start
+        self._queue.put_nowait((frame, ps))
+
+    def stop(self):
+        self._exit_event.clear()
+        self._thread.join()
+
+    def play_time(self):
+        elapsed = time.time() - self._display_time
+        return self._current_time + elapsed
+
 
 
-class AudioRender(BaseRender):
-    def __init__(self):
-        super().__init__()
 

render/base_render.py

@@ -4,7 +4,15 @@ from abc import ABC, abstractmethod
 
 
 class BaseRender(ABC):
-    def __init__(self):
+    def __init__(self, start):
+        self._start = start
+
+    @abstractmethod
+    def put(self, frame):
+        pass
+
+    @abstractmethod
+    def stop(self):
         pass
 
 

render/video_render.py

@@ -2,6 +2,6 @@
 from base_render import BaseRender
 
 
-class VideoRender(BaseRender):
+class VideoRenderImpl(BaseRender):
-    def __init__(self):
+    def __init__(self, start):
-        super().__init__()
+        super().__init__(start)

test/test_render_queue.py

@@ -1,3 +1,4 @@
+'''
 import pygame
 import time
 from pydub import AudioSegment
@@ -93,3 +94,70 @@ if __name__ == "__main__":
     audio_file = "your_audio_file.mp3"  # 替换为你的音频文件
     image_files = ["image1.png", "image2.png", "image3.png"]  # 替换为你的图像文件
     sync_controller = AudioVisualSync(audio_file, image_files)
+'''
+
+import threading
+import time
+import queue
+
+
+class MediaPlayer:
+    def __init__(self, audio_queue, video_queue):
+        self.audio_queue = audio_queue
+        self.video_queue = video_queue
+        self.sync_threshold = 0.01  # 10ms 同步阈值
+        self.audio_playing = True
+        self.video_playing = True
+
+    def play_audio(self):
+        while self.audio_playing:
+            if not self.audio_queue.empty():
+                audio_frame = self.audio_queue.get()  # 获取音频帧
+                audio_timestamp = audio_frame['timestamp']  # 获取音频时间戳
+                print(f"Playing audio frame with timestamp: {audio_timestamp}")
+                time.sleep(0.02)  # 假设每帧播放时间20ms
+
+    def play_video(self):
+        while self.video_playing:
+            if not self.video_queue.empty():
+                video_frame = self.video_queue.queue[0]  # 获取当前视频帧但不出队
+                video_timestamp = video_frame['timestamp']
+
+                if not self.audio_queue.empty():
+                    audio_frame = self.audio_queue.queue[0]  # 获取音频队列中的第一个音频帧
+                    audio_timestamp = audio_frame['timestamp']
+
+                    # 视频快了，等待
+                    if video_timestamp - audio_timestamp > self.sync_threshold:
+                        time.sleep(0.01)
+
+                    # 视频慢了，丢弃帧
+                    elif audio_timestamp - video_timestamp > self.sync_threshold:
+                        print(f"Dropping video frame with timestamp: {video_timestamp}")
+                        self.video_queue.get()  # 丢弃当前帧
+                    else:
+                        self.video_queue.get()  # 播放当前帧
+                        print(f"Playing video frame with timestamp: {video_timestamp}")
+                        time.sleep(0.02)  # 假设每帧播放时间20ms
+
+    def start(self):
+        audio_thread = threading.Thread(target=self.play_audio)
+        video_thread = threading.Thread(target=self.play_video)
+        audio_thread.start()
+        video_thread.start()
+        audio_thread.join()
+        video_thread.join()
+
+
+if __name__ == "__main__":
+    # 初始化音频和视频队列，每个帧包含时间戳
+    audio_queue = queue.Queue()
+    video_queue = queue.Queue()
+
+    # 填充一些模拟数据
+    for i in range(100):
+        audio_queue.put({'timestamp': i * 0.02})
+        video_queue.put({'timestamp': i * 0.02})
+
+    player = MediaPlayer(audio_queue, video_queue)
+    player.start()