From b55bf34a74b59d0e70a127dc8aeb23cc17706be7 Mon Sep 17 00:00:00 2001 From: brige Date: Mon, 21 Oct 2024 19:55:04 +0800 Subject: [PATCH] modify audio render --- human/human_render.py | 4 ++- render/__init__.py | 2 ++ render/audio_render.py | 69 ++++++++++++++++++++++++++++++++++++--- render/base_render.py | 10 +++++- render/video_render.py | 6 ++-- test/test_render_queue.py | 68 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 150 insertions(+), 9 deletions(-) diff --git a/human/human_render.py b/human/human_render.py index 9ff5f40..9f0dd9f 100644 --- a/human/human_render.py +++ b/human/human_render.py @@ -11,6 +11,8 @@ import numpy as np from human_handler import AudioHandler +logger = logging.getLogger(__name__) + class HumanRender(AudioHandler): def __init__(self, context, handler): @@ -29,7 +31,7 @@ class HumanRender(AudioHandler): logging.info('human render run') while self._exit_event.is_set(): self._run_step() - time.sleep(0.002) + time.sleep(0.02) logging.info('human render exit') diff --git a/render/__init__.py b/render/__init__.py index 6bb50fc..ef7d113 100644 --- a/render/__init__.py +++ b/render/__init__.py @@ -1,2 +1,4 @@ #encoding = utf8 +from .audio_render import AudioRenderImpl +from .video_render import VideoRenderImpl diff --git a/render/audio_render.py b/render/audio_render.py index 2fb5f3b..6ac145b 100644 --- a/render/audio_render.py +++ b/render/audio_render.py @@ -1,8 +1,69 @@ #encoding = utf8 -from .base_render import BaseRender +import logging +import time +from queue import Queue, Empty +from threading import Event, Thread + +import numpy as np + +from audio_render import AudioRender +from base_render import BaseRender + +logger = logging.getLogger(__name__) + + +class AudioRenderImpl(BaseRender): + def __init__(self, start): + super().__init__(start) + + self._queue = Queue() + self._exit_event = Event() + self._thread = Thread(target=self._on_run) + self._exit_event.set() + self._thread.start() + self._audio_render = AudioRender() + self._current_time = 0 + self._display_time = 0 + + def _on_run(self): + logging.info('Audio render run') + while self._exit_event.is_set(): + self._run_step() + time.sleep(0.02) + + logging.info('Audio render exit') + + def _run_step(self): + try: + audio_frames, ps = self._queue.get(block=True, timeout=0.01) + except Empty: + return + + self._display_time = time.time() + self._current_time = ps + + for audio_frame in audio_frames: + frame, type_ = audio_frame + frame = (frame * 32767).astype(np.int16) + + if self._audio_render is not None: + try: + self._audio_render.write(frame.tobytes(), int(frame.shape[0] * 2)) + except Exception as e: + logging.error(f'Error writing audio frame: {e}') + + def put(self, frame): + ps = time.time() - self._start + self._queue.put_nowait((frame, ps)) + + def stop(self): + self._exit_event.clear() + self._thread.join() + + def play_time(self): + elapsed = time.time() - self._display_time + return self._current_time + elapsed + -class AudioRender(BaseRender): - def __init__(self): - super().__init__() diff --git a/render/base_render.py b/render/base_render.py index f1f946e..fa8f7fa 100644 --- a/render/base_render.py +++ b/render/base_render.py @@ -4,7 +4,15 @@ from abc import ABC, abstractmethod class BaseRender(ABC): - def __init__(self): + def __init__(self, start): + self._start = start + + @abstractmethod + def put(self, frame): + pass + + @abstractmethod + def stop(self): pass diff --git a/render/video_render.py b/render/video_render.py index 9c5950f..a862f60 100644 --- a/render/video_render.py +++ b/render/video_render.py @@ -2,6 +2,6 @@ from base_render import BaseRender -class VideoRender(BaseRender): - def __init__(self): - super().__init__() +class VideoRenderImpl(BaseRender): + def __init__(self, start): + super().__init__(start) diff --git a/test/test_render_queue.py b/test/test_render_queue.py index 9fdc21b..3153640 100644 --- a/test/test_render_queue.py +++ b/test/test_render_queue.py @@ -1,3 +1,4 @@ +''' import pygame import time from pydub import AudioSegment @@ -93,3 +94,70 @@ if __name__ == "__main__": audio_file = "your_audio_file.mp3" # 替换为你的音频文件 image_files = ["image1.png", "image2.png", "image3.png"] # 替换为你的图像文件 sync_controller = AudioVisualSync(audio_file, image_files) +''' + +import threading +import time +import queue + + +class MediaPlayer: + def __init__(self, audio_queue, video_queue): + self.audio_queue = audio_queue + self.video_queue = video_queue + self.sync_threshold = 0.01 # 10ms 同步阈值 + self.audio_playing = True + self.video_playing = True + + def play_audio(self): + while self.audio_playing: + if not self.audio_queue.empty(): + audio_frame = self.audio_queue.get() # 获取音频帧 + audio_timestamp = audio_frame['timestamp'] # 获取音频时间戳 + print(f"Playing audio frame with timestamp: {audio_timestamp}") + time.sleep(0.02) # 假设每帧播放时间20ms + + def play_video(self): + while self.video_playing: + if not self.video_queue.empty(): + video_frame = self.video_queue.queue[0] # 获取当前视频帧但不出队 + video_timestamp = video_frame['timestamp'] + + if not self.audio_queue.empty(): + audio_frame = self.audio_queue.queue[0] # 获取音频队列中的第一个音频帧 + audio_timestamp = audio_frame['timestamp'] + + # 视频快了，等待 + if video_timestamp - audio_timestamp > self.sync_threshold: + time.sleep(0.01) + + # 视频慢了，丢弃帧 + elif audio_timestamp - video_timestamp > self.sync_threshold: + print(f"Dropping video frame with timestamp: {video_timestamp}") + self.video_queue.get() # 丢弃当前帧 + else: + self.video_queue.get() # 播放当前帧 + print(f"Playing video frame with timestamp: {video_timestamp}") + time.sleep(0.02) # 假设每帧播放时间20ms + + def start(self): + audio_thread = threading.Thread(target=self.play_audio) + video_thread = threading.Thread(target=self.play_video) + audio_thread.start() + video_thread.start() + audio_thread.join() + video_thread.join() + + +if __name__ == "__main__": + # 初始化音频和视频队列，每个帧包含时间戳 + audio_queue = queue.Queue() + video_queue = queue.Queue() + + # 填充一些模拟数据 + for i in range(100): + audio_queue.put({'timestamp': i * 0.02}) + video_queue.put({'timestamp': i * 0.02}) + + player = MediaPlayer(audio_queue, video_queue) + player.start()