brige/human
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 2 Wiki Activity

modify render

Browse Source
This commit is contained in:
brige 2024-10-27 13:51:22 +08:00
parent 53edda7ebe
commit 3f833a3c42
6 changed files with 31 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
asr/sherpa_ncnn_asr.py
View File

@ -57,12 +57,13 @@ class SherpaNcnnAsr(AsrBase):
def _recognize_loop(self): def _recognize_loop(self):
segment_id = 0 segment_id = 0
time.sleep(3)
last_result = "" last_result = ""
logger.info(f'_recognize_loop') logger.info(f'_recognize_loop')
while not self._stop_event.is_set(): while not self._stop_event.is_set():
self._notify_complete('中国人民万岁') self._notify_complete('中国人民万岁')
segment_id += 1 segment_id += 1
time.sleep(10) time.sleep(60)
# #
# with sd.InputStream(channels=1, dtype="float32", samplerate=self._sample_rate) as s: # with sd.InputStream(channels=1, dtype="float32", samplerate=self._sample_rate) as s:
# while not self._stop_event.is_set(): # while not self._stop_event.is_set():

1
human/audio_inference_handler.py
View File

@ -36,7 +36,6 @@ class AudioInferenceHandler(AudioHandler):
self._audio_queue.put(stream) self._audio_queue.put(stream)
def on_message(self, message): def on_message(self, message):
print('human render notify:', message)
super().on_message(message) super().on_message(message)
def __on_run(self): def __on_run(self):

1
human/human_render.py
View File

@ -34,7 +34,6 @@ class HumanRender(AudioHandler):
self._image_render.on_render(image) self._image_render.on_render(image)
def on_message(self, message): def on_message(self, message):
print('human render notify:', message)
super().on_message(message) super().on_message(message)
def on_handle(self, stream, index): def on_handle(self, stream, index):

27
render/video_render.py
View File

@ -15,30 +15,37 @@ class VideoRender(BaseRender):
def __init__(self, play_clock, context, human_render): def __init__(self, play_clock, context, human_render):
super().__init__(play_clock, context, 'Video') super().__init__(play_clock, context, 'Video')
self._human_render = human_render self._human_render = human_render
self._diff_avg_count = 0
def _run_step(self): def _run_step(self):
while self._exit_event.is_set(): while self._exit_event.is_set():
try: try:
frame, ps = self._queue.get(block=True, timeout=0.01) frame, ps = self._queue.get(block=True, timeout=0.02)
res_frame, idx, type_ = frame res_frame, idx, type_ = frame
except Empty: except Empty:
return return
clock_time = self._play_clock.clock_time() clock_time = self._play_clock.clock_time()
time_difference = clock_time - ps time_difference = clock_time - ps
if abs(time_difference) > self._play_clock.audio_diff_threshold:
print('video render:', ps, ' ', clock_time, ' ', time_difference) if self._diff_avg_count < 10:
self._diff_avg_count += 1
else:
if time_difference < -self._play_clock.audio_diff_threshold: if time_difference < -self._play_clock.audio_diff_threshold:
sleep_time = abs(time_difference + self._play_clock.audio_diff_threshold) sleep_time = abs(time_difference )
print("Video frame waiting to catch up with audio", sleep_time) # print("Video frame waiting to catch up with audio", sleep_time)
if sleep_time > 0: if sleep_time <= 1.0:
time.sleep(sleep_time) time.sleep(sleep_time)
elif time_difference > self._play_clock.audio_diff_threshold: # 视频比音频快超过10ms # elif time_difference > self._play_clock.audio_diff_threshold: # 视频比音频快超过10ms
print("Video frame dropped to catch up with audio") # print("Video frame dropped to catch up with audio")
continue # continue
print('get face', self._queue.qsize()) else:
self._diff_avg_count = 0
print('video render:', ps, ' ', clock_time, ' ', time_difference,
'get face', self._queue.qsize(), self._diff_avg_count)
if type_ == 0: if type_ == 0:
combine_frame = self._context.frame_list_cycle[idx] combine_frame = self._context.frame_list_cycle[idx]

12
render/voice_render.py
View File

@ -14,8 +14,9 @@ logger = logging.getLogger(__name__)
class VoiceRender(BaseRender): class VoiceRender(BaseRender):
def __init__(self, play_clock, context): def __init__(self, play_clock, context):
super().__init__(play_clock, context, 'Voice')
self._audio_render = AudioRender() self._audio_render = AudioRender()
self._is_empty = True
super().__init__(play_clock, context, 'Voice')
def is_full(self): def is_full(self):
return self._queue.qsize() >= self._context.render_batch * 2 return self._queue.qsize() >= self._context.render_batch * 2
@ -23,11 +24,18 @@ class VoiceRender(BaseRender):
def _run_step(self): def _run_step(self):
try: try:
audio_frames, ps = self._queue.get(block=True, timeout=0.01) audio_frames, ps = self._queue.get(block=True, timeout=0.01)
print('voice render queue size', self._queue.qsize()) # print('voice render queue size', self._queue.qsize())
except Empty: except Empty:
self._context.notify({'msg_id': MessageType.Video_Render_Queue_Empty}) self._context.notify({'msg_id': MessageType.Video_Render_Queue_Empty})
if not self._is_empty:
print('voice render queue empty')
self._is_empty = True
return return
if self._is_empty:
print('voice render queue not empty')
self._is_empty = False
status = MessageType.Video_Render_Queue_Not_Empty status = MessageType.Video_Render_Queue_Not_Empty
if self._queue.qsize() < self._context.render_batch: if self._queue.qsize() < self._context.render_batch:
status = MessageType.Video_Render_Queue_Empty status = MessageType.Video_Render_Queue_Empty