brige/human
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 2 Wiki Activity

modify code

Browse Source
This commit is contained in:
brige 2024-11-05 19:40:03 +08:00
parent a92789c3b5
commit 4998d0b422
7 changed files with 22 additions and 159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
human/human_render.py
View File

@ -33,7 +33,7 @@ class HumanRender(AudioHandler):
logging.info('human render run')
while self._exit_event.is_set():
self._run_step()
time.sleep(0.037)
time.sleep(0.038)
logging.info('human render exit')
@ -82,12 +82,13 @@ class HumanRender(AudioHandler):
# if self._voice_render.is_full():
# self._context.notify({'msg_id': MessageType.Video_Render_Queue_Full})
def get_audio_queue_size(self):
return self._voice_render.size()
# def get_audio_queue_size(self):
# return self._voice_render.size()
def pause_talk(self):
self._voice_render.pause_talk()
self._video_render.pause_talk()
pass
# self._voice_render.pause_talk()
# self._video_render.pause_talk()
def stop(self):
logging.info('hunan render stop')

36
render/base_render.py
View File

@ -11,11 +11,9 @@ logger = logging.getLogger(__name__)
class BaseRender(ABC):
def __init__(self, play_clock, context, type_, delay=0.02, thread_name="BaseRenderThread"):
def __init__(self, play_clock, context, type_):
self._play_clock = play_clock
self._context = context
self._type = type_
self._delay = delay
# self._queue = SyncQueue(context.batch_size, f'{type_}RenderQueue')
# self._exit_event = Event()
# self._thread = Thread(target=self._on_run, name=thread_name)
@ -25,35 +23,3 @@ class BaseRender(ABC):
@abstractmethod
def render(self, frame, ps):
pass
def _on_run(self):
logging.info(f'{self._type} render run')
# while self._exit_event.is_set():
# self._run_step()
# time.sleep(self._delay)
logging.info(f'{self._type} render exit')
def put(self, frame, ps):
pass
# self._queue.put((frame, ps))
def size(self):
pass
# return self._queue.size()
def pause_talk(self):
pass
# self._queue.clear()
def stop(self):
pass
# self._queue.clear()
# self._exit_event.clear()
# self._thread.join()
@abstractmethod
def _run_step(self):
pass

57
render/video_render.py
View File

@ -13,7 +13,7 @@ from human.message_type import MessageType
class VideoRender(BaseRender):
def __init__(self, play_clock, context, human_render):
super().__init__(play_clock, context, 'Video', 0.038, "VideoRenderThread")
super().__init__(play_clock, context, 'Video')
self._human_render = human_render
self._diff_avg_count = 0
@ -21,7 +21,6 @@ class VideoRender(BaseRender):
res_frame, idx, type_ = frame
clock_time = self._play_clock.clock_time()
time_difference = clock_time - ps
print("Video frame time", clock_time, ps, time_difference)
if abs(time_difference) > self._play_clock.audio_diff_threshold:
if self._diff_avg_count < 5:
self._diff_avg_count += 1
@ -56,57 +55,3 @@ class VideoRender(BaseRender):
# image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
if self._human_render is not None:
self._human_render.put_image(image)
def _run_step(self):
while self._exit_event.is_set():
try:
value = self._queue.get()
if value is None:
return
frame, ps = value
res_frame, idx, type_ = frame
except Empty:
return
clock_time = self._play_clock.clock_time()
time_difference = clock_time - ps
print("Video frame time", clock_time, ps, time_difference)
if abs(time_difference) > self._play_clock.audio_diff_threshold:
if self._diff_avg_count < 5:
self._diff_avg_count += 1
else:
if time_difference < -self._play_clock.audio_diff_threshold:
sleep_time = abs(time_difference)
print("Video frame waiting to catch up with audio", sleep_time)
if sleep_time <= 1.0:
time.sleep(sleep_time)
# elif time_difference > self._play_clock.audio_diff_threshold: # 视频比音频快超过10ms
# print("Video frame dropped to catch up with audio")
# continue
else:
self._diff_avg_count = 0
print('video render:',
'get face', self._queue.size(),
'audio queue', self._human_render.get_audio_queue_size())
if type_ == 0:
combine_frame = self._context.frame_list_cycle[idx]
else:
bbox = self._context.coord_list_cycle[idx]
combine_frame = copy.deepcopy(self._context.frame_list_cycle[idx])
y1, y2, x1, x2 = bbox
try:
res_frame = cv2.resize(res_frame.astype(np.uint8), (x2 - x1, y2 - y1))
except:
print('resize error')
return
combine_frame[y1:y2, x1:x2, :3] = res_frame
image = combine_frame
# image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
if self._human_render is not None:
self._human_render.put_image(image)
return

46
render/voice_render.py
View File

@ -15,11 +15,7 @@ logger = logging.getLogger(__name__)
class VoiceRender(BaseRender):
def __init__(self, play_clock, context):
self._audio_render = AudioRender()
self._is_empty = True
super().__init__(play_clock, context, 'Voice', 0.03, "VoiceRenderThread")
def is_full(self):
return self._queue.size() >= self._context.render_batch * 2
super().__init__(play_clock, context, 'Voice')
def render(self, frame, ps):
self._play_clock.update_display_time()
@ -36,43 +32,3 @@ class VoiceRender(BaseRender):
self._audio_render.write(frame.tobytes(), chunk_len)
except Exception as e:
logging.error(f'Error writing audio frame: {e}')
def _run_step(self):
try:
value = self._queue.get()
if value is None:
return
audio_frames, ps = value
# print('voice render queue size', self._queue.size())
except Empty:
self._context.notify({'msg_id': MessageType.Video_Render_Queue_Empty})
if not self._is_empty:
print('voice render queue empty')
self._is_empty = True
return
if self._is_empty:
print('voice render queue not empty')
self._is_empty = False
status = MessageType.Video_Render_Queue_Not_Empty
if self._queue.size() < self._context.render_batch:
status = MessageType.Video_Render_Queue_Empty
elif self._queue.size() >= self._context.render_batch * 2:
status = MessageType.Video_Render_Queue_Full
self._context.notify({'msg_id': status})
self._play_clock.update_display_time()
self._play_clock.current_time = ps
for audio_frame in audio_frames:
frame, type_ = audio_frame
frame = (frame * 32767).astype(np.int16)
if self._audio_render is not None:
try:
chunk_len = int(frame.shape[0] * 2)
# print('audio frame:', frame.shape, chunk_len)
self._audio_render.write(frame.tobytes(), chunk_len)
except Exception as e:
logging.error(f'Error writing audio frame: {e}')

2
ui/pygame_ui.py
View File

@ -44,9 +44,9 @@ class PyGameUI:
elif event.type == VIDEORESIZE:
self.background_display_ = pygame.transform.scale(self._background, event.dict['size'])
self.screen_.blit(self.background_display_, (0, 0))
self._update_human()
if self._human_image is not None:
self.screen_.blit(self._human_image, (0, 0))
self._update_human()
pygame.display.flip()
self.stop()
pygame.quit()

19
utils/async_task_queue.py
View File

@ -9,7 +9,6 @@ class AsyncTaskQueue:
self._queue = asyncio.Queue()
self._worker_num = work_num
self._current_worker_num = work_num
self._condition = threading.Condition()
self._thread = threading.Thread(target=self._run_loop)
self._thread.start()
self.__loop = None
@ -27,35 +26,31 @@ class AsyncTaskQueue:
async def _worker(self):
print('_worker')
while True:
with self._condition:
self._condition.wait_for(lambda: not self._queue.empty())
task = await self._queue.get()
func, *args = task # 解包任务
if func is None: # None 作为结束信号
if task is None: # None as a stop signal
break
func, *args = task # Unpack task
print(f"Executing task with args: {args}")
await func(*args) # 执行异步函数
await func(*args) # Execute async function
self._queue.task_done()
print('_worker finish')
self._current_worker_num = self._current_worker_num - 1
self._current_worker_num -= 1
if self._current_worker_num == 0:
print('loop stop')
self.__loop.stop()
def add_task(self, func, *args):
with self._condition:
self._queue.put_nowait((func, *args))
self._condition.notify()
self.__loop.call_soon_threadsafe(self._queue.put_nowait, (func, *args))
def stop_workers(self):
for _ in range(self._worker_num):
self.add_task(None) # 发送结束信号
self.add_task(None) # Send stop signal
def clear(self):
while not self._queue.empty():
self._queue.get()
self._queue.get_nowait()
self._queue.task_done()
def stop(self):

2
utils/utils.py
View File

@ -178,7 +178,7 @@ def load_avatar(path, img_size, device):
face_frames = []
coord_frames = []
for face, coord in face_det_results:
resized_crop_frame = cv2.resize(face, (img_size, img_size))
resized_crop_frame = cv2.resize(face[:, :, :3], (img_size, img_size))
face_frames.append(resized_crop_frame)
coord_frames.append(coord)