diff --git a/asr/sherpa_ncnn_asr.py b/asr/sherpa_ncnn_asr.py index 1cc81d3..4124e76 100644 --- a/asr/sherpa_ncnn_asr.py +++ b/asr/sherpa_ncnn_asr.py @@ -63,15 +63,6 @@ class SherpaNcnnAsr(AsrBase): logger.info(f'_recognize_loop') print(f'_recognize_loop') - while self._stop_event.is_set(): - logger.info(f'_recognize_loop000') - self._notify_complete('介绍中国5000年历史文学') - logger.info(f'_recognize_loop111') - segment_id += 1 - time.sleep(150) - logger.info(f'_recognize_loop222') - logger.info(f'_recognize_loop exit') -''' with sd.InputStream(channels=1, dtype="float32", samplerate=self._sample_rate) as s: while self._stop_event.is_set(): samples, _ = s.read(self._samples_per_read) # a blocking read @@ -94,3 +85,13 @@ class SherpaNcnnAsr(AsrBase): self._recognizer.reset() ''' + while self._stop_event.is_set(): + logger.info(f'_recognize_loop000') + self._notify_complete('介绍中国5000年历史文学') + logger.info(f'_recognize_loop111') + segment_id += 1 + time.sleep(150) + logger.info(f'_recognize_loop222') + logger.info(f'_recognize_loop exit') +''' + diff --git a/human/human_render.py b/human/human_render.py index cc8061e..186ddbc 100644 --- a/human/human_render.py +++ b/human/human_render.py @@ -48,7 +48,7 @@ class HumanRender(AudioHandler): logging.info('human render run') while self._exit_event.is_set() and self._is_running: self._run_step() - delay = 0.03 + delay = 0.04 time.sleep(delay) logging.info('human render exit') diff --git a/render/video_render.py b/render/video_render.py index 605859f..91426bd 100644 --- a/render/video_render.py +++ b/render/video_render.py @@ -9,23 +9,6 @@ import numpy as np from .base_render import BaseRender -def img_warp_back_inv_m(img, img_to, inv_m): - h_up, w_up, c = img_to.shape - t = time.perf_counter() - mask = np.ones_like(img).astype(np.float32) - inv_mask = cv2.warpAffine(mask, inv_m, (w_up, h_up)) - inv_img = cv2.warpAffine(img, inv_m, (w_up, h_up)) - mask_indices = inv_mask == 1 - print(f'time1: {time.perf_counter() - t}') - if 4 == c: - t = time.perf_counter() - img_to[:, :, :3][mask_indices] = inv_img[mask_indices] - print(f'time2: {time.perf_counter() - t}') - else: - img_to[inv_mask == 1] = inv_img[inv_mask == 1] - return img_to - - class VideoRender(BaseRender): def __init__(self, play_clock, context, human_render): super().__init__(play_clock, context, 'Video') @@ -33,27 +16,8 @@ class VideoRender(BaseRender): self.index = 0 def render(self, frame, ps): - res_frame, idx, type_ = frame - - if type_ == 0: - combine_frame = self._context.frame_list_cycle[idx] - else: - bbox = self._context.coord_list_cycle[idx] - combine_frame = copy.deepcopy(self._context.frame_list_cycle[idx]) - af = self._context.align_frames[idx] - inv_m = self._context.inv_m_frames[idx] - y1, y2, x1, x2 = bbox - try: - t = time.perf_counter() - res_frame = cv2.resize(res_frame.astype(np.uint8), (x2 - x1, y2 - y1)) - af[y1:y2, x1:x2] = res_frame - combine_frame = img_warp_back_inv_m(af, combine_frame, inv_m) - print(time.perf_counter() - t) - except Exception as e: - logging.error(f'resize error:{e}') - return - - image = combine_frame - # image = cv2.cvtColor(image, cv2.COLOR_BGRA2RGBA) if self._human_render is not None: - self._human_render.put_image(image) + self._human_render.put_image(frame) + + # image = cv2.cvtColor(image, cv2.COLOR_BGRA2RGBA) + diff --git a/ui/pygame_ui.py b/ui/pygame_ui.py index f3890fe..134b580 100644 --- a/ui/pygame_ui.py +++ b/ui/pygame_ui.py @@ -1,6 +1,8 @@ #encoding = utf8 +import copy import logging import os +import time from queue import Queue import cv2 @@ -15,17 +17,54 @@ logger = logging.getLogger(__name__) current_file_path = os.path.dirname(os.path.abspath(__file__)) +def img_warp_back_inv_m(img, img_to, inv_m): + h_up, w_up, c = img_to.shape + mask = np.ones_like(img).astype(np.float32) + inv_mask = cv2.warpAffine(mask, inv_m, (w_up, h_up)) + inv_img = cv2.warpAffine(img, inv_m, (w_up, h_up)) + mask_indices = inv_mask == 1 + if 4 == c: + img_to[:, :, :3][mask_indices] = inv_img[mask_indices] + else: + img_to[inv_mask == 1] = inv_img[inv_mask == 1] + return img_to + + +def render_image(context, frame): + res_frame, idx, type_ = frame + + if type_ == 0: + combine_frame = context.frame_list_cycle[idx] + else: + bbox = context.coord_list_cycle[idx] + combine_frame = copy.deepcopy(context.frame_list_cycle[idx]) + af = context.align_frames[idx] + inv_m = context.inv_m_frames[idx] + y1, y2, x1, x2 = bbox + try: + t = time.perf_counter() + res_frame = cv2.resize(res_frame.astype(np.uint8), (x2 - x1, y2 - y1)) + af[y1:y2, x1:x2] = res_frame + combine_frame = img_warp_back_inv_m(af, combine_frame, inv_m) + except Exception as e: + logging.error(f'resize error:{e}') + return + + image = combine_frame + return image + + class PyGameUI: def __init__(self): self._human_context = None self._queue = None - self.screen_ = pygame.display.set_mode((800, 600), HWSURFACE | DOUBLEBUF | RESIZABLE) + self.screen_ = pygame.display.set_mode((1920, 1080), HWSURFACE | DOUBLEBUF | RESIZABLE) self.clock = pygame.time.Clock() background = os.path.join(current_file_path, '..', 'data', 'background', 'background.jpg') logger.info(f'background: {background}') self._background = pygame.image.load(background).convert() - self.background_display_ = pygame.transform.scale(self._background, (800, 600)) + self.background_display_ = pygame.transform.scale(self._background, (1920, 1080)) self._human_image = None self.running = True @@ -48,7 +87,7 @@ class PyGameUI: self.screen_.blit(self.background_display_, (0, 0)) self._update_human() if self._human_image is not None: - self.screen_.blit(self._human_image, (0, -200)) + self.screen_.blit(self._human_image, (760, -300)) fps = self.clock.get_fps() pygame.display.set_caption('fps:{:.2f}'.format(fps)) @@ -59,7 +98,12 @@ class PyGameUI: def _update_human(self): if self._queue.empty(): return + + if self._queue.qsize() > 5: + print('render queue is slower') + image = self._queue.get() + image = render_image(self._human_context, image) color_format = "RGB" if 4 == image.shape[2]: color_format = "RGBA"