modify render and ncnn asr to ask

asr/sherpa_ncnn_asr.py

@@ -63,15 +63,6 @@ class SherpaNcnnAsr(AsrBase):
         logger.info(f'_recognize_loop')
         print(f'_recognize_loop')
 
-        while self._stop_event.is_set():
-            logger.info(f'_recognize_loop000')
-            self._notify_complete('介绍中国5000年历史文学')
-            logger.info(f'_recognize_loop111')
-            segment_id += 1
-            time.sleep(150)
-            logger.info(f'_recognize_loop222')
-        logger.info(f'_recognize_loop exit')
-'''
         with sd.InputStream(channels=1, dtype="float32", samplerate=self._sample_rate) as s:
             while self._stop_event.is_set():
                 samples, _ = s.read(self._samples_per_read)  # a blocking read
@@ -94,3 +85,13 @@ class SherpaNcnnAsr(AsrBase):
                     self._recognizer.reset()
 '''
 
+        while self._stop_event.is_set():
+            logger.info(f'_recognize_loop000')
+            self._notify_complete('介绍中国5000年历史文学')
+            logger.info(f'_recognize_loop111')
+            segment_id += 1
+            time.sleep(150)
+            logger.info(f'_recognize_loop222')
+        logger.info(f'_recognize_loop exit')
+'''
+

human/human_render.py

@@ -48,7 +48,7 @@ class HumanRender(AudioHandler):
         logging.info('human render run')
         while self._exit_event.is_set() and self._is_running:
             self._run_step()
-            delay = 0.03
+            delay = 0.04
             time.sleep(delay)
 
         logging.info('human render exit')

render/video_render.py

@@ -9,23 +9,6 @@ import numpy as np
 from .base_render import BaseRender
 
 
-def img_warp_back_inv_m(img, img_to, inv_m):
-    h_up, w_up, c = img_to.shape
-    t = time.perf_counter()
-    mask = np.ones_like(img).astype(np.float32)
-    inv_mask = cv2.warpAffine(mask, inv_m, (w_up, h_up))
-    inv_img = cv2.warpAffine(img, inv_m, (w_up, h_up))
-    mask_indices = inv_mask == 1
-    print(f'time1: {time.perf_counter() - t}')
-    if 4 == c:
-        t = time.perf_counter()
-        img_to[:, :, :3][mask_indices] = inv_img[mask_indices]
-        print(f'time2: {time.perf_counter() - t}')
-    else:
-        img_to[inv_mask == 1] = inv_img[inv_mask == 1]
-    return img_to
-
-
 class VideoRender(BaseRender):
     def __init__(self, play_clock, context, human_render):
         super().__init__(play_clock, context, 'Video')
@@ -33,27 +16,8 @@ class VideoRender(BaseRender):
         self.index = 0
 
     def render(self, frame, ps):
-        res_frame, idx, type_ = frame
-
-        if type_ == 0:
-            combine_frame = self._context.frame_list_cycle[idx]
-        else:
-            bbox = self._context.coord_list_cycle[idx]
-            combine_frame = copy.deepcopy(self._context.frame_list_cycle[idx])
-            af = self._context.align_frames[idx]
-            inv_m = self._context.inv_m_frames[idx]
-            y1, y2, x1, x2 = bbox
-            try:
-                t = time.perf_counter()
-                res_frame = cv2.resize(res_frame.astype(np.uint8), (x2 - x1, y2 - y1))
-                af[y1:y2, x1:x2] = res_frame
-                combine_frame = img_warp_back_inv_m(af, combine_frame, inv_m)
-                print(time.perf_counter() - t)
-            except Exception as e:
-                logging.error(f'resize error:{e}')
-                return
-
-        image = combine_frame
-        # image = cv2.cvtColor(image, cv2.COLOR_BGRA2RGBA)
         if self._human_render is not None:
-            self._human_render.put_image(image)
+            self._human_render.put_image(frame)
+
+        # image = cv2.cvtColor(image, cv2.COLOR_BGRA2RGBA)
+

ui/pygame_ui.py

@@ -1,6 +1,8 @@
 #encoding = utf8
+import copy
 import logging
 import os
+import time
 from queue import Queue
 
 import cv2
@@ -15,17 +17,54 @@ logger = logging.getLogger(__name__)
 current_file_path = os.path.dirname(os.path.abspath(__file__))
 
 
+def img_warp_back_inv_m(img, img_to, inv_m):
+    h_up, w_up, c = img_to.shape
+    mask = np.ones_like(img).astype(np.float32)
+    inv_mask = cv2.warpAffine(mask, inv_m, (w_up, h_up))
+    inv_img = cv2.warpAffine(img, inv_m, (w_up, h_up))
+    mask_indices = inv_mask == 1
+    if 4 == c:
+        img_to[:, :, :3][mask_indices] = inv_img[mask_indices]
+    else:
+        img_to[inv_mask == 1] = inv_img[inv_mask == 1]
+    return img_to
+
+
+def render_image(context, frame):
+    res_frame, idx, type_ = frame
+
+    if type_ == 0:
+        combine_frame = context.frame_list_cycle[idx]
+    else:
+        bbox = context.coord_list_cycle[idx]
+        combine_frame = copy.deepcopy(context.frame_list_cycle[idx])
+        af = context.align_frames[idx]
+        inv_m = context.inv_m_frames[idx]
+        y1, y2, x1, x2 = bbox
+        try:
+            t = time.perf_counter()
+            res_frame = cv2.resize(res_frame.astype(np.uint8), (x2 - x1, y2 - y1))
+            af[y1:y2, x1:x2] = res_frame
+            combine_frame = img_warp_back_inv_m(af, combine_frame, inv_m)
+        except Exception as e:
+            logging.error(f'resize error:{e}')
+            return
+
+    image = combine_frame
+    return image
+
+
 class PyGameUI:
     def __init__(self):
         self._human_context = None
         self._queue = None
-        self.screen_ = pygame.display.set_mode((800, 600), HWSURFACE | DOUBLEBUF | RESIZABLE)
+        self.screen_ = pygame.display.set_mode((1920, 1080), HWSURFACE | DOUBLEBUF | RESIZABLE)
         self.clock = pygame.time.Clock()
 
         background = os.path.join(current_file_path, '..', 'data', 'background', 'background.jpg')
         logger.info(f'background: {background}')
         self._background = pygame.image.load(background).convert()
-        self.background_display_ = pygame.transform.scale(self._background, (800, 600))
+        self.background_display_ = pygame.transform.scale(self._background, (1920, 1080))
         self._human_image = None
         self.running = True
 
@@ -48,7 +87,7 @@ class PyGameUI:
             self.screen_.blit(self.background_display_, (0, 0))
             self._update_human()
             if self._human_image is not None:
-                self.screen_.blit(self._human_image, (0, -200))
+                self.screen_.blit(self._human_image, (760, -300))
 
             fps = self.clock.get_fps()
             pygame.display.set_caption('fps:{:.2f}'.format(fps))
@@ -59,7 +98,12 @@ class PyGameUI:
     def _update_human(self):
         if self._queue.empty():
             return
+
+        if self._queue.qsize() > 5:
+            print('render queue is slower')
+
         image = self._queue.get()
+        image = render_image(self._human_context, image)
         color_format = "RGB"
         if 4 == image.shape[2]:
             color_format = "RGBA"