use pygame to render

asr/asr_base.py

@@ -13,6 +13,7 @@ class AsrBase:
         self._observers = []
 
         self._stop_event = threading.Event()
+        self._stop_event.set()
         self._thread = threading.Thread(target=self._recognize_loop)
         self._thread.start()
 
@@ -28,7 +29,7 @@ class AsrBase:
             observer.completed(message)
 
     def stop(self):
-        self._stop_event.set()
+        self._stop_event.clear()
         self._thread.join()
 
     def attach(self, observer: AsrObserver):

asr/sherpa_ncnn_asr.py

@@ -60,10 +60,14 @@ class SherpaNcnnAsr(AsrBase):
         time.sleep(3)
         last_result = ""
         logger.info(f'_recognize_loop')
-        while not self._stop_event.is_set():
+        while self._stop_event.is_set():
+            logger.info(f'_recognize_loop000')
             self._notify_complete('介绍中国5000年历史文学')
+            logger.info(f'_recognize_loop111')
             segment_id += 1
-            time.sleep(60)
+            time.sleep(5)
+            logger.info(f'_recognize_loop222')
+        logger.info(f'_recognize_loop exit')
 '''
         with sd.InputStream(channels=1, dtype="float32", samplerate=self._sample_rate) as s:
             while not self._stop_event.is_set():

human/audio_inference_handler.py

@@ -133,6 +133,6 @@ class AudioInferenceHandler(AudioHandler):
         self._run_thread.join()
 
     def pause_talk(self):
-        print('AudioInferenceHandler pause_talk', self._audio_queue.size(), self._mal_queue.qsize())
+        print('AudioInferenceHandler pause_talk', self._audio_queue.size(), self._mal_queue.size())
         self._audio_queue.clear()
-        self._mal_queue.queue.clear()
+        self._mal_queue.clear()

human/audio_mal_handler.py

@@ -80,7 +80,7 @@ class AudioMalHandler(AudioHandler):
     def get_audio_frame(self):
         try:
             # print('AudioMalHandler get_audio_frame')
-            frame = self._queue.get(timeout=0.02)
+            frame = self._queue.get()
             type_ = 0
         except queue.Empty:
             frame = np.zeros(self.chunk, dtype=np.float32)

human/human_context.py

@@ -46,13 +46,6 @@ class HumanContext:
 
     def __del__(self):
         print(f'HumanContext: __del__')
-        object_stop(self._asr)
-        object_stop(self._nlp)
-        object_stop(self._tts)
-        object_stop(self._tts_handle)
-        object_stop(self._mal_handler)
-        object_stop(self._infer_handler)
-        object_stop(self._render_handler)
 
     @property
     def fps(self):
@@ -119,6 +112,15 @@ class HumanContext:
         self._asr = SherpaNcnnAsr()
         self._asr.attach(self._nlp)
 
+    def stop(self):
+        object_stop(self._asr)
+        object_stop(self._nlp)
+        object_stop(self._tts)
+        object_stop(self._tts_handle)
+        object_stop(self._mal_handler)
+        object_stop(self._infer_handler)
+        object_stop(self._render_handler)
+
     def pause_talk(self):
         self._nlp.pause_talk()
         self._tts.pause_talk()

human/human_render.py

@@ -33,7 +33,7 @@ class HumanRender(AudioHandler):
         logging.info('human render run')
         while self._exit_event.is_set():
             self._run_step()
-            time.sleep(0.035)
+            time.sleep(0.037)
 
         logging.info('human render exit')
 

nlp/nlp_base.py

@@ -43,8 +43,10 @@ class NLPBase(AsrObserver):
     def ask(self, question):
         logger.info(f'ask:{question}')
         self._ask_queue.add_task(self._request, question)
+        logger.info(f'ask:{question} completed')
 
     def stop(self):
+        logger.info('NLPBase stop')
         self._ask_queue.add_task(self._on_close)
         self._ask_queue.stop()
 

render/base_render.py

@@ -35,18 +35,22 @@ class BaseRender(ABC):
         logging.info(f'{self._type} render exit')
 
     def put(self, frame, ps):
-        self._queue.put((frame, ps))
+        pass
+        # self._queue.put((frame, ps))
 
     def size(self):
-        return self._queue.size()
+        pass
+        # return self._queue.size()
 
     def pause_talk(self):
-        self._queue.clear()
+        pass
+        # self._queue.clear()
 
     def stop(self):
-        self._queue.clear()
+        pass
-        self._exit_event.clear()
+        # self._queue.clear()
-        self._thread.join()
+        # self._exit_event.clear()
+        # self._thread.join()
 
     @abstractmethod
     def _run_step(self):

ui.py

@@ -57,7 +57,7 @@ class App(customtkinter.CTk):
         # self.main_button_1.grid(row=2, column=2, padx=(20, 20), pady=(20, 20), sticky="nsew")
         background = os.path.join(current_file_path, 'data', 'background', 'background.webp')
         logger.info(f'background: {background}')
-        self._background = read_image(background).convert("RGBA")
+        self._background = read_image(background).convert("RGB")
 
         self._init_image_canvas()
 
@@ -105,13 +105,13 @@ class App(customtkinter.CTk):
             image = cv2.resize(image, (int(iwidth * height / iheight), int(height)), interpolation=cv2.INTER_AREA)
         img = Image.fromarray(image)
 
-        bg_width, bg_height = self._background.size
+        # bg_width, bg_height = self._background.size
-        fg_width, fg_height = img.size
+        # fg_width, fg_height = img.size
-        x = (bg_width - fg_width) // 2
+        # x = (bg_width - fg_width) // 2
-        y = (bg_height - fg_height) // 2
+        # y = (bg_height - fg_height) // 2
-        self._background.paste(img, (x, y), img)
+        # self._background.paste(img, (x, y))
 
-        imgtk = ImageTk.PhotoImage(self._background)
+        imgtk = ImageTk.PhotoImage(img)
 
         self._canvas.delete("all")
 

ui/pygame_ui.py

@@ -0,0 +1,76 @@
+#encoding = utf8
+import logging
+import os
+from queue import Queue
+
+import pygame
+from pygame.locals import *
+
+from human import HumanContext
+from utils import config_logging, read_image
+
+logger = logging.getLogger(__name__)
+current_file_path = os.path.dirname(os.path.abspath(__file__))
+
+
+class PyGameUI:
+    def __init__(self):
+        self._human_context = None
+        self._queue = None
+        self.screen_ = pygame.display.set_mode((800, 600), HWSURFACE | DOUBLEBUF | RESIZABLE)
+        self.clock = pygame.time.Clock()
+
+        background = os.path.join(current_file_path, '..', 'data', 'background', 'background.jpg')
+        logger.info(f'background: {background}')
+        self._background = pygame.image.load(background).convert()
+        self.background_display_ = pygame.transform.scale(self._background, (800, 600))
+        self._human_image = None
+        self.running = True
+
+    def start(self):
+        self._queue = Queue()
+        self._human_context = HumanContext()
+        self._human_context.build()
+        render = self._human_context.render_handler
+        render.set_image_render(self)
+
+    def run(self):
+        self.start()
+        while self.running:
+            self.clock.tick(60)
+            for event in pygame.event.get():
+                if event.type == pygame.QUIT:
+                    self.running = False
+                elif event.type == VIDEORESIZE:
+                    self.background_display_ = pygame.transform.scale(self._background, event.dict['size'])
+            self.screen_.blit(self.background_display_, (0, 0))
+            if self._human_image is not None:
+                self.screen_.blit(self._human_image, (0, 0))
+            self._update_human()
+            pygame.display.flip()
+        self.stop()
+        pygame.quit()
+
+    def _update_human(self):
+        if self._queue.empty():
+            return
+        image = self._queue.get()
+        self._human_image = pygame.image.frombuffer(image.tobytes(), image.shape[1::-1], "RGB")
+
+    def stop(self):
+        logger.info('stop')
+        if self._human_context is not None:
+            self._human_context.pause_talk()
+            self._human_context.stop()
+
+    def on_render(self, image):
+        self._queue.put(image)
+
+
+if __name__ == '__main__':
+    config_logging('../logs/info.log', logging.INFO, logging.INFO)
+
+    logger.info('------------start------------')
+    ui = PyGameUI()
+    ui.run()
+    logger.info('------------finish------------')

utils/utils.py

@@ -36,7 +36,6 @@ def read_images(img_list):
         print(f'read image path:{img_path}')
         # frame = cv2.imread(img_path, cv2.IMREAD_UNCHANGED)
         frame = Image.open(img_path)
-        # frame = frame.convert("RGBA")
         frame = np.array(frame)
         frames.append(frame)
     return frames
@@ -179,7 +178,7 @@ def load_avatar(path, img_size, device):
     face_frames = []
     coord_frames = []
     for face, coord in face_det_results:
-        resized_crop_frame = cv2.resize(face[:, :, :3], (img_size, img_size))
+        resized_crop_frame = cv2.resize(face, (img_size, img_size))
         face_frames.append(resized_crop_frame)
         coord_frames.append(coord)