diff --git a/asr/asr_base.py b/asr/asr_base.py index 0b69cd9..0528093 100644 --- a/asr/asr_base.py +++ b/asr/asr_base.py @@ -13,6 +13,7 @@ class AsrBase: self._observers = [] self._stop_event = threading.Event() + self._stop_event.set() self._thread = threading.Thread(target=self._recognize_loop) self._thread.start() @@ -28,7 +29,7 @@ class AsrBase: observer.completed(message) def stop(self): - self._stop_event.set() + self._stop_event.clear() self._thread.join() def attach(self, observer: AsrObserver): diff --git a/asr/sherpa_ncnn_asr.py b/asr/sherpa_ncnn_asr.py index d329193..8abda87 100644 --- a/asr/sherpa_ncnn_asr.py +++ b/asr/sherpa_ncnn_asr.py @@ -60,10 +60,14 @@ class SherpaNcnnAsr(AsrBase): time.sleep(3) last_result = "" logger.info(f'_recognize_loop') - while not self._stop_event.is_set(): + while self._stop_event.is_set(): + logger.info(f'_recognize_loop000') self._notify_complete('介绍中国5000年历史文学') + logger.info(f'_recognize_loop111') segment_id += 1 - time.sleep(60) + time.sleep(5) + logger.info(f'_recognize_loop222') + logger.info(f'_recognize_loop exit') ''' with sd.InputStream(channels=1, dtype="float32", samplerate=self._sample_rate) as s: while not self._stop_event.is_set(): diff --git a/face/img00016.jpg.bak b/face/img00016.jpg.bak deleted file mode 100644 index 3bc141f..0000000 Binary files a/face/img00016.jpg.bak and /dev/null differ diff --git a/face/img00020.png b/face/img00020.png deleted file mode 100644 index 22d9e15..0000000 Binary files a/face/img00020.png and /dev/null differ diff --git a/human/audio_inference_handler.py b/human/audio_inference_handler.py index 57cf16a..0a67f79 100644 --- a/human/audio_inference_handler.py +++ b/human/audio_inference_handler.py @@ -133,6 +133,6 @@ class AudioInferenceHandler(AudioHandler): self._run_thread.join() def pause_talk(self): - print('AudioInferenceHandler pause_talk', self._audio_queue.size(), self._mal_queue.qsize()) + print('AudioInferenceHandler pause_talk', self._audio_queue.size(), self._mal_queue.size()) self._audio_queue.clear() - self._mal_queue.queue.clear() + self._mal_queue.clear() diff --git a/human/audio_mal_handler.py b/human/audio_mal_handler.py index 4190692..796ad25 100644 --- a/human/audio_mal_handler.py +++ b/human/audio_mal_handler.py @@ -80,7 +80,7 @@ class AudioMalHandler(AudioHandler): def get_audio_frame(self): try: # print('AudioMalHandler get_audio_frame') - frame = self._queue.get(timeout=0.02) + frame = self._queue.get() type_ = 0 except queue.Empty: frame = np.zeros(self.chunk, dtype=np.float32) diff --git a/human/human_context.py b/human/human_context.py index 4a69304..cbefb1f 100644 --- a/human/human_context.py +++ b/human/human_context.py @@ -46,13 +46,6 @@ class HumanContext: def __del__(self): print(f'HumanContext: __del__') - object_stop(self._asr) - object_stop(self._nlp) - object_stop(self._tts) - object_stop(self._tts_handle) - object_stop(self._mal_handler) - object_stop(self._infer_handler) - object_stop(self._render_handler) @property def fps(self): @@ -119,6 +112,15 @@ class HumanContext: self._asr = SherpaNcnnAsr() self._asr.attach(self._nlp) + def stop(self): + object_stop(self._asr) + object_stop(self._nlp) + object_stop(self._tts) + object_stop(self._tts_handle) + object_stop(self._mal_handler) + object_stop(self._infer_handler) + object_stop(self._render_handler) + def pause_talk(self): self._nlp.pause_talk() self._tts.pause_talk() diff --git a/human/human_render.py b/human/human_render.py index 68b561b..673ad66 100644 --- a/human/human_render.py +++ b/human/human_render.py @@ -33,7 +33,7 @@ class HumanRender(AudioHandler): logging.info('human render run') while self._exit_event.is_set(): self._run_step() - time.sleep(0.035) + time.sleep(0.037) logging.info('human render exit') diff --git a/nlp/nlp_base.py b/nlp/nlp_base.py index bfe9fce..7ef3942 100644 --- a/nlp/nlp_base.py +++ b/nlp/nlp_base.py @@ -43,8 +43,10 @@ class NLPBase(AsrObserver): def ask(self, question): logger.info(f'ask:{question}') self._ask_queue.add_task(self._request, question) + logger.info(f'ask:{question} completed') def stop(self): + logger.info('NLPBase stop') self._ask_queue.add_task(self._on_close) self._ask_queue.stop() diff --git a/render/base_render.py b/render/base_render.py index 7534cb1..59090a0 100644 --- a/render/base_render.py +++ b/render/base_render.py @@ -35,18 +35,22 @@ class BaseRender(ABC): logging.info(f'{self._type} render exit') def put(self, frame, ps): - self._queue.put((frame, ps)) + pass + # self._queue.put((frame, ps)) def size(self): - return self._queue.size() + pass + # return self._queue.size() def pause_talk(self): - self._queue.clear() + pass + # self._queue.clear() def stop(self): - self._queue.clear() - self._exit_event.clear() - self._thread.join() + pass + # self._queue.clear() + # self._exit_event.clear() + # self._thread.join() @abstractmethod def _run_step(self): diff --git a/ui.py b/ui.py index 2587a48..70730c8 100644 --- a/ui.py +++ b/ui.py @@ -57,7 +57,7 @@ class App(customtkinter.CTk): # self.main_button_1.grid(row=2, column=2, padx=(20, 20), pady=(20, 20), sticky="nsew") background = os.path.join(current_file_path, 'data', 'background', 'background.webp') logger.info(f'background: {background}') - self._background = read_image(background).convert("RGBA") + self._background = read_image(background).convert("RGB") self._init_image_canvas() @@ -105,13 +105,13 @@ class App(customtkinter.CTk): image = cv2.resize(image, (int(iwidth * height / iheight), int(height)), interpolation=cv2.INTER_AREA) img = Image.fromarray(image) - bg_width, bg_height = self._background.size - fg_width, fg_height = img.size - x = (bg_width - fg_width) // 2 - y = (bg_height - fg_height) // 2 - self._background.paste(img, (x, y), img) + # bg_width, bg_height = self._background.size + # fg_width, fg_height = img.size + # x = (bg_width - fg_width) // 2 + # y = (bg_height - fg_height) // 2 + # self._background.paste(img, (x, y)) - imgtk = ImageTk.PhotoImage(self._background) + imgtk = ImageTk.PhotoImage(img) self._canvas.delete("all") diff --git a/ui/pygame_ui.py b/ui/pygame_ui.py new file mode 100644 index 0000000..7910329 --- /dev/null +++ b/ui/pygame_ui.py @@ -0,0 +1,76 @@ +#encoding = utf8 +import logging +import os +from queue import Queue + +import pygame +from pygame.locals import * + +from human import HumanContext +from utils import config_logging, read_image + +logger = logging.getLogger(__name__) +current_file_path = os.path.dirname(os.path.abspath(__file__)) + + +class PyGameUI: + def __init__(self): + self._human_context = None + self._queue = None + self.screen_ = pygame.display.set_mode((800, 600), HWSURFACE | DOUBLEBUF | RESIZABLE) + self.clock = pygame.time.Clock() + + background = os.path.join(current_file_path, '..', 'data', 'background', 'background.jpg') + logger.info(f'background: {background}') + self._background = pygame.image.load(background).convert() + self.background_display_ = pygame.transform.scale(self._background, (800, 600)) + self._human_image = None + self.running = True + + def start(self): + self._queue = Queue() + self._human_context = HumanContext() + self._human_context.build() + render = self._human_context.render_handler + render.set_image_render(self) + + def run(self): + self.start() + while self.running: + self.clock.tick(60) + for event in pygame.event.get(): + if event.type == pygame.QUIT: + self.running = False + elif event.type == VIDEORESIZE: + self.background_display_ = pygame.transform.scale(self._background, event.dict['size']) + self.screen_.blit(self.background_display_, (0, 0)) + if self._human_image is not None: + self.screen_.blit(self._human_image, (0, 0)) + self._update_human() + pygame.display.flip() + self.stop() + pygame.quit() + + def _update_human(self): + if self._queue.empty(): + return + image = self._queue.get() + self._human_image = pygame.image.frombuffer(image.tobytes(), image.shape[1::-1], "RGB") + + def stop(self): + logger.info('stop') + if self._human_context is not None: + self._human_context.pause_talk() + self._human_context.stop() + + def on_render(self, image): + self._queue.put(image) + + +if __name__ == '__main__': + config_logging('../logs/info.log', logging.INFO, logging.INFO) + + logger.info('------------start------------') + ui = PyGameUI() + ui.run() + logger.info('------------finish------------') diff --git a/utils/utils.py b/utils/utils.py index 8a688cc..82208f9 100644 --- a/utils/utils.py +++ b/utils/utils.py @@ -36,7 +36,6 @@ def read_images(img_list): print(f'read image path:{img_path}') # frame = cv2.imread(img_path, cv2.IMREAD_UNCHANGED) frame = Image.open(img_path) - # frame = frame.convert("RGBA") frame = np.array(frame) frames.append(frame) return frames @@ -179,7 +178,7 @@ def load_avatar(path, img_size, device): face_frames = [] coord_frames = [] for face, coord in face_det_results: - resized_crop_frame = cv2.resize(face[:, :, :3], (img_size, img_size)) + resized_crop_frame = cv2.resize(face, (img_size, img_size)) face_frames.append(resized_crop_frame) coord_frames.append(coord)