brige/human
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 2 Wiki Activity

use pygame to render

Browse Source
This commit is contained in:
brige 2024-11-04 21:44:51 +08:00
parent 4495f5182b
commit a92789c3b5
13 changed files with 117 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
asr/asr_base.py
View File

@ -13,6 +13,7 @@ class AsrBase:
self._observers = []
self._stop_event = threading.Event()
self._stop_event.set()
self._thread = threading.Thread(target=self._recognize_loop)
self._thread.start()
@ -28,7 +29,7 @@ class AsrBase:
observer.completed(message)
def stop(self):
self._stop_event.set()
self._stop_event.clear()
self._thread.join()
def attach(self, observer: AsrObserver):

8
asr/sherpa_ncnn_asr.py
View File

@ -60,10 +60,14 @@ class SherpaNcnnAsr(AsrBase):
time.sleep(3)
last_result = ""
logger.info(f'_recognize_loop')
while not self._stop_event.is_set():
while self._stop_event.is_set():
logger.info(f'_recognize_loop000')
self._notify_complete('介绍中国5000年历史文学')
logger.info(f'_recognize_loop111')
segment_id += 1
time.sleep(60)
time.sleep(5)
logger.info(f'_recognize_loop222')
logger.info(f'_recognize_loop exit')
'''
with sd.InputStream(channels=1, dtype="float32", samplerate=self._sample_rate) as s:
while not self._stop_event.is_set():

BIN
face/img00016.jpg.bak
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 452 KiB

BIN
face/img00020.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 258 KiB

4
human/audio_inference_handler.py
View File

@ -133,6 +133,6 @@ class AudioInferenceHandler(AudioHandler):
self._run_thread.join()
def pause_talk(self):
print('AudioInferenceHandler pause_talk', self._audio_queue.size(), self._mal_queue.qsize())
print('AudioInferenceHandler pause_talk', self._audio_queue.size(), self._mal_queue.size())
self._audio_queue.clear()
self._mal_queue.queue.clear()
self._mal_queue.clear()

2
human/audio_mal_handler.py
View File

@ -80,7 +80,7 @@ class AudioMalHandler(AudioHandler):
def get_audio_frame(self):
try:
# print('AudioMalHandler get_audio_frame')
frame = self._queue.get(timeout=0.02)
frame = self._queue.get()
type_ = 0
except queue.Empty:
frame = np.zeros(self.chunk, dtype=np.float32)

16
human/human_context.py
View File

@ -46,13 +46,6 @@ class HumanContext:
def __del__(self):
print(f'HumanContext: __del__')
object_stop(self._asr)
object_stop(self._nlp)
object_stop(self._tts)
object_stop(self._tts_handle)
object_stop(self._mal_handler)
object_stop(self._infer_handler)
object_stop(self._render_handler)
@property
def fps(self):
@ -119,6 +112,15 @@ class HumanContext:
self._asr = SherpaNcnnAsr()
self._asr.attach(self._nlp)
def stop(self):
object_stop(self._asr)
object_stop(self._nlp)
object_stop(self._tts)
object_stop(self._tts_handle)
object_stop(self._mal_handler)
object_stop(self._infer_handler)
object_stop(self._render_handler)
def pause_talk(self):
self._nlp.pause_talk()
self._tts.pause_talk()

2
human/human_render.py
View File

@ -33,7 +33,7 @@ class HumanRender(AudioHandler):
logging.info('human render run')
while self._exit_event.is_set():
self._run_step()
time.sleep(0.035)
time.sleep(0.037)
logging.info('human render exit')

2
nlp/nlp_base.py
View File

@ -43,8 +43,10 @@ class NLPBase(AsrObserver):
def ask(self, question):
logger.info(f'ask:{question}')
self._ask_queue.add_task(self._request, question)
logger.info(f'ask:{question} completed')
def stop(self):
logger.info('NLPBase stop')
self._ask_queue.add_task(self._on_close)
self._ask_queue.stop()

16
render/base_render.py
View File

@ -35,18 +35,22 @@ class BaseRender(ABC):
logging.info(f'{self._type} render exit')
def put(self, frame, ps):
self._queue.put((frame, ps))
pass
# self._queue.put((frame, ps))
def size(self):
return self._queue.size()
pass
# return self._queue.size()
def pause_talk(self):
self._queue.clear()
pass
# self._queue.clear()
def stop(self):
self._queue.clear()
self._exit_event.clear()
self._thread.join()
pass
# self._queue.clear()
# self._exit_event.clear()
# self._thread.join()
@abstractmethod
def _run_step(self):

14
ui.py
View File

@ -57,7 +57,7 @@ class App(customtkinter.CTk):
# self.main_button_1.grid(row=2, column=2, padx=(20, 20), pady=(20, 20), sticky="nsew")
background = os.path.join(current_file_path, 'data', 'background', 'background.webp')
logger.info(f'background: {background}')
self._background = read_image(background).convert("RGBA")
self._background = read_image(background).convert("RGB")
self._init_image_canvas()
@ -105,13 +105,13 @@ class App(customtkinter.CTk):
image = cv2.resize(image, (int(iwidth * height / iheight), int(height)), interpolation=cv2.INTER_AREA)
img = Image.fromarray(image)
bg_width, bg_height = self._background.size
fg_width, fg_height = img.size
x = (bg_width - fg_width) // 2
y = (bg_height - fg_height) // 2
self._background.paste(img, (x, y), img)
# bg_width, bg_height = self._background.size
# fg_width, fg_height = img.size
# x = (bg_width - fg_width) // 2
# y = (bg_height - fg_height) // 2
# self._background.paste(img, (x, y))
imgtk = ImageTk.PhotoImage(self._background)
imgtk = ImageTk.PhotoImage(img)
self._canvas.delete("all")

76
ui/pygame_ui.py Normal file
View File

@ -0,0 +1,76 @@
#encoding = utf8
import logging
import os
from queue import Queue
import pygame
from pygame.locals import *
from human import HumanContext
from utils import config_logging, read_image
logger = logging.getLogger(__name__)
current_file_path = os.path.dirname(os.path.abspath(__file__))
class PyGameUI:
def __init__(self):
self._human_context = None
self._queue = None
self.screen_ = pygame.display.set_mode((800, 600), HWSURFACE | DOUBLEBUF | RESIZABLE)
self.clock = pygame.time.Clock()
background = os.path.join(current_file_path, '..', 'data', 'background', 'background.jpg')
logger.info(f'background: {background}')
self._background = pygame.image.load(background).convert()
self.background_display_ = pygame.transform.scale(self._background, (800, 600))
self._human_image = None
self.running = True
def start(self):
self._queue = Queue()
self._human_context = HumanContext()
self._human_context.build()
render = self._human_context.render_handler
render.set_image_render(self)
def run(self):
self.start()
while self.running:
self.clock.tick(60)
for event in pygame.event.get():
if event.type == pygame.QUIT:
self.running = False
elif event.type == VIDEORESIZE:
self.background_display_ = pygame.transform.scale(self._background, event.dict['size'])
self.screen_.blit(self.background_display_, (0, 0))
if self._human_image is not None:
self.screen_.blit(self._human_image, (0, 0))
self._update_human()
pygame.display.flip()
self.stop()
pygame.quit()
def _update_human(self):
if self._queue.empty():
return
image = self._queue.get()
self._human_image = pygame.image.frombuffer(image.tobytes(), image.shape[1::-1], "RGB")
def stop(self):
logger.info('stop')
if self._human_context is not None:
self._human_context.pause_talk()
self._human_context.stop()
def on_render(self, image):
self._queue.put(image)
if __name__ == '__main__':
config_logging('../logs/info.log', logging.INFO, logging.INFO)
logger.info('------------start------------')
ui = PyGameUI()
ui.run()
logger.info('------------finish------------')

3
utils/utils.py
View File

@ -36,7 +36,6 @@ def read_images(img_list):
print(f'read image path:{img_path}')
# frame = cv2.imread(img_path, cv2.IMREAD_UNCHANGED)
frame = Image.open(img_path)
# frame = frame.convert("RGBA")
frame = np.array(frame)
frames.append(frame)
return frames
@ -179,7 +178,7 @@ def load_avatar(path, img_size, device):
face_frames = []
coord_frames = []
for face, coord in face_det_results:
resized_crop_frame = cv2.resize(face[:, :, :3], (img_size, img_size))
resized_crop_frame = cv2.resize(face, (img_size, img_size))
face_frames.append(resized_crop_frame)
coord_frames.append(coord)