brige/human
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 2 Wiki Activity

modify render and ncnn asr to ask

Browse Source
This commit is contained in:
jiegeaiai 2024-11-25 01:03:29 +08:00
parent 7ae1432312
commit 7c1104a9ac
4 changed files with 62 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
asr/sherpa_ncnn_asr.py
View File

@ -63,15 +63,6 @@ class SherpaNcnnAsr(AsrBase):
logger.info(f'_recognize_loop')
print(f'_recognize_loop')
while self._stop_event.is_set():
logger.info(f'_recognize_loop000')
self._notify_complete('介绍中国5000年历史文学')
logger.info(f'_recognize_loop111')
segment_id += 1
time.sleep(150)
logger.info(f'_recognize_loop222')
logger.info(f'_recognize_loop exit')
'''
with sd.InputStream(channels=1, dtype="float32", samplerate=self._sample_rate) as s:
while self._stop_event.is_set():
samples, _ = s.read(self._samples_per_read) # a blocking read
@ -94,3 +85,13 @@ class SherpaNcnnAsr(AsrBase):
self._recognizer.reset()
'''
while self._stop_event.is_set():
logger.info(f'_recognize_loop000')
self._notify_complete('介绍中国5000年历史文学')
logger.info(f'_recognize_loop111')
segment_id += 1
time.sleep(150)
logger.info(f'_recognize_loop222')
logger.info(f'_recognize_loop exit')
'''

2
human/human_render.py
View File

@ -48,7 +48,7 @@ class HumanRender(AudioHandler):
logging.info('human render run')
while self._exit_event.is_set() and self._is_running:
self._run_step()
delay = 0.03
delay = 0.04
time.sleep(delay)
logging.info('human render exit')

44
render/video_render.py
View File

@ -9,23 +9,6 @@ import numpy as np
from .base_render import BaseRender
def img_warp_back_inv_m(img, img_to, inv_m):
h_up, w_up, c = img_to.shape
t = time.perf_counter()
mask = np.ones_like(img).astype(np.float32)
inv_mask = cv2.warpAffine(mask, inv_m, (w_up, h_up))
inv_img = cv2.warpAffine(img, inv_m, (w_up, h_up))
mask_indices = inv_mask == 1
print(f'time1: {time.perf_counter() - t}')
if 4 == c:
t = time.perf_counter()
img_to[:, :, :3][mask_indices] = inv_img[mask_indices]
print(f'time2: {time.perf_counter() - t}')
else:
img_to[inv_mask == 1] = inv_img[inv_mask == 1]
return img_to
class VideoRender(BaseRender):
def __init__(self, play_clock, context, human_render):
super().__init__(play_clock, context, 'Video')
@ -33,27 +16,8 @@ class VideoRender(BaseRender):
self.index = 0
def render(self, frame, ps):
res_frame, idx, type_ = frame
if type_ == 0:
combine_frame = self._context.frame_list_cycle[idx]
else:
bbox = self._context.coord_list_cycle[idx]
combine_frame = copy.deepcopy(self._context.frame_list_cycle[idx])
af = self._context.align_frames[idx]
inv_m = self._context.inv_m_frames[idx]
y1, y2, x1, x2 = bbox
try:
t = time.perf_counter()
res_frame = cv2.resize(res_frame.astype(np.uint8), (x2 - x1, y2 - y1))
af[y1:y2, x1:x2] = res_frame
combine_frame = img_warp_back_inv_m(af, combine_frame, inv_m)
print(time.perf_counter() - t)
except Exception as e:
logging.error(f'resize error:{e}')
return
image = combine_frame
# image = cv2.cvtColor(image, cv2.COLOR_BGRA2RGBA)
if self._human_render is not None:
self._human_render.put_image(image)
self._human_render.put_image(frame)
# image = cv2.cvtColor(image, cv2.COLOR_BGRA2RGBA)

50
ui/pygame_ui.py
View File

@ -1,6 +1,8 @@
#encoding = utf8
import copy
import logging
import os
import time
from queue import Queue
import cv2
@ -15,17 +17,54 @@ logger = logging.getLogger(__name__)
current_file_path = os.path.dirname(os.path.abspath(__file__))
def img_warp_back_inv_m(img, img_to, inv_m):
h_up, w_up, c = img_to.shape
mask = np.ones_like(img).astype(np.float32)
inv_mask = cv2.warpAffine(mask, inv_m, (w_up, h_up))
inv_img = cv2.warpAffine(img, inv_m, (w_up, h_up))
mask_indices = inv_mask == 1
if 4 == c:
img_to[:, :, :3][mask_indices] = inv_img[mask_indices]
else:
img_to[inv_mask == 1] = inv_img[inv_mask == 1]
return img_to
def render_image(context, frame):
res_frame, idx, type_ = frame
if type_ == 0:
combine_frame = context.frame_list_cycle[idx]
else:
bbox = context.coord_list_cycle[idx]
combine_frame = copy.deepcopy(context.frame_list_cycle[idx])
af = context.align_frames[idx]
inv_m = context.inv_m_frames[idx]
y1, y2, x1, x2 = bbox
try:
t = time.perf_counter()
res_frame = cv2.resize(res_frame.astype(np.uint8), (x2 - x1, y2 - y1))
af[y1:y2, x1:x2] = res_frame
combine_frame = img_warp_back_inv_m(af, combine_frame, inv_m)
except Exception as e:
logging.error(f'resize error:{e}')
return
image = combine_frame
return image
class PyGameUI:
def __init__(self):
self._human_context = None
self._queue = None
self.screen_ = pygame.display.set_mode((800, 600), HWSURFACE | DOUBLEBUF | RESIZABLE)
self.screen_ = pygame.display.set_mode((1920, 1080), HWSURFACE | DOUBLEBUF | RESIZABLE)
self.clock = pygame.time.Clock()
background = os.path.join(current_file_path, '..', 'data', 'background', 'background.jpg')
logger.info(f'background: {background}')
self._background = pygame.image.load(background).convert()
self.background_display_ = pygame.transform.scale(self._background, (800, 600))
self.background_display_ = pygame.transform.scale(self._background, (1920, 1080))
self._human_image = None
self.running = True
@ -48,7 +87,7 @@ class PyGameUI:
self.screen_.blit(self.background_display_, (0, 0))
self._update_human()
if self._human_image is not None:
self.screen_.blit(self._human_image, (0, -200))
self.screen_.blit(self._human_image, (760, -300))
fps = self.clock.get_fps()
pygame.display.set_caption('fps:{:.2f}'.format(fps))
@ -59,7 +98,12 @@ class PyGameUI:
def _update_human(self):
if self._queue.empty():
return
if self._queue.qsize() > 5:
print('render queue is slower')
image = self._queue.get()
image = render_image(self._human_context, image)
color_format = "RGB"
if 4 == image.shape[2]:
color_format = "RGBA"