human/utils/utils.py

#encoding = utf8
import glob
import logging
import os
import pickle

import cv2
import numpy as np
import torch
from tqdm import tqdm
from PIL import Image

import face_detection
from models import Wav2Lip, Wav2LipV2

logger = logging.getLogger(__name__)


def mirror_index(size, index):
    # size = len(self.coord_list_cycle)
    turn = index // size
    res = index % size
    if turn % 2 == 0:
        return res
    else:
        return size - res - 1


def read_image(path):
    image = Image.open(path)
    return image


def read_images(img_list):
    frames = []
    print('reading images...')
    for img_path in tqdm(img_list):
        # frame = cv2.imread(img_path, cv2.IMREAD_UNCHANGED)
        frame = Image.open(img_path)
        frame = np.array(frame)
        frames.append(frame)
    return frames


def read_files_path(path):
    file_paths = []
    files = os.listdir(path)
    for file in files:
        if not os.path.isdir(file) and file.endswith('.png') or file.endswith('.jpg'):
            file_paths.append(os.path.join(path, file))
    return file_paths


def get_smoothened_boxes(boxes, t):
    for i in range(len(boxes)):
        if i + t > len(boxes):
            window = boxes[len(boxes) - t:]
        else:
            window = boxes[i: i + t]
        boxes[i] = np.mean(window, axis=0)
    return boxes


def datagen_signal(frame, mel, face_det_results, img_size, wav2lip_batch_size=128):
    img_batch, mel_batch, frame_batch, coord_batch = [], [], [], []

    idx = 0
    frame_to_save = frame.copy()
    face, coord = face_det_results[idx].copy()

    face = cv2.resize(face, (img_size, img_size))

    for i, m in enumerate(mel):
        img_batch.append(face)
        mel_batch.append(m)
        frame_batch.append(frame_to_save)
        coord_batch.append(coord)

    if len(img_batch) >= wav2lip_batch_size:
        img_batch, mel_batch = np.asarray(img_batch), np.asarray(mel_batch)

        img_masked = img_batch.copy()
        img_masked[:, img_size // 2:] = 0
        img_batch = np.concatenate((img_masked, img_batch), axis=3) / 255.
        mel_batch = np.reshape(mel_batch, [len(mel_batch), mel_batch.shape[1], mel_batch.shape[2], 1])

        return img_batch, mel_batch, frame_batch, coord_batch

    if len(img_batch) > 0:
        img_batch, mel_batch = np.asarray(img_batch), np.asarray(mel_batch)
        img_masked = img_batch.copy()
        img_masked[:, img_size//2:] = 0

        img_batch = np.concatenate((img_masked, img_batch), axis=3) / 255.
        mel_batch = np.reshape(mel_batch, [len(mel_batch), mel_batch.shape[1], mel_batch.shape[2], 1])

        return img_batch, mel_batch, frame_batch, coord_batch


def face_detect(images, device):
    detector = face_detection.FaceAlignment(face_detection.LandmarksType._2D,
                                            flip_input=False, device=device)
    batch_size = 16

    while 1:
        predictions = []
        try:
            for i in range(0, len(images), batch_size):
                predictions.extend(detector.get_detections_for_batch(np.array(images[i:i + batch_size])))
        except RuntimeError:
            if batch_size == 1:
                raise RuntimeError(
                    'Image too big to run face detection on GPU. Please use the --resize_factor argument')
            batch_size //= 2
            print('Recovering from OOM error; New batch size: {}'.format(batch_size))
            continue
        break

    results = []
    pad_y1, pad_y2, pad_x1, pad_x2 = [0, 10, 0, 0]
    for rect, image in zip(predictions, images):
        if rect is None:
            cv2.imwrite('temp/faulty_frame.jpg', image)  # check this frame where the face was not detected.
            raise ValueError('Face not detected! Ensure the video contains a face in all the frames.')

        y1 = max(0, rect[1] - pad_y1)
        y2 = min(image.shape[0], rect[3] + pad_y2)
        x1 = max(0, rect[0] - pad_x1)
        x2 = min(image.shape[1], rect[2] + pad_x2)

        results.append([x1, y1, x2, y2])

    boxes = np.array(results)
    if not False:
        boxes = get_smoothened_boxes(boxes, t=5)
    results = [[image[y1: y2, x1:x2], (y1, y2, x1, x2)] for image, (x1, y1, x2, y2) in zip(images, boxes)]

    del detector
    return results


def get_device():
    return 'cuda' if torch.cuda.is_available() else 'cpu'


def _load(checkpoint_path):
    device = get_device()
    if device == 'cuda':
        checkpoint = torch.load(checkpoint_path)
    else:
        checkpoint = torch.load(checkpoint_path,
                                map_location=lambda storage, loc: storage)
    return checkpoint


def load_model(path):
    model = Wav2LipV2()
    print("Load checkpoint from: {}".format(path))
    logging.info(f'Load checkpoint from {path}')
    checkpoint = _load(path)
    s = checkpoint["state_dict"]
    new_s = {}
    for k, v in s.items():
        new_s[k.replace('module.', '')] = v
    model.load_state_dict(new_s)
    device = get_device()
    model = model.to(device)
    return model.eval()


def load_avatar(path, img_size, device):
    print(f'load avatar:{path}')
    face_images_path = os.path.join(path, 'face')
    face_images_path = read_files_path(face_images_path)
    full_list_cycle = read_images(face_images_path)

    face_det_results = face_detect(full_list_cycle, device)

    face_frames = []
    coord_frames = []
    for face, coord in face_det_results:
        resized_crop_frame = cv2.resize(face[:, :, :3], (img_size, img_size))
        face_frames.append(resized_crop_frame)
        coord_frames.append(coord)

    return full_list_cycle, face_frames, coord_frames


def load_avatar_from_processed(base_path, avatar_name):
    avatar_path = os.path.join(base_path, 'data', 'avatars', avatar_name)
    print(f'load avatar from processed:{avatar_path}')
    coord_path = os.path.join(avatar_path, 'coords.pkl')
    print(f'load avatar_path from processed:{avatar_path}')
    face_image_path = os.path.join(avatar_path, 'face_imgs')
    print(f'load face_image_path from processed:{face_image_path}')
    full_image_path = os.path.join(avatar_path, 'full_imgs')
    print(f'load full_image_path from processed:{full_image_path}')

    with open(coord_path, 'rb') as f:
        coord_list_frames = pickle.load(f)

    face_image_list = glob.glob(os.path.join(face_image_path, '*.[jpJP][pnPN]*[gG]'))
    face_image_list = sorted(face_image_list, key=lambda x: int(os.path.splitext(os.path.basename(x))[0]))
    face_list_cycle = read_images(face_image_list)

    full_image_list = glob.glob(os.path.join(full_image_path, '*.[jpJP][pnPN]*[gG]'))
    full_image_list = sorted(full_image_list, key=lambda x: int(os.path.splitext(os.path.basename(x))[0]))
    frame_list_cycle = read_images(full_image_list)

    return frame_list_cycle, face_list_cycle, coord_list_frames


def jpeg_to_png(image):
    min_green = np.array([50, 100, 100])
    max_green = np.array([70, 255, 255])

    hsv = cv2.cvtColor(image, cv2.COLOR_RGB2HSV)
    mask = cv2.inRange(hsv, min_green, max_green)
    mask_not = cv2.bitwise_not(mask)
    green_not = cv2.bitwise_and(image, image, mask=mask_not)
    b, g, r = cv2.split(green_not)

    # todo 合成四通道
    image = cv2.merge([b, g, r, mask_not])
    return image


def load_avatar_from_256_processed(base_path, avatar_name, pkl):
    avatar_path = os.path.join(base_path, 'data', 'avatars', avatar_name, pkl)
    print(f'load avatar from processed:{avatar_path}')

    with open(avatar_path, "rb") as f:
        avatar_data = pickle.load(f)

    face_list_cycle = []
    frame_list_cycle = []
    coord_list_frames = []
    align_frames = []
    m_frames = []
    inv_m_frames = []

    frame_info_list = avatar_data['frame_info_list']
    for frame_info in tqdm(frame_info_list):
        face_list_cycle.append(frame_info['img'])
        frame_list_cycle.append(jpeg_to_png(frame_info['frame']))
        coord_list_frames.append(frame_info['coords'])
        align_frames.append(frame_info['align_frame'])
        m_frames.append(frame_info['m'])
        inv_m_frames.append(frame_info['inv_m'])

    return frame_list_cycle, face_list_cycle, coord_list_frames, align_frames, m_frames, inv_m_frames


def config_logging(file_name: str, console_level: int = logging.INFO, file_level: int = logging.DEBUG):
    file_handler = logging.FileHandler(file_name, mode='a', encoding="utf8")
    file_handler.setFormatter(logging.Formatter(
        '%(asctime)s [%(levelname)s] %(module)s.%(lineno)d %(name)s:\t%(message)s'
        ))
    file_handler.setLevel(file_level)

    console_handler = logging.StreamHandler()
    console_handler.setFormatter(logging.Formatter(
        '[%(asctime)s.%(msecs)03d %(levelname)s] %(message)s',
        datefmt="%Y/%m/%d %H:%M:%S"
        ))
    console_handler.setLevel(console_level)

    logging.basicConfig(
        level=min(console_level, file_level),
        handlers=[file_handler, console_handler],
    )


def object_stop(obj):
    if obj is not None:
        obj.stop()