diff --git a/mefamo/blendshapes/blendshape_calculator.py b/mefamo/blendshapes/blendshape_calculator.py index 6608f02..98df4db 100644 --- a/mefamo/blendshapes/blendshape_calculator.py +++ b/mefamo/blendshapes/blendshape_calculator.py @@ -103,14 +103,18 @@ def _calculate_mouth_landmarks(self): jaw_open_ratio = jaw_nose_dist / head_height # self._live_link_face.set_blendshape(ARKitFace.MouthFrownRight, max(min(mouth_frown_right, 1), 0)) + jaw_open = self._remap_blendshape( FaceBlendShape.JawOpen, jaw_open_ratio) self._live_link_face.set_blendshape(FaceBlendShape.JawOpen, jaw_open) - + mouth_open = self._remap_blendshape( FaceBlendShape.MouthClose, mouth_center_nose_dist - mouth_open_dist) self._live_link_face.set_blendshape( FaceBlendShape.MouthClose, mouth_open) + + print(mouth_open) + # TODO mouth open but teeth closed smile_left = upper_lip[1] - mouth_corner_left[1] diff --git a/mefamo/blendshapes/blendshape_config.py b/mefamo/blendshapes/blendshape_config.py index ed29f47..98a7dcb 100644 --- a/mefamo/blendshapes/blendshape_config.py +++ b/mefamo/blendshapes/blendshape_config.py @@ -96,7 +96,7 @@ class CanonicalPpoints: # FaceBlendShape.NoseSneerLeft : (-0.4, 0.0), # FaceBlendShape.NoseSneerRight : (-0.4, 0.0), # FaceBlendShape.TongueOut : (-0.4, 0.0), - # FaceBlendShape.HeadYaw : (-0.4, 0.0), + # FaceBlendShape.67 : (-0.4, 0.0), # FaceBlendShape.HeadPitch : (-0.4, 0.0), # FaceBlendShape.HeadRoll : (-0.4, 0.0), # FaceBlendShape.LeftEyeYaw : (-0.4, 0.0), diff --git a/mefamo/custom/__init__.py b/mefamo/custom/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/mefamo/custom/__init__.pyo b/mefamo/custom/__init__.pyo deleted file mode 100644 index 5456f4a..0000000 Binary files a/mefamo/custom/__init__.pyo and /dev/null differ diff --git a/mefamo/custom/face_geometry.pyo b/mefamo/custom/face_geometry.pyo deleted file mode 100644 index 52c140e..0000000 Binary files a/mefamo/custom/face_geometry.pyo and /dev/null differ diff --git a/mefamo/mefamo.py b/mefamo/mefamo.py index 240b316..247e9ab 100644 --- a/mefamo/mefamo.py +++ b/mefamo/mefamo.py @@ -10,7 +10,7 @@ import transforms3d import open3d as o3d -from pylivelinkface import PyLiveLinkFace, FaceBlendShape +from mefamo.pylivelinkface.pylivelinkface import PyLiveLinkFace, FaceBlendShape from mefamo.utils.drawing import Drawing from mefamo.blendshapes.blendshape_calculator import BlendshapeCalculator @@ -58,7 +58,7 @@ def calculate_rotation(face_landmarks, pcf: PCF, image_shape): ) success, rotation_vector, translation_vector = cv2.solvePnP( - model_points, + model_points, image_points, camera_matrix, dist_coeff, diff --git a/mefamo/pylivelinkface/__init__.py b/mefamo/pylivelinkface/__init__.py new file mode 100644 index 0000000..214db5f --- /dev/null +++ b/mefamo/pylivelinkface/__init__.py @@ -0,0 +1 @@ +from .pylivelinkface import PyLiveLinkFace, FaceBlendShape \ No newline at end of file diff --git a/mefamo/pylivelinkface/pylivelinkface.py b/mefamo/pylivelinkface/pylivelinkface.py new file mode 100644 index 0000000..ef1abeb --- /dev/null +++ b/mefamo/pylivelinkface/pylivelinkface.py @@ -0,0 +1,248 @@ +from __future__ import annotations +from collections import deque +from statistics import mean +from enum import Enum +import struct +from typing import Tuple +import datetime +import uuid +import numpy as np +from timecode import Timecode + +class FaceBlendShape(Enum): + EyeBlinkLeft = 0 + EyeLookDownLeft = 1 + EyeLookInLeft = 2 + EyeLookOutLeft = 3 + EyeLookUpLeft = 4 + EyeSquintLeft = 5 + EyeWideLeft = 6 + EyeBlinkRight = 7 + EyeLookDownRight = 8 + EyeLookInRight = 9 + EyeLookOutRight = 10 + EyeLookUpRight = 11 + EyeSquintRight = 12 + EyeWideRight = 13 + JawForward = 14 + JawLeft = 15 + JawRight = 16 + JawOpen = 17 + MouthClose = 18 + MouthFunnel = 19 + MouthPucker = 20 + MouthLeft = 21 + MouthRight = 22 + MouthSmileLeft = 23 + MouthSmileRight = 24 + MouthFrownLeft = 25 + MouthFrownRight = 26 + MouthDimpleLeft = 27 + MouthDimpleRight = 28 + MouthStretchLeft = 29 + MouthStretchRight = 30 + MouthRollLower = 31 + MouthRollUpper = 32 + MouthShrugLower = 33 + MouthShrugUpper = 34 + MouthPressLeft = 35 + MouthPressRight = 36 + MouthLowerDownLeft = 37 + MouthLowerDownRight = 38 + MouthUpperUpLeft = 39 + MouthUpperUpRight = 40 + BrowDownLeft = 41 + BrowDownRight = 42 + BrowInnerUp = 43 + BrowOuterUpLeft = 44 + BrowOuterUpRight = 45 + CheekPuff = 46 + CheekSquintLeft = 47 + CheekSquintRight = 48 + NoseSneerLeft = 49 + NoseSneerRight = 50 + TongueOut = 51 + HeadYaw = 52 + HeadPitch = 53 + HeadRoll = 54 + LeftEyeYaw = 55 + LeftEyePitch = 56 + LeftEyeRoll = 57 + RightEyeYaw = 58 + RightEyePitch = 59 + RightEyeRoll = 60 + + +class PyLiveLinkFace: + """PyLiveLinkFace class + + Can be used to receive PyLiveLinkFace from the PyLiveLinkFace IPhone app or + other PyLiveLinkFace compatible programs like this library. + """ + + def __init__(self, name: str = "Python_LiveLinkFace", + uuid: str = str(uuid.uuid1()), fps=60, + filter_size: int = 5) -> None: + + # properties + self.uuid = uuid + self.name = name + self.fps = fps + self._filter_size = filter_size + + self._version = 6 + now = datetime.datetime.now() + timcode = Timecode( + self._fps, f'{now.hour}:{now.minute}:{now.second}:{now.microsecond * 0.001}') + self._frames = timcode.frames + self._sub_frame = 1056060032 # I don't know how to calculate this + self._denominator = int(self._fps / 60) # 1 most of the time + self._blend_shapes = [0.000] * 61 + self._old_blend_shapes = [] # used for filtering + for i in range(61): + self._old_blend_shapes.append(deque([0.0], maxlen = self._filter_size)) + + @property + def uuid(self) -> str: + return self._uuid + + @uuid.setter + def uuid(self, value: str) -> None: + # uuid needs to start with a $, if it doesn't add it + if not value.startswith("$"): + self._uuid = '$' + value + else: + self._uuid = value + + @property + def name(self) -> str: + return self._name + + @name.setter + def name(self, value: str) -> None: + self._name = value + + @property + def fps(self) -> int: + return self._fps + + @fps.setter + def fps(self, value: int) -> None: + if value < 1: + raise ValueError("Only fps values greater than 1 are allowed.") + self._fps = value + + def encode(self) -> bytes: + """ Encodes the PyLiveLinkFace object into a bytes object so it can be + send over a network. """ + + version_packed = struct.pack(' float: + """ Get the current value of the blend shape. + + Parameters + ---------- + index : FaceBlendShape + Index of the BlendShape to get the value from. + + Returns + ------- + float + The value of the BlendShape. + """ + return self._blend_shapes[index.value] + + def set_blendshape(self, index: FaceBlendShape, value: float, + no_filter: bool = False) -> None: + """ Sets the value of the blendshape. + + The function will use mean to filter between the old and the new + values, unless `no_filter` is set to True. + + Parameters + ---------- + index : FaceBlendShape + Index of the BlendShape to get the value from. + value: float + Value to set the BlendShape to, should be in the range of 0 - 1 for + the blendshapes and between -1 and 1 for the head rotation + (yaw, pitch, roll). + no_filter: bool + If set to True, the blendshape will be set to the value without + filtering. + + Returns + ---------- + None + """ + + if no_filter: + self._blend_shapes[index.value] = value + else: + self._old_blend_shapes[index.value].append(value) + filterd_value = mean(self._old_blend_shapes[index.value]) + self._blend_shapes[index.value] = filterd_value + + @staticmethod + def decode(bytes_data: bytes) -> Tuple[bool, PyLiveLinkFace]: + """ Decodes the given bytes (send from an PyLiveLinkFace App or from + this library) and creates a new PyLiveLinkFace object. + Returns True and the generated object if a face was found in the data, + False an a new empty PyLiveLinkFace otherwise. + + Parameters + ---------- + bytes_data : bytes + Bytes input to create the PyLiveLinkFace object from. + + Returns + ------- + bool + True if the bytes data contained a face, False if not. + PyLiveLinkFace + The PyLiveLinkFace object. + + """ + version = struct.unpack(' name_end_pos + 16: + + #FFrameTime, FFrameRate and data length + frame_number, sub_frame, fps, denominator, data_length = struct.unpack( + "!if2ib", bytes_data[name_end_pos:name_end_pos + 17]) + + if data_length != 61: + raise ValueError( + f'Blend shape length is {data_length} but should be 61, something is wrong with the data.') + + data = struct.unpack( + "!61f", bytes_data[name_end_pos + 17:]) + + live_link_face = PyLiveLinkFace(name, uuid, fps) + live_link_face._version = version + live_link_face._frames = frame_number + live_link_face._sub_frame = sub_frame + live_link_face._denominator = denominator + live_link_face._blend_shapes = data + + return True, live_link_face + else: + #print("Data does not contain a face, returning default empty face.") + return False, PyLiveLinkFace() diff --git a/examples/mefamo_cli.py b/mefamo_cli.py similarity index 96% rename from examples/mefamo_cli.py rename to mefamo_cli.py index 2fa6047..adf5257 100644 --- a/examples/mefamo_cli.py +++ b/mefamo_cli.py @@ -1,4 +1,4 @@ -from mefamo import Mefamo +from mefamo.mefamo import Mefamo from argparse import ArgumentParser if __name__ == "__main__":