diff --git a/.gitignore b/.gitignore index d9005f2..f4ebf71 100644 --- a/.gitignore +++ b/.gitignore @@ -150,3 +150,5 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +/.vs diff --git a/README.md b/README.md index 8896c49..637511e 100644 --- a/README.md +++ b/README.md @@ -7,14 +7,15 @@ Currently the architecture is a moddified mobilenetv2 training on blenshapes fro 1. The current version of the model may not work for your mouth as it has been trained on a not very diverse dataset. Expect this to change in newer versions. 2. Not all blendshapes may work as there isnt enough data collected in the dataset. 3. No detailed setup guide will be provided until later more stable versions. -4. The (ML) model is currently un optimized so expect large amounts of CPU or GPU usage. This will be worked on in later versions - +4. This model is not a 1 to 1 translation to SRanipal blenshapes. While most blendshapes do cross over with SRanipal cheek and tounge movements are limited. +### Warranty +Lol ### Current setup guide 1. Clone the repo 2. Install the dependencies in requirements.txt with python 3.8 (or newer). -3. run model_loader_no_crop.py +3. run onnxinference.py 4. run the unity build in the unity build folder 5. make a netural face and move the camera around your mouth area until the avatar in the unity app makes a netural face too. 6. Enjoy mouth tracking! diff --git a/onnxinference.py b/onnxinference.py index ba7ff0e..9520fb6 100644 --- a/onnxinference.py +++ b/onnxinference.py @@ -1,76 +1,46 @@ -import onnx +import os +import time import torch +import onnx import onnxruntime as ort import numpy as np import cv2 -import time from pythonosc import udp_client -OSCip="127.0.0.1" -OSCport= 9000 #VR Chat OSC port +OSCip = "127.0.0.1" +OSCport = 9000 #VR Chat OSC port client = udp_client.SimpleUDPClient(OSCip, OSCport) -classes = ["cheekPuff", "cheekSquint_L", "cheekSquint_R", "noseSneer_L", "noseSneer_R", "jawOpen", "jawForward", "jawLeft", "jawRight", "mouthFunnel", "mouthPucker", "mouthLeft", "mouthRight", "mouthRollUpper", "mouthRollLower", "mouthShrugUpper", "mouthShrugLower", "mouthClose", "mouthSmile_L", "mouthSmile_R", "mouthFrown_L", "mouthFrown_R", "mouthDimple_L", "mouthDimple_R", "mouthUpperUp_L", "mouthUpperUp_R", "mouthLowerDown_L", "mouthLowerDown_R", "mouthPress_L", "mouthPress_R", "mouthStretch_L", "mouthStretch_R", "tongueOut"] +classes = ["cheekPuff", "cheekSquint_L", "cheekSquint_R", "noseSneer_L", "noseSneer_R", "jawOpen", "jawForward", "jawLeft", "jawRight", "mouthFunnel", "mouthPucker", "mouthLeft", "mouthRight", "mouthRollUpper", "mouthRollLower", "mouthShrugUpper", "mouthShrugLower", "mouthClose", "mouthSmile_L", "mouthSmile_R", "mouthFrown_L", "mouthFrown_R", "mouthDimple_L", "mouthDimple_R", "mouthUpperUp_L", "mouthUpperUp_R", "mouthLowerDown_L", "mouthLowerDown_R", "mouthPress_L", "mouthPress_R", "mouthStretch_L", "mouthStretch_R", "tongueOut"] cap = cv2.VideoCapture(0) sessionOptions = ort.SessionOptions() ort_sess = ort.InferenceSession("modelv2.onnx", providers = ['CPUExecutionProvider']) while True: - #read the frame ret, frame = cap.read() - #convert the frame to gr ayscale frame = cv2.resize(frame, (100,100)) - #remove 10 pixels from the top frame = frame[10:,:,:] img = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) img = cv2.resize(img, (100,100)) gray = cv2.GaussianBlur(img, (3, 3), 0) image = gray.reshape(1,1, 100, 100) image = torch.from_numpy(image).float() - #make numpy array image = image.numpy() outputs = ort_sess.run(None, {'input': image}) end = time.time() output = outputs[0][0] output = output.tolist() - print(output[0]) - output = [x * 100 for x in output] - client.send_message("/cheekPuff", output[0]) - client.send_message("/cheekSquintLeft", output[1]) - client.send_message("/cheekSquintRight", output[2]) - client.send_message("/noseSneerLeft", output[3]) - client.send_message("/noseSneerRight", output[4]) - client.send_message("/jawOpen", output[5]) - client.send_message("/jawForward", output[6]) - client.send_message("/jawLeft", output[7]) - client.send_message("/jawRight", output[8]) - client.send_message("/mouthFunnel", output[9]) - client.send_message("/mouthPucker", output[10]) - client.send_message("/mouthLeft", output[11]) - client.send_message("/mouthRight", output[12]) - client.send_message("/mouthRollUpper", output[13]) - client.send_message("/mouthRollLower", output[14]) - client.send_message("/mouthShrugUpper", output[15]) - client.send_message("/mouthShrugLower", output[16]) - client.send_message("/mouthClose", output[17]) - client.send_message("/mouthSmileLeft", output[18]) - client.send_message("/mouthSmileRight", output[19]) - client.send_message("/mouthFrownLeft", output[20]) - client.send_message("/mouthFrownRight", output[21]) - client.send_message("/mouthDimpleLeft", output[22]) - client.send_message("/mouthDimpleRight", output[23]) - client.send_message("/mouthUpperUpLeft", output[24]) - client.send_message("/mouthUpperUpRight", output[25]) - client.send_message("/mouthLowerDownLeft", output[26]) - client.send_message("/mouthLowerDownRight", output[27]) - client.send_message("/mouthPressLeft", output[28]) - client.send_message("/mouthPressRight", output[29]) - client.send_message("/mouthStretchLeft", output[30]) - client.send_message("/mouthStretchRight", output[31]) - client.send_message("/tongueOut", output[32]) - #show image - cv2.imshow("frame", frame) + + # For Unity, we need to multiply the output by 100 + output = [x * 100 for x in output] + + for i in range(len(output)): + client.send_message(f"/{classes[i]}", output[i]) + print(f"{classes[i]} : {output[i]}") + + cv2.imshow("Frame", frame) if cv2.waitKey(1) & 0xFF == ord('q'): break + cap.release() cv2.destroyAllWindows() \ No newline at end of file diff --git a/run.bat b/run.bat new file mode 100644 index 0000000..1415003 --- /dev/null +++ b/run.bat @@ -0,0 +1,2 @@ +python onnxinference.py +pause \ No newline at end of file