From d94bf39e144e634603bee079203924199162b5ea Mon Sep 17 00:00:00 2001 From: dfgHiatus <51272212+dfgHiatus@users.noreply.github.com> Date: Wed, 10 Aug 2022 16:49:11 -0500 Subject: [PATCH 1/2] Refactor --- .gitignore | 2 ++ onnxinference.py | 63 +++++++++++++----------------------------------- run.bat | 2 ++ 3 files changed, 21 insertions(+), 46 deletions(-) create mode 100644 run.bat diff --git a/.gitignore b/.gitignore index d9005f2..f4ebf71 100644 --- a/.gitignore +++ b/.gitignore @@ -150,3 +150,5 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +/.vs diff --git a/onnxinference.py b/onnxinference.py index 1c54616..e7068ab 100644 --- a/onnxinference.py +++ b/onnxinference.py @@ -1,76 +1,47 @@ -import onnx +import os +import time import torch +import onnx import onnxruntime as ort import numpy as np import cv2 -import time from pythonosc import udp_client -OSCip="127.0.0.1" -OSCport= 9000 #VR Chat OSC port +OSCip = "127.0.0.1" +OSCport = 9000 #VR Chat OSC port client = udp_client.SimpleUDPClient(OSCip, OSCport) -classes = ["cheekPuff", "cheekSquint_L", "cheekSquint_R", "noseSneer_L", "noseSneer_R", "jawOpen", "jawForward", "jawLeft", "jawRight", "mouthFunnel", "mouthPucker", "mouthLeft", "mouthRight", "mouthRollUpper", "mouthRollLower", "mouthShrugUpper", "mouthShrugLower", "mouthClose", "mouthSmile_L", "mouthSmile_R", "mouthFrown_L", "mouthFrown_R", "mouthDimple_L", "mouthDimple_R", "mouthUpperUp_L", "mouthUpperUp_R", "mouthLowerDown_L", "mouthLowerDown_R", "mouthPress_L", "mouthPress_R", "mouthStretch_L", "mouthStretch_R", "tongueOut"] +classes = ["cheekPuff", "cheekSquint_L", "cheekSquint_R", "noseSneer_L", "noseSneer_R", "jawOpen", "jawForward", "jawLeft", "jawRight", "mouthFunnel", "mouthPucker", "mouthLeft", "mouthRight", "mouthRollUpper", "mouthRollLower", "mouthShrugUpper", "mouthShrugLower", "mouthClose", "mouthSmile_L", "mouthSmile_R", "mouthFrown_L", "mouthFrown_R", "mouthDimple_L", "mouthDimple_R", "mouthUpperUp_L", "mouthUpperUp_R", "mouthLowerDown_L", "mouthLowerDown_R", "mouthPress_L", "mouthPress_R", "mouthStretch_L", "mouthStretch_R", "tongueOut"] cap = cv2.VideoCapture(0) sessionOptions = ort.SessionOptions() ort_sess = ort.InferenceSession("modelv2.2.onnx", providers = ['CPUExecutionProvider']) + while True: - #read the frame ret, frame = cap.read() - #convert the frame to gr ayscale frame = cv2.resize(frame, (100,100)) - #remove 10 pixels from the top frame = frame[10:,:,:] img = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) img = cv2.resize(img, (100,100)) gray = cv2.GaussianBlur(img, (3, 3), 0) image = gray.reshape(1,1, 100, 100) image = torch.from_numpy(image).float() - #make numpy array image = image.numpy() outputs = ort_sess.run(None, {'input': image}) end = time.time() output = outputs[0][0] output = output.tolist() - print(output[0]) - output = [x * 100 for x in output] - client.send_message("/cheekPuff", output[0]) - client.send_message("/cheekSquintLeft", output[1]) - client.send_message("/cheekSquintRight", output[2]) - client.send_message("/noseSneerLeft", output[3]) - client.send_message("/noseSneerRight", output[4]) - client.send_message("/jawOpen", output[5]) - client.send_message("/jawForward", output[6]) - client.send_message("/jawLeft", output[7]) - client.send_message("/jawRight", output[8]) - client.send_message("/mouthFunnel", output[9]) - client.send_message("/mouthPucker", output[10]) - client.send_message("/mouthLeft", output[11]) - client.send_message("/mouthRight", output[12]) - client.send_message("/mouthRollUpper", output[13]) - client.send_message("/mouthRollLower", output[14]) - client.send_message("/mouthShrugUpper", output[15]) - client.send_message("/mouthShrugLower", output[16]) - client.send_message("/mouthClose", output[17]) - client.send_message("/mouthSmileLeft", output[18]) - client.send_message("/mouthSmileRight", output[19]) - client.send_message("/mouthFrownLeft", output[20]) - client.send_message("/mouthFrownRight", output[21]) - client.send_message("/mouthDimpleLeft", output[22]) - client.send_message("/mouthDimpleRight", output[23]) - client.send_message("/mouthUpperUpLeft", output[24]) - client.send_message("/mouthUpperUpRight", output[25]) - client.send_message("/mouthLowerDownLeft", output[26]) - client.send_message("/mouthLowerDownRight", output[27]) - client.send_message("/mouthPressLeft", output[28]) - client.send_message("/mouthPressRight", output[29]) - client.send_message("/mouthStretchLeft", output[30]) - client.send_message("/mouthStretchRight", output[31]) - client.send_message("/tongueOut", output[32]) - #show image - cv2.imshow("frame", frame) + + # For Unity, we need to multiply the output by 100 + output = [x * 100 for x in output] + + for i in range(len(output)): + client.send_message(f"/{classes[i]}", output[i]) + print(f"{classes[i]} : {output[i]}") + + cv2.imshow("Frame", frame) if cv2.waitKey(1) & 0xFF == ord('q'): break + cap.release() cv2.destroyAllWindows() \ No newline at end of file diff --git a/run.bat b/run.bat new file mode 100644 index 0000000..1415003 --- /dev/null +++ b/run.bat @@ -0,0 +1,2 @@ +python onnxinference.py +pause \ No newline at end of file From eaeaeae8e1f8ef4a6549aeecc99a7402a86b6653 Mon Sep 17 00:00:00 2001 From: Summer <71572746+SummerSigh@users.noreply.github.com> Date: Thu, 11 Aug 2022 12:26:26 -0700 Subject: [PATCH 2/2] Update README.md --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8896c49..637511e 100644 --- a/README.md +++ b/README.md @@ -7,14 +7,15 @@ Currently the architecture is a moddified mobilenetv2 training on blenshapes fro 1. The current version of the model may not work for your mouth as it has been trained on a not very diverse dataset. Expect this to change in newer versions. 2. Not all blendshapes may work as there isnt enough data collected in the dataset. 3. No detailed setup guide will be provided until later more stable versions. -4. The (ML) model is currently un optimized so expect large amounts of CPU or GPU usage. This will be worked on in later versions - +4. This model is not a 1 to 1 translation to SRanipal blenshapes. While most blendshapes do cross over with SRanipal cheek and tounge movements are limited. +### Warranty +Lol ### Current setup guide 1. Clone the repo 2. Install the dependencies in requirements.txt with python 3.8 (or newer). -3. run model_loader_no_crop.py +3. run onnxinference.py 4. run the unity build in the unity build folder 5. make a netural face and move the camera around your mouth area until the avatar in the unity app makes a netural face too. 6. Enjoy mouth tracking!