This commit is contained in:
Summer committed 2022-08-19 08:02:25 -07:00
commit eb5ea7ba14
4 files changed
+24 -49

No files matched your search

+2
View File
@@ -150,3 +150,5 @@ cython_debug/
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
/.vs
+4 -3
View File
@@ -7,14 +7,15 @@ Currently the architecture is a moddified mobilenetv2 training on blenshapes fro
1. The current version of the model may not work for your mouth as it has been trained on a not very diverse dataset. Expect this to change in newer versions.
2. Not all blendshapes may work as there isnt enough data collected in the dataset.
3. No detailed setup guide will be provided until later more stable versions.
4. The (ML) model is currently un optimized so expect large amounts of CPU or GPU usage. This will be worked on in later versions
4. This model is not a 1 to 1 translation to SRanipal blenshapes. While most blendshapes do cross over with SRanipal cheek and tounge movements are limited.
### Warranty
Lol
### Current setup guide
1. Clone the repo
2. Install the dependencies in requirements.txt with python 3.8 (or newer).
3. run model_loader_no_crop.py
3. run onnxinference.py
4. run the unity build in the unity build folder
5. make a netural face and move the camera around your mouth area until the avatar in the unity app makes a netural face too.
6. Enjoy mouth tracking!
+16 -46
View File
@@ -1,76 +1,46 @@
import onnx
import os
import time
import torch
import onnx
import onnxruntime as ort
import numpy as np
import cv2
import time
from pythonosc import udp_client
OSCip="127.0.0.1"
OSCport= 9000 #VR Chat OSC port
OSCip = "127.0.0.1"
OSCport = 9000 #VR Chat OSC port
client = udp_client.SimpleUDPClient(OSCip, OSCport)
classes = ["cheekPuff", "cheekSquint_L", "cheekSquint_R", "noseSneer_L", "noseSneer_R", "jawOpen", "jawForward", "jawLeft", "jawRight", "mouthFunnel", "mouthPucker", "mouthLeft", "mouthRight", "mouthRollUpper", "mouthRollLower", "mouthShrugUpper", "mouthShrugLower", "mouthClose", "mouthSmile_L", "mouthSmile_R", "mouthFrown_L", "mouthFrown_R", "mouthDimple_L", "mouthDimple_R", "mouthUpperUp_L", "mouthUpperUp_R", "mouthLowerDown_L", "mouthLowerDown_R", "mouthPress_L", "mouthPress_R", "mouthStretch_L", "mouthStretch_R", "tongueOut"]
classes = ["cheekPuff", "cheekSquint_L", "cheekSquint_R", "noseSneer_L", "noseSneer_R", "jawOpen", "jawForward", "jawLeft", "jawRight", "mouthFunnel", "mouthPucker", "mouthLeft", "mouthRight", "mouthRollUpper", "mouthRollLower", "mouthShrugUpper", "mouthShrugLower", "mouthClose", "mouthSmile_L", "mouthSmile_R", "mouthFrown_L", "mouthFrown_R", "mouthDimple_L", "mouthDimple_R", "mouthUpperUp_L", "mouthUpperUp_R", "mouthLowerDown_L", "mouthLowerDown_R", "mouthPress_L", "mouthPress_R", "mouthStretch_L", "mouthStretch_R", "tongueOut"]
cap = cv2.VideoCapture(0)
sessionOptions = ort.SessionOptions()
ort_sess = ort.InferenceSession("modelv2.onnx", providers = ['CPUExecutionProvider'])
while True:
#read the frame
ret, frame = cap.read()
#convert the frame to gr ayscale
frame = cv2.resize(frame, (100,100))
#remove 10 pixels from the top
frame = frame[10:,:,:]
img = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
img = cv2.resize(img, (100,100))
gray = cv2.GaussianBlur(img, (3, 3), 0)
image = gray.reshape(1,1, 100, 100)
image = torch.from_numpy(image).float()
#make numpy array
image = image.numpy()
outputs = ort_sess.run(None, {'input': image})
end = time.time()
output = outputs[0][0]
output = output.tolist()
print(output[0])
output = [x * 100 for x in output]
client.send_message("/cheekPuff", output[0])
client.send_message("/cheekSquintLeft", output[1])
client.send_message("/cheekSquintRight", output[2])
client.send_message("/noseSneerLeft", output[3])
client.send_message("/noseSneerRight", output[4])
client.send_message("/jawOpen", output[5])
client.send_message("/jawForward", output[6])
client.send_message("/jawLeft", output[7])
client.send_message("/jawRight", output[8])
client.send_message("/mouthFunnel", output[9])
client.send_message("/mouthPucker", output[10])
client.send_message("/mouthLeft", output[11])
client.send_message("/mouthRight", output[12])
client.send_message("/mouthRollUpper", output[13])
client.send_message("/mouthRollLower", output[14])
client.send_message("/mouthShrugUpper", output[15])
client.send_message("/mouthShrugLower", output[16])
client.send_message("/mouthClose", output[17])
client.send_message("/mouthSmileLeft", output[18])
client.send_message("/mouthSmileRight", output[19])
client.send_message("/mouthFrownLeft", output[20])
client.send_message("/mouthFrownRight", output[21])
client.send_message("/mouthDimpleLeft", output[22])
client.send_message("/mouthDimpleRight", output[23])
client.send_message("/mouthUpperUpLeft", output[24])
client.send_message("/mouthUpperUpRight", output[25])
client.send_message("/mouthLowerDownLeft", output[26])
client.send_message("/mouthLowerDownRight", output[27])
client.send_message("/mouthPressLeft", output[28])
client.send_message("/mouthPressRight", output[29])
client.send_message("/mouthStretchLeft", output[30])
client.send_message("/mouthStretchRight", output[31])
client.send_message("/tongueOut", output[32])
#show image
cv2.imshow("frame", frame)
# For Unity, we need to multiply the output by 100
output = [x * 100 for x in output]
for i in range(len(output)):
client.send_message(f"/{classes[i]}", output[i])
print(f"{classes[i]} : {output[i]}")
cv2.imshow("Frame", frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cap.release()
cv2.destroyAllWindows()
+2
View File
@@ -0,0 +1,2 @@
python onnxinference.py
pause