From d94bf39e144e634603bee079203924199162b5ea Mon Sep 17 00:00:00 2001 From: dfgHiatus <51272212+dfgHiatus@users.noreply.github.com> Date: Wed, 10 Aug 2022 16:49:11 -0500 Subject: [PATCH] Refactor --- .gitignore | 2 ++ onnxinference.py | 63 +++++++++++++----------------------------------- run.bat | 2 ++ 3 files changed, 21 insertions(+), 46 deletions(-) create mode 100644 run.bat diff --git a/.gitignore b/.gitignore index d9005f2..f4ebf71 100644 --- a/.gitignore +++ b/.gitignore @@ -150,3 +150,5 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +/.vs diff --git a/onnxinference.py b/onnxinference.py index 1c54616..e7068ab 100644 --- a/onnxinference.py +++ b/onnxinference.py @@ -1,76 +1,47 @@ -import onnx +import os +import time import torch +import onnx import onnxruntime as ort import numpy as np import cv2 -import time from pythonosc import udp_client -OSCip="127.0.0.1" -OSCport= 9000 #VR Chat OSC port +OSCip = "127.0.0.1" +OSCport = 9000 #VR Chat OSC port client = udp_client.SimpleUDPClient(OSCip, OSCport) -classes = ["cheekPuff", "cheekSquint_L", "cheekSquint_R", "noseSneer_L", "noseSneer_R", "jawOpen", "jawForward", "jawLeft", "jawRight", "mouthFunnel", "mouthPucker", "mouthLeft", "mouthRight", "mouthRollUpper", "mouthRollLower", "mouthShrugUpper", "mouthShrugLower", "mouthClose", "mouthSmile_L", "mouthSmile_R", "mouthFrown_L", "mouthFrown_R", "mouthDimple_L", "mouthDimple_R", "mouthUpperUp_L", "mouthUpperUp_R", "mouthLowerDown_L", "mouthLowerDown_R", "mouthPress_L", "mouthPress_R", "mouthStretch_L", "mouthStretch_R", "tongueOut"] +classes = ["cheekPuff", "cheekSquint_L", "cheekSquint_R", "noseSneer_L", "noseSneer_R", "jawOpen", "jawForward", "jawLeft", "jawRight", "mouthFunnel", "mouthPucker", "mouthLeft", "mouthRight", "mouthRollUpper", "mouthRollLower", "mouthShrugUpper", "mouthShrugLower", "mouthClose", "mouthSmile_L", "mouthSmile_R", "mouthFrown_L", "mouthFrown_R", "mouthDimple_L", "mouthDimple_R", "mouthUpperUp_L", "mouthUpperUp_R", "mouthLowerDown_L", "mouthLowerDown_R", "mouthPress_L", "mouthPress_R", "mouthStretch_L", "mouthStretch_R", "tongueOut"] cap = cv2.VideoCapture(0) sessionOptions = ort.SessionOptions() ort_sess = ort.InferenceSession("modelv2.2.onnx", providers = ['CPUExecutionProvider']) + while True: - #read the frame ret, frame = cap.read() - #convert the frame to gr ayscale frame = cv2.resize(frame, (100,100)) - #remove 10 pixels from the top frame = frame[10:,:,:] img = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) img = cv2.resize(img, (100,100)) gray = cv2.GaussianBlur(img, (3, 3), 0) image = gray.reshape(1,1, 100, 100) image = torch.from_numpy(image).float() - #make numpy array image = image.numpy() outputs = ort_sess.run(None, {'input': image}) end = time.time() output = outputs[0][0] output = output.tolist() - print(output[0]) - output = [x * 100 for x in output] - client.send_message("/cheekPuff", output[0]) - client.send_message("/cheekSquintLeft", output[1]) - client.send_message("/cheekSquintRight", output[2]) - client.send_message("/noseSneerLeft", output[3]) - client.send_message("/noseSneerRight", output[4]) - client.send_message("/jawOpen", output[5]) - client.send_message("/jawForward", output[6]) - client.send_message("/jawLeft", output[7]) - client.send_message("/jawRight", output[8]) - client.send_message("/mouthFunnel", output[9]) - client.send_message("/mouthPucker", output[10]) - client.send_message("/mouthLeft", output[11]) - client.send_message("/mouthRight", output[12]) - client.send_message("/mouthRollUpper", output[13]) - client.send_message("/mouthRollLower", output[14]) - client.send_message("/mouthShrugUpper", output[15]) - client.send_message("/mouthShrugLower", output[16]) - client.send_message("/mouthClose", output[17]) - client.send_message("/mouthSmileLeft", output[18]) - client.send_message("/mouthSmileRight", output[19]) - client.send_message("/mouthFrownLeft", output[20]) - client.send_message("/mouthFrownRight", output[21]) - client.send_message("/mouthDimpleLeft", output[22]) - client.send_message("/mouthDimpleRight", output[23]) - client.send_message("/mouthUpperUpLeft", output[24]) - client.send_message("/mouthUpperUpRight", output[25]) - client.send_message("/mouthLowerDownLeft", output[26]) - client.send_message("/mouthLowerDownRight", output[27]) - client.send_message("/mouthPressLeft", output[28]) - client.send_message("/mouthPressRight", output[29]) - client.send_message("/mouthStretchLeft", output[30]) - client.send_message("/mouthStretchRight", output[31]) - client.send_message("/tongueOut", output[32]) - #show image - cv2.imshow("frame", frame) + + # For Unity, we need to multiply the output by 100 + output = [x * 100 for x in output] + + for i in range(len(output)): + client.send_message(f"/{classes[i]}", output[i]) + print(f"{classes[i]} : {output[i]}") + + cv2.imshow("Frame", frame) if cv2.waitKey(1) & 0xFF == ord('q'): break + cap.release() cv2.destroyAllWindows() \ No newline at end of file diff --git a/run.bat b/run.bat new file mode 100644 index 0000000..1415003 --- /dev/null +++ b/run.bat @@ -0,0 +1,2 @@ +python onnxinference.py +pause \ No newline at end of file