Merge pull request #20 from RamesTheGeneric/main

OpenVino for CPU inference
This commit is contained in:
Summer authored and GitHub committed 2023-07-24 13:32:46 -07:00
commit 664b24151c
8 files changed
+9863 -27

No files matched your search

+4
View File
@@ -9,3 +9,7 @@ BabbleApp/dist
/vivetest
/training
.vscode
/testing
scripts/example_build_app_and_installer.bat
scripts/example_build_app_and_installer.bat
scripts/installer.iss
Binary file not shown.
Binary file not shown.
File diff suppressed because it is too large. Load diff
+43 -19
View File
@@ -1,6 +1,7 @@
import os
import json
os.environ["OMP_NUM_THREADS"] = "1"
from openvino.inference_engine import IECore
import onnxruntime as ort
import time
import PySimpleGUI as sg
@@ -13,25 +14,48 @@ from torchvision import transforms
from threading import Thread
from one_euro_filter import OneEuroFilter
def run_model(self):
if not self.use_gpu:
frame = cv2.resize(self.current_image_gray, (256, 256))
# make it pil
frame = Image.fromarray(frame)
# make it grayscale
frame = to_grayscale(frame)
# make it a tensor
frame = transforms.ToTensor()(frame)
# make it a batch
frame = frame.unsqueeze(0)
# make it a numpy array
frame = frame.numpy()
out = self.sess.infer(inputs={self.input_name: frame})
#out = self.sess.run([self.output_name], {self.input_name: frame})
#end = time.time()
output = out[self.output_name]
output = output[0]
output = self.one_euro_filter(output)
for i in range(len(output)): # Clip values between 0 - 1
output[i] = max(min(output[i], 1), 0)
self.output = output
frame = cv2.resize(self.current_image_gray, (256, 256))
# make it pil
frame = Image.fromarray(frame)
# make it grayscale
frame = to_grayscale(frame)
# make it a tensor
frame = transforms.ToTensor()(frame)
# make it a batch
frame = frame.unsqueeze(0)
# make it a numpy array
frame = frame.numpy()
else:
frame = cv2.resize(self.current_image_gray, (256, 256))
# make it pil
frame = Image.fromarray(frame)
# make it grayscale
frame = to_grayscale(frame)
# make it a tensor
frame = transforms.ToTensor()(frame)
# make it a batch
frame = frame.unsqueeze(0)
# make it a numpy array
frame = frame.numpy()
out = self.sess.run([self.output_name], {self.input_name: frame})
#end = time.time()
output = out[0]
output = output[0]
output = self.one_euro_filter(output)
for i in range(len(output)): # Clip values between 0 - 1
output[i] = max(min(output[i], 1), 0)
self.output = output
out = self.sess.run([self.output_name], {self.input_name: frame})
#end = time.time()
output = out[0]
output = output[0]
output = self.one_euro_filter(output)
for i in range(len(output)): # Clip values between 0 - 1
output[i] = max(min(output[i], 1), 0)
self.output = output
+10 -6
View File
@@ -89,11 +89,15 @@ class BabbleProcessor:
self.opts.inter_op_num_threads = settings.gui_inference_threads
self.opts.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
if not self.use_gpu:
self.sess = ort.InferenceSession(self.model, self.opts, providers=['CPUExecutionProvider'])
ie = IECore()
net = ie.read_network(model=f'{self.model}openvino/model.xml', weights=f'{self.model}openvino/model.bin')
self.sess = ie.load_network(network=net, device_name='CPU')
self.input_name = next(iter(net.input_info))
self.output_name = next(iter(net.outputs))
else:
self.sess = ort.InferenceSession(self.model, self.opts, providers=['DmlExecutionProvider'])
self.input_name = self.sess.get_inputs()[0].name
self.output_name = self.sess.get_outputs()[0].name
self.sess = ort.InferenceSession(f'{self.model}onnx/model.onnx', self.opts, providers=['DmlExecutionProvider'])
self.input_name = self.sess.get_inputs()[0].name
self.output_name = self.sess.get_outputs()[0].name
try:
@@ -227,8 +231,8 @@ class BabbleProcessor:
run_model(self)
#if self.config.use_calibration:
cal.cal_osc(self, self.output)
if self.config.use_calibration:
self.output = cal.cal_osc(self, self.output)
#else:
# pass
+1 -1
View File
@@ -32,7 +32,7 @@ class BabbleSettingsConfig(BaseModel):
gui_ROSC: bool = False
gui_osc_location: str = ""
gui_multiply: int = 1
gui_model_file: str = 'Models/EFV2300K45E100P2.onnx'
gui_model_file: str = 'Models/Effv2E44TA/'
gui_use_gpu: bool = False
gui_inference_threads: int = 1
calib_array: str = None
+1 -1
View File
@@ -1,7 +1,7 @@
# ProjectBabble
A Mouth tracking system for VR, 100% Opensource!
Currently the architecture is a modified efficientnetv2-b0 training on blenshapes from ARKit! The model outputs 31 blenshapes of which most cross over with SRanipals blendshapes.
Currently the architecture is a modified efficientnetv2-b0 training on blenshapes from ARKit! The model outputs 45 blenshapes of which most cross over with SRanipals blendshapes.
### One thing to note
1. This model is not a 1 to 1 translation to SRanipal blenshapes. While most blendshapes do cross over with SRanipal cheek and tounge movements are limited.