mirror of
https://github.com/Project-Babble/ProjectBabble.git
synced 2026-11-20 17:11:16 -08:00
Merge pull request #20 from RamesTheGeneric/main
OpenVino for CPU inference
This commit is contained in:
8 files changed
+9863
-27
No files matched your search
@@ -9,3 +9,7 @@ BabbleApp/dist
|
||||
/vivetest
|
||||
/training
|
||||
.vscode
|
||||
/testing
|
||||
scripts/example_build_app_and_installer.bat
|
||||
scripts/example_build_app_and_installer.bat
|
||||
scripts/installer.iss
|
||||
Binary file not shown.
Binary file not shown.
File diff suppressed because it is too large.
Load diff
@@ -1,6 +1,7 @@
|
||||
import os
|
||||
import json
|
||||
os.environ["OMP_NUM_THREADS"] = "1"
|
||||
from openvino.inference_engine import IECore
|
||||
import onnxruntime as ort
|
||||
import time
|
||||
import PySimpleGUI as sg
|
||||
@@ -13,25 +14,48 @@ from torchvision import transforms
|
||||
from threading import Thread
|
||||
from one_euro_filter import OneEuroFilter
|
||||
|
||||
|
||||
def run_model(self):
|
||||
if not self.use_gpu:
|
||||
frame = cv2.resize(self.current_image_gray, (256, 256))
|
||||
# make it pil
|
||||
frame = Image.fromarray(frame)
|
||||
# make it grayscale
|
||||
frame = to_grayscale(frame)
|
||||
# make it a tensor
|
||||
frame = transforms.ToTensor()(frame)
|
||||
# make it a batch
|
||||
frame = frame.unsqueeze(0)
|
||||
# make it a numpy array
|
||||
frame = frame.numpy()
|
||||
out = self.sess.infer(inputs={self.input_name: frame})
|
||||
#out = self.sess.run([self.output_name], {self.input_name: frame})
|
||||
#end = time.time()
|
||||
output = out[self.output_name]
|
||||
output = output[0]
|
||||
output = self.one_euro_filter(output)
|
||||
for i in range(len(output)): # Clip values between 0 - 1
|
||||
output[i] = max(min(output[i], 1), 0)
|
||||
self.output = output
|
||||
|
||||
frame = cv2.resize(self.current_image_gray, (256, 256))
|
||||
# make it pil
|
||||
frame = Image.fromarray(frame)
|
||||
# make it grayscale
|
||||
frame = to_grayscale(frame)
|
||||
# make it a tensor
|
||||
frame = transforms.ToTensor()(frame)
|
||||
# make it a batch
|
||||
frame = frame.unsqueeze(0)
|
||||
# make it a numpy array
|
||||
frame = frame.numpy()
|
||||
else:
|
||||
frame = cv2.resize(self.current_image_gray, (256, 256))
|
||||
# make it pil
|
||||
frame = Image.fromarray(frame)
|
||||
# make it grayscale
|
||||
frame = to_grayscale(frame)
|
||||
# make it a tensor
|
||||
frame = transforms.ToTensor()(frame)
|
||||
# make it a batch
|
||||
frame = frame.unsqueeze(0)
|
||||
# make it a numpy array
|
||||
frame = frame.numpy()
|
||||
|
||||
out = self.sess.run([self.output_name], {self.input_name: frame})
|
||||
#end = time.time()
|
||||
output = out[0]
|
||||
output = output[0]
|
||||
output = self.one_euro_filter(output)
|
||||
for i in range(len(output)): # Clip values between 0 - 1
|
||||
output[i] = max(min(output[i], 1), 0)
|
||||
self.output = output
|
||||
out = self.sess.run([self.output_name], {self.input_name: frame})
|
||||
#end = time.time()
|
||||
output = out[0]
|
||||
output = output[0]
|
||||
output = self.one_euro_filter(output)
|
||||
for i in range(len(output)): # Clip values between 0 - 1
|
||||
output[i] = max(min(output[i], 1), 0)
|
||||
self.output = output
|
||||
@@ -89,11 +89,15 @@ class BabbleProcessor:
|
||||
self.opts.inter_op_num_threads = settings.gui_inference_threads
|
||||
self.opts.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
if not self.use_gpu:
|
||||
self.sess = ort.InferenceSession(self.model, self.opts, providers=['CPUExecutionProvider'])
|
||||
ie = IECore()
|
||||
net = ie.read_network(model=f'{self.model}openvino/model.xml', weights=f'{self.model}openvino/model.bin')
|
||||
self.sess = ie.load_network(network=net, device_name='CPU')
|
||||
self.input_name = next(iter(net.input_info))
|
||||
self.output_name = next(iter(net.outputs))
|
||||
else:
|
||||
self.sess = ort.InferenceSession(self.model, self.opts, providers=['DmlExecutionProvider'])
|
||||
self.input_name = self.sess.get_inputs()[0].name
|
||||
self.output_name = self.sess.get_outputs()[0].name
|
||||
self.sess = ort.InferenceSession(f'{self.model}onnx/model.onnx', self.opts, providers=['DmlExecutionProvider'])
|
||||
self.input_name = self.sess.get_inputs()[0].name
|
||||
self.output_name = self.sess.get_outputs()[0].name
|
||||
|
||||
|
||||
try:
|
||||
@@ -227,8 +231,8 @@ class BabbleProcessor:
|
||||
|
||||
|
||||
run_model(self)
|
||||
#if self.config.use_calibration:
|
||||
cal.cal_osc(self, self.output)
|
||||
if self.config.use_calibration:
|
||||
self.output = cal.cal_osc(self, self.output)
|
||||
|
||||
#else:
|
||||
# pass
|
||||
|
||||
+1
-1
@@ -32,7 +32,7 @@ class BabbleSettingsConfig(BaseModel):
|
||||
gui_ROSC: bool = False
|
||||
gui_osc_location: str = ""
|
||||
gui_multiply: int = 1
|
||||
gui_model_file: str = 'Models/EFV2300K45E100P2.onnx'
|
||||
gui_model_file: str = 'Models/Effv2E44TA/'
|
||||
gui_use_gpu: bool = False
|
||||
gui_inference_threads: int = 1
|
||||
calib_array: str = None
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# ProjectBabble
|
||||
A Mouth tracking system for VR, 100% Opensource!
|
||||
|
||||
Currently the architecture is a modified efficientnetv2-b0 training on blenshapes from ARKit! The model outputs 31 blenshapes of which most cross over with SRanipals blendshapes.
|
||||
Currently the architecture is a modified efficientnetv2-b0 training on blenshapes from ARKit! The model outputs 45 blenshapes of which most cross over with SRanipals blendshapes.
|
||||
|
||||
### One thing to note
|
||||
1. This model is not a 1 to 1 translation to SRanipal blenshapes. While most blendshapes do cross over with SRanipal cheek and tounge movements are limited.
|
||||
|
||||
Reference in new issue
Block a user