netural-input code

This commit is contained in:
Summer committed 2024-11-12 12:50:13 -08:00
1 parent e24de3b21b
commit 3f92864d35
13 files changed
+117 -22

No files matched your search

+1
View File
@@ -6,6 +6,7 @@ BabbleApp/babble_settings.json
BabbleApp/babble_settings.backup
BabbleApp/build
BabbleApp/dist
BabbleApp/UserData
/vivetest
/training
.vscode
+1
View File
@@ -113,6 +113,7 @@
"camera.waitingOnMouthCrop": "Awating Mouth Crop",
"camera.calibration": "Calibration",
"camera.tracking": "Tracking",
"camera.neutralface": "Model Face Calibraion",
"config.noSettingsFile": "No settings file, using base settings",
"config.failedToLoadSettings": "Failed to load settings file",
"config.usingBackupSettings": "Using backup settings",
+1
View File
@@ -113,6 +113,7 @@
"camera.waitingOnMouthCrop": "Esperando Recorte de Boca",
"camera.calibration": "Calibración",
"camera.tracking": "Seguimiento",
"camera.neutralface": "Capturando la cara",
"config.noSettingsFile": "No hay archivo de configuración, usando configuración base",
"config.failedToLoadSettings": "Error al cargar el archivo de configuración",
"config.usingBackupSettings": "Usando configuración de respaldo",
+2 -1
View File
@@ -59,7 +59,7 @@
"info.exitCaptureThread": "Exiting capture thread",
"info.exitTrackingThread": "Exiting tracking thread",
"info.serialCapture": "Fawled to gwet seiewal pictower 3:",
"log.info": "INFO",
"log.info": "INFO",
"log.warn": "WARN",
"log.error": "ERROR",
"algorithm.header": "Modew Settings :3",
@@ -113,6 +113,7 @@
"camera.waitingOnMouthCrop": "Awaiting Mouth Cwop :3",
"camera.calibration": "Cawibwation OwO",
"camera.tracking": "Twacking UwU",
"camera.neutralface": "Modew Face nya~",
"config.noSettingsFile": "No settings file, using base settings",
"config.failedToLoadSettings": "Failed to load settings file",
"config.usingBackupSettings": "Using backup settings",
Binary file not shown.
+50 -10
View File
@@ -1,27 +1,35 @@
import os
import json
os.environ["OMP_NUM_THREADS"] = "1"
import onnxruntime as ort
import time
import PySimpleGUI as sg
import cv2
import numpy as np
from pythonosc import udp_client
import utils.image_transforms as transforms
import PIL.Image as Image
from threading import Thread
from one_euro_filter import OneEuroFilter
def run_model(self):
def run_model(self, embedding, netural_blends):
if self.runtime in ("ONNX", "Default (ONNX)"):
frame = cv2.resize(self.current_image_gray, (256, 256))
frame = transforms.to_tensor(frame)
frame = transforms.unsqueeze(frame, 0)
out = self.sess.run([self.output_name], {self.input_name: frame})
output = out[0][0]
frame = transforms.normalize(frame)
print(frame)
frame = frame * 2 - 1
#make the image 3 channels but concat
frame = np.concatenate((frame, frame, frame), axis=1)
output_emb = self.sess_encoder.run([self.output_name_encoder], {self.input_name_encoder: frame})
output_emb = output_emb[0]
#concat output_emb, and embedding from 1,128 in both to 1.256
output_emb = np.concatenate((output_emb, embedding), axis=-1)
#pass it to the model head
output = self.sess_head.run([self.output_name_head], {self.input_name_head: output_emb})
output = output[0][0] - netural_blends
output = self.one_euro_filter(output)
# for i in range(len(output)): # Clip values between 0 - 1
@@ -29,3 +37,35 @@ def run_model(self):
## Clip values between 0 - 1
output = np.clip(output, 0, 1)
self.output = output
def run_model_embeding(self):
if self.runtime in ("ONNX", "Default (ONNX)"):
#load BabbleApp\UserData\neutral_frame.jpg
frame = cv2.imread("UserData/neutral_frame.jpg")
#convert to black and white
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
#convert back to color
#frame = cv2.cvtColor(frame, cv2.COLOR_GRAY2BGR)
if frame is None:
print("No frame")
frame = np.zeros((256, 256, 3), dtype=np.uint8)
frame = cv2.resize(frame, (256, 256))
frame = transforms.to_tensor(frame)
frame = transforms.unsqueeze(frame, 0)
frame = transforms.normalize(frame)
frame = frame * 2 - 1
frame = np.concatenate((frame, frame, frame), axis=1)
#convert to color by duplicating the channels
out = self.sess_encoder.run([self.output_name_encoder], {self.input_name_encoder: frame})
out = out[0]
output_emb = np.concatenate((out, out), axis=-1)
output_netural = self.sess_head.run([self.output_name_head], {self.input_name_head: output_emb})
output = out
output_netural = output_netural[0][0]
#save to np array in UserData
np.save("UserData/model_embeding.npy", output) #TODO: Expose this in settings
np.save("UserData/netural.npy", output_netural) #TODO: Expose this in settings
+18 -6
View File
@@ -103,14 +103,23 @@ class BabbleProcessor:
provider = "DmlExecutionProvider"
else:
provider = "CPUExecutionProvider" # Build onnxruntime to get both DML and OpenVINO
self.sess = ort.InferenceSession(
f"{self.model}onnx/model.onnx",
self.sess_encoder = ort.InferenceSession(
f"{self.model}onnx/encoder.onnx",
self.opts,
providers=[provider],
provider_options=[{"device_id": self.gpu_index}],
)
self.input_name = self.sess.get_inputs()[0].name
self.output_name = self.sess.get_outputs()[0].name
self.sess_head = ort.InferenceSession(
f"{self.model}onnx/head_with_selu.onnx",
self.opts,
providers=[provider],
provider_options=[{"device_id": self.gpu_index}],
)
self.input_name_encoder = self.sess_encoder.get_inputs()[0].name
self.output_name_encoder = self.sess_encoder.get_outputs()[0].name
self.input_name_head = self.sess_head.get_inputs()[0].name
self.output_name_head = self.sess_head.get_outputs()[0].name
try:
min_cutoff = float(self.settings.gui_min_cutoff)
beta = float(self.settings.gui_speed_coefficient)
@@ -207,7 +216,10 @@ class BabbleProcessor:
pass
def run(self):
run_model_embeding(self)
#load the model embeding. It's a NPY
embed_netural = np.load('UserData/model_embeding.npy')
netural_blends = np.load('UserData/netural.npy')
while True:
# Check to make sure we haven't been requested to close
if self.cancellation_event.is_set():
@@ -253,7 +265,7 @@ class BabbleProcessor:
self.current_image_gray.copy()
) # copy this frame to have a clean image for blink algo
run_model(self)
run_model(self, embed_netural, netural_blends)
if self.settings.use_calibration:
self.output = cal.cal_osc(self, self.output)
# else:
+1 -1
View File
@@ -264,7 +264,7 @@ def main():
try:
# If window isn't in focus increase timeout and stop loop early
if window.TKroot.focus_get():
if window.TKroot.focus_get():
if fs:
fs = False
tint = 33
+11
View File
@@ -13,6 +13,7 @@ from config import BabbleConfig, BabbleSettingsConfig
from utils.misc_utils import get_camera_index_by_name, list_camera_names
from enum import Enum
import sys
import os
WAIT_TIME = 0.1
BUFFER_SIZE = 32768
@@ -302,3 +303,13 @@ class Camera:
)
self.camera_output_outgoing.put((self.clamp_max_res(image), frame_number, fps))
self.capture_event.clear()
def save_frame(self, destination_folder, frame_name, window_x, window_y, window_w, window_h):
#get camera data from buffer
image = self.camera_output_outgoing.get()[0] #this feels wrong but it's easy
#crop image to ROI
image = image[window_y:window_y+window_h, window_x:window_x+window_w]
if not os.path.exists(destination_folder):
os.makedirs(destination_folder)
cv2.imwrite(os.path.join(destination_folder, frame_name), image)
+28 -3
View File
@@ -43,6 +43,8 @@ class CameraWidget:
self.gui_horizontal_flip = f"-HORIZONTALFLIP{widget_id}-"
self.use_calibration = f"-USECALIBRATION{widget_id}-"
self.gui_refresh_button = f"-REFRESHCAMLIST{widget_id}-"
self.neural_face_frame_calibration_button = f"-FACEID{widget_id}-"
self.osc_queue = osc_queue
self.main_config = main_config
self.cam_id = widget_id
@@ -197,8 +199,10 @@ class CameraWidget:
f'{lang._instance.get_string("camera.crop")}:',
key=self.gui_roi_message,
background_color=bg_color_highlight,
font=("Helvetica", 16, "bold"),
text_color="red", # Use a color that contrasts with bg_color_highlight
visible=False,
),
)
],
]
@@ -242,6 +246,11 @@ class CameraWidget:
button_color=button_color,
tooltip=f'{lang._instance.get_string("camera.croppingModeToolTip")}.',
),
sg.Button(
lang._instance.get_string("camera.neutralface"),
key=self.neural_face_frame_calibration_button,
button_color=button_color,
),
],
[
sg.Column(
@@ -298,6 +307,7 @@ class CameraWidget:
self.camera_thread.join()
def render(self, window, event, values):
if self.image_queue.qsize() > 2:
with self.image_queue.mutex:
self.image_queue.queue.clear()
@@ -368,6 +378,8 @@ class CameraWidget:
if self.settings_config.use_calibration != values[self.use_calibration]:
self.settings_config.use_calibration = values[self.use_calibration]
changed = True
if changed:
self.main_config.save()
@@ -416,7 +428,7 @@ class CameraWidget:
if event == self.gui_roi_selection:
# Event for mouse button down or mouse drag in ROI mode
if self.is_mouse_up:
if self.is_mouse_up:
self.is_mouse_up = False
self.x0, self.y0 = values[self.gui_roi_selection]
self.x1, self.y1 = values[self.gui_roi_selection]
@@ -453,6 +465,19 @@ class CameraWidget:
needs_roi_set = self.config.roi_window_h <= 0 or self.config.roi_window_w <= 0
if event == self.neural_face_frame_calibration_button:
if needs_roi_set:
return
if self.config.neutral_frame == None:
self.camera.save_frame("UserData", "neutral_frame.jpg", self.config.roi_window_x, self.config.roi_window_y, self.config.roi_window_w, self.config.roi_window_h)
self.config.neutral_frame = "UserData/neutral_frame.jpg" #TODO: Have this exposed in the settings widget
else:
self.camera.save_frame("UserData", "neutral_frame.jpg", self.config.roi_window_x, self.config.roi_window_y, self.config.roi_window_w, self.config.roi_window_h)
changed = True
# TODO: Refactor if statements below...
window[self.gui_tracking_fps].update("")
window[self.gui_tracking_bps].update("")
@@ -478,7 +503,7 @@ class CameraWidget:
window[self.gui_mode_readout].update(
lang._instance.get_string("camera.calibration")
)
else:
else:
window[self.gui_mode_readout].update(
lang._instance.get_string("camera.tracking")
)
+3
View File
@@ -22,6 +22,8 @@ class BabbleCameraConfig(BaseModel):
gui_vertical_flip: bool = False
gui_horizontal_flip: bool = False
use_ffmpeg: bool = False
neutral_frame: str = None
class BabbleSettingsConfig(BaseModel):
@@ -53,6 +55,7 @@ class BabbleSettingsConfig(BaseModel):
gui_language: str = "English"
class BabbleConfig(BaseModel):
version: int = 1
cam: BabbleCameraConfig = BabbleCameraConfig()
+1 -1
View File
@@ -38,7 +38,7 @@ def to_tensor(numpy_array, dtype=np.float32):
numpy_array = numpy_array[:, :, np.newaxis]
# Transpose the array to match PyTorch tensor format (C x H x W)
tensor = normalize(np.transpose(numpy_array, (2, 0, 1)))
tensor = np.transpose(numpy_array, (2, 0, 1))
return tensor