mirror of
https://github.com/Project-Babble/ProjectBabble.git
synced 2026-11-20 09:01:16 -08:00
netural-input code
This commit is contained in:
1 parent
e24de3b21b
commit
3f92864d35
13 files changed
+117
-22
No files matched your search
@@ -6,6 +6,7 @@ BabbleApp/babble_settings.json
|
||||
BabbleApp/babble_settings.backup
|
||||
BabbleApp/build
|
||||
BabbleApp/dist
|
||||
BabbleApp/UserData
|
||||
/vivetest
|
||||
/training
|
||||
.vscode
|
||||
|
||||
@@ -113,6 +113,7 @@
|
||||
"camera.waitingOnMouthCrop": "Awating Mouth Crop",
|
||||
"camera.calibration": "Calibration",
|
||||
"camera.tracking": "Tracking",
|
||||
"camera.neutralface": "Model Face Calibraion",
|
||||
"config.noSettingsFile": "No settings file, using base settings",
|
||||
"config.failedToLoadSettings": "Failed to load settings file",
|
||||
"config.usingBackupSettings": "Using backup settings",
|
||||
|
||||
@@ -113,6 +113,7 @@
|
||||
"camera.waitingOnMouthCrop": "Esperando Recorte de Boca",
|
||||
"camera.calibration": "Calibración",
|
||||
"camera.tracking": "Seguimiento",
|
||||
"camera.neutralface": "Capturando la cara",
|
||||
"config.noSettingsFile": "No hay archivo de configuración, usando configuración base",
|
||||
"config.failedToLoadSettings": "Error al cargar el archivo de configuración",
|
||||
"config.usingBackupSettings": "Usando configuración de respaldo",
|
||||
|
||||
@@ -59,7 +59,7 @@
|
||||
"info.exitCaptureThread": "Exiting capture thread",
|
||||
"info.exitTrackingThread": "Exiting tracking thread",
|
||||
"info.serialCapture": "Fawled to gwet seiewal pictower 3:",
|
||||
"log.info": "INFO",
|
||||
"log.info": "INFO",
|
||||
"log.warn": "WARN",
|
||||
"log.error": "ERROR",
|
||||
"algorithm.header": "Modew Settings :3",
|
||||
@@ -113,6 +113,7 @@
|
||||
"camera.waitingOnMouthCrop": "Awaiting Mouth Cwop :3",
|
||||
"camera.calibration": "Cawibwation OwO",
|
||||
"camera.tracking": "Twacking UwU",
|
||||
"camera.neutralface": "Modew Face nya~",
|
||||
"config.noSettingsFile": "No settings file, using base settings",
|
||||
"config.failedToLoadSettings": "Failed to load settings file",
|
||||
"config.usingBackupSettings": "Using backup settings",
|
||||
|
||||
Binary file not shown.
Binary file not shown.
@@ -1,27 +1,35 @@
|
||||
import os
|
||||
import json
|
||||
|
||||
os.environ["OMP_NUM_THREADS"] = "1"
|
||||
import onnxruntime as ort
|
||||
import time
|
||||
import PySimpleGUI as sg
|
||||
import cv2
|
||||
import numpy as np
|
||||
from pythonosc import udp_client
|
||||
import utils.image_transforms as transforms
|
||||
import PIL.Image as Image
|
||||
from threading import Thread
|
||||
from one_euro_filter import OneEuroFilter
|
||||
|
||||
|
||||
def run_model(self):
|
||||
def run_model(self, embedding, netural_blends):
|
||||
if self.runtime in ("ONNX", "Default (ONNX)"):
|
||||
frame = cv2.resize(self.current_image_gray, (256, 256))
|
||||
frame = transforms.to_tensor(frame)
|
||||
frame = transforms.unsqueeze(frame, 0)
|
||||
out = self.sess.run([self.output_name], {self.input_name: frame})
|
||||
output = out[0][0]
|
||||
frame = transforms.normalize(frame)
|
||||
print(frame)
|
||||
frame = frame * 2 - 1
|
||||
#make the image 3 channels but concat
|
||||
frame = np.concatenate((frame, frame, frame), axis=1)
|
||||
|
||||
|
||||
|
||||
output_emb = self.sess_encoder.run([self.output_name_encoder], {self.input_name_encoder: frame})
|
||||
output_emb = output_emb[0]
|
||||
#concat output_emb, and embedding from 1,128 in both to 1.256
|
||||
output_emb = np.concatenate((output_emb, embedding), axis=-1)
|
||||
#pass it to the model head
|
||||
|
||||
output = self.sess_head.run([self.output_name_head], {self.input_name_head: output_emb})
|
||||
|
||||
output = output[0][0] - netural_blends
|
||||
|
||||
output = self.one_euro_filter(output)
|
||||
|
||||
# for i in range(len(output)): # Clip values between 0 - 1
|
||||
@@ -29,3 +37,35 @@ def run_model(self):
|
||||
## Clip values between 0 - 1
|
||||
output = np.clip(output, 0, 1)
|
||||
self.output = output
|
||||
|
||||
def run_model_embeding(self):
|
||||
if self.runtime in ("ONNX", "Default (ONNX)"):
|
||||
#load BabbleApp\UserData\neutral_frame.jpg
|
||||
frame = cv2.imread("UserData/neutral_frame.jpg")
|
||||
|
||||
#convert to black and white
|
||||
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
|
||||
#convert back to color
|
||||
#frame = cv2.cvtColor(frame, cv2.COLOR_GRAY2BGR)
|
||||
if frame is None:
|
||||
print("No frame")
|
||||
frame = np.zeros((256, 256, 3), dtype=np.uint8)
|
||||
frame = cv2.resize(frame, (256, 256))
|
||||
frame = transforms.to_tensor(frame)
|
||||
frame = transforms.unsqueeze(frame, 0)
|
||||
frame = transforms.normalize(frame)
|
||||
frame = frame * 2 - 1
|
||||
frame = np.concatenate((frame, frame, frame), axis=1)
|
||||
|
||||
#convert to color by duplicating the channels
|
||||
out = self.sess_encoder.run([self.output_name_encoder], {self.input_name_encoder: frame})
|
||||
out = out[0]
|
||||
output_emb = np.concatenate((out, out), axis=-1)
|
||||
output_netural = self.sess_head.run([self.output_name_head], {self.input_name_head: output_emb})
|
||||
|
||||
|
||||
output = out
|
||||
output_netural = output_netural[0][0]
|
||||
#save to np array in UserData
|
||||
np.save("UserData/model_embeding.npy", output) #TODO: Expose this in settings
|
||||
np.save("UserData/netural.npy", output_netural) #TODO: Expose this in settings
|
||||
@@ -103,14 +103,23 @@ class BabbleProcessor:
|
||||
provider = "DmlExecutionProvider"
|
||||
else:
|
||||
provider = "CPUExecutionProvider" # Build onnxruntime to get both DML and OpenVINO
|
||||
self.sess = ort.InferenceSession(
|
||||
f"{self.model}onnx/model.onnx",
|
||||
self.sess_encoder = ort.InferenceSession(
|
||||
f"{self.model}onnx/encoder.onnx",
|
||||
self.opts,
|
||||
providers=[provider],
|
||||
provider_options=[{"device_id": self.gpu_index}],
|
||||
)
|
||||
self.input_name = self.sess.get_inputs()[0].name
|
||||
self.output_name = self.sess.get_outputs()[0].name
|
||||
self.sess_head = ort.InferenceSession(
|
||||
f"{self.model}onnx/head_with_selu.onnx",
|
||||
self.opts,
|
||||
providers=[provider],
|
||||
provider_options=[{"device_id": self.gpu_index}],
|
||||
)
|
||||
self.input_name_encoder = self.sess_encoder.get_inputs()[0].name
|
||||
self.output_name_encoder = self.sess_encoder.get_outputs()[0].name
|
||||
|
||||
self.input_name_head = self.sess_head.get_inputs()[0].name
|
||||
self.output_name_head = self.sess_head.get_outputs()[0].name
|
||||
try:
|
||||
min_cutoff = float(self.settings.gui_min_cutoff)
|
||||
beta = float(self.settings.gui_speed_coefficient)
|
||||
@@ -207,7 +216,10 @@ class BabbleProcessor:
|
||||
pass
|
||||
|
||||
def run(self):
|
||||
|
||||
run_model_embeding(self)
|
||||
#load the model embeding. It's a NPY
|
||||
embed_netural = np.load('UserData/model_embeding.npy')
|
||||
netural_blends = np.load('UserData/netural.npy')
|
||||
while True:
|
||||
# Check to make sure we haven't been requested to close
|
||||
if self.cancellation_event.is_set():
|
||||
@@ -253,7 +265,7 @@ class BabbleProcessor:
|
||||
self.current_image_gray.copy()
|
||||
) # copy this frame to have a clean image for blink algo
|
||||
|
||||
run_model(self)
|
||||
run_model(self, embed_netural, netural_blends)
|
||||
if self.settings.use_calibration:
|
||||
self.output = cal.cal_osc(self, self.output)
|
||||
# else:
|
||||
|
||||
@@ -264,7 +264,7 @@ def main():
|
||||
|
||||
try:
|
||||
# If window isn't in focus increase timeout and stop loop early
|
||||
if window.TKroot.focus_get():
|
||||
if window.TKroot.focus_get():
|
||||
if fs:
|
||||
fs = False
|
||||
tint = 33
|
||||
|
||||
@@ -13,6 +13,7 @@ from config import BabbleConfig, BabbleSettingsConfig
|
||||
from utils.misc_utils import get_camera_index_by_name, list_camera_names
|
||||
from enum import Enum
|
||||
import sys
|
||||
import os
|
||||
|
||||
WAIT_TIME = 0.1
|
||||
BUFFER_SIZE = 32768
|
||||
@@ -302,3 +303,13 @@ class Camera:
|
||||
)
|
||||
self.camera_output_outgoing.put((self.clamp_max_res(image), frame_number, fps))
|
||||
self.capture_event.clear()
|
||||
|
||||
def save_frame(self, destination_folder, frame_name, window_x, window_y, window_w, window_h):
|
||||
#get camera data from buffer
|
||||
image = self.camera_output_outgoing.get()[0] #this feels wrong but it's easy
|
||||
#crop image to ROI
|
||||
image = image[window_y:window_y+window_h, window_x:window_x+window_w]
|
||||
|
||||
if not os.path.exists(destination_folder):
|
||||
os.makedirs(destination_folder)
|
||||
cv2.imwrite(os.path.join(destination_folder, frame_name), image)
|
||||
@@ -43,6 +43,8 @@ class CameraWidget:
|
||||
self.gui_horizontal_flip = f"-HORIZONTALFLIP{widget_id}-"
|
||||
self.use_calibration = f"-USECALIBRATION{widget_id}-"
|
||||
self.gui_refresh_button = f"-REFRESHCAMLIST{widget_id}-"
|
||||
self.neural_face_frame_calibration_button = f"-FACEID{widget_id}-"
|
||||
|
||||
self.osc_queue = osc_queue
|
||||
self.main_config = main_config
|
||||
self.cam_id = widget_id
|
||||
@@ -197,8 +199,10 @@ class CameraWidget:
|
||||
f'{lang._instance.get_string("camera.crop")}:',
|
||||
key=self.gui_roi_message,
|
||||
background_color=bg_color_highlight,
|
||||
font=("Helvetica", 16, "bold"),
|
||||
text_color="red", # Use a color that contrasts with bg_color_highlight
|
||||
visible=False,
|
||||
),
|
||||
)
|
||||
],
|
||||
]
|
||||
|
||||
@@ -242,6 +246,11 @@ class CameraWidget:
|
||||
button_color=button_color,
|
||||
tooltip=f'{lang._instance.get_string("camera.croppingModeToolTip")}.',
|
||||
),
|
||||
sg.Button(
|
||||
lang._instance.get_string("camera.neutralface"),
|
||||
key=self.neural_face_frame_calibration_button,
|
||||
button_color=button_color,
|
||||
),
|
||||
],
|
||||
[
|
||||
sg.Column(
|
||||
@@ -298,6 +307,7 @@ class CameraWidget:
|
||||
self.camera_thread.join()
|
||||
|
||||
def render(self, window, event, values):
|
||||
|
||||
if self.image_queue.qsize() > 2:
|
||||
with self.image_queue.mutex:
|
||||
self.image_queue.queue.clear()
|
||||
@@ -368,6 +378,8 @@ class CameraWidget:
|
||||
if self.settings_config.use_calibration != values[self.use_calibration]:
|
||||
self.settings_config.use_calibration = values[self.use_calibration]
|
||||
changed = True
|
||||
|
||||
|
||||
|
||||
if changed:
|
||||
self.main_config.save()
|
||||
@@ -416,7 +428,7 @@ class CameraWidget:
|
||||
|
||||
if event == self.gui_roi_selection:
|
||||
# Event for mouse button down or mouse drag in ROI mode
|
||||
if self.is_mouse_up:
|
||||
if self.is_mouse_up:
|
||||
self.is_mouse_up = False
|
||||
self.x0, self.y0 = values[self.gui_roi_selection]
|
||||
self.x1, self.y1 = values[self.gui_roi_selection]
|
||||
@@ -453,6 +465,19 @@ class CameraWidget:
|
||||
|
||||
needs_roi_set = self.config.roi_window_h <= 0 or self.config.roi_window_w <= 0
|
||||
|
||||
if event == self.neural_face_frame_calibration_button:
|
||||
if needs_roi_set:
|
||||
return
|
||||
if self.config.neutral_frame == None:
|
||||
self.camera.save_frame("UserData", "neutral_frame.jpg", self.config.roi_window_x, self.config.roi_window_y, self.config.roi_window_w, self.config.roi_window_h)
|
||||
self.config.neutral_frame = "UserData/neutral_frame.jpg" #TODO: Have this exposed in the settings widget
|
||||
else:
|
||||
self.camera.save_frame("UserData", "neutral_frame.jpg", self.config.roi_window_x, self.config.roi_window_y, self.config.roi_window_w, self.config.roi_window_h)
|
||||
changed = True
|
||||
|
||||
|
||||
|
||||
|
||||
# TODO: Refactor if statements below...
|
||||
window[self.gui_tracking_fps].update("")
|
||||
window[self.gui_tracking_bps].update("")
|
||||
@@ -478,7 +503,7 @@ class CameraWidget:
|
||||
window[self.gui_mode_readout].update(
|
||||
lang._instance.get_string("camera.calibration")
|
||||
)
|
||||
else:
|
||||
else:
|
||||
window[self.gui_mode_readout].update(
|
||||
lang._instance.get_string("camera.tracking")
|
||||
)
|
||||
|
||||
@@ -22,6 +22,8 @@ class BabbleCameraConfig(BaseModel):
|
||||
gui_vertical_flip: bool = False
|
||||
gui_horizontal_flip: bool = False
|
||||
use_ffmpeg: bool = False
|
||||
neutral_frame: str = None
|
||||
|
||||
|
||||
|
||||
class BabbleSettingsConfig(BaseModel):
|
||||
@@ -53,6 +55,7 @@ class BabbleSettingsConfig(BaseModel):
|
||||
gui_language: str = "English"
|
||||
|
||||
|
||||
|
||||
class BabbleConfig(BaseModel):
|
||||
version: int = 1
|
||||
cam: BabbleCameraConfig = BabbleCameraConfig()
|
||||
|
||||
@@ -38,7 +38,7 @@ def to_tensor(numpy_array, dtype=np.float32):
|
||||
numpy_array = numpy_array[:, :, np.newaxis]
|
||||
|
||||
# Transpose the array to match PyTorch tensor format (C x H x W)
|
||||
tensor = normalize(np.transpose(numpy_array, (2, 0, 1)))
|
||||
tensor = np.transpose(numpy_array, (2, 0, 1))
|
||||
|
||||
return tensor
|
||||
|
||||
|
||||
Reference in new issue
Block a user