mirror of
https://github.com/Project-Babble/ProjectBabble.git
synced 2026-11-20 09:01:16 -08:00
Landmark Inference for ROI
Methods for running the landmark model either continuously or for a single frame. landmark_processor and landmark_model_loader are ready to be modified with the proper preprocessing and outputs for landmarks.
This commit is contained in:
1 parent
d071dcc5a7
commit
1cc5e5feb7
5 files changed
+327
-14
No files matched your search
+1
-1
@@ -12,8 +12,8 @@ BabbleApp/dist
|
||||
/testing
|
||||
/old_models
|
||||
/Lib
|
||||
/scripts
|
||||
/share
|
||||
/BabbleApp/Models/dev
|
||||
|
||||
scripts/example_build_app_and_installer.bat
|
||||
scripts/example_build_app_and_installer.bat
|
||||
|
||||
@@ -46,7 +46,7 @@ SETTINGS_RADIO_NAME = "-SETTINGSRADIO-"
|
||||
ALGO_SETTINGS_RADIO_NAME = "-ALGOSETTINGSRADIO-"
|
||||
|
||||
page_url = "https://github.com/SummerSigh/ProjectBabble/releases/latest"
|
||||
appversion = "Babble v2.0.5"
|
||||
appversion = "Babble v2.0.6 Alpha"
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
+37
-12
@@ -4,6 +4,7 @@ from config import BabbleSettingsConfig
|
||||
from collections import deque
|
||||
from threading import Event, Thread
|
||||
from babble_processor import BabbleProcessor, CamInfoOrigin
|
||||
from landmark_processor import LandmarkProcessor
|
||||
from enum import Enum
|
||||
from queue import Queue, Empty
|
||||
from camera import Camera, CameraState
|
||||
@@ -56,7 +57,18 @@ class CameraWidget:
|
||||
self.roi_queue = Queue(maxsize=2)
|
||||
self.image_queue = Queue(maxsize=500)
|
||||
|
||||
self.ransac = BabbleProcessor(
|
||||
self.babble_cnn = BabbleProcessor(
|
||||
self.config,
|
||||
self.settings_config,
|
||||
self.main_config,
|
||||
self.cancellation_event,
|
||||
self.capture_event,
|
||||
self.capture_queue,
|
||||
self.image_queue,
|
||||
self.cam_id,
|
||||
)
|
||||
|
||||
self.babble_landmark = LandmarkProcessor(
|
||||
self.config,
|
||||
self.settings_config,
|
||||
self.main_config,
|
||||
@@ -114,7 +126,7 @@ class CameraWidget:
|
||||
],
|
||||
[
|
||||
sg.Checkbox(
|
||||
"Use Neurtal Calibration:",
|
||||
"Use Neutral Calibration:",
|
||||
default=self.config.use_n_calibration,
|
||||
key=self.use_n_calibration,
|
||||
background_color='#424042',
|
||||
@@ -200,8 +212,10 @@ class CameraWidget:
|
||||
if not self.cancellation_event.is_set():
|
||||
return
|
||||
self.cancellation_event.clear()
|
||||
self.ransac_thread = Thread(target=self.ransac.run)
|
||||
self.ransac_thread.start()
|
||||
self.babble_cnn_thread = Thread(target=self.babble_cnn.run)
|
||||
self.babble_cnn_thread.start()
|
||||
self.babble_landmark_thread = Thread(target=self.babble_landmark.run)
|
||||
self.babble_landmark_thread.start()
|
||||
self.camera_thread = Thread(target=self.camera.run)
|
||||
self.camera_thread.start()
|
||||
|
||||
@@ -210,7 +224,8 @@ class CameraWidget:
|
||||
if self.cancellation_event.is_set():
|
||||
return
|
||||
self.cancellation_event.set()
|
||||
self.ransac_thread.join()
|
||||
self.babble_cnn_thread.join()
|
||||
self.babble_landmark_thread.join()
|
||||
self.camera_thread.join()
|
||||
|
||||
def render(self, window, event, values):
|
||||
@@ -292,12 +307,22 @@ class CameraWidget:
|
||||
self.config.roi_window_w = abs(self.x0 - self.x1)
|
||||
self.config.roi_window_h = abs(self.y0 - self.y1)
|
||||
self.main_config.save()
|
||||
|
||||
if event == self.gui_autoroi:
|
||||
print("Auto ROI")
|
||||
self.x1 = 120
|
||||
self.y1 = 120
|
||||
self.x0 = 20
|
||||
self.y0 = 20
|
||||
#image = self.image_queue.get()
|
||||
#image = self.babble_landmark.get_frame() # Get image for pfld
|
||||
#print(image)
|
||||
#print(len(image))
|
||||
#print(image)
|
||||
#cv2.imwrite("yeah.png", image)
|
||||
self.babble_landmark.infer_frame()
|
||||
output = self.babble_landmark.output
|
||||
print(f"Output: {output}")
|
||||
self.x1 = output[2]
|
||||
self.y1 = output[3]
|
||||
self.x0 = output[0]
|
||||
self.y0 = output[1]
|
||||
self.config.roi_window_x = min([self.x0, self.x1])
|
||||
self.config.roi_window_y = min([self.y0, self.y1])
|
||||
self.config.roi_window_w = abs(self.x0 - self.x1)
|
||||
@@ -312,11 +337,11 @@ class CameraWidget:
|
||||
self.x1, self.y1 = values[self.gui_roi_selection]
|
||||
|
||||
if event == self.gui_restart_calibration:
|
||||
self.ransac.calibration_frame_counter = 1500
|
||||
self.babble_cnn.calibration_frame_counter = 1500
|
||||
PlaySound('Audio/start.wav', SND_FILENAME | SND_ASYNC)
|
||||
|
||||
if event == self.gui_stop_calibration:
|
||||
self.ransac.calibration_frame_counter = 0
|
||||
self.babble_cnn.calibration_frame_counter = 0
|
||||
|
||||
needs_roi_set = self.config.roi_window_h <= 0 or self.config.roi_window_w <= 0
|
||||
|
||||
@@ -333,7 +358,7 @@ class CameraWidget:
|
||||
window[self.gui_mode_readout].update("Camera Reconnecting...")
|
||||
elif needs_roi_set:
|
||||
window[self.gui_mode_readout].update("Awaiting Mouth Crop")
|
||||
elif self.ransac.calibration_frame_counter != None:
|
||||
elif self.babble_cnn.calibration_frame_counter != None:
|
||||
window[self.gui_mode_readout].update("Calibration")
|
||||
else:
|
||||
window[self.gui_mode_readout].update("Tracking")
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
import os
|
||||
import json
|
||||
os.environ["OMP_NUM_THREADS"] = "1"
|
||||
import onnxruntime as ort
|
||||
import time
|
||||
import PySimpleGUI as sg
|
||||
import cv2
|
||||
import numpy as np
|
||||
from pythonosc import udp_client
|
||||
import utils.image_transforms as transforms
|
||||
import PIL.Image as Image
|
||||
from threading import Thread
|
||||
from one_euro_filter import OneEuroFilter
|
||||
|
||||
def run_model(self): # Replace transforms n shit for the pfld model
|
||||
if self.runtime == "ONNX" or self.runtime == "Default (ONNX)":
|
||||
frame = cv2.resize(self.current_image_gray, (256, 256))
|
||||
frame = transforms.to_tensor(frame)
|
||||
frame = transforms.unsqueeze(frame,0)
|
||||
out = self.sess.run([self.output_name], {self.input_name: frame})
|
||||
output = out[0][0]
|
||||
|
||||
output = self.one_euro_filter(output)
|
||||
|
||||
for i in range(len(output)): # Clip values between 0 - 1
|
||||
output[i] = max(min(output[i], 1), 0)
|
||||
|
||||
self.output = output
|
||||
|
||||
|
||||
def write_image(self): # Debug function for development, remove once pfld is implemented.
|
||||
frame = cv2.resize(self.current_image_gray, (256, 256))
|
||||
cv2.imwrite("yeah.png", frame)
|
||||
print("Image Wrote")
|
||||
self.output = (0, 0, 100, 100)
|
||||
@@ -0,0 +1,253 @@
|
||||
|
||||
from operator import truth
|
||||
from dataclasses import dataclass
|
||||
import sys
|
||||
import asyncio
|
||||
sys.path.append(".")
|
||||
from config import BabbleCameraConfig, BabbleSettingsConfig, BabbleConfig
|
||||
import queue
|
||||
import threading
|
||||
import numpy as np
|
||||
import cv2
|
||||
from enum import Enum
|
||||
from one_euro_filter import OneEuroFilter
|
||||
from utils.misc_utils import PlaySound, SND_FILENAME, SND_ASYNC
|
||||
#import importlib
|
||||
#from osc import Tab
|
||||
from osc_calibrate_filter import *
|
||||
from tab import CamInfo, CamInfoOrigin
|
||||
from landmark_model_loader import *
|
||||
import os
|
||||
os.environ["OMP_NUM_THREADS"] = "1"
|
||||
import onnxruntime as ort
|
||||
|
||||
def run_once(f):
|
||||
def wrapper(*args, **kwargs):
|
||||
if not wrapper.has_run:
|
||||
wrapper.has_run = True
|
||||
return f(*args, **kwargs)
|
||||
|
||||
wrapper.has_run = False
|
||||
return wrapper
|
||||
|
||||
|
||||
async def delayed_setting_change(setting, value):
|
||||
await asyncio.sleep(5)
|
||||
setting = value
|
||||
PlaySound('Audio/completed.wav', SND_FILENAME | SND_ASYNC)
|
||||
|
||||
|
||||
|
||||
class LandmarkProcessor:
|
||||
def __init__(
|
||||
self,
|
||||
config: "BabbleCameraConfig",
|
||||
settings: "BabbleSettingsConfig",
|
||||
fullconfig: "BabbleConfig",
|
||||
cancellation_event: "threading.Event",
|
||||
capture_event: "threading.Event",
|
||||
capture_queue_incoming: "queue.Queue(maxsize=2)",
|
||||
image_queue_outgoing: "queue.Queue(maxsize=2)",
|
||||
cam_id,
|
||||
):
|
||||
self.main_config = BabbleSettingsConfig
|
||||
self.config = config
|
||||
self.settings = settings
|
||||
self.cam_id = cam_id
|
||||
self.config_class = fullconfig
|
||||
# Cross-thread communication management
|
||||
self.capture_queue_incoming = capture_queue_incoming
|
||||
self.image_queue_outgoing = image_queue_outgoing
|
||||
self.cancellation_event = cancellation_event
|
||||
self.capture_event = capture_event
|
||||
self.cam_id = cam_id
|
||||
|
||||
# Image state
|
||||
self.previous_image = None
|
||||
self.current_image = None
|
||||
self.current_image_gray_clean = None
|
||||
self.current_image_gray = None
|
||||
self.current_frame_number = None
|
||||
self.current_fps = None
|
||||
self.infer = False
|
||||
|
||||
self.calibration_frame_counter = None
|
||||
|
||||
self.cccs = False
|
||||
self.ts = 10
|
||||
self.previous_rotation = self.config.rotation_angle
|
||||
self.roi_include_set = {"rotation_angle", "roi_window_x", "roi_window_y"}
|
||||
|
||||
self.current_algo = CamInfoOrigin.MODEL
|
||||
self.model = self.settings.gui_model_file
|
||||
self.runtime = self.settings.gui_runtime
|
||||
self.use_gpu = self.settings.gui_use_gpu
|
||||
self.gpu_index = self.settings.gui_gpu_index
|
||||
self.output = []
|
||||
self.val_list = []
|
||||
#self.calibrate_config = np.empty((1, 45))
|
||||
#self.min_max_array = np.empty((2, 45))
|
||||
|
||||
|
||||
self.opts = ort.SessionOptions()
|
||||
self.opts.intra_op_num_threads = settings.gui_inference_threads
|
||||
self.opts.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
if self.runtime == "ONNX" or self.runtime == "Default (ONNX)": # ONNX
|
||||
if self.use_gpu: provider = 'DmlExecutionProvider'
|
||||
else: provider = "CPUExecutionProvider" # Build onnxruntime to get both DML and OpenVINO
|
||||
self.sess = ort.InferenceSession(f'{self.model}onnx/model.onnx', self.opts, providers=[provider, ], provider_options=[{'device_id': self.gpu_index}]) # Load Babble CNN until PFLD has been converted
|
||||
self.input_name = self.sess.get_inputs()[0].name
|
||||
self.output_name = self.sess.get_outputs()[0].name
|
||||
try:
|
||||
min_cutoff = float(self.settings.gui_min_cutoff)
|
||||
beta = float(self.settings.gui_speed_coefficient)
|
||||
except:
|
||||
print('\033[93m[WARN] OneEuroFilter values must be a legal number.\033[0m')
|
||||
min_cutoff = 0.9
|
||||
beta = 0.9
|
||||
noisy_point = np.array([45])
|
||||
self.one_euro_filter = OneEuroFilter(
|
||||
noisy_point,
|
||||
min_cutoff=min_cutoff,
|
||||
beta=beta
|
||||
)
|
||||
|
||||
def get_frame(self):
|
||||
return self.current_image_gray_clean
|
||||
|
||||
def infer_frame(self):
|
||||
return write_image(self)
|
||||
|
||||
def output_images_and_update(self, output_information: CamInfo):
|
||||
try:
|
||||
image_stack = np.concatenate(
|
||||
(
|
||||
cv2.cvtColor(self.current_image_gray, cv2.COLOR_GRAY2BGR),
|
||||
),
|
||||
axis=1,
|
||||
)
|
||||
self.image_queue_outgoing.put((image_stack, output_information))
|
||||
self.previous_image = self.current_image
|
||||
self.previous_rotation = self.config.rotation_angle
|
||||
except: # If this fails it likely means that the images are not the same size for some reason.
|
||||
print('\033[91m[ERROR] Size of frames to display are of unequal sizes.\033[0m')
|
||||
|
||||
pass
|
||||
def capture_crop_rotate_image(self):
|
||||
# Get our current frame
|
||||
|
||||
try:
|
||||
# Get frame from capture source, crop to ROI
|
||||
self.current_image = self.current_image[
|
||||
int(self.config.roi_window_y): int(
|
||||
self.config.roi_window_y + self.config.roi_window_h
|
||||
),
|
||||
int(self.config.roi_window_x): int(
|
||||
self.config.roi_window_x + self.config.roi_window_w
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
except:
|
||||
# Failure to process frame, reuse previous frame.
|
||||
self.current_image = self.previous_image
|
||||
print("\033[91m[ERROR] Frame capture issue detected.\033[0m")
|
||||
|
||||
try:
|
||||
# Apply rotation to cropped area. For any rotation area outside of the bounds of the image,
|
||||
# fill with white.
|
||||
try:
|
||||
rows, cols, _ = self.current_image.shape
|
||||
except:
|
||||
rows, cols, _ = self.previous_image.shape
|
||||
|
||||
if self.config.gui_vertical_flip:
|
||||
self.current_image = cv2.flip(self.current_image, 0)
|
||||
|
||||
if self.config.gui_horizontal_flip:
|
||||
self.current_image = cv2.flip(self.current_image, 1)
|
||||
|
||||
img_center = (cols / 2, rows / 2)
|
||||
rotation_matrix = cv2.getRotationMatrix2D(
|
||||
img_center, self.config.rotation_angle, 1
|
||||
)
|
||||
avg_color_per_row = np.average(self.current_image, axis=0)
|
||||
avg_color = np.average(avg_color_per_row, axis=0)
|
||||
ar, ag, ab = avg_color
|
||||
self.current_image = cv2.warpAffine(
|
||||
self.current_image,
|
||||
rotation_matrix,
|
||||
(cols, rows),
|
||||
borderMode=cv2.BORDER_CONSTANT,
|
||||
borderValue=(ar + 10, ag + 10, ab + 10),#(255, 255, 255),
|
||||
)
|
||||
self.current_image_white = cv2.warpAffine(
|
||||
self.current_image,
|
||||
rotation_matrix,
|
||||
(cols, rows),
|
||||
borderMode=cv2.BORDER_CONSTANT,
|
||||
borderValue=(255, 255, 255),
|
||||
)
|
||||
return True
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
def run(self):
|
||||
|
||||
while True:
|
||||
# Check to make sure we haven't been requested to close
|
||||
if self.cancellation_event.is_set():
|
||||
print("\033[94m[INFO] Exiting Tracking thread\033[0m")
|
||||
return
|
||||
|
||||
|
||||
if self.config.roi_window_w <= 0 or self.config.roi_window_h <= 0:
|
||||
# At this point, we're waiting for the user to set up the ROI window in the GUI.
|
||||
# Sleep a bit while we wait.
|
||||
if self.cancellation_event.wait(0.1):
|
||||
return
|
||||
continue
|
||||
|
||||
|
||||
|
||||
try:
|
||||
if self.capture_queue_incoming.empty():
|
||||
self.capture_event.set()
|
||||
# Wait a bit for images here. If we don't get one, just try again.
|
||||
(
|
||||
self.current_image,
|
||||
self.current_frame_number,
|
||||
self.current_fps,
|
||||
) = self.capture_queue_incoming.get(block=True, timeout=0.2)
|
||||
except queue.Empty:
|
||||
# print("No image available")
|
||||
continue
|
||||
|
||||
if not self.capture_crop_rotate_image():
|
||||
continue
|
||||
|
||||
if self.settings.gui_use_red_channel: # Make G and B channels equal to red.
|
||||
blue_channel, green_channel, red_channel = cv2.split(self.current_image)
|
||||
new_blue_channel = red_channel
|
||||
new_green_channel = red_channel
|
||||
self.current_image = cv2.merge((new_blue_channel, new_green_channel, red_channel))
|
||||
|
||||
self.current_image_gray = cv2.cvtColor(
|
||||
self.current_image, cv2.COLOR_BGR2GRAY
|
||||
)
|
||||
self.current_image_gray_clean = self.current_image_gray.copy() #copy this frame to have a clean image for blink algo
|
||||
if self.infer == True:
|
||||
'''
|
||||
run_model(self)
|
||||
if self.config.use_calibration:
|
||||
self.output = cal.cal_osc(self, self.output)
|
||||
'''
|
||||
self.output = write_image(self)
|
||||
|
||||
|
||||
#else:
|
||||
# pass
|
||||
#print(self.output)
|
||||
#self.output_images_and_update(CamInfo(self.current_algo, self.output))
|
||||
|
||||
Reference in new issue
Block a user