From 1cc5e5feb72e690db4a4ab0dec3beddc92be0653 Mon Sep 17 00:00:00 2001 From: Rames The Generic <53163624+RamesTheGeneric@users.noreply.github.com> Date: Sat, 13 Jan 2024 20:04:30 -0500 Subject: [PATCH] Landmark Inference for ROI Methods for running the landmark model either continuously or for a single frame. landmark_processor and landmark_model_loader are ready to be modified with the proper preprocessing and outputs for landmarks. --- .gitignore | 2 +- BabbleApp/babbleapp.py | 2 +- BabbleApp/camera_widget.py | 49 ++++-- BabbleApp/landmark_model_loader.py | 35 ++++ BabbleApp/landmark_processor.py | 253 +++++++++++++++++++++++++++++ 5 files changed, 327 insertions(+), 14 deletions(-) create mode 100644 BabbleApp/landmark_model_loader.py create mode 100644 BabbleApp/landmark_processor.py diff --git a/.gitignore b/.gitignore index 479f677..5ea6d78 100644 --- a/.gitignore +++ b/.gitignore @@ -12,8 +12,8 @@ BabbleApp/dist /testing /old_models /Lib -/scripts /share +/BabbleApp/Models/dev scripts/example_build_app_and_installer.bat scripts/example_build_app_and_installer.bat diff --git a/BabbleApp/babbleapp.py b/BabbleApp/babbleapp.py index b508f47..a5abead 100644 --- a/BabbleApp/babbleapp.py +++ b/BabbleApp/babbleapp.py @@ -46,7 +46,7 @@ SETTINGS_RADIO_NAME = "-SETTINGSRADIO-" ALGO_SETTINGS_RADIO_NAME = "-ALGOSETTINGSRADIO-" page_url = "https://github.com/SummerSigh/ProjectBabble/releases/latest" -appversion = "Babble v2.0.5" +appversion = "Babble v2.0.6 Alpha" def main(): diff --git a/BabbleApp/camera_widget.py b/BabbleApp/camera_widget.py index 0dc3cc8..598df65 100644 --- a/BabbleApp/camera_widget.py +++ b/BabbleApp/camera_widget.py @@ -4,6 +4,7 @@ from config import BabbleSettingsConfig from collections import deque from threading import Event, Thread from babble_processor import BabbleProcessor, CamInfoOrigin +from landmark_processor import LandmarkProcessor from enum import Enum from queue import Queue, Empty from camera import Camera, CameraState @@ -56,7 +57,18 @@ class CameraWidget: self.roi_queue = Queue(maxsize=2) self.image_queue = Queue(maxsize=500) - self.ransac = BabbleProcessor( + self.babble_cnn = BabbleProcessor( + self.config, + self.settings_config, + self.main_config, + self.cancellation_event, + self.capture_event, + self.capture_queue, + self.image_queue, + self.cam_id, + ) + + self.babble_landmark = LandmarkProcessor( self.config, self.settings_config, self.main_config, @@ -114,7 +126,7 @@ class CameraWidget: ], [ sg.Checkbox( - "Use Neurtal Calibration:", + "Use Neutral Calibration:", default=self.config.use_n_calibration, key=self.use_n_calibration, background_color='#424042', @@ -200,8 +212,10 @@ class CameraWidget: if not self.cancellation_event.is_set(): return self.cancellation_event.clear() - self.ransac_thread = Thread(target=self.ransac.run) - self.ransac_thread.start() + self.babble_cnn_thread = Thread(target=self.babble_cnn.run) + self.babble_cnn_thread.start() + self.babble_landmark_thread = Thread(target=self.babble_landmark.run) + self.babble_landmark_thread.start() self.camera_thread = Thread(target=self.camera.run) self.camera_thread.start() @@ -210,7 +224,8 @@ class CameraWidget: if self.cancellation_event.is_set(): return self.cancellation_event.set() - self.ransac_thread.join() + self.babble_cnn_thread.join() + self.babble_landmark_thread.join() self.camera_thread.join() def render(self, window, event, values): @@ -292,12 +307,22 @@ class CameraWidget: self.config.roi_window_w = abs(self.x0 - self.x1) self.config.roi_window_h = abs(self.y0 - self.y1) self.main_config.save() + if event == self.gui_autoroi: print("Auto ROI") - self.x1 = 120 - self.y1 = 120 - self.x0 = 20 - self.y0 = 20 + #image = self.image_queue.get() + #image = self.babble_landmark.get_frame() # Get image for pfld + #print(image) + #print(len(image)) + #print(image) + #cv2.imwrite("yeah.png", image) + self.babble_landmark.infer_frame() + output = self.babble_landmark.output + print(f"Output: {output}") + self.x1 = output[2] + self.y1 = output[3] + self.x0 = output[0] + self.y0 = output[1] self.config.roi_window_x = min([self.x0, self.x1]) self.config.roi_window_y = min([self.y0, self.y1]) self.config.roi_window_w = abs(self.x0 - self.x1) @@ -312,11 +337,11 @@ class CameraWidget: self.x1, self.y1 = values[self.gui_roi_selection] if event == self.gui_restart_calibration: - self.ransac.calibration_frame_counter = 1500 + self.babble_cnn.calibration_frame_counter = 1500 PlaySound('Audio/start.wav', SND_FILENAME | SND_ASYNC) if event == self.gui_stop_calibration: - self.ransac.calibration_frame_counter = 0 + self.babble_cnn.calibration_frame_counter = 0 needs_roi_set = self.config.roi_window_h <= 0 or self.config.roi_window_w <= 0 @@ -333,7 +358,7 @@ class CameraWidget: window[self.gui_mode_readout].update("Camera Reconnecting...") elif needs_roi_set: window[self.gui_mode_readout].update("Awaiting Mouth Crop") - elif self.ransac.calibration_frame_counter != None: + elif self.babble_cnn.calibration_frame_counter != None: window[self.gui_mode_readout].update("Calibration") else: window[self.gui_mode_readout].update("Tracking") diff --git a/BabbleApp/landmark_model_loader.py b/BabbleApp/landmark_model_loader.py new file mode 100644 index 0000000..e391f37 --- /dev/null +++ b/BabbleApp/landmark_model_loader.py @@ -0,0 +1,35 @@ +import os +import json +os.environ["OMP_NUM_THREADS"] = "1" +import onnxruntime as ort +import time +import PySimpleGUI as sg +import cv2 +import numpy as np +from pythonosc import udp_client +import utils.image_transforms as transforms +import PIL.Image as Image +from threading import Thread +from one_euro_filter import OneEuroFilter + +def run_model(self): # Replace transforms n shit for the pfld model + if self.runtime == "ONNX" or self.runtime == "Default (ONNX)": + frame = cv2.resize(self.current_image_gray, (256, 256)) + frame = transforms.to_tensor(frame) + frame = transforms.unsqueeze(frame,0) + out = self.sess.run([self.output_name], {self.input_name: frame}) + output = out[0][0] + + output = self.one_euro_filter(output) + + for i in range(len(output)): # Clip values between 0 - 1 + output[i] = max(min(output[i], 1), 0) + + self.output = output + + +def write_image(self): # Debug function for development, remove once pfld is implemented. + frame = cv2.resize(self.current_image_gray, (256, 256)) + cv2.imwrite("yeah.png", frame) + print("Image Wrote") + self.output = (0, 0, 100, 100) \ No newline at end of file diff --git a/BabbleApp/landmark_processor.py b/BabbleApp/landmark_processor.py new file mode 100644 index 0000000..14568a1 --- /dev/null +++ b/BabbleApp/landmark_processor.py @@ -0,0 +1,253 @@ + +from operator import truth +from dataclasses import dataclass +import sys +import asyncio +sys.path.append(".") +from config import BabbleCameraConfig, BabbleSettingsConfig, BabbleConfig +import queue +import threading +import numpy as np +import cv2 +from enum import Enum +from one_euro_filter import OneEuroFilter +from utils.misc_utils import PlaySound, SND_FILENAME, SND_ASYNC +#import importlib +#from osc import Tab +from osc_calibrate_filter import * +from tab import CamInfo, CamInfoOrigin +from landmark_model_loader import * +import os +os.environ["OMP_NUM_THREADS"] = "1" +import onnxruntime as ort + +def run_once(f): + def wrapper(*args, **kwargs): + if not wrapper.has_run: + wrapper.has_run = True + return f(*args, **kwargs) + + wrapper.has_run = False + return wrapper + + +async def delayed_setting_change(setting, value): + await asyncio.sleep(5) + setting = value + PlaySound('Audio/completed.wav', SND_FILENAME | SND_ASYNC) + + + +class LandmarkProcessor: + def __init__( + self, + config: "BabbleCameraConfig", + settings: "BabbleSettingsConfig", + fullconfig: "BabbleConfig", + cancellation_event: "threading.Event", + capture_event: "threading.Event", + capture_queue_incoming: "queue.Queue(maxsize=2)", + image_queue_outgoing: "queue.Queue(maxsize=2)", + cam_id, + ): + self.main_config = BabbleSettingsConfig + self.config = config + self.settings = settings + self.cam_id = cam_id + self.config_class = fullconfig + # Cross-thread communication management + self.capture_queue_incoming = capture_queue_incoming + self.image_queue_outgoing = image_queue_outgoing + self.cancellation_event = cancellation_event + self.capture_event = capture_event + self.cam_id = cam_id + + # Image state + self.previous_image = None + self.current_image = None + self.current_image_gray_clean = None + self.current_image_gray = None + self.current_frame_number = None + self.current_fps = None + self.infer = False + + self.calibration_frame_counter = None + + self.cccs = False + self.ts = 10 + self.previous_rotation = self.config.rotation_angle + self.roi_include_set = {"rotation_angle", "roi_window_x", "roi_window_y"} + + self.current_algo = CamInfoOrigin.MODEL + self.model = self.settings.gui_model_file + self.runtime = self.settings.gui_runtime + self.use_gpu = self.settings.gui_use_gpu + self.gpu_index = self.settings.gui_gpu_index + self.output = [] + self.val_list = [] + #self.calibrate_config = np.empty((1, 45)) + #self.min_max_array = np.empty((2, 45)) + + + self.opts = ort.SessionOptions() + self.opts.intra_op_num_threads = settings.gui_inference_threads + self.opts.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL + if self.runtime == "ONNX" or self.runtime == "Default (ONNX)": # ONNX + if self.use_gpu: provider = 'DmlExecutionProvider' + else: provider = "CPUExecutionProvider" # Build onnxruntime to get both DML and OpenVINO + self.sess = ort.InferenceSession(f'{self.model}onnx/model.onnx', self.opts, providers=[provider, ], provider_options=[{'device_id': self.gpu_index}]) # Load Babble CNN until PFLD has been converted + self.input_name = self.sess.get_inputs()[0].name + self.output_name = self.sess.get_outputs()[0].name + try: + min_cutoff = float(self.settings.gui_min_cutoff) + beta = float(self.settings.gui_speed_coefficient) + except: + print('\033[93m[WARN] OneEuroFilter values must be a legal number.\033[0m') + min_cutoff = 0.9 + beta = 0.9 + noisy_point = np.array([45]) + self.one_euro_filter = OneEuroFilter( + noisy_point, + min_cutoff=min_cutoff, + beta=beta + ) + + def get_frame(self): + return self.current_image_gray_clean + + def infer_frame(self): + return write_image(self) + + def output_images_and_update(self, output_information: CamInfo): + try: + image_stack = np.concatenate( + ( + cv2.cvtColor(self.current_image_gray, cv2.COLOR_GRAY2BGR), + ), + axis=1, + ) + self.image_queue_outgoing.put((image_stack, output_information)) + self.previous_image = self.current_image + self.previous_rotation = self.config.rotation_angle + except: # If this fails it likely means that the images are not the same size for some reason. + print('\033[91m[ERROR] Size of frames to display are of unequal sizes.\033[0m') + + pass + def capture_crop_rotate_image(self): + # Get our current frame + + try: + # Get frame from capture source, crop to ROI + self.current_image = self.current_image[ + int(self.config.roi_window_y): int( + self.config.roi_window_y + self.config.roi_window_h + ), + int(self.config.roi_window_x): int( + self.config.roi_window_x + self.config.roi_window_w + ), + ] + + + except: + # Failure to process frame, reuse previous frame. + self.current_image = self.previous_image + print("\033[91m[ERROR] Frame capture issue detected.\033[0m") + + try: + # Apply rotation to cropped area. For any rotation area outside of the bounds of the image, + # fill with white. + try: + rows, cols, _ = self.current_image.shape + except: + rows, cols, _ = self.previous_image.shape + + if self.config.gui_vertical_flip: + self.current_image = cv2.flip(self.current_image, 0) + + if self.config.gui_horizontal_flip: + self.current_image = cv2.flip(self.current_image, 1) + + img_center = (cols / 2, rows / 2) + rotation_matrix = cv2.getRotationMatrix2D( + img_center, self.config.rotation_angle, 1 + ) + avg_color_per_row = np.average(self.current_image, axis=0) + avg_color = np.average(avg_color_per_row, axis=0) + ar, ag, ab = avg_color + self.current_image = cv2.warpAffine( + self.current_image, + rotation_matrix, + (cols, rows), + borderMode=cv2.BORDER_CONSTANT, + borderValue=(ar + 10, ag + 10, ab + 10),#(255, 255, 255), + ) + self.current_image_white = cv2.warpAffine( + self.current_image, + rotation_matrix, + (cols, rows), + borderMode=cv2.BORDER_CONSTANT, + borderValue=(255, 255, 255), + ) + return True + except: + pass + + + def run(self): + + while True: + # Check to make sure we haven't been requested to close + if self.cancellation_event.is_set(): + print("\033[94m[INFO] Exiting Tracking thread\033[0m") + return + + + if self.config.roi_window_w <= 0 or self.config.roi_window_h <= 0: + # At this point, we're waiting for the user to set up the ROI window in the GUI. + # Sleep a bit while we wait. + if self.cancellation_event.wait(0.1): + return + continue + + + + try: + if self.capture_queue_incoming.empty(): + self.capture_event.set() + # Wait a bit for images here. If we don't get one, just try again. + ( + self.current_image, + self.current_frame_number, + self.current_fps, + ) = self.capture_queue_incoming.get(block=True, timeout=0.2) + except queue.Empty: + # print("No image available") + continue + + if not self.capture_crop_rotate_image(): + continue + + if self.settings.gui_use_red_channel: # Make G and B channels equal to red. + blue_channel, green_channel, red_channel = cv2.split(self.current_image) + new_blue_channel = red_channel + new_green_channel = red_channel + self.current_image = cv2.merge((new_blue_channel, new_green_channel, red_channel)) + + self.current_image_gray = cv2.cvtColor( + self.current_image, cv2.COLOR_BGR2GRAY + ) + self.current_image_gray_clean = self.current_image_gray.copy() #copy this frame to have a clean image for blink algo + if self.infer == True: + ''' + run_model(self) + if self.config.use_calibration: + self.output = cal.cal_osc(self, self.output) + ''' + self.output = write_image(self) + + + #else: + # pass + #print(self.output) + #self.output_images_and_update(CamInfo(self.current_algo, self.output)) +