Landmark Inference for ROI

Methods for running the landmark model either continuously or for a single frame. landmark_processor and landmark_model_loader are ready to be modified with the proper preprocessing and outputs for landmarks.
This commit is contained in:
Rames The Generic committed 2024-01-13 20:04:30 -05:00
1 parent d071dcc5a7
commit 1cc5e5feb7
5 files changed
+327 -14

No files matched your search

+1 -1
View File
@@ -12,8 +12,8 @@ BabbleApp/dist
/testing
/old_models
/Lib
/scripts
/share
/BabbleApp/Models/dev
scripts/example_build_app_and_installer.bat
scripts/example_build_app_and_installer.bat
+1 -1
View File
@@ -46,7 +46,7 @@ SETTINGS_RADIO_NAME = "-SETTINGSRADIO-"
ALGO_SETTINGS_RADIO_NAME = "-ALGOSETTINGSRADIO-"
page_url = "https://github.com/SummerSigh/ProjectBabble/releases/latest"
appversion = "Babble v2.0.5"
appversion = "Babble v2.0.6 Alpha"
def main():
+37 -12
View File
@@ -4,6 +4,7 @@ from config import BabbleSettingsConfig
from collections import deque
from threading import Event, Thread
from babble_processor import BabbleProcessor, CamInfoOrigin
from landmark_processor import LandmarkProcessor
from enum import Enum
from queue import Queue, Empty
from camera import Camera, CameraState
@@ -56,7 +57,18 @@ class CameraWidget:
self.roi_queue = Queue(maxsize=2)
self.image_queue = Queue(maxsize=500)
self.ransac = BabbleProcessor(
self.babble_cnn = BabbleProcessor(
self.config,
self.settings_config,
self.main_config,
self.cancellation_event,
self.capture_event,
self.capture_queue,
self.image_queue,
self.cam_id,
)
self.babble_landmark = LandmarkProcessor(
self.config,
self.settings_config,
self.main_config,
@@ -114,7 +126,7 @@ class CameraWidget:
],
[
sg.Checkbox(
"Use Neurtal Calibration:",
"Use Neutral Calibration:",
default=self.config.use_n_calibration,
key=self.use_n_calibration,
background_color='#424042',
@@ -200,8 +212,10 @@ class CameraWidget:
if not self.cancellation_event.is_set():
return
self.cancellation_event.clear()
self.ransac_thread = Thread(target=self.ransac.run)
self.ransac_thread.start()
self.babble_cnn_thread = Thread(target=self.babble_cnn.run)
self.babble_cnn_thread.start()
self.babble_landmark_thread = Thread(target=self.babble_landmark.run)
self.babble_landmark_thread.start()
self.camera_thread = Thread(target=self.camera.run)
self.camera_thread.start()
@@ -210,7 +224,8 @@ class CameraWidget:
if self.cancellation_event.is_set():
return
self.cancellation_event.set()
self.ransac_thread.join()
self.babble_cnn_thread.join()
self.babble_landmark_thread.join()
self.camera_thread.join()
def render(self, window, event, values):
@@ -292,12 +307,22 @@ class CameraWidget:
self.config.roi_window_w = abs(self.x0 - self.x1)
self.config.roi_window_h = abs(self.y0 - self.y1)
self.main_config.save()
if event == self.gui_autoroi:
print("Auto ROI")
self.x1 = 120
self.y1 = 120
self.x0 = 20
self.y0 = 20
#image = self.image_queue.get()
#image = self.babble_landmark.get_frame() # Get image for pfld
#print(image)
#print(len(image))
#print(image)
#cv2.imwrite("yeah.png", image)
self.babble_landmark.infer_frame()
output = self.babble_landmark.output
print(f"Output: {output}")
self.x1 = output[2]
self.y1 = output[3]
self.x0 = output[0]
self.y0 = output[1]
self.config.roi_window_x = min([self.x0, self.x1])
self.config.roi_window_y = min([self.y0, self.y1])
self.config.roi_window_w = abs(self.x0 - self.x1)
@@ -312,11 +337,11 @@ class CameraWidget:
self.x1, self.y1 = values[self.gui_roi_selection]
if event == self.gui_restart_calibration:
self.ransac.calibration_frame_counter = 1500
self.babble_cnn.calibration_frame_counter = 1500
PlaySound('Audio/start.wav', SND_FILENAME | SND_ASYNC)
if event == self.gui_stop_calibration:
self.ransac.calibration_frame_counter = 0
self.babble_cnn.calibration_frame_counter = 0
needs_roi_set = self.config.roi_window_h <= 0 or self.config.roi_window_w <= 0
@@ -333,7 +358,7 @@ class CameraWidget:
window[self.gui_mode_readout].update("Camera Reconnecting...")
elif needs_roi_set:
window[self.gui_mode_readout].update("Awaiting Mouth Crop")
elif self.ransac.calibration_frame_counter != None:
elif self.babble_cnn.calibration_frame_counter != None:
window[self.gui_mode_readout].update("Calibration")
else:
window[self.gui_mode_readout].update("Tracking")
+35
View File
@@ -0,0 +1,35 @@
import os
import json
os.environ["OMP_NUM_THREADS"] = "1"
import onnxruntime as ort
import time
import PySimpleGUI as sg
import cv2
import numpy as np
from pythonosc import udp_client
import utils.image_transforms as transforms
import PIL.Image as Image
from threading import Thread
from one_euro_filter import OneEuroFilter
def run_model(self): # Replace transforms n shit for the pfld model
if self.runtime == "ONNX" or self.runtime == "Default (ONNX)":
frame = cv2.resize(self.current_image_gray, (256, 256))
frame = transforms.to_tensor(frame)
frame = transforms.unsqueeze(frame,0)
out = self.sess.run([self.output_name], {self.input_name: frame})
output = out[0][0]
output = self.one_euro_filter(output)
for i in range(len(output)): # Clip values between 0 - 1
output[i] = max(min(output[i], 1), 0)
self.output = output
def write_image(self): # Debug function for development, remove once pfld is implemented.
frame = cv2.resize(self.current_image_gray, (256, 256))
cv2.imwrite("yeah.png", frame)
print("Image Wrote")
self.output = (0, 0, 100, 100)
+253
View File
@@ -0,0 +1,253 @@
from operator import truth
from dataclasses import dataclass
import sys
import asyncio
sys.path.append(".")
from config import BabbleCameraConfig, BabbleSettingsConfig, BabbleConfig
import queue
import threading
import numpy as np
import cv2
from enum import Enum
from one_euro_filter import OneEuroFilter
from utils.misc_utils import PlaySound, SND_FILENAME, SND_ASYNC
#import importlib
#from osc import Tab
from osc_calibrate_filter import *
from tab import CamInfo, CamInfoOrigin
from landmark_model_loader import *
import os
os.environ["OMP_NUM_THREADS"] = "1"
import onnxruntime as ort
def run_once(f):
def wrapper(*args, **kwargs):
if not wrapper.has_run:
wrapper.has_run = True
return f(*args, **kwargs)
wrapper.has_run = False
return wrapper
async def delayed_setting_change(setting, value):
await asyncio.sleep(5)
setting = value
PlaySound('Audio/completed.wav', SND_FILENAME | SND_ASYNC)
class LandmarkProcessor:
def __init__(
self,
config: "BabbleCameraConfig",
settings: "BabbleSettingsConfig",
fullconfig: "BabbleConfig",
cancellation_event: "threading.Event",
capture_event: "threading.Event",
capture_queue_incoming: "queue.Queue(maxsize=2)",
image_queue_outgoing: "queue.Queue(maxsize=2)",
cam_id,
):
self.main_config = BabbleSettingsConfig
self.config = config
self.settings = settings
self.cam_id = cam_id
self.config_class = fullconfig
# Cross-thread communication management
self.capture_queue_incoming = capture_queue_incoming
self.image_queue_outgoing = image_queue_outgoing
self.cancellation_event = cancellation_event
self.capture_event = capture_event
self.cam_id = cam_id
# Image state
self.previous_image = None
self.current_image = None
self.current_image_gray_clean = None
self.current_image_gray = None
self.current_frame_number = None
self.current_fps = None
self.infer = False
self.calibration_frame_counter = None
self.cccs = False
self.ts = 10
self.previous_rotation = self.config.rotation_angle
self.roi_include_set = {"rotation_angle", "roi_window_x", "roi_window_y"}
self.current_algo = CamInfoOrigin.MODEL
self.model = self.settings.gui_model_file
self.runtime = self.settings.gui_runtime
self.use_gpu = self.settings.gui_use_gpu
self.gpu_index = self.settings.gui_gpu_index
self.output = []
self.val_list = []
#self.calibrate_config = np.empty((1, 45))
#self.min_max_array = np.empty((2, 45))
self.opts = ort.SessionOptions()
self.opts.intra_op_num_threads = settings.gui_inference_threads
self.opts.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
if self.runtime == "ONNX" or self.runtime == "Default (ONNX)": # ONNX
if self.use_gpu: provider = 'DmlExecutionProvider'
else: provider = "CPUExecutionProvider" # Build onnxruntime to get both DML and OpenVINO
self.sess = ort.InferenceSession(f'{self.model}onnx/model.onnx', self.opts, providers=[provider, ], provider_options=[{'device_id': self.gpu_index}]) # Load Babble CNN until PFLD has been converted
self.input_name = self.sess.get_inputs()[0].name
self.output_name = self.sess.get_outputs()[0].name
try:
min_cutoff = float(self.settings.gui_min_cutoff)
beta = float(self.settings.gui_speed_coefficient)
except:
print('\033[93m[WARN] OneEuroFilter values must be a legal number.\033[0m')
min_cutoff = 0.9
beta = 0.9
noisy_point = np.array([45])
self.one_euro_filter = OneEuroFilter(
noisy_point,
min_cutoff=min_cutoff,
beta=beta
)
def get_frame(self):
return self.current_image_gray_clean
def infer_frame(self):
return write_image(self)
def output_images_and_update(self, output_information: CamInfo):
try:
image_stack = np.concatenate(
(
cv2.cvtColor(self.current_image_gray, cv2.COLOR_GRAY2BGR),
),
axis=1,
)
self.image_queue_outgoing.put((image_stack, output_information))
self.previous_image = self.current_image
self.previous_rotation = self.config.rotation_angle
except: # If this fails it likely means that the images are not the same size for some reason.
print('\033[91m[ERROR] Size of frames to display are of unequal sizes.\033[0m')
pass
def capture_crop_rotate_image(self):
# Get our current frame
try:
# Get frame from capture source, crop to ROI
self.current_image = self.current_image[
int(self.config.roi_window_y): int(
self.config.roi_window_y + self.config.roi_window_h
),
int(self.config.roi_window_x): int(
self.config.roi_window_x + self.config.roi_window_w
),
]
except:
# Failure to process frame, reuse previous frame.
self.current_image = self.previous_image
print("\033[91m[ERROR] Frame capture issue detected.\033[0m")
try:
# Apply rotation to cropped area. For any rotation area outside of the bounds of the image,
# fill with white.
try:
rows, cols, _ = self.current_image.shape
except:
rows, cols, _ = self.previous_image.shape
if self.config.gui_vertical_flip:
self.current_image = cv2.flip(self.current_image, 0)
if self.config.gui_horizontal_flip:
self.current_image = cv2.flip(self.current_image, 1)
img_center = (cols / 2, rows / 2)
rotation_matrix = cv2.getRotationMatrix2D(
img_center, self.config.rotation_angle, 1
)
avg_color_per_row = np.average(self.current_image, axis=0)
avg_color = np.average(avg_color_per_row, axis=0)
ar, ag, ab = avg_color
self.current_image = cv2.warpAffine(
self.current_image,
rotation_matrix,
(cols, rows),
borderMode=cv2.BORDER_CONSTANT,
borderValue=(ar + 10, ag + 10, ab + 10),#(255, 255, 255),
)
self.current_image_white = cv2.warpAffine(
self.current_image,
rotation_matrix,
(cols, rows),
borderMode=cv2.BORDER_CONSTANT,
borderValue=(255, 255, 255),
)
return True
except:
pass
def run(self):
while True:
# Check to make sure we haven't been requested to close
if self.cancellation_event.is_set():
print("\033[94m[INFO] Exiting Tracking thread\033[0m")
return
if self.config.roi_window_w <= 0 or self.config.roi_window_h <= 0:
# At this point, we're waiting for the user to set up the ROI window in the GUI.
# Sleep a bit while we wait.
if self.cancellation_event.wait(0.1):
return
continue
try:
if self.capture_queue_incoming.empty():
self.capture_event.set()
# Wait a bit for images here. If we don't get one, just try again.
(
self.current_image,
self.current_frame_number,
self.current_fps,
) = self.capture_queue_incoming.get(block=True, timeout=0.2)
except queue.Empty:
# print("No image available")
continue
if not self.capture_crop_rotate_image():
continue
if self.settings.gui_use_red_channel: # Make G and B channels equal to red.
blue_channel, green_channel, red_channel = cv2.split(self.current_image)
new_blue_channel = red_channel
new_green_channel = red_channel
self.current_image = cv2.merge((new_blue_channel, new_green_channel, red_channel))
self.current_image_gray = cv2.cvtColor(
self.current_image, cv2.COLOR_BGR2GRAY
)
self.current_image_gray_clean = self.current_image_gray.copy() #copy this frame to have a clean image for blink algo
if self.infer == True:
'''
run_model(self)
if self.config.use_calibration:
self.output = cal.cal_osc(self, self.output)
'''
self.output = write_image(self)
#else:
# pass
#print(self.output)
#self.output_images_and_update(CamInfo(self.current_algo, self.output))