From 015253fda0525a9c82e3debb322de2053fd369fd Mon Sep 17 00:00:00 2001 From: Rames The Generic <53163624+RamesTheGeneric@users.noreply.github.com> Date: Sat, 20 Jan 2024 15:55:31 -0500 Subject: [PATCH 1/7] Fixed Auto ROI not getting an image when no ROI set. Auto ROI can now run without an roi being set initially. Also fixed a bug with the osc receiver that prevented it from initializing. --- BabbleApp/camera_widget.py | 20 ++++++++++---------- BabbleApp/landmark_model_loader.py | 8 ++++---- BabbleApp/landmark_processor.py | 4 ++-- BabbleApp/osc.py | 7 +++++-- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/BabbleApp/camera_widget.py b/BabbleApp/camera_widget.py index 598df65..93f180e 100644 --- a/BabbleApp/camera_widget.py +++ b/BabbleApp/camera_widget.py @@ -44,6 +44,7 @@ class CameraWidget: self.settings_config = main_config.settings self.config = main_config.cam self.settings = main_config.settings + self.maybe_image = None if self.cam_id == Tab.CAM: self.config = main_config.cam else: @@ -255,7 +256,7 @@ class CameraWidget: changed = True - + if self.config.rotation_angle != values[self.gui_rotation_slider]: self.config.rotation_angle = int(values[self.gui_rotation_slider]) changed = True @@ -310,19 +311,17 @@ class CameraWidget: if event == self.gui_autoroi: print("Auto ROI") - #image = self.image_queue.get() - #image = self.babble_landmark.get_frame() # Get image for pfld - #print(image) - #print(len(image)) - #print(image) - #cv2.imwrite("yeah.png", image) - self.babble_landmark.infer_frame() - output = self.babble_landmark.output + #needs_roi_set = self.config.roi_window_h <= 0 or self.config.roi_window_w <= 0 + self.babble_landmark.infer_frame(image = self.maybe_image[0]) + output = self.babble_landmark.output[0] + rotation = self.babble_landmark.output[1] + print(f'Rotation: {rotation}') print(f"Output: {output}") self.x1 = output[2] self.y1 = output[3] self.x0 = output[0] self.y0 = output[1] + window[self.gui_rotation_slider].update(value = rotation) self.config.roi_window_x = min([self.x0, self.x1]) self.config.roi_window_y = min([self.y0, self.y1]) self.config.roi_window_w = abs(self.x0 - self.x1) @@ -370,6 +369,7 @@ class CameraWidget: if self.roi_queue.empty(): self.capture_event.set() maybe_image = self.roi_queue.get(block=False) + self.maybe_image = maybe_image imgbytes = cv2.imencode(".ppm", maybe_image[0])[1].tobytes() graph = window[self.gui_roi_selection] if self.figure: @@ -400,4 +400,4 @@ class CameraWidget: if cam_info.info_type != CamInfoOrigin.FAILURE: self.osc_queue.put((self.cam_id, cam_info)) except Empty: - pass + pass \ No newline at end of file diff --git a/BabbleApp/landmark_model_loader.py b/BabbleApp/landmark_model_loader.py index e391f37..1da5f19 100644 --- a/BabbleApp/landmark_model_loader.py +++ b/BabbleApp/landmark_model_loader.py @@ -28,8 +28,8 @@ def run_model(self): # Replace transforms n shit for the pfld model self.output = output -def write_image(self): # Debug function for development, remove once pfld is implemented. - frame = cv2.resize(self.current_image_gray, (256, 256)) - cv2.imwrite("yeah.png", frame) +def write_image(self, image): # Placeholder function for development, replace all instances with run_model() once pfld is implemented. + frame = cv2.resize(image, (256, 256)) + cv2.imwrite("yeah.png", frame) # Write the clean, untransformed frame to show that the frame is clean for pfld print("Image Wrote") - self.output = (0, 0, 100, 100) \ No newline at end of file + self.output = ((0, 0, 100, 100), 25) # Return ROI box and Rotation information. \ No newline at end of file diff --git a/BabbleApp/landmark_processor.py b/BabbleApp/landmark_processor.py index 14568a1..d642a47 100644 --- a/BabbleApp/landmark_processor.py +++ b/BabbleApp/landmark_processor.py @@ -115,8 +115,8 @@ class LandmarkProcessor: def get_frame(self): return self.current_image_gray_clean - def infer_frame(self): - return write_image(self) + def infer_frame(self, image): + return write_image(self, image) def output_images_and_update(self, output_information: CamInfo): try: diff --git a/BabbleApp/osc.py b/BabbleApp/osc.py index 6f3bf7d..30b7fcb 100644 --- a/BabbleApp/osc.py +++ b/BabbleApp/osc.py @@ -5,6 +5,7 @@ import threading from enum import IntEnum import time from config import BabbleConfig +import traceback class Tab(IntEnum): CAM = 0 @@ -90,6 +91,7 @@ class VRChatOSCReceiver: self.cancellation_event = cancellation_event self.dispatcher = dispatcher.Dispatcher() self.cams = cams # we cant import CameraWidget so any type it is + print("OSC_INIT") try: self.server = osc_server.OSCUDPServer((self.config.gui_osc_address, int(self.config.gui_osc_receiver_port)), self.dispatcher) except: @@ -106,17 +108,18 @@ class VRChatOSCReceiver: if type(osc_value) != bool: return # just incase we get anything other than bool if osc_value: for cam in self.cams: - cam.ransac.calibration_frame_counter = 300 + cam.babble_cnn.calibration_frame_counter = 300 PlaySound('Audio/start.wav', SND_FILENAME | SND_ASYNC) def run(self): # bind what function to run when specified OSC message is received try: - self.dispatcher.map(self.config.gui_osc_recalibrate_address, self.recalibrate_cams) + self.dispatcher.map(self.config.gui_osc_recalibrate_address, self.recalibrate_mouth) # start the server print("\033[92m[INFO] VRChatOSCReceiver serving on {}\033[0m".format(self.server.server_address)) self.server.serve_forever() except: + traceback.print_exc() print(f"\033[91m[ERROR] OSC Receive port: {self.config.gui_osc_receiver_port} occupied.\033[0m") \ No newline at end of file From 0c9584849357f39f2d402b7fb1718e4779e256f3 Mon Sep 17 00:00:00 2001 From: Rames The Generic <53163624+RamesTheGeneric@users.noreply.github.com> Date: Wed, 7 Feb 2024 01:24:09 -0500 Subject: [PATCH 2/7] Added Calibration Tab Contains Min and Max values for manually calibrating model output. Also moved the calibration mode setting from the camera tab to avoid confusion. --- BabbleApp/babble_processor.py | 2 +- BabbleApp/babbleapp.py | 38 +++++ BabbleApp/calib_settings_values.py | 108 ++++++++++++++ BabbleApp/calib_settings_widget.py | 219 +++++++++++++++++++++++++++++ BabbleApp/camera_widget.py | 22 +-- BabbleApp/config.py | 6 +- BabbleApp/osc.py | 1 + BabbleApp/osc_calibrate_filter.py | 40 ++++-- BabbleApp/tab.py | 1 + 9 files changed, 405 insertions(+), 32 deletions(-) create mode 100644 BabbleApp/calib_settings_values.py create mode 100644 BabbleApp/calib_settings_widget.py diff --git a/BabbleApp/babble_processor.py b/BabbleApp/babble_processor.py index 615e460..33762c1 100644 --- a/BabbleApp/babble_processor.py +++ b/BabbleApp/babble_processor.py @@ -231,7 +231,7 @@ class BabbleProcessor: run_model(self) - if self.config.use_calibration: + if self.settings.use_calibration: self.output = cal.cal_osc(self, self.output) #else: diff --git a/BabbleApp/babbleapp.py b/BabbleApp/babbleapp.py index a5abead..0f1854f 100644 --- a/BabbleApp/babbleapp.py +++ b/BabbleApp/babbleapp.py @@ -28,6 +28,7 @@ from tab import CamInfo, Tab from osc import VRChatOSCReceiver, VRChatOSC from general_settings_widget import SettingsWidget from algo_settings_widget import AlgoSettingsWidget +from calib_settings_widget import CalibSettingsWidget from utils.misc_utils import is_nt if is_nt: from winotify import Notification @@ -41,9 +42,11 @@ WINDOW_NAME = "Babble sApp" CAM_NAME = "-CAMWIDGET-" SETTINGS_NAME = "-SETTINGSWIDGET-" ALGO_SETTINGS_NAME = "-ALGOSETTINGSWIDGET-" +CALIB_SETTINGS_NAME = "-CALIBSETTINGSWIDGET-" CAM_RADIO_NAME = "-CAMRADIO-" SETTINGS_RADIO_NAME = "-SETTINGSRADIO-" ALGO_SETTINGS_RADIO_NAME = "-ALGOSETTINGSRADIO-" +CALIB_SETTINGS_RADIO_NAME = "-CALIBSETTINGSRADIO-" page_url = "https://github.com/SummerSigh/ProjectBabble/releases/latest" appversion = "Babble v2.0.6 Alpha" @@ -108,6 +111,7 @@ def main(): settings = [ SettingsWidget(Tab.SETTINGS, config, osc_queue), AlgoSettingsWidget(Tab.ALGOSETTINGS, config, osc_queue), + CalibSettingsWidget(Tab.CALIBRATION, config, osc_queue), ] layout = [ @@ -133,6 +137,13 @@ def main(): default=(config.cam_display_id == Tab.ALGOSETTINGS), key=ALGO_SETTINGS_RADIO_NAME, ), + sg.Radio( + "Calibration", + "TABSELECTRADIO", + background_color="#292929", + default=(config.cam_display_id == Tab.CALIBRATION), + key=CALIB_SETTINGS_RADIO_NAME, + ), ], [ @@ -157,6 +168,13 @@ def main(): visible=(config.cam_display_id in [Tab.ALGOSETTINGS]), background_color="#424042", ), + sg.Column( + settings[2].widget_layout, + vertical_alignment="top", + key=CALIB_SETTINGS_NAME, + visible=(config.cam_display_id in [Tab.CALIBRATION]), + background_color="#424042", + ), ], ] @@ -167,6 +185,8 @@ def main(): settings[0].start() if config.cam_display_id in [Tab.ALGOSETTINGS]: settings[1].start() + if config.cam_display_id in [Tab.CALIBRATION]: + settings[2].start() # the cam needs to be running before it is passed to the OSC if config.settings.gui_ROSC: @@ -206,9 +226,11 @@ def main(): cams[0].start() settings[0].stop() settings[1].stop() + settings[2].stop() window[CAM_NAME].update(visible=True) window[SETTINGS_NAME].update(visible=False) window[ALGO_SETTINGS_NAME].update(visible=False) + window[CALIB_SETTINGS_NAME].update(visible=False) config.cam_display_id = Tab.CAM config.save() @@ -217,10 +239,12 @@ def main(): elif values[SETTINGS_RADIO_NAME] and config.cam_display_id != Tab.SETTINGS: cams[0].stop() settings[1].stop() + settings[2].stop() settings[0].start() window[CAM_NAME].update(visible=False) window[SETTINGS_NAME].update(visible=True) window[ALGO_SETTINGS_NAME].update(visible=False) + window[CALIB_SETTINGS_NAME].update(visible=False) config.cam_display_id = Tab.SETTINGS config.save() @@ -228,12 +252,26 @@ def main(): elif values[ALGO_SETTINGS_RADIO_NAME] and config.cam_display_id != Tab.ALGOSETTINGS: cams[0].stop() settings[0].stop() + settings[2].stop() settings[1].start() window[CAM_NAME].update(visible=False) window[SETTINGS_NAME].update(visible=False) window[ALGO_SETTINGS_NAME].update(visible=True) + window[CALIB_SETTINGS_NAME].update(visible=False) config.cam_display_id = Tab.ALGOSETTINGS config.save() + + elif values[CALIB_SETTINGS_RADIO_NAME] and config.cam_display_id != Tab.CALIBRATION: + cams[0].stop() + settings[0].stop() + settings[1].stop() + settings[2].start() + window[CAM_NAME].update(visible=False) + window[SETTINGS_NAME].update(visible=False) + window[ALGO_SETTINGS_NAME].update(visible=False) + window[CALIB_SETTINGS_NAME].update(visible=True) + config.cam_display_id = Tab.CALIBRATION + config.save() # Otherwise, render all diff --git a/BabbleApp/calib_settings_values.py b/BabbleApp/calib_settings_values.py new file mode 100644 index 0000000..453ea6a --- /dev/null +++ b/BabbleApp/calib_settings_values.py @@ -0,0 +1,108 @@ + +def set_shapes(widget_id): + shape_index = ["cheekPuffLeft", "cheekPuffRight", "cheekSuckLeft", "cheekSuckRight", "jawOpen", "jawForward", "jawLeft", "jawRight", "noseSneerLeft", "noseSneerRight", "mouthFunnel", "mouthPucker", "mouthLeft", "mouthRight", + "mouthRollUpper", "mouthRollLower", "mouthShrugUpper", "mouthShrugLower", "mouthClose", "mouthSmileLeft", + "mouthSmileRight", "mouthFrownLeft", "mouthFrownRight", "mouthDimpleLeft", "mouthDimpleRight", "mouthUpperUpLeft", + "mouthUpperUpRight", "mouthLowerDownLeft", "mouthLowerDownRight", "mouthPressLeft", "mouthPressRight", "mouthStretchLeft", + "mouthStretchRight", "tongueOut", "tongueUp", "tongueDown", "tongueLeft", "tongueRight", "tongueRoll", "tongueBendDown", "tongueCurlUp", "tongueSquish", "tongueFlat", "tongueTwistLeft", "tongueTwistRight"] + + shape = [ + [ + f"-CHEEKPUFFLEFTMIN{widget_id}-", + f"-CHEEKPUFFRIGHTMIN{widget_id}-", + f"-CHEEKSUCKLEFTMIN{widget_id}-", + f"-CHEEKSUCKRIGHTMIN{widget_id}-", + f"-JAWOPENMIN{widget_id}-", + f"-JAWFORWARDMIN{widget_id}-", + f"-JAWLEFTMIN{widget_id}-", + f"-JAWRIGHTMIN{widget_id}-", + f"-NOSESNEERLEFTMIN{widget_id}-", + f"-NOSESNEERRIGHTMIN{widget_id}-", + f"-MOUTHFUNNELMIN{widget_id}-", + f"-MOUTHPUCKERMIN{widget_id}-", + f"-MOUTHLEFTMIN{widget_id}-", + f"-MOUTHRIGHTMIN{widget_id}-", + f"-MOUTHROLLUPPERMIN{widget_id}-", + f"-MOUTHROLLLOWERMIN{widget_id}-", + f"-MOUTHSHRUGUPPERMIN{widget_id}-", + f"-MOUTHSHRUGLOWERMIN{widget_id}-", + f"-MOUTHCLOSEMIN{widget_id}-", + f"-MOUTHSMILELEFTMIN{widget_id}-", + f"-MOUTHSMILERIGHTMIN{widget_id}-", + f"-MOUTHFROWNLEFTMIN{widget_id}-", + f"-MOUTHFROWNRIGHTMIN{widget_id}-", + f"-MOUTHDIMPLELEFTMIN{widget_id}-", + f"-MOUTHDIMPLERIGHTMIN{widget_id}-", + f"-MOUTHUPPERUPLEFTMIN{widget_id}-", + f"-MOUTHUPPERUPRIGHTMIN{widget_id}-", + f"-MOUTHLOWERDOWNLEFTMIN{widget_id}-", + f"-MOUTHLOWERDOWNRIGHTMIN{widget_id}-", + f"-MOUTHPRESSLEFTMIN{widget_id}-", + f"-MOUTHPRESSRIGHTMIN{widget_id}-", + f"-MOUTHSTRETCHLEFTMIN{widget_id}-", + f"-MOUTHSTRETCHRIGHTMIN{widget_id}-", + f"-TONGUEOUTMIN{widget_id}-", + f"-TONGUEUPMIN{widget_id}-", + f"-TONGUEDOWNMIN{widget_id}-", + f"-TONGUELEFTMIN{widget_id}-", + f"-TONGUERIGHTMIN{widget_id}-", + f"-TONGUEROLLMIN{widget_id}-", + f"-TONGUEBENDDOWNMIN{widget_id}-", + f"-TONGUECURLUPMIN{widget_id}-", + f"-TONGUESQUISHMIN{widget_id}-", + f"-TONGUEFLATMIN{widget_id}-", + f"-TONGUETWISTLEFTMIN{widget_id}-", + f"-TONGUETWISTRIGHTMIN{widget_id}-", + ], + [ + f"-CHEEKPUFFLEFTMAX{widget_id}-", + f"-CHEEKPUFFRIGHTMAX{widget_id}-", + f"-CHEEKSUCKLEFTMAX{widget_id}-", + f"-CHEEKSUCKRIGHTMAX{widget_id}-", + f"-JAWOPENMAX{widget_id}-", + f"-JAWFORWARDMAX{widget_id}-", + f"-JAWLEFTMAX{widget_id}-", + f"-JAWRIGHTMAX{widget_id}-", + f"-NOSESNEERLEFTMAX{widget_id}-", + f"-NOSESNEERRIGHTMAX{widget_id}-", + f"-MOUTHFUNNELMAX{widget_id}-", + f"-MOUTHPUCKERMAX{widget_id}-", + f"-MOUTHLEFTMAX{widget_id}-", + f"-MOUTHRIGHTMAX{widget_id}-", + f"-MOUTHROLLUPPERMAX{widget_id}-", + f"-MOUTHROLLLOWERMAX{widget_id}-", + f"-MOUTHSHRUGUPPERMAX{widget_id}-", + f"-MOUTHSHRUGLOWERMAX{widget_id}-", + f"-MOUTHCLOSEMAX{widget_id}-", + f"-MOUTHSMILELEFTMAX{widget_id}-", + f"-MOUTHSMILERIGHTMAX{widget_id}-", + f"-MOUTHFROWNLEFTMAX{widget_id}-", + f"-MOUTHFROWNRIGHTMAX{widget_id}-", + f"-MOUTHDIMPLELEFTMAX{widget_id}-", + f"-MOUTHDIMPLERIGHTMAX{widget_id}-", + f"-MOUTHUPPERUPLEFTMAX{widget_id}-", + f"-MOUTHUPPERUPRIGHTMAX{widget_id}-", + f"-MOUTHLOWERDOWNLEFTMAX{widget_id}-", + f"-MOUTHLOWERDOWNRIGHTMAX{widget_id}-", + f"-MOUTHPRESSLEFTMAX{widget_id}-", + f"-MOUTHPRESSRIGHTMAX{widget_id}-", + f"-MOUTHSTRETCHLEFTMAX{widget_id}-", + f"-MOUTHSTRETCHRIGHTMAX{widget_id}-", + f"-TONGUEOUTMAX{widget_id}-", + f"-TONGUEUPMAX{widget_id}-", + f"-TONGUEDOWNMAX{widget_id}-", + f"-TONGUELEFTMAX{widget_id}-", + f"-TONGUERIGHTMAX{widget_id}-", + f"-TONGUEROLLMAX{widget_id}-", + f"-TONGUEBENDDOWNMAX{widget_id}-", + f"-TONGUECURLUPMAX{widget_id}-", + f"-TONGUESQUISHMAX{widget_id}-", + f"-TONGUEFLATMAX{widget_id}-", + f"-TONGUETWISTLEFTMAX{widget_id}-", + f"-TONGUETWISTRIGHTMAX{widget_id}-", + ], + + ] + return shape_index, shape + + diff --git a/BabbleApp/calib_settings_widget.py b/BabbleApp/calib_settings_widget.py new file mode 100644 index 0000000..852e22f --- /dev/null +++ b/BabbleApp/calib_settings_widget.py @@ -0,0 +1,219 @@ +import PySimpleGUI as sg + +from config import BabbleSettingsConfig +from osc import Tab +from queue import Queue +from threading import Event +import numpy as np +from calib_settings_values import set_shapes + + +class CalibSettingsWidget: + def __init__(self, widget_id: Tab, main_config: BabbleSettingsConfig, osc_queue: Queue): + + + self.gui_general_settings_layout = f"-GENERALSETTINGSLAYOUT{widget_id}-" + self.gui_reset_min = f"-RESETMIN{widget_id}-" + self.gui_reset_max = f"-RESETMAX{widget_id}-" + self.gui_multiply = f"-MULTIPLY{widget_id}-" + self.gui_calibration_mode = f"-CALIBRATIONMODE{widget_id}-" + self.main_config = main_config + self.config = main_config.settings + self.array = np.fromstring(self.config.calib_array.replace('[', '').replace(']', ''), sep=',').reshape(2, 45) + self.calibration_list = ['Neutral', 'Full'] + self.osc_queue = osc_queue + self.shape_index, self.shape = set_shapes(widget_id) + self.refreshed = False + + + + # Define the window's contents + s = self.single_shape + d = self.double_shape + self.general_settings_layout = [ + + d('cheekPuff'), + d('cheekSuck'), + s('jawOpen'), + s('jawForward'), + d('jaw'), + d('noseSneer'), + s('mouthFunnel'), + s('mouthPucker'), + d('mouth'), + s('mouthRollUpper'), + s('mouthRollLower'), + s('mouthShrugUpper'), + s('mouthShrugLower'), + s('mouthClose'), + d('mouthSmile'), + d('mouthFrown'), + d('mouthDimple'), + d('mouthUpperUp'), + d('mouthLowerDown'), + d('mouthPress'), + d('mouthStretch'), + s('tongueOut'), + s('tongueUp'), + s('tongueDown'), + d('tongue'), + s('tongueRoll'), + s('tongueBendDown'), + s('tongueCurlUp'), + s('tongueSquish'), + s('tongueFlat'), + d('tongueTwist'), + + + ] + + self.widget_layout = [ + [ + sg.Text("Calibration Settings:", background_color='#242224'), + sg.Text("Calibration Mode:", background_color='#424042'), + sg.OptionMenu( + self.calibration_list, + self.config.calibration_mode, + key=self.gui_calibration_mode, + tooltip='Neutral = Only Min values are set when starting and stopping calibration. Full = Min and Max values are set based on recorded values when starting calibration.', + ), + ], + [ + sg.Text("Left ", expand_x=True, justification='left'), + sg.Text("Shape", expand_x=True, justification='center'), + sg.Text("Right", expand_x=True, justification='right') + ], + [ + sg.Text("Min", expand_x=True, justification='center'), + sg.Text("Max", expand_x=True, justification='center'), + sg.HSeparator(pad=(50,0)), + sg.Text("Max", expand_x=True, justification='center'), + sg.Text("Min", expand_x=True, justification='center'), + ], + [ + sg.Column( + self.general_settings_layout, + key=self.gui_general_settings_layout, + scrollable=True, + vertical_scroll_only=True, + element_justification='center', + background_color='#424042' ), + ], + [ + sg.Button("Reset Min", key=self.gui_reset_min, button_color='#FF0000', tooltip = "Reset minimum values",), + sg.Button("Reset Max", key=self.gui_reset_max, button_color='#FF0000', tooltip = "Reset maximum values",), + ], + ] + + self.cancellation_event = Event() # Set the event until start is called, otherwise we can block if shutdown is called. + self.cancellation_event.set() + self.image_queue = Queue(maxsize=2) + + def double_shape(self, shapename): + indexl = self.shape_index.index(f"{shapename}Left") + indexr = self.shape_index.index(f"{shapename}Right") + double_shape = [ + sg.InputText( + default_text=self.array[0][indexl], + key=self.shape[0][indexl], + size=(8), + tooltip="Min Left Value", + ), + sg.InputText( + default_text=self.array[1][indexl], + key=self.shape[1][indexl], + size=(8), + tooltip="Max Left Value", + ), + sg.Text(f"{shapename}Left/Right", background_color='#424042', expand_x=True), + sg.InputText( + default_text=self.array[1][indexr], + key=self.shape[1][indexr], + size=(8), + tooltip="Max Right Value", + ), + sg.InputText( + default_text=self.array[0][indexr], + key=self.shape[0][indexr], + size=(8), + tooltip="Min Right Value", + ), + ] + return double_shape + + def single_shape(self, shapename): + index = self.shape_index.index(f"{shapename}") + single_shape = [ + sg.InputText( + default_text=self.array[0][index], + key=self.shape[0][index], + size=(8), + tooltip="Min Left Value", + ), + sg.InputText( + default_text=self.array[1][index], + key=self.shape[1][index], + size=(8), + tooltip="Max Left Value", + ), + sg.Text(f"{shapename}", background_color='#424042', expand_x=True), + ] + return single_shape + + def started(self): + return not self.cancellation_event.is_set() + + def start(self): + # If we're already running, bail + if not self.cancellation_event.is_set(): + return + self.cancellation_event.clear() + self.array = np.fromstring(self.config.calib_array.replace('[', '').replace(']', ''), sep=',').reshape(2, 45) # Reload the array from the config + self.refreshed = False + + def stop(self): + # If we're not running yet, bail + if self.cancellation_event.is_set(): + return + self.cancellation_event.set() + + def render(self, window, event, values): + # If anything has changed in our configuration settings, change/update those. + changed = False + if not self.refreshed: + for count1, element1 in enumerate(self.shape): + for count2, element2 in enumerate(element1): + window[element2].update(float(self.array[count1][count2])) + #values[element2] = float(self.array[count1][count2]) + self.refreshed = True + print('DEBUG: Refreshed') + + if self.config.calibration_mode != str(values[self.gui_calibration_mode]): + self.config.calibration_mode = str(values[self.gui_calibration_mode]) + changed = True + + for count1, element1 in enumerate(self.shape): + for count2, element2 in enumerate(element1): + if values[element2] != '': + if float(self.array[count1][count2]) != float(values[element2]): + self.array[count1][count2] = float(values[element2]) + changed = True + + if event == self.gui_reset_min: + for count1, element1 in enumerate(self.shape): + for count2, element2 in enumerate(element1): + self.array[0][count2] = float(0) + changed = True + self.refreshed = False + elif event == self.gui_reset_max: + for count1, element1 in enumerate(self.shape): + for count2, element2 in enumerate(element1): + self.array[1][count2] = float(1) + changed = True + self.refreshed = False + + if changed: + self.config.calib_array = np.array2string(self.array, separator=',') + self.main_config.save() + #print(self.main_config) + self.osc_queue.put(Tab.CALIBRATION) diff --git a/BabbleApp/camera_widget.py b/BabbleApp/camera_widget.py index 93f180e..aa96684 100644 --- a/BabbleApp/camera_widget.py +++ b/BabbleApp/camera_widget.py @@ -37,7 +37,6 @@ class CameraWidget: self.gui_vertical_flip = f"-VERTICALFLIP{widget_id}-" self.gui_horizontal_flip = f"-HORIZONTALFLIP{widget_id}-" self.use_calibration = f"-USECALIBRATION{widget_id}-" - self.use_n_calibration = f"-USENCALIBRATION{widget_id}-" self.osc_queue = osc_queue self.main_config = main_config self.cam_id = widget_id @@ -127,18 +126,11 @@ class CameraWidget: ], [ sg.Checkbox( - "Use Neutral Calibration:", - default=self.config.use_n_calibration, - key=self.use_n_calibration, - background_color='#424042', - tooltip="Toggle use of calibration using minimum values found during a neutral pose calibration step.", - ), - sg.Checkbox( - "Use Full Calibration:", - default=self.config.use_calibration, + "Enable Calibration:", + default=self.settings_config.use_calibration, key=self.use_calibration, background_color='#424042', - tooltip="Toggle use of calibration.", + tooltip="Checked = Calibrated model output. Unchecked = Raw model output", ), ], [ @@ -270,12 +262,8 @@ class CameraWidget: self.config.gui_horizontal_flip = values[self.gui_horizontal_flip] changed = True - if self.config.use_calibration != values[self.use_calibration]: - self.config.use_calibration = values[self.use_calibration] - changed = True - - if self.config.use_n_calibration != values[self.use_n_calibration]: - self.config.use_n_calibration = values[self.use_n_calibration] + if self.settings_config.use_calibration != values[self.use_calibration]: + self.settings_config.use_calibration = values[self.use_calibration] changed = True if changed: diff --git a/BabbleApp/config.py b/BabbleApp/config.py index 163d97a..72c9244 100644 --- a/BabbleApp/config.py +++ b/BabbleApp/config.py @@ -16,8 +16,6 @@ class BabbleCameraConfig(BaseModel): roi_window_w: int = 0 roi_window_h: int = 0 capture_source: Union[int, str, None] = None - use_calibration: bool = False - use_n_calibration: bool = True gui_vertical_flip: bool = False gui_horizontal_flip: bool = False @@ -43,6 +41,8 @@ class BabbleSettingsConfig(BaseModel): gui_cam_resolution_x: int = 0 gui_cam_resolution_y: int = 0 gui_cam_framerate: int = 0 + use_calibration: bool = False + calibration_mode: str = 'Neutral' class BabbleConfig(BaseModel): version: int = 1 @@ -86,5 +86,5 @@ class BabbleConfig(BaseModel): # No backup because the saved settings file is broken. pass with open(CONFIG_FILE_NAME, "w") as settings_file: - json.dump(obj=self.dict(), fp=settings_file) + json.dump(obj=self.dict(), fp=settings_file, indent=2) print("[INFO] Config Saved Successfully") diff --git a/BabbleApp/osc.py b/BabbleApp/osc.py index 30b7fcb..d5bdc72 100644 --- a/BabbleApp/osc.py +++ b/BabbleApp/osc.py @@ -11,6 +11,7 @@ class Tab(IntEnum): CAM = 0 SETTINGS = 1 ALGOSETTINGS = 2 + CALIBRATION = 3 def output_osc(array, self): diff --git a/BabbleApp/osc_calibrate_filter.py b/BabbleApp/osc_calibrate_filter.py index 195e9a3..714eaa3 100644 --- a/BabbleApp/osc_calibrate_filter.py +++ b/BabbleApp/osc_calibrate_filter.py @@ -9,10 +9,15 @@ class CamId(IntEnum): class cal(): + def __init__(self): + self.calibrated_array = None + self.raw_array = None + def cal_osc(self, array): + self.raw_array = array #print(self.calibration_frame_counter) if self.calibration_frame_counter == 0: - if not self.config.use_n_calibration: + if self.settings.calibration_mode == 'Full': self.calibration_frame_counter = None values = np.array(self.val_list) @@ -42,19 +47,27 @@ class cal(): PlaySound('Audio/completed.wav', SND_FILENAME | SND_ASYNC) - if self.config.use_n_calibration: + if self.settings.calibration_mode == 'Neutral': + self.min_max_array = np.fromstring(self.settings.calib_array.replace('[', '').replace(']', ''), sep=',') + self.min_max_array = self.min_max_array.reshape((2, 45)) + print(f'minmax {self.min_max_array}') + max_values = self.min_max_array[1] self.calibration_frame_counter = None values = np.array(self.val_list) + #print(f'values: {values}') deadzone_value = self.settings.calib_deadzone # Initialize the min_max_array with shape (2, num_outputs) num_outputs = values.shape[1] self.min_max_array = np.zeros((2, num_outputs)) - lower_threshold = np.clip([np.mean(values, axis = 0) + deadzone_value], 0, 1) - upper_threshold = np.ones((1, num_outputs)) # We don't need to adjust the max values. + lower_threshold = np.clip([np.mean(values, axis = 0) + deadzone_value], 0, 1).tolist()[0] + upper_threshold = max_values.tolist() + #upper_threshold = np.ones((1, num_outputs)) # We don't need to adjust the max values. + #upper_threshold = self.min_max_array[1] print(lower_threshold) print(upper_threshold) - self.min_max_array = np.array([lower_threshold, upper_threshold]) - self.settings.calib_array = np.array2string(self.min_max_array, separator=',') + self.min_max_array = np.append(lower_threshold, upper_threshold) + #self.min_max_array = np.array([lower_threshold.tolist(), upper_threshold.tolist()]) + self.settings.calib_array = str([lower_threshold, upper_threshold]) self.config_class.save() print("[INFO] Calibration completed.") @@ -67,8 +80,8 @@ class cal(): self.val_list.append(array) self.calibration_frame_counter -= 1 - - if self.settings.calib_array is not None and self.config.use_calibration: + self.raw_array = array + if self.settings.calib_array is not None and self.settings.use_calibration and self.settings.calibration_mode == 'Full': self.min_max_array = np.fromstring(self.settings.calib_array.replace('[', '').replace(']', ''), sep=',') self.min_max_array = self.min_max_array.reshape((2, 45)) @@ -85,17 +98,22 @@ class cal(): calibrated_array[i] = calibrated_value array = calibrated_array - if self.settings.calib_array is not None and self.config.use_n_calibration: + if self.settings.calib_array is not None and self.settings.use_calibration and self.settings.calibration_mode == 'Neutral': self.min_max_array = np.fromstring(self.settings.calib_array.replace('[', '').replace(']', ''), sep=',') self.min_max_array = self.min_max_array.reshape((2, 45)) calibrated_array = np.zeros_like(array) for i, value in enumerate(array): min_value = self.min_max_array[0, i] + max_value = self.min_max_array[1, i] - calibrated_value = (value - min_value) / (1.0 - min_value) + calibrated_value = (value - min_value) / (max_value - min_value) calibrated_array[i] = calibrated_value array = calibrated_array #array[4] = log((np.clip(array[4]*10,0,10))+1.0, 11) Log Filter: Move to filter system. - return np.clip(array,0,1) # Clamp outputs between 0-1 + array = np.clip(array,0,1) + return array # Clamp outputs between 0-1 + + def get_outputs(self): + return self.calibrated_array, self.raw_array diff --git a/BabbleApp/tab.py b/BabbleApp/tab.py index e57d414..21074d3 100644 --- a/BabbleApp/tab.py +++ b/BabbleApp/tab.py @@ -5,6 +5,7 @@ class Tab(IntEnum): CAM = 0 SETTINGS = 1 ALGOSETTINGS = 2 + CALIBRATION = 3 class CamInfoOrigin(Enum): From f4c3dc0f857ad749d525baca8e4a1565829c87a4 Mon Sep 17 00:00:00 2001 From: Rames The Generic Date: Tue, 13 Aug 2024 13:40:46 -0400 Subject: [PATCH 3/7] Fix Calibration Tab Crashing Added check so we don't attempt to convert a non-float str to a float. --- BabbleApp/babble_processor.py | 3 +-- BabbleApp/calib_settings_widget.py | 9 ++++++--- BabbleApp/camera_widget.py | 25 +++---------------------- 3 files changed, 10 insertions(+), 27 deletions(-) diff --git a/BabbleApp/babble_processor.py b/BabbleApp/babble_processor.py index 33762c1..f4190f1 100644 --- a/BabbleApp/babble_processor.py +++ b/BabbleApp/babble_processor.py @@ -223,9 +223,8 @@ class BabbleProcessor: new_blue_channel = red_channel new_green_channel = red_channel self.current_image = cv2.merge((new_blue_channel, new_green_channel, red_channel)) - self.current_image_gray = cv2.cvtColor( - self.current_image, cv2.COLOR_BGR2GRAY + self.current_image, cv2.COLOR_BGR2GRAY ) self.current_image_gray_clean = self.current_image_gray.copy() #copy this frame to have a clean image for blink algo diff --git a/BabbleApp/calib_settings_widget.py b/BabbleApp/calib_settings_widget.py index 852e22f..eac445d 100644 --- a/BabbleApp/calib_settings_widget.py +++ b/BabbleApp/calib_settings_widget.py @@ -195,9 +195,12 @@ class CalibSettingsWidget: for count1, element1 in enumerate(self.shape): for count2, element2 in enumerate(element1): if values[element2] != '': - if float(self.array[count1][count2]) != float(values[element2]): - self.array[count1][count2] = float(values[element2]) - changed = True + try: + float(values[element2]) + if float(self.array[count1][count2]) != float(values[element2]): + self.array[count1][count2] = float(values[element2]) + changed = True + except: print("Not a float") if event == self.gui_reset_min: for count1, element1 in enumerate(self.shape): diff --git a/BabbleApp/camera_widget.py b/BabbleApp/camera_widget.py index aa96684..9ba1b81 100644 --- a/BabbleApp/camera_widget.py +++ b/BabbleApp/camera_widget.py @@ -207,8 +207,8 @@ class CameraWidget: self.cancellation_event.clear() self.babble_cnn_thread = Thread(target=self.babble_cnn.run) self.babble_cnn_thread.start() - self.babble_landmark_thread = Thread(target=self.babble_landmark.run) - self.babble_landmark_thread.start() + #self.babble_landmark_thread = Thread(target=self.babble_landmark.run) + #self.babble_landmark_thread.start() self.camera_thread = Thread(target=self.camera.run) self.camera_thread.start() @@ -218,7 +218,7 @@ class CameraWidget: return self.cancellation_event.set() self.babble_cnn_thread.join() - self.babble_landmark_thread.join() + #self.babble_landmark_thread.join() self.camera_thread.join() def render(self, window, event, values): @@ -297,25 +297,6 @@ class CameraWidget: self.config.roi_window_h = abs(self.y0 - self.y1) self.main_config.save() - if event == self.gui_autoroi: - print("Auto ROI") - #needs_roi_set = self.config.roi_window_h <= 0 or self.config.roi_window_w <= 0 - self.babble_landmark.infer_frame(image = self.maybe_image[0]) - output = self.babble_landmark.output[0] - rotation = self.babble_landmark.output[1] - print(f'Rotation: {rotation}') - print(f"Output: {output}") - self.x1 = output[2] - self.y1 = output[3] - self.x0 = output[0] - self.y0 = output[1] - window[self.gui_rotation_slider].update(value = rotation) - self.config.roi_window_x = min([self.x0, self.x1]) - self.config.roi_window_y = min([self.y0, self.y1]) - self.config.roi_window_w = abs(self.x0 - self.x1) - self.config.roi_window_h = abs(self.y0 - self.y1) - self.main_config.save() - if event == self.gui_roi_selection: # Event for mouse button down or mouse drag in ROI mode if self.is_mouse_up: From 857626a549dedf1d2a61ab31113809148020186d Mon Sep 17 00:00:00 2001 From: Rames The Generic Date: Thu, 22 Aug 2024 14:05:03 -0400 Subject: [PATCH 4/7] Fix UVC Camera's not changing + List COM ports --- BabbleApp/camera.py | 9 ++++---- BabbleApp/camera_widget.py | 10 ++++++--- BabbleApp/utils/misc_utils.py | 42 +++++++++++++++++++++++++++++++++-- 3 files changed, 51 insertions(+), 10 deletions(-) diff --git a/BabbleApp/camera.py b/BabbleApp/camera.py index 839e6b1..d11aee4 100644 --- a/BabbleApp/camera.py +++ b/BabbleApp/camera.py @@ -87,9 +87,7 @@ class Camera: if ( self.config.capture_source is not None and self.config.capture_source != "" ): - self.current_capture_source = self.config.capture_source - - if "COM" in str(self.config.capture_source) and self.config.capture_source not in self.camera_list: + if "COM" in str(self.config.capture_source): if ( self.serial_connection is None or self.camera_status == CameraState.DISCONNECTED @@ -114,7 +112,7 @@ class Camera: if self.config.capture_source not in self.camera_list: self.current_capture_source = self.config.capture_source else: - self.current_capture_source = get_camera_index_by_name(self.current_capture_source) + self.current_capture_source = get_camera_index_by_name(self.config.capture_source) if self.config.use_ffmpeg: self.cv2_camera = cv2.VideoCapture(self.current_capture_source, cv2.CAP_FFMPEG) @@ -139,7 +137,8 @@ class Camera: if should_push and not self.capture_event.wait(timeout=0.02): continue if self.config.capture_source is not None: - if "COM" in str(self.config.capture_source): + ports = ("COM", "/dev/tty") + if any(x in str(self.config.capture_source) for x in ports): self.get_serial_camera_picture(should_push) else: self.get_cv2_camera_picture(should_push) diff --git a/BabbleApp/camera_widget.py b/BabbleApp/camera_widget.py index fc3e1ab..21158c4 100644 --- a/BabbleApp/camera_widget.py +++ b/BabbleApp/camera_widget.py @@ -10,7 +10,7 @@ from camera import Camera, CameraState from config import BabbleConfig from landmark_processor import LandmarkProcessor from osc import Tab -from utils.misc_utils import PlaySound, SND_FILENAME, SND_ASYNC, list_camera_names +from utils.misc_utils import PlaySound, SND_FILENAME, SND_ASYNC, list_camera_names, get_camera_index_by_name class CameraWidget: @@ -245,9 +245,13 @@ class CameraWidget: self.config.use_ffmpeg = False # Try storing ints as ints, for those using wired cameras. if value not in self.camera_list: - self.config.capture_source = int(value) - else: self.config.capture_source = value + else: + #if "COM" not in value: + ports = ("COM", "/dev/tty") + if any(x in str(self.config.capture_source) for x in ports): + self.config.capture_source = get_camera_index_by_name(value) + else: self.config.capture_source = value except ValueError: if value == "": self.config.capture_source = None diff --git a/BabbleApp/utils/misc_utils.py b/BabbleApp/utils/misc_utils.py index d1e5785..be9c536 100644 --- a/BabbleApp/utils/misc_utils.py +++ b/BabbleApp/utils/misc_utils.py @@ -1,8 +1,10 @@ -import os import typing +import serial +import sys +import glob from pygrabber.dshow_graph import FilterGraph -is_nt = True if os.name == "nt" else False +is_nt = True if sys.platform.startswith('win') else False graph = FilterGraph() @@ -11,11 +13,43 @@ def list_camera_names(): cam_names = [] for index, name in enumerate(cam_list): cam_names.append(name) + cam_names = cam_names + list_serial_ports() return cam_names +def list_serial_ports(): + """ Lists serial port names + + :raises EnvironmentError: + On unsupported or unknown platforms + :returns: + A list of the serial ports available on the system + """ + if sys.platform.startswith('win'): + ports = ['COM%s' % (i + 1) for i in range(256)] + elif sys.platform.startswith('linux') or sys.platform.startswith('cygwin'): + # this excludes your current terminal "/dev/tty" + ports = glob.glob('/dev/tty[A-Za-z]*') + elif sys.platform.startswith('darwin'): + ports = glob.glob('/dev/tty.*') + else: + raise EnvironmentError('Unsupported platform') + + result = [] + for port in ports: + try: + s = serial.Serial(port) + s.close() + result.append(port) + except (OSError, serial.SerialException): + pass + return result + def get_camera_index_by_name(name): + #cam_list = list_camera_names() cam_list = graph.get_input_devices() + #rint(f"name: {name}") + #rint(f"cam_list: {cam_list}") for i, device_name in enumerate(cam_list): if device_name == name: @@ -23,6 +57,10 @@ def get_camera_index_by_name(name): return None +#def get_serial_port(name): +# for i, device in enumerate(cam_list): + + def PlaySound(*args, **kwargs): pass From 15fd99c134d225d745919cde42a5b05689b437c9 Mon Sep 17 00:00:00 2001 From: Rames The Generic Date: Thu, 22 Aug 2024 14:06:25 -0400 Subject: [PATCH 5/7] Fixed type checks across the app --- BabbleApp/algo_settings_widget.py | 32 +++++++++++++++++----------- BabbleApp/calib_settings_widget.py | 1 - BabbleApp/general_settings_widget.py | 24 +++++++++++++-------- 3 files changed, 35 insertions(+), 22 deletions(-) diff --git a/BabbleApp/algo_settings_widget.py b/BabbleApp/algo_settings_widget.py index fe253f4..eb6a1f5 100644 --- a/BabbleApp/algo_settings_widget.py +++ b/BabbleApp/algo_settings_widget.py @@ -147,34 +147,42 @@ class AlgoSettingsWidget: # If anything has changed in our configuration settings, change/update those. changed = False - if self.config.gui_multiply != float(values[self.gui_multiply]): - self.config.gui_multiply = float(values[self.gui_multiply]) - changed = True + if str(self.config.gui_multiply) != values[self.gui_multiply]: + try: + self.config.gui_multiply = float(values[self.gui_multiply]) + changed = True + except: print("Not a Float") if self.config.gui_model_file != values[self.gui_model_file]: self.config.gui_model_file = values[self.gui_model_file] changed = True - if self.config.calib_deadzone != float(values[self.calib_deadzone]): - self.config.calib_deadzone = float(values[self.calib_deadzone]) - changed = True + if str(self.config.calib_deadzone) != values[self.calib_deadzone]: + try: + self.config.calib_deadzone = float(values[self.calib_deadzone]) + changed = True + except: print("Not a Float") if self.config.gui_use_gpu != values[self.gui_use_gpu]: self.config.gui_use_gpu = values[self.gui_use_gpu] changed = True if values[self.gui_gpu_index] != '': - if self.config.gui_gpu_index != int(values[self.gui_gpu_index]): - self.config.gui_gpu_index = int(values[self.gui_gpu_index]) - changed = True + if str(self.config.gui_gpu_index) != values[self.gui_gpu_index]: + try: + self.config.gui_gpu_index = int(values[self.gui_gpu_index]) + changed = True + except: print("Not an Int") if self.config.gui_runtime != str(values[self.gui_runtime]): self.config.gui_runtime = str(values[self.gui_runtime]) changed = True if values[self.gui_inference_threads] != '': - if self.config.gui_inference_threads != int(values[self.gui_inference_threads]): - self.config.gui_inference_threads = int(values[self.gui_inference_threads]) - changed = True + if str(self.config.gui_inference_threads) != values[self.gui_inference_threads]: + try: + self.config.gui_inference_threads = int(values[self.gui_inference_threads]) + changed = True + except: print("Not an Int") if values[self.gui_min_cutoff] != '': if self.config.gui_min_cutoff != values[self.gui_min_cutoff]: self.config.gui_min_cutoff = values[self.gui_min_cutoff] diff --git a/BabbleApp/calib_settings_widget.py b/BabbleApp/calib_settings_widget.py index eac445d..3c2df43 100644 --- a/BabbleApp/calib_settings_widget.py +++ b/BabbleApp/calib_settings_widget.py @@ -196,7 +196,6 @@ class CalibSettingsWidget: for count2, element2 in enumerate(element1): if values[element2] != '': try: - float(values[element2]) if float(self.array[count1][count2]) != float(values[element2]): self.array[count1][count2] = float(values[element2]) changed = True diff --git a/BabbleApp/general_settings_widget.py b/BabbleApp/general_settings_widget.py index b7714e5..c2e8e6f 100644 --- a/BabbleApp/general_settings_widget.py +++ b/BabbleApp/general_settings_widget.py @@ -225,17 +225,23 @@ class SettingsWidget: self.config.gui_osc_location = values[self.gui_osc_location] changed = True if values[self.gui_cam_resolution_x] != '': - if self.config.gui_cam_resolution_x != int(values[self.gui_cam_resolution_x]): - self.config.gui_cam_resolution_x = int(values[self.gui_cam_resolution_x]) - changed = True + if str(self.config.gui_cam_resolution_x) != values[self.gui_cam_resolution_x]: + try: + self.config.gui_cam_resolution_x = int(values[self.gui_cam_resolution_x]) + changed = True + except: print("Not an Int") if values[self.gui_cam_resolution_y] != '': - if self.config.gui_cam_resolution_y != int(values[self.gui_cam_resolution_y]): - self.config.gui_cam_resolution_y = int(values[self.gui_cam_resolution_y]) - changed = True + if str(self.config.gui_cam_resolution_y) != values[self.gui_cam_resolution_y]: + try: + self.config.gui_cam_resolution_y = int(values[self.gui_cam_resolution_y]) + changed = True + except: print("Not an Int") if values[self.gui_cam_framerate] != '': - if self.config.gui_cam_framerate != int(values[self.gui_cam_framerate]): - self.config.gui_cam_framerate = int(values[self.gui_cam_framerate]) - changed = True + if str(self.config.gui_cam_framerate) != values[self.gui_cam_framerate]: + try: + self.config.gui_cam_framerate = int(values[self.gui_cam_framerate]) + changed = True + except: print("Not an Int") if self.config.gui_use_red_channel != bool(values[self.gui_use_red_channel]): self.config.gui_use_red_channel = bool(values[self.gui_use_red_channel]) From 17c68cec6eac54c84b1b79ff0e24a082f9f90f3d Mon Sep 17 00:00:00 2001 From: Rames The Generic Date: Mon, 23 Sep 2024 13:57:22 -0400 Subject: [PATCH 6/7] Fixed UVC Cams after breaking them again :P --- BabbleApp/camera.py | 4 ++-- BabbleApp/camera_widget.py | 31 +++++++++++++++++++++---------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/BabbleApp/camera.py b/BabbleApp/camera.py index d11aee4..c34dd77 100644 --- a/BabbleApp/camera.py +++ b/BabbleApp/camera.py @@ -83,7 +83,7 @@ class Camera: return should_push = True # If things aren't open, retry until they are. Don't let read requests come in any earlier - # than this, otherwise we can deadlock ourselves. + # than this, otherwise we can deadlock (valve reference) ourselves. if ( self.config.capture_source is not None and self.config.capture_source != "" ): @@ -174,7 +174,7 @@ class Camera: #self.bps = image.nbytes if should_push: self.push_image_to_queue(image, frame_number, self.fps) - except: + except Exception as e: print( f"{Fore.YELLOW}[WARN] Capture source problem, assuming camera disconnected, waiting for reconnect.{Fore.RESET}") self.camera_status = CameraState.DISCONNECTED diff --git a/BabbleApp/camera_widget.py b/BabbleApp/camera_widget.py index 21158c4..1ad7569 100644 --- a/BabbleApp/camera_widget.py +++ b/BabbleApp/camera_widget.py @@ -35,6 +35,7 @@ class CameraWidget: self.gui_vertical_flip = f"-VERTICALFLIP{widget_id}-" self.gui_horizontal_flip = f"-HORIZONTALFLIP{widget_id}-" self.use_calibration = f"-USECALIBRATION{widget_id}-" + self.gui_refresh_button = f"-REFRESHCAMLIST{widget_id}-" self.osc_queue = osc_queue self.main_config = main_config self.cam_id = widget_id @@ -165,10 +166,11 @@ class CameraWidget: self.widget_layout = [ [ sg.Text("Camera Address", background_color='#424042'), - sg.InputCombo(self.camera_list, default_value=self.config.capture_source, + sg.InputCombo(values=self.camera_list, default_value=self.config.capture_source, key=self.gui_camera_addr, tooltip="Enter the IP address or UVC port of your camera. (Include the 'http://')", - enable_events=True) + enable_events=True), + sg.Button("Refresh List", key=self.gui_refresh_button, button_color='#539e8a') ], [ sg.Button("Save and Restart Tracking", key=self.gui_save_tracking_button, button_color='#539e8a'), @@ -244,14 +246,18 @@ class CameraWidget: try: self.config.use_ffmpeg = False # Try storing ints as ints, for those using wired cameras. - if value not in self.camera_list: - self.config.capture_source = value - else: + #if value not in self.camera_list: + # self.config.capture_source = value #if "COM" not in value: - ports = ("COM", "/dev/tty") - if any(x in str(self.config.capture_source) for x in ports): + ports = ("COM", "/dev/tty") + if any(x in str(value) for x in ports): + self.config.capture_source = value + else: + cam = get_camera_index_by_name(value) # Set capture_source to the UVC index. Otherwise treat value like an ipcam if we return none + if cam != None: self.config.capture_source = get_camera_index_by_name(value) - else: self.config.capture_source = value + else: + self.config.capture_source = value except ValueError: if value == "": self.config.capture_source = None @@ -273,8 +279,6 @@ class CameraWidget: changed = True - - if self.config.rotation_angle != values[self.gui_rotation_slider]: self.config.rotation_angle = int(values[self.gui_rotation_slider]) changed = True @@ -330,6 +334,13 @@ class CameraWidget: self.x0, self.y0 = values[self.gui_roi_selection] self.x1, self.y1 = values[self.gui_roi_selection] + if (event == self.gui_refresh_button): + print("Refreshed Cameralist") + self.camera_list = list_camera_names() + print(self.camera_list) + window[self.gui_camera_addr].update(values=self.camera_list) + + if event == self.gui_restart_calibration: self.babble_cnn.calibration_frame_counter = 1500 PlaySound('Audio/start.wav', SND_FILENAME | SND_ASYNC) From d6bca7cd8ee4d116b228fede84719707f251ad50 Mon Sep 17 00:00:00 2001 From: Rames The Generic Date: Wed, 25 Sep 2024 20:09:44 -0400 Subject: [PATCH 7/7] List serial cams and fix bugs --- BabbleApp/babble_processor.py | 4 +++ BabbleApp/babbleapp.py | 2 +- BabbleApp/camera.py | 3 ++ BabbleApp/camera_widget.py | 52 +++++++++++++++++++++-------------- BabbleApp/utils/misc_utils.py | 7 +++-- 5 files changed, 44 insertions(+), 24 deletions(-) diff --git a/BabbleApp/babble_processor.py b/BabbleApp/babble_processor.py index f4190f1..2eddb4c 100644 --- a/BabbleApp/babble_processor.py +++ b/BabbleApp/babble_processor.py @@ -68,6 +68,7 @@ class BabbleProcessor: self.current_image_gray = None self.current_frame_number = None self.current_fps = None + self.FRAMESIZE = [0,0,1] self.calibration_frame_counter = None @@ -129,6 +130,7 @@ class BabbleProcessor: try: # Get frame from capture source, crop to ROI + self.FRAMESIZE = self.current_image.shape self.current_image = self.current_image[ int(self.config.roi_window_y): int( self.config.roi_window_y + self.config.roi_window_h @@ -238,3 +240,5 @@ class BabbleProcessor: #print(self.output) self.output_images_and_update(CamInfo(self.current_algo, self.output)) + def get_framesize(self): + return self.FRAMESIZE \ No newline at end of file diff --git a/BabbleApp/babbleapp.py b/BabbleApp/babbleapp.py index 0f1854f..e979dcd 100644 --- a/BabbleApp/babbleapp.py +++ b/BabbleApp/babbleapp.py @@ -262,7 +262,7 @@ def main(): config.save() elif values[CALIB_SETTINGS_RADIO_NAME] and config.cam_display_id != Tab.CALIBRATION: - cams[0].stop() + cams[0].start() # Allow tracking to continue in calibration tab settings[0].stop() settings[1].stop() settings[2].start() diff --git a/BabbleApp/camera.py b/BabbleApp/camera.py index c34dd77..213a26c 100644 --- a/BabbleApp/camera.py +++ b/BabbleApp/camera.py @@ -66,6 +66,7 @@ class Camera: self.prevft = 0 self.newft = 0 self.fl = [0] + self.FRAME_SIZE = [0,0] self.error_message = f"{Fore.YELLOW}[WARN] Capture source {{}} not found, retrying...{Fore.RESET}" @@ -141,6 +142,7 @@ class Camera: if any(x in str(self.config.capture_source) for x in ports): self.get_serial_camera_picture(should_push) else: + self.__del__() self.get_cv2_camera_picture(should_push) if not should_push: # if we get all the way down here, consider ourselves connected @@ -152,6 +154,7 @@ class Camera: if not ret: self.cv2_camera.set(cv2.CAP_PROP_POS_FRAMES, 0) raise RuntimeError("Problem while getting frame") + self.FRAME_SIZE = image.shape frame_number = self.cv2_camera.get(cv2.CAP_PROP_POS_FRAMES) # Calculate the fps. yeah = time.time() diff --git a/BabbleApp/camera_widget.py b/BabbleApp/camera_widget.py index 1ad7569..66a3c47 100644 --- a/BabbleApp/camera_widget.py +++ b/BabbleApp/camera_widget.py @@ -8,7 +8,6 @@ import cv2 from babble_processor import BabbleProcessor, CamInfoOrigin from camera import Camera, CameraState from config import BabbleConfig -from landmark_processor import LandmarkProcessor from osc import Tab from utils.misc_utils import PlaySound, SND_FILENAME, SND_ASYNC, list_camera_names, get_camera_index_by_name @@ -68,16 +67,6 @@ class CameraWidget: self.cam_id, ) - self.babble_landmark = LandmarkProcessor( - self.config, - self.settings_config, - self.main_config, - self.cancellation_event, - self.capture_event, - self.capture_queue, - self.image_queue, - self.cam_id, - ) self.camera_status_queue = Queue(maxsize=2) self.camera = Camera( @@ -92,7 +81,7 @@ class CameraWidget: self.roi_layout = [ [ - sg.Button("Auto ROI", key=self.gui_autoroi, button_color='#539e8a', tooltip="Automatically set ROI", ), + sg.Button("Select Entire Frame", key=self.gui_autoroi, button_color='#539e8a', tooltip="Automatically set ROI", ), ], [ sg.Graph( @@ -122,9 +111,10 @@ class CameraWidget: ], [ sg.Button("Start Calibration", key=self.gui_restart_calibration, button_color='#539e8a', - tooltip="Start calibration. Look all arround to all extreams without blinking until sound is heard.", ), + tooltip="Neutural Calibration: Hold a relaxed face, press [Start Calibration] and then press [Stop Calibraion]. \nFull Calibration: Press [Start Calibration] and make as many face movements as you can until it switches back to tracking mode or press [Stop Calibration]", disabled=True), + sg.Button("Stop Calibration", key=self.gui_stop_calibration, button_color='#539e8a', - tooltip="Stop calibration manualy.", ), + tooltip="Stop calibration manualy.", disabled=True), ], [ sg.Checkbox( @@ -133,6 +123,7 @@ class CameraWidget: key=self.use_calibration, background_color='#424042', tooltip="Checked = Calibrated model output. Unchecked = Raw model output", + enable_events=True ), ], [ @@ -197,7 +188,7 @@ class CameraWidget: def _movavg_fps(self, next_fps): self.movavg_fps_queue.append(next_fps) - fps = round(sum(self.movavg_fps_queue) / len(self.movavg_fps_queue)) + fps = round(sum(self.movavg_fps_queue) / len(self.movavg_fps_queue)) millisec = round((1 / fps if fps else 0) * 1000) return f"{fps} Fps {millisec} ms" @@ -283,7 +274,6 @@ class CameraWidget: self.config.rotation_angle = int(values[self.gui_rotation_slider]) changed = True - #print(self.config.gui_vertical_flip) if self.config.gui_vertical_flip != values[self.gui_vertical_flip]: self.config.gui_vertical_flip = values[self.gui_vertical_flip] changed = True @@ -313,6 +303,17 @@ class CameraWidget: window[self.gui_roi_layout].update(visible=True) window[self.gui_tracking_layout].update(visible=False) + if event == self.use_calibration: + print("toggle event") + if self.settings_config.use_calibration == True: + window[self.gui_restart_calibration].update(disabled = False) + window[self.gui_stop_calibration].update(disabled = False) + print("Enabled") + else: + window[self.gui_restart_calibration].update(disabled = True) + window[self.gui_stop_calibration].update(disabled = True) + print("Disabled") + if event == "{}+UP".format(self.gui_roi_selection): # Event for mouse button up in ROI mode self.is_mouse_up = True @@ -334,19 +335,30 @@ class CameraWidget: self.x0, self.y0 = values[self.gui_roi_selection] self.x1, self.y1 = values[self.gui_roi_selection] + if event == self.gui_autoroi: + print("Set ROI") + output = self.babble_cnn.get_framesize() + self.config.roi_window_x = 0 + self.config.roi_window_y = 0 + self.config.roi_window_w = output[0] + self.config.roi_window_h = output[1] + self.main_config.save() + if (event == self.gui_refresh_button): - print("Refreshed Cameralist") + print("\033[94m[INFO] Refreshed Camera List\033[0m") self.camera_list = list_camera_names() print(self.camera_list) window[self.gui_camera_addr].update(values=self.camera_list) if event == self.gui_restart_calibration: - self.babble_cnn.calibration_frame_counter = 1500 - PlaySound('Audio/start.wav', SND_FILENAME | SND_ASYNC) + if values[self.use_calibration] == True: # Don't start recording if the calibration filter is disabled. + self.babble_cnn.calibration_frame_counter = 1500 + PlaySound('Audio/start.wav', SND_FILENAME | SND_ASYNC) if event == self.gui_stop_calibration: - self.babble_cnn.calibration_frame_counter = 0 + if self.babble_cnn.calibration_frame_counter != None: # Only assign the variable if we are in calibration mode. + self.babble_cnn.calibration_frame_counter = 0 needs_roi_set = self.config.roi_window_h <= 0 or self.config.roi_window_w <= 0 diff --git a/BabbleApp/utils/misc_utils.py b/BabbleApp/utils/misc_utils.py index e5227a7..a69823b 100644 --- a/BabbleApp/utils/misc_utils.py +++ b/BabbleApp/utils/misc_utils.py @@ -90,20 +90,21 @@ def list_camera_names(): if is_nt: # On Windows, use pygrabber to list devices cam_list = graph.get_input_devices() - return cam_list + return cam_list + list_serial_ports() elif os_type == "Linux": # On Linux, return UVC device paths like '/dev/video0' - return list_linux_uvc_devices() + return list_linux_uvc_devices() + list_serial_ports() elif os_type == "Darwin": # On macOS, fallback to OpenCV (device names aren't fetched) - return list_cameras_opencv() + return list_cameras_opencv() + list_serial_ports() else: return ["Unsupported operating system"] def list_serial_ports(): + print("DEBUG: Listed Serial Ports") """ Lists serial port names :raises EnvironmentError: