Merge pull request #42 from RamesTheGeneric/main

Completed calibration tab
This commit is contained in:
Summer authored and GitHub committed 2024-09-26 09:01:38 -07:00
commit fc56aba7ef
15 files changed
+563 -111

No files matched your search

+3
View File
@@ -198,6 +198,7 @@ class AlgoSettingsWidget:
except ValueError:
pass # Ignore invalid float conversion
if self.config.gui_model_file != values[self.gui_model_file]:
self.config.gui_model_file = values[self.gui_model_file]
changed = True
@@ -210,6 +211,7 @@ class AlgoSettingsWidget:
except ValueError:
pass # Ignore invalid float conversion
if self.config.gui_use_gpu != values[self.gui_use_gpu]:
self.config.gui_use_gpu = values[self.gui_use_gpu]
changed = True
@@ -222,6 +224,7 @@ class AlgoSettingsWidget:
except ValueError:
pass # Ignore invalid int conversion
if self.config.gui_runtime != str(values[self.gui_runtime]):
self.config.gui_runtime = str(values[self.gui_runtime])
changed = True
+6 -3
View File
@@ -68,6 +68,7 @@ class BabbleProcessor:
self.current_image_gray = None
self.current_frame_number = None
self.current_fps = None
self.FRAMESIZE = [0,0,1]
self.calibration_frame_counter = None
@@ -129,6 +130,7 @@ class BabbleProcessor:
try:
# Get frame from capture source, crop to ROI
self.FRAMESIZE = self.current_image.shape
self.current_image = self.current_image[
int(self.config.roi_window_y): int(
self.config.roi_window_y + self.config.roi_window_h
@@ -223,15 +225,14 @@ class BabbleProcessor:
new_blue_channel = red_channel
new_green_channel = red_channel
self.current_image = cv2.merge((new_blue_channel, new_green_channel, red_channel))
self.current_image_gray = cv2.cvtColor(
self.current_image, cv2.COLOR_BGR2GRAY
self.current_image, cv2.COLOR_BGR2GRAY
)
self.current_image_gray_clean = self.current_image_gray.copy() #copy this frame to have a clean image for blink algo
run_model(self)
if self.config.use_calibration:
if self.settings.use_calibration:
self.output = cal.cal_osc(self, self.output)
#else:
@@ -239,3 +240,5 @@ class BabbleProcessor:
#print(self.output)
self.output_images_and_update(CamInfo(self.current_algo, self.output))
def get_framesize(self):
return self.FRAMESIZE
+38
View File
@@ -28,6 +28,7 @@ from tab import CamInfo, Tab
from osc import VRChatOSCReceiver, VRChatOSC
from general_settings_widget import SettingsWidget
from algo_settings_widget import AlgoSettingsWidget
from calib_settings_widget import CalibSettingsWidget
from utils.misc_utils import is_nt
if is_nt:
from winotify import Notification
@@ -41,9 +42,11 @@ WINDOW_NAME = "Babble sApp"
CAM_NAME = "-CAMWIDGET-"
SETTINGS_NAME = "-SETTINGSWIDGET-"
ALGO_SETTINGS_NAME = "-ALGOSETTINGSWIDGET-"
CALIB_SETTINGS_NAME = "-CALIBSETTINGSWIDGET-"
CAM_RADIO_NAME = "-CAMRADIO-"
SETTINGS_RADIO_NAME = "-SETTINGSRADIO-"
ALGO_SETTINGS_RADIO_NAME = "-ALGOSETTINGSRADIO-"
CALIB_SETTINGS_RADIO_NAME = "-CALIBSETTINGSRADIO-"
page_url = "https://github.com/SummerSigh/ProjectBabble/releases/latest"
appversion = "Babble v2.0.6 Alpha"
@@ -108,6 +111,7 @@ def main():
settings = [
SettingsWidget(Tab.SETTINGS, config, osc_queue),
AlgoSettingsWidget(Tab.ALGOSETTINGS, config, osc_queue),
CalibSettingsWidget(Tab.CALIBRATION, config, osc_queue),
]
layout = [
@@ -133,6 +137,13 @@ def main():
default=(config.cam_display_id == Tab.ALGOSETTINGS),
key=ALGO_SETTINGS_RADIO_NAME,
),
sg.Radio(
"Calibration",
"TABSELECTRADIO",
background_color="#292929",
default=(config.cam_display_id == Tab.CALIBRATION),
key=CALIB_SETTINGS_RADIO_NAME,
),
],
[
@@ -157,6 +168,13 @@ def main():
visible=(config.cam_display_id in [Tab.ALGOSETTINGS]),
background_color="#424042",
),
sg.Column(
settings[2].widget_layout,
vertical_alignment="top",
key=CALIB_SETTINGS_NAME,
visible=(config.cam_display_id in [Tab.CALIBRATION]),
background_color="#424042",
),
],
]
@@ -167,6 +185,8 @@ def main():
settings[0].start()
if config.cam_display_id in [Tab.ALGOSETTINGS]:
settings[1].start()
if config.cam_display_id in [Tab.CALIBRATION]:
settings[2].start()
# the cam needs to be running before it is passed to the OSC
if config.settings.gui_ROSC:
@@ -206,9 +226,11 @@ def main():
cams[0].start()
settings[0].stop()
settings[1].stop()
settings[2].stop()
window[CAM_NAME].update(visible=True)
window[SETTINGS_NAME].update(visible=False)
window[ALGO_SETTINGS_NAME].update(visible=False)
window[CALIB_SETTINGS_NAME].update(visible=False)
config.cam_display_id = Tab.CAM
config.save()
@@ -217,10 +239,12 @@ def main():
elif values[SETTINGS_RADIO_NAME] and config.cam_display_id != Tab.SETTINGS:
cams[0].stop()
settings[1].stop()
settings[2].stop()
settings[0].start()
window[CAM_NAME].update(visible=False)
window[SETTINGS_NAME].update(visible=True)
window[ALGO_SETTINGS_NAME].update(visible=False)
window[CALIB_SETTINGS_NAME].update(visible=False)
config.cam_display_id = Tab.SETTINGS
config.save()
@@ -228,12 +252,26 @@ def main():
elif values[ALGO_SETTINGS_RADIO_NAME] and config.cam_display_id != Tab.ALGOSETTINGS:
cams[0].stop()
settings[0].stop()
settings[2].stop()
settings[1].start()
window[CAM_NAME].update(visible=False)
window[SETTINGS_NAME].update(visible=False)
window[ALGO_SETTINGS_NAME].update(visible=True)
window[CALIB_SETTINGS_NAME].update(visible=False)
config.cam_display_id = Tab.ALGOSETTINGS
config.save()
elif values[CALIB_SETTINGS_RADIO_NAME] and config.cam_display_id != Tab.CALIBRATION:
cams[0].start() # Allow tracking to continue in calibration tab
settings[0].stop()
settings[1].stop()
settings[2].start()
window[CAM_NAME].update(visible=False)
window[SETTINGS_NAME].update(visible=False)
window[ALGO_SETTINGS_NAME].update(visible=False)
window[CALIB_SETTINGS_NAME].update(visible=True)
config.cam_display_id = Tab.CALIBRATION
config.save()
# Otherwise, render all
+108
View File
@@ -0,0 +1,108 @@
def set_shapes(widget_id):
shape_index = ["cheekPuffLeft", "cheekPuffRight", "cheekSuckLeft", "cheekSuckRight", "jawOpen", "jawForward", "jawLeft", "jawRight", "noseSneerLeft", "noseSneerRight", "mouthFunnel", "mouthPucker", "mouthLeft", "mouthRight",
"mouthRollUpper", "mouthRollLower", "mouthShrugUpper", "mouthShrugLower", "mouthClose", "mouthSmileLeft",
"mouthSmileRight", "mouthFrownLeft", "mouthFrownRight", "mouthDimpleLeft", "mouthDimpleRight", "mouthUpperUpLeft",
"mouthUpperUpRight", "mouthLowerDownLeft", "mouthLowerDownRight", "mouthPressLeft", "mouthPressRight", "mouthStretchLeft",
"mouthStretchRight", "tongueOut", "tongueUp", "tongueDown", "tongueLeft", "tongueRight", "tongueRoll", "tongueBendDown", "tongueCurlUp", "tongueSquish", "tongueFlat", "tongueTwistLeft", "tongueTwistRight"]
shape = [
[
f"-CHEEKPUFFLEFTMIN{widget_id}-",
f"-CHEEKPUFFRIGHTMIN{widget_id}-",
f"-CHEEKSUCKLEFTMIN{widget_id}-",
f"-CHEEKSUCKRIGHTMIN{widget_id}-",
f"-JAWOPENMIN{widget_id}-",
f"-JAWFORWARDMIN{widget_id}-",
f"-JAWLEFTMIN{widget_id}-",
f"-JAWRIGHTMIN{widget_id}-",
f"-NOSESNEERLEFTMIN{widget_id}-",
f"-NOSESNEERRIGHTMIN{widget_id}-",
f"-MOUTHFUNNELMIN{widget_id}-",
f"-MOUTHPUCKERMIN{widget_id}-",
f"-MOUTHLEFTMIN{widget_id}-",
f"-MOUTHRIGHTMIN{widget_id}-",
f"-MOUTHROLLUPPERMIN{widget_id}-",
f"-MOUTHROLLLOWERMIN{widget_id}-",
f"-MOUTHSHRUGUPPERMIN{widget_id}-",
f"-MOUTHSHRUGLOWERMIN{widget_id}-",
f"-MOUTHCLOSEMIN{widget_id}-",
f"-MOUTHSMILELEFTMIN{widget_id}-",
f"-MOUTHSMILERIGHTMIN{widget_id}-",
f"-MOUTHFROWNLEFTMIN{widget_id}-",
f"-MOUTHFROWNRIGHTMIN{widget_id}-",
f"-MOUTHDIMPLELEFTMIN{widget_id}-",
f"-MOUTHDIMPLERIGHTMIN{widget_id}-",
f"-MOUTHUPPERUPLEFTMIN{widget_id}-",
f"-MOUTHUPPERUPRIGHTMIN{widget_id}-",
f"-MOUTHLOWERDOWNLEFTMIN{widget_id}-",
f"-MOUTHLOWERDOWNRIGHTMIN{widget_id}-",
f"-MOUTHPRESSLEFTMIN{widget_id}-",
f"-MOUTHPRESSRIGHTMIN{widget_id}-",
f"-MOUTHSTRETCHLEFTMIN{widget_id}-",
f"-MOUTHSTRETCHRIGHTMIN{widget_id}-",
f"-TONGUEOUTMIN{widget_id}-",
f"-TONGUEUPMIN{widget_id}-",
f"-TONGUEDOWNMIN{widget_id}-",
f"-TONGUELEFTMIN{widget_id}-",
f"-TONGUERIGHTMIN{widget_id}-",
f"-TONGUEROLLMIN{widget_id}-",
f"-TONGUEBENDDOWNMIN{widget_id}-",
f"-TONGUECURLUPMIN{widget_id}-",
f"-TONGUESQUISHMIN{widget_id}-",
f"-TONGUEFLATMIN{widget_id}-",
f"-TONGUETWISTLEFTMIN{widget_id}-",
f"-TONGUETWISTRIGHTMIN{widget_id}-",
],
[
f"-CHEEKPUFFLEFTMAX{widget_id}-",
f"-CHEEKPUFFRIGHTMAX{widget_id}-",
f"-CHEEKSUCKLEFTMAX{widget_id}-",
f"-CHEEKSUCKRIGHTMAX{widget_id}-",
f"-JAWOPENMAX{widget_id}-",
f"-JAWFORWARDMAX{widget_id}-",
f"-JAWLEFTMAX{widget_id}-",
f"-JAWRIGHTMAX{widget_id}-",
f"-NOSESNEERLEFTMAX{widget_id}-",
f"-NOSESNEERRIGHTMAX{widget_id}-",
f"-MOUTHFUNNELMAX{widget_id}-",
f"-MOUTHPUCKERMAX{widget_id}-",
f"-MOUTHLEFTMAX{widget_id}-",
f"-MOUTHRIGHTMAX{widget_id}-",
f"-MOUTHROLLUPPERMAX{widget_id}-",
f"-MOUTHROLLLOWERMAX{widget_id}-",
f"-MOUTHSHRUGUPPERMAX{widget_id}-",
f"-MOUTHSHRUGLOWERMAX{widget_id}-",
f"-MOUTHCLOSEMAX{widget_id}-",
f"-MOUTHSMILELEFTMAX{widget_id}-",
f"-MOUTHSMILERIGHTMAX{widget_id}-",
f"-MOUTHFROWNLEFTMAX{widget_id}-",
f"-MOUTHFROWNRIGHTMAX{widget_id}-",
f"-MOUTHDIMPLELEFTMAX{widget_id}-",
f"-MOUTHDIMPLERIGHTMAX{widget_id}-",
f"-MOUTHUPPERUPLEFTMAX{widget_id}-",
f"-MOUTHUPPERUPRIGHTMAX{widget_id}-",
f"-MOUTHLOWERDOWNLEFTMAX{widget_id}-",
f"-MOUTHLOWERDOWNRIGHTMAX{widget_id}-",
f"-MOUTHPRESSLEFTMAX{widget_id}-",
f"-MOUTHPRESSRIGHTMAX{widget_id}-",
f"-MOUTHSTRETCHLEFTMAX{widget_id}-",
f"-MOUTHSTRETCHRIGHTMAX{widget_id}-",
f"-TONGUEOUTMAX{widget_id}-",
f"-TONGUEUPMAX{widget_id}-",
f"-TONGUEDOWNMAX{widget_id}-",
f"-TONGUELEFTMAX{widget_id}-",
f"-TONGUERIGHTMAX{widget_id}-",
f"-TONGUEROLLMAX{widget_id}-",
f"-TONGUEBENDDOWNMAX{widget_id}-",
f"-TONGUECURLUPMAX{widget_id}-",
f"-TONGUESQUISHMAX{widget_id}-",
f"-TONGUEFLATMAX{widget_id}-",
f"-TONGUETWISTLEFTMAX{widget_id}-",
f"-TONGUETWISTRIGHTMAX{widget_id}-",
],
]
return shape_index, shape
+221
View File
@@ -0,0 +1,221 @@
import PySimpleGUI as sg
from config import BabbleSettingsConfig
from osc import Tab
from queue import Queue
from threading import Event
import numpy as np
from calib_settings_values import set_shapes
class CalibSettingsWidget:
def __init__(self, widget_id: Tab, main_config: BabbleSettingsConfig, osc_queue: Queue):
self.gui_general_settings_layout = f"-GENERALSETTINGSLAYOUT{widget_id}-"
self.gui_reset_min = f"-RESETMIN{widget_id}-"
self.gui_reset_max = f"-RESETMAX{widget_id}-"
self.gui_multiply = f"-MULTIPLY{widget_id}-"
self.gui_calibration_mode = f"-CALIBRATIONMODE{widget_id}-"
self.main_config = main_config
self.config = main_config.settings
self.array = np.fromstring(self.config.calib_array.replace('[', '').replace(']', ''), sep=',').reshape(2, 45)
self.calibration_list = ['Neutral', 'Full']
self.osc_queue = osc_queue
self.shape_index, self.shape = set_shapes(widget_id)
self.refreshed = False
# Define the window's contents
s = self.single_shape
d = self.double_shape
self.general_settings_layout = [
d('cheekPuff'),
d('cheekSuck'),
s('jawOpen'),
s('jawForward'),
d('jaw'),
d('noseSneer'),
s('mouthFunnel'),
s('mouthPucker'),
d('mouth'),
s('mouthRollUpper'),
s('mouthRollLower'),
s('mouthShrugUpper'),
s('mouthShrugLower'),
s('mouthClose'),
d('mouthSmile'),
d('mouthFrown'),
d('mouthDimple'),
d('mouthUpperUp'),
d('mouthLowerDown'),
d('mouthPress'),
d('mouthStretch'),
s('tongueOut'),
s('tongueUp'),
s('tongueDown'),
d('tongue'),
s('tongueRoll'),
s('tongueBendDown'),
s('tongueCurlUp'),
s('tongueSquish'),
s('tongueFlat'),
d('tongueTwist'),
]
self.widget_layout = [
[
sg.Text("Calibration Settings:", background_color='#242224'),
sg.Text("Calibration Mode:", background_color='#424042'),
sg.OptionMenu(
self.calibration_list,
self.config.calibration_mode,
key=self.gui_calibration_mode,
tooltip='Neutral = Only Min values are set when starting and stopping calibration. Full = Min and Max values are set based on recorded values when starting calibration.',
),
],
[
sg.Text("Left ", expand_x=True, justification='left'),
sg.Text("Shape", expand_x=True, justification='center'),
sg.Text("Right", expand_x=True, justification='right')
],
[
sg.Text("Min", expand_x=True, justification='center'),
sg.Text("Max", expand_x=True, justification='center'),
sg.HSeparator(pad=(50,0)),
sg.Text("Max", expand_x=True, justification='center'),
sg.Text("Min", expand_x=True, justification='center'),
],
[
sg.Column(
self.general_settings_layout,
key=self.gui_general_settings_layout,
scrollable=True,
vertical_scroll_only=True,
element_justification='center',
background_color='#424042' ),
],
[
sg.Button("Reset Min", key=self.gui_reset_min, button_color='#FF0000', tooltip = "Reset minimum values",),
sg.Button("Reset Max", key=self.gui_reset_max, button_color='#FF0000', tooltip = "Reset maximum values",),
],
]
self.cancellation_event = Event() # Set the event until start is called, otherwise we can block if shutdown is called.
self.cancellation_event.set()
self.image_queue = Queue(maxsize=2)
def double_shape(self, shapename):
indexl = self.shape_index.index(f"{shapename}Left")
indexr = self.shape_index.index(f"{shapename}Right")
double_shape = [
sg.InputText(
default_text=self.array[0][indexl],
key=self.shape[0][indexl],
size=(8),
tooltip="Min Left Value",
),
sg.InputText(
default_text=self.array[1][indexl],
key=self.shape[1][indexl],
size=(8),
tooltip="Max Left Value",
),
sg.Text(f"{shapename}Left/Right", background_color='#424042', expand_x=True),
sg.InputText(
default_text=self.array[1][indexr],
key=self.shape[1][indexr],
size=(8),
tooltip="Max Right Value",
),
sg.InputText(
default_text=self.array[0][indexr],
key=self.shape[0][indexr],
size=(8),
tooltip="Min Right Value",
),
]
return double_shape
def single_shape(self, shapename):
index = self.shape_index.index(f"{shapename}")
single_shape = [
sg.InputText(
default_text=self.array[0][index],
key=self.shape[0][index],
size=(8),
tooltip="Min Left Value",
),
sg.InputText(
default_text=self.array[1][index],
key=self.shape[1][index],
size=(8),
tooltip="Max Left Value",
),
sg.Text(f"{shapename}", background_color='#424042', expand_x=True),
]
return single_shape
def started(self):
return not self.cancellation_event.is_set()
def start(self):
# If we're already running, bail
if not self.cancellation_event.is_set():
return
self.cancellation_event.clear()
self.array = np.fromstring(self.config.calib_array.replace('[', '').replace(']', ''), sep=',').reshape(2, 45) # Reload the array from the config
self.refreshed = False
def stop(self):
# If we're not running yet, bail
if self.cancellation_event.is_set():
return
self.cancellation_event.set()
def render(self, window, event, values):
# If anything has changed in our configuration settings, change/update those.
changed = False
if not self.refreshed:
for count1, element1 in enumerate(self.shape):
for count2, element2 in enumerate(element1):
window[element2].update(float(self.array[count1][count2]))
#values[element2] = float(self.array[count1][count2])
self.refreshed = True
print('DEBUG: Refreshed')
if self.config.calibration_mode != str(values[self.gui_calibration_mode]):
self.config.calibration_mode = str(values[self.gui_calibration_mode])
changed = True
for count1, element1 in enumerate(self.shape):
for count2, element2 in enumerate(element1):
if values[element2] != '':
try:
if float(self.array[count1][count2]) != float(values[element2]):
self.array[count1][count2] = float(values[element2])
changed = True
except: print("Not a float")
if event == self.gui_reset_min:
for count1, element1 in enumerate(self.shape):
for count2, element2 in enumerate(element1):
self.array[0][count2] = float(0)
changed = True
self.refreshed = False
elif event == self.gui_reset_max:
for count1, element1 in enumerate(self.shape):
for count2, element2 in enumerate(element1):
self.array[1][count2] = float(1)
changed = True
self.refreshed = False
if changed:
self.config.calib_array = np.array2string(self.array, separator=',')
self.main_config.save()
#print(self.main_config)
self.osc_queue.put(Tab.CALIBRATION)
+9 -7
View File
@@ -66,6 +66,7 @@ class Camera:
self.prevft = 0
self.newft = 0
self.fl = [0]
self.FRAME_SIZE = [0,0]
self.error_message = f"{Fore.YELLOW}[WARN] Capture source {{}} not found, retrying...{Fore.RESET}"
@@ -83,13 +84,11 @@ class Camera:
return
should_push = True
# If things aren't open, retry until they are. Don't let read requests come in any earlier
# than this, otherwise we can deadlock ourselves.
# than this, otherwise we can deadlock (valve reference) ourselves.
if (
self.config.capture_source is not None and self.config.capture_source != ""
):
self.current_capture_source = self.config.capture_source
if "COM" in str(self.config.capture_source) and self.config.capture_source not in self.camera_list:
if "COM" in str(self.config.capture_source):
if (
self.serial_connection is None
or self.camera_status == CameraState.DISCONNECTED
@@ -114,7 +113,7 @@ class Camera:
if self.config.capture_source not in self.camera_list:
self.current_capture_source = self.config.capture_source
else:
self.current_capture_source = get_camera_index_by_name(self.current_capture_source)
self.current_capture_source = get_camera_index_by_name(self.config.capture_source)
if self.config.use_ffmpeg:
self.cv2_camera = cv2.VideoCapture(self.current_capture_source, cv2.CAP_FFMPEG)
@@ -139,9 +138,11 @@ class Camera:
if should_push and not self.capture_event.wait(timeout=0.02):
continue
if self.config.capture_source is not None:
if "COM" in str(self.config.capture_source):
ports = ("COM", "/dev/tty")
if any(x in str(self.config.capture_source) for x in ports):
self.get_serial_camera_picture(should_push)
else:
self.__del__()
self.get_cv2_camera_picture(should_push)
if not should_push:
# if we get all the way down here, consider ourselves connected
@@ -153,6 +154,7 @@ class Camera:
if not ret:
self.cv2_camera.set(cv2.CAP_PROP_POS_FRAMES, 0)
raise RuntimeError("Problem while getting frame")
self.FRAME_SIZE = image.shape
frame_number = self.cv2_camera.get(cv2.CAP_PROP_POS_FRAMES)
# Calculate the fps.
yeah = time.time()
@@ -175,7 +177,7 @@ class Camera:
#self.bps = image.nbytes
if should_push:
self.push_image_to_queue(image, frame_number, self.fps)
except:
except Exception as e:
print(
f"{Fore.YELLOW}[WARN] Capture source problem, assuming camera disconnected, waiting for reconnect.{Fore.RESET}")
self.camera_status = CameraState.DISCONNECTED
+65 -67
View File
@@ -8,9 +8,8 @@ import cv2
from babble_processor import BabbleProcessor, CamInfoOrigin
from camera import Camera, CameraState
from config import BabbleConfig
from landmark_processor import LandmarkProcessor
from osc import Tab
from utils.misc_utils import PlaySound, SND_FILENAME, SND_ASYNC, list_camera_names
from utils.misc_utils import PlaySound, SND_FILENAME, SND_ASYNC, list_camera_names, get_camera_index_by_name
class CameraWidget:
@@ -35,7 +34,7 @@ class CameraWidget:
self.gui_vertical_flip = f"-VERTICALFLIP{widget_id}-"
self.gui_horizontal_flip = f"-HORIZONTALFLIP{widget_id}-"
self.use_calibration = f"-USECALIBRATION{widget_id}-"
self.use_n_calibration = f"-USENCALIBRATION{widget_id}-"
self.gui_refresh_button = f"-REFRESHCAMLIST{widget_id}-"
self.osc_queue = osc_queue
self.main_config = main_config
self.cam_id = widget_id
@@ -43,6 +42,7 @@ class CameraWidget:
self.config = main_config.cam
self.settings = main_config.settings
self.camera_list = list_camera_names()
self.maybe_image = None
if self.cam_id == Tab.CAM:
self.config = main_config.cam
else:
@@ -67,16 +67,6 @@ class CameraWidget:
self.cam_id,
)
self.babble_landmark = LandmarkProcessor(
self.config,
self.settings_config,
self.main_config,
self.cancellation_event,
self.capture_event,
self.capture_queue,
self.image_queue,
self.cam_id,
)
self.camera_status_queue = Queue(maxsize=2)
self.camera = Camera(
@@ -91,7 +81,7 @@ class CameraWidget:
self.roi_layout = [
[
sg.Button("Auto ROI", key=self.gui_autoroi, button_color='#539e8a', tooltip="Automatically set ROI", ),
sg.Button("Select Entire Frame", key=self.gui_autoroi, button_color='#539e8a', tooltip="Automatically set ROI", ),
],
[
sg.Graph(
@@ -121,24 +111,19 @@ class CameraWidget:
],
[
sg.Button("Start Calibration", key=self.gui_restart_calibration, button_color='#539e8a',
tooltip="Start calibration. Look all arround to all extreams without blinking until sound is heard.", ),
tooltip="Neutural Calibration: Hold a relaxed face, press [Start Calibration] and then press [Stop Calibraion]. \nFull Calibration: Press [Start Calibration] and make as many face movements as you can until it switches back to tracking mode or press [Stop Calibration]", disabled=True),
sg.Button("Stop Calibration", key=self.gui_stop_calibration, button_color='#539e8a',
tooltip="Stop calibration manualy.", ),
tooltip="Stop calibration manualy.", disabled=True),
],
[
sg.Checkbox(
"Use Neutral Calibration:",
default=self.config.use_n_calibration,
key=self.use_n_calibration,
background_color='#424042',
tooltip="Toggle use of calibration using minimum values found during a neutral pose calibration step.",
),
sg.Checkbox(
"Use Full Calibration:",
default=self.config.use_calibration,
"Enable Calibration:",
default=self.settings_config.use_calibration,
key=self.use_calibration,
background_color='#424042',
tooltip="Toggle use of calibration.",
tooltip="Checked = Calibrated model output. Unchecked = Raw model output",
enable_events=True
),
],
[
@@ -172,10 +157,11 @@ class CameraWidget:
self.widget_layout = [
[
sg.Text("Camera Address", background_color='#424042'),
sg.InputCombo(self.camera_list, default_value=self.config.capture_source,
sg.InputCombo(values=self.camera_list, default_value=self.config.capture_source,
key=self.gui_camera_addr,
tooltip="Enter the IP address or UVC port of your camera. (Include the 'http://')",
enable_events=True)
enable_events=True),
sg.Button("Refresh List", key=self.gui_refresh_button, button_color='#539e8a')
],
[
sg.Button("Save and Restart Tracking", key=self.gui_save_tracking_button, button_color='#539e8a'),
@@ -202,7 +188,7 @@ class CameraWidget:
def _movavg_fps(self, next_fps):
self.movavg_fps_queue.append(next_fps)
fps = round(sum(self.movavg_fps_queue) / len(self.movavg_fps_queue))
fps = round(sum(self.movavg_fps_queue) / len(self.movavg_fps_queue))
millisec = round((1 / fps if fps else 0) * 1000)
return f"{fps} Fps {millisec} ms"
@@ -220,8 +206,8 @@ class CameraWidget:
self.cancellation_event.clear()
self.babble_cnn_thread = Thread(target=self.babble_cnn.run)
self.babble_cnn_thread.start()
self.babble_landmark_thread = Thread(target=self.babble_landmark.run)
self.babble_landmark_thread.start()
#self.babble_landmark_thread = Thread(target=self.babble_landmark.run)
#self.babble_landmark_thread.start()
self.camera_thread = Thread(target=self.camera.run)
self.camera_thread.start()
@@ -231,7 +217,7 @@ class CameraWidget:
return
self.cancellation_event.set()
self.babble_cnn_thread.join()
self.babble_landmark_thread.join()
#self.babble_landmark_thread.join()
self.camera_thread.join()
def render(self, window, event, values):
@@ -251,10 +237,18 @@ class CameraWidget:
try:
self.config.use_ffmpeg = False
# Try storing ints as ints, for those using wired cameras.
if value not in self.camera_list:
self.config.capture_source = int(value)
else:
#if value not in self.camera_list:
# self.config.capture_source = value
#if "COM" not in value:
ports = ("COM", "/dev/tty")
if any(x in str(value) for x in ports):
self.config.capture_source = value
else:
cam = get_camera_index_by_name(value) # Set capture_source to the UVC index. Otherwise treat value like an ipcam if we return none
if cam != None:
self.config.capture_source = get_camera_index_by_name(value)
else:
self.config.capture_source = value
except ValueError:
if value == "":
self.config.capture_source = None
@@ -280,7 +274,6 @@ class CameraWidget:
self.config.rotation_angle = int(values[self.gui_rotation_slider])
changed = True
#print(self.config.gui_vertical_flip)
if self.config.gui_vertical_flip != values[self.gui_vertical_flip]:
self.config.gui_vertical_flip = values[self.gui_vertical_flip]
changed = True
@@ -289,12 +282,8 @@ class CameraWidget:
self.config.gui_horizontal_flip = values[self.gui_horizontal_flip]
changed = True
if self.config.use_calibration != values[self.use_calibration]:
self.config.use_calibration = values[self.use_calibration]
changed = True
if self.config.use_n_calibration != values[self.use_n_calibration]:
self.config.use_n_calibration = values[self.use_n_calibration]
if self.settings_config.use_calibration != values[self.use_calibration]:
self.settings_config.use_calibration = values[self.use_calibration]
changed = True
if changed:
@@ -314,6 +303,17 @@ class CameraWidget:
window[self.gui_roi_layout].update(visible=True)
window[self.gui_tracking_layout].update(visible=False)
if event == self.use_calibration:
print("toggle event")
if self.settings_config.use_calibration == True:
window[self.gui_restart_calibration].update(disabled = False)
window[self.gui_stop_calibration].update(disabled = False)
print("Enabled")
else:
window[self.gui_restart_calibration].update(disabled = True)
window[self.gui_stop_calibration].update(disabled = True)
print("Disabled")
if event == "{}+UP".format(self.gui_roi_selection):
# Event for mouse button up in ROI mode
self.is_mouse_up = True
@@ -328,27 +328,6 @@ class CameraWidget:
self.config.roi_window_h = abs(self.y0 - self.y1)
self.main_config.save()
if event == self.gui_autoroi:
print("Auto ROI")
#image = self.image_queue.get()
#image = self.babble_landmark.get_frame() # Get image for pfld
#print(image)
#print(len(image))
#print(image)
#cv2.imwrite("yeah.png", image)
self.babble_landmark.infer_frame()
output = self.babble_landmark.output
print(f"Output: {output}")
self.x1 = output[2]
self.y1 = output[3]
self.x0 = output[0]
self.y0 = output[1]
self.config.roi_window_x = min([self.x0, self.x1])
self.config.roi_window_y = min([self.y0, self.y1])
self.config.roi_window_w = abs(self.x0 - self.x1)
self.config.roi_window_h = abs(self.y0 - self.y1)
self.main_config.save()
if event == self.gui_roi_selection:
# Event for mouse button down or mouse drag in ROI mode
if self.is_mouse_up:
@@ -356,12 +335,30 @@ class CameraWidget:
self.x0, self.y0 = values[self.gui_roi_selection]
self.x1, self.y1 = values[self.gui_roi_selection]
if event == self.gui_autoroi:
print("Set ROI")
output = self.babble_cnn.get_framesize()
self.config.roi_window_x = 0
self.config.roi_window_y = 0
self.config.roi_window_w = output[0]
self.config.roi_window_h = output[1]
self.main_config.save()
if (event == self.gui_refresh_button):
print("\033[94m[INFO] Refreshed Camera List\033[0m")
self.camera_list = list_camera_names()
print(self.camera_list)
window[self.gui_camera_addr].update(values=self.camera_list)
if event == self.gui_restart_calibration:
self.babble_cnn.calibration_frame_counter = 1500
PlaySound('Audio/start.wav', SND_FILENAME | SND_ASYNC)
if values[self.use_calibration] == True: # Don't start recording if the calibration filter is disabled.
self.babble_cnn.calibration_frame_counter = 1500
PlaySound('Audio/start.wav', SND_FILENAME | SND_ASYNC)
if event == self.gui_stop_calibration:
self.babble_cnn.calibration_frame_counter = 0
if self.babble_cnn.calibration_frame_counter != None: # Only assign the variable if we are in calibration mode.
self.babble_cnn.calibration_frame_counter = 0
needs_roi_set = self.config.roi_window_h <= 0 or self.config.roi_window_w <= 0
@@ -390,6 +387,7 @@ class CameraWidget:
if self.roi_queue.empty():
self.capture_event.set()
maybe_image = self.roi_queue.get(block=False)
self.maybe_image = maybe_image
imgbytes = cv2.imencode(".ppm", maybe_image[0])[1].tobytes()
graph = window[self.gui_roi_selection]
if self.figure:
@@ -419,4 +417,4 @@ class CameraWidget:
if cam_info.info_type != CamInfoOrigin.FAILURE:
self.osc_queue.put((self.cam_id, cam_info))
except Empty:
pass
pass
+3 -3
View File
@@ -16,8 +16,6 @@ class BabbleCameraConfig(BaseModel):
roi_window_w: int = 0
roi_window_h: int = 0
capture_source: Union[int, str, None] = None
use_calibration: bool = False
use_n_calibration: bool = True
gui_vertical_flip: bool = False
gui_horizontal_flip: bool = False
use_ffmpeg: bool = False
@@ -44,6 +42,8 @@ class BabbleSettingsConfig(BaseModel):
gui_cam_resolution_x: int = 0
gui_cam_resolution_y: int = 0
gui_cam_framerate: int = 0
use_calibration: bool = False
calibration_mode: str = 'Neutral'
class BabbleConfig(BaseModel):
version: int = 1
@@ -87,5 +87,5 @@ class BabbleConfig(BaseModel):
# No backup because the saved settings file is broken.
pass
with open(CONFIG_FILE_NAME, "w") as settings_file:
json.dump(obj=self.dict(), fp=settings_file)
json.dump(obj=self.dict(), fp=settings_file, indent=2)
print("[INFO] Config Saved Successfully")
+15 -9
View File
@@ -225,17 +225,23 @@ class SettingsWidget:
self.config.gui_osc_location = values[self.gui_osc_location]
changed = True
if values[self.gui_cam_resolution_x] != '':
if self.config.gui_cam_resolution_x != int(values[self.gui_cam_resolution_x]):
self.config.gui_cam_resolution_x = int(values[self.gui_cam_resolution_x])
changed = True
if str(self.config.gui_cam_resolution_x) != values[self.gui_cam_resolution_x]:
try:
self.config.gui_cam_resolution_x = int(values[self.gui_cam_resolution_x])
changed = True
except: print("Not an Int")
if values[self.gui_cam_resolution_y] != '':
if self.config.gui_cam_resolution_y != int(values[self.gui_cam_resolution_y]):
self.config.gui_cam_resolution_y = int(values[self.gui_cam_resolution_y])
changed = True
if str(self.config.gui_cam_resolution_y) != values[self.gui_cam_resolution_y]:
try:
self.config.gui_cam_resolution_y = int(values[self.gui_cam_resolution_y])
changed = True
except: print("Not an Int")
if values[self.gui_cam_framerate] != '':
if self.config.gui_cam_framerate != int(values[self.gui_cam_framerate]):
self.config.gui_cam_framerate = int(values[self.gui_cam_framerate])
changed = True
if str(self.config.gui_cam_framerate) != values[self.gui_cam_framerate]:
try:
self.config.gui_cam_framerate = int(values[self.gui_cam_framerate])
changed = True
except: print("Not an Int")
if self.config.gui_use_red_channel != bool(values[self.gui_use_red_channel]):
self.config.gui_use_red_channel = bool(values[self.gui_use_red_channel])
+4 -4
View File
@@ -28,8 +28,8 @@ def run_model(self): # Replace transforms n shit for the pfld model
self.output = output
def write_image(self): # Debug function for development, remove once pfld is implemented.
frame = cv2.resize(self.current_image_gray, (256, 256))
cv2.imwrite("yeah.png", frame)
def write_image(self, image): # Placeholder function for development, replace all instances with run_model() once pfld is implemented.
frame = cv2.resize(image, (256, 256))
cv2.imwrite("yeah.png", frame) # Write the clean, untransformed frame to show that the frame is clean for pfld
print("Image Wrote")
self.output = (0, 0, 100, 100)
self.output = ((0, 0, 100, 100), 25) # Return ROI box and Rotation information.
+2 -2
View File
@@ -115,8 +115,8 @@ class LandmarkProcessor:
def get_frame(self):
return self.current_image_gray_clean
def infer_frame(self):
return write_image(self)
def infer_frame(self, image):
return write_image(self, image)
def output_images_and_update(self, output_information: CamInfo):
try:
+6 -2
View File
@@ -5,12 +5,14 @@ import threading
from enum import IntEnum
import time
from config import BabbleConfig
import traceback
import math
class Tab(IntEnum):
CAM = 0
SETTINGS = 1
ALGOSETTINGS = 2
CALIBRATION = 3
import numpy as np
@@ -98,6 +100,7 @@ class VRChatOSCReceiver:
self.cancellation_event = cancellation_event
self.dispatcher = dispatcher.Dispatcher()
self.cams = cams # we cant import CameraWidget so any type it is
print("OSC_INIT")
try:
self.server = osc_server.OSCUDPServer((self.config.gui_osc_address, int(self.config.gui_osc_receiver_port)), self.dispatcher)
except:
@@ -114,17 +117,18 @@ class VRChatOSCReceiver:
if type(osc_value) != bool: return # just incase we get anything other than bool
if osc_value:
for cam in self.cams:
cam.ransac.calibration_frame_counter = 300
cam.babble_cnn.calibration_frame_counter = 300
PlaySound('Audio/start.wav', SND_FILENAME | SND_ASYNC)
def run(self):
# bind what function to run when specified OSC message is received
try:
self.dispatcher.map(self.config.gui_osc_recalibrate_address, self.recalibrate_cams)
self.dispatcher.map(self.config.gui_osc_recalibrate_address, self.recalibrate_mouth)
# start the server
print("\033[92m[INFO] VRChatOSCReceiver serving on {}\033[0m".format(self.server.server_address))
self.server.serve_forever()
except:
traceback.print_exc()
print(f"\033[91m[ERROR] OSC Receive port: {self.config.gui_osc_receiver_port} occupied.\033[0m")
+29 -11
View File
@@ -9,10 +9,15 @@ class CamId(IntEnum):
class cal():
def __init__(self):
self.calibrated_array = None
self.raw_array = None
def cal_osc(self, array):
self.raw_array = array
#print(self.calibration_frame_counter)
if self.calibration_frame_counter == 0:
if not self.config.use_n_calibration:
if self.settings.calibration_mode == 'Full':
self.calibration_frame_counter = None
values = np.array(self.val_list)
@@ -42,19 +47,27 @@ class cal():
PlaySound('Audio/completed.wav', SND_FILENAME | SND_ASYNC)
if self.config.use_n_calibration:
if self.settings.calibration_mode == 'Neutral':
self.min_max_array = np.fromstring(self.settings.calib_array.replace('[', '').replace(']', ''), sep=',')
self.min_max_array = self.min_max_array.reshape((2, 45))
print(f'minmax {self.min_max_array}')
max_values = self.min_max_array[1]
self.calibration_frame_counter = None
values = np.array(self.val_list)
#print(f'values: {values}')
deadzone_value = self.settings.calib_deadzone
# Initialize the min_max_array with shape (2, num_outputs)
num_outputs = values.shape[1]
self.min_max_array = np.zeros((2, num_outputs))
lower_threshold = np.clip([np.mean(values, axis = 0) + deadzone_value], 0, 1)
upper_threshold = np.ones((1, num_outputs)) # We don't need to adjust the max values.
lower_threshold = np.clip([np.mean(values, axis = 0) + deadzone_value], 0, 1).tolist()[0]
upper_threshold = max_values.tolist()
#upper_threshold = np.ones((1, num_outputs)) # We don't need to adjust the max values.
#upper_threshold = self.min_max_array[1]
print(lower_threshold)
print(upper_threshold)
self.min_max_array = np.array([lower_threshold, upper_threshold])
self.settings.calib_array = np.array2string(self.min_max_array, separator=',')
self.min_max_array = np.append(lower_threshold, upper_threshold)
#self.min_max_array = np.array([lower_threshold.tolist(), upper_threshold.tolist()])
self.settings.calib_array = str([lower_threshold, upper_threshold])
self.config_class.save()
print("[INFO] Calibration completed.")
@@ -67,8 +80,8 @@ class cal():
self.val_list.append(array)
self.calibration_frame_counter -= 1
if self.settings.calib_array is not None and self.config.use_calibration:
self.raw_array = array
if self.settings.calib_array is not None and self.settings.use_calibration and self.settings.calibration_mode == 'Full':
self.min_max_array = np.fromstring(self.settings.calib_array.replace('[', '').replace(']', ''), sep=',')
self.min_max_array = self.min_max_array.reshape((2, 45))
@@ -85,17 +98,22 @@ class cal():
calibrated_array[i] = calibrated_value
array = calibrated_array
if self.settings.calib_array is not None and self.config.use_n_calibration:
if self.settings.calib_array is not None and self.settings.use_calibration and self.settings.calibration_mode == 'Neutral':
self.min_max_array = np.fromstring(self.settings.calib_array.replace('[', '').replace(']', ''), sep=',')
self.min_max_array = self.min_max_array.reshape((2, 45))
calibrated_array = np.zeros_like(array)
for i, value in enumerate(array):
min_value = self.min_max_array[0, i]
max_value = self.min_max_array[1, i]
calibrated_value = (value - min_value) / (1.0 - min_value)
calibrated_value = (value - min_value) / (max_value - min_value)
calibrated_array[i] = calibrated_value
array = calibrated_array
#array[4] = log((np.clip(array[4]*10,0,10))+1.0, 11) Log Filter: Move to filter system.
return np.clip(array,0,1) # Clamp outputs between 0-1
array = np.clip(array,0,1)
return array # Clamp outputs between 0-1
def get_outputs(self):
return self.calibrated_array, self.raw_array
+1
View File
@@ -5,6 +5,7 @@ class Tab(IntEnum):
CAM = 0
SETTINGS = 1
ALGOSETTINGS = 2
CALIBRATION = 3
class CamInfoOrigin(Enum):
+53 -3
View File
@@ -1,7 +1,24 @@
import typing
import serial
import sys
import glob
import os
import platform
import cv2
import subprocess
from pygrabber.dshow_graph import FilterGraph
is_nt = True if sys.platform.startswith('win') else False
graph = FilterGraph()
def list_camera_names():
cam_list = graph.get_input_devices()
cam_names = []
for index, name in enumerate(cam_list):
cam_names.append(name)
cam_names = cam_names + list_serial_ports()
return cam_names
# Detect the operating system
is_nt = True if os.name == "nt" else False
@@ -73,19 +90,48 @@ def list_camera_names():
if is_nt:
# On Windows, use pygrabber to list devices
cam_list = graph.get_input_devices()
return cam_list
return cam_list + list_serial_ports()
elif os_type == "Linux":
# On Linux, return UVC device paths like '/dev/video0'
return list_linux_uvc_devices()
return list_linux_uvc_devices() + list_serial_ports()
elif os_type == "Darwin":
# On macOS, fallback to OpenCV (device names aren't fetched)
return list_cameras_opencv()
return list_cameras_opencv() + list_serial_ports()
else:
return ["Unsupported operating system"]
def list_serial_ports():
print("DEBUG: Listed Serial Ports")
""" Lists serial port names
:raises EnvironmentError:
On unsupported or unknown platforms
:returns:
A list of the serial ports available on the system
"""
if sys.platform.startswith('win'):
ports = ['COM%s' % (i + 1) for i in range(256)]
elif sys.platform.startswith('linux') or sys.platform.startswith('cygwin'):
# this excludes your current terminal "/dev/tty"
ports = glob.glob('/dev/tty[A-Za-z]*')
elif sys.platform.startswith('darwin'):
ports = glob.glob('/dev/tty.*')
else:
raise EnvironmentError('Unsupported platform')
result = []
for port in ports:
try:
s = serial.Serial(port)
s.close()
result.append(port)
except (OSError, serial.SerialException):
pass
return result
def get_camera_index_by_name(name):
""" Cross-platform function to get the camera index by its name or path """
@@ -111,6 +157,10 @@ def get_camera_index_by_name(name):
return None
#def get_serial_port(name):
# for i, device in enumerate(cam_list):
# Placeholder for sound functions on Windows
def PlaySound(*args, **kwargs):