mirror of
https://github.com/Project-Babble/ProjectBabble.git
synced 2026-11-20 00:51:16 -08:00
423 lines
18 KiB
Python
423 lines
18 KiB
Python
from collections import deque
|
|
from queue import Queue, Empty
|
|
from threading import Event, Thread
|
|
|
|
import PySimpleGUI as sg
|
|
|
|
import cv2
|
|
from babble_processor import BabbleProcessor, CamInfoOrigin
|
|
from camera import Camera, CameraState
|
|
from config import BabbleConfig
|
|
from landmark_processor import LandmarkProcessor
|
|
from osc import Tab
|
|
from utils.misc_utils import PlaySound, SND_FILENAME, SND_ASYNC, list_camera_names
|
|
|
|
|
|
class CameraWidget:
|
|
def __init__(self, widget_id: Tab, main_config: BabbleConfig, osc_queue: Queue):
|
|
self.gui_camera_addr = f"-CAMERAADDR{widget_id}-"
|
|
self.gui_rotation_slider = f"-ROTATIONSLIDER{widget_id}-"
|
|
self.gui_roi_button = f"-ROIMODE{widget_id}-"
|
|
self.gui_roi_layout = f"-ROILAYOUT{widget_id}-"
|
|
self.gui_roi_selection = f"-GRAPH{widget_id}-"
|
|
self.gui_tracking_button = f"-TRACKINGMODE{widget_id}-"
|
|
self.gui_autoroi = f"-AUTOROI{widget_id}-"
|
|
self.gui_save_tracking_button = f"-SAVETRACKINGBUTTON{widget_id}-"
|
|
self.gui_tracking_layout = f"-TRACKINGLAYOUT{widget_id}-"
|
|
self.gui_tracking_image = f"-IMAGE{widget_id}-"
|
|
self.gui_tracking_fps = f"-TRACKINGFPS{widget_id}-"
|
|
self.gui_tracking_bps = f"-TRACKINGBPS{widget_id}-"
|
|
self.gui_output_graph = f"-OUTPUTGRAPH{widget_id}-"
|
|
self.gui_restart_calibration = f"-RESTARTCALIBRATION{widget_id}-"
|
|
self.gui_stop_calibration = f"-STOPCALIBRATION{widget_id}-"
|
|
self.gui_mode_readout = f"-APPMODE{widget_id}-"
|
|
self.gui_roi_message = f"-ROIMESSAGE{widget_id}-"
|
|
self.gui_vertical_flip = f"-VERTICALFLIP{widget_id}-"
|
|
self.gui_horizontal_flip = f"-HORIZONTALFLIP{widget_id}-"
|
|
self.use_calibration = f"-USECALIBRATION{widget_id}-"
|
|
self.use_n_calibration = f"-USENCALIBRATION{widget_id}-"
|
|
self.osc_queue = osc_queue
|
|
self.main_config = main_config
|
|
self.cam_id = widget_id
|
|
self.settings_config = main_config.settings
|
|
self.config = main_config.cam
|
|
self.settings = main_config.settings
|
|
self.camera_list = list_camera_names()
|
|
if self.cam_id == Tab.CAM:
|
|
self.config = main_config.cam
|
|
else:
|
|
raise RuntimeError("\033[91m[WARN] Improper tab value!\033[0m")
|
|
|
|
self.cancellation_event = Event()
|
|
# Set the event until start is called, otherwise we can block if shutdown is called.
|
|
self.cancellation_event.set()
|
|
self.capture_event = Event()
|
|
self.capture_queue = Queue(maxsize=2)
|
|
self.roi_queue = Queue(maxsize=2)
|
|
self.image_queue = Queue(maxsize=500)
|
|
|
|
self.babble_cnn = BabbleProcessor(
|
|
self.config,
|
|
self.settings_config,
|
|
self.main_config,
|
|
self.cancellation_event,
|
|
self.capture_event,
|
|
self.capture_queue,
|
|
self.image_queue,
|
|
self.cam_id,
|
|
)
|
|
|
|
self.babble_landmark = LandmarkProcessor(
|
|
self.config,
|
|
self.settings_config,
|
|
self.main_config,
|
|
self.cancellation_event,
|
|
self.capture_event,
|
|
self.capture_queue,
|
|
self.image_queue,
|
|
self.cam_id,
|
|
)
|
|
|
|
self.camera_status_queue = Queue(maxsize=2)
|
|
self.camera = Camera(
|
|
self.config,
|
|
0,
|
|
self.cancellation_event,
|
|
self.capture_event,
|
|
self.camera_status_queue,
|
|
self.capture_queue,
|
|
self.settings
|
|
)
|
|
|
|
self.roi_layout = [
|
|
[
|
|
sg.Button("Auto ROI", key=self.gui_autoroi, button_color='#539e8a', tooltip="Automatically set ROI", ),
|
|
],
|
|
[
|
|
sg.Graph(
|
|
(640, 480),
|
|
(0, 480),
|
|
(640, 0),
|
|
key=self.gui_roi_selection,
|
|
drag_submits=True,
|
|
enable_events=True,
|
|
background_color='#424042',
|
|
)
|
|
]
|
|
]
|
|
|
|
# Define the window's contents
|
|
self.tracking_layout = [
|
|
[
|
|
sg.Text("Rotation", background_color='#424042'),
|
|
sg.Slider(
|
|
range=(0, 360),
|
|
default_value=self.config.rotation_angle,
|
|
orientation="h",
|
|
key=self.gui_rotation_slider,
|
|
background_color='#424042',
|
|
tooltip="Adjust the rotation of your cameras, make them level.",
|
|
),
|
|
],
|
|
[
|
|
sg.Button("Start Calibration", key=self.gui_restart_calibration, button_color='#539e8a',
|
|
tooltip="Start calibration. Look all arround to all extreams without blinking until sound is heard.", ),
|
|
sg.Button("Stop Calibration", key=self.gui_stop_calibration, button_color='#539e8a',
|
|
tooltip="Stop calibration manualy.", ),
|
|
],
|
|
[
|
|
sg.Checkbox(
|
|
"Use Neutral Calibration:",
|
|
default=self.config.use_n_calibration,
|
|
key=self.use_n_calibration,
|
|
background_color='#424042',
|
|
tooltip="Toggle use of calibration using minimum values found during a neutral pose calibration step.",
|
|
),
|
|
sg.Checkbox(
|
|
"Use Full Calibration:",
|
|
default=self.config.use_calibration,
|
|
key=self.use_calibration,
|
|
background_color='#424042',
|
|
tooltip="Toggle use of calibration.",
|
|
),
|
|
],
|
|
[
|
|
sg.Text("Mode:", background_color='#424042'),
|
|
sg.Text("Calibrating", key=self.gui_mode_readout, background_color='#539e8a'),
|
|
sg.Text("", key=self.gui_tracking_fps, background_color='#424042'),
|
|
sg.Text("", key=self.gui_tracking_bps, background_color='#424042'),
|
|
],
|
|
[
|
|
sg.Checkbox(
|
|
"Vertical Flip:",
|
|
default=self.config.gui_vertical_flip,
|
|
key=self.gui_vertical_flip,
|
|
background_color='#424042',
|
|
tooltip="Vertically flip camera feed.",
|
|
),
|
|
sg.Checkbox(
|
|
"Horizontal Flip:",
|
|
default=self.config.gui_horizontal_flip,
|
|
key=self.gui_horizontal_flip,
|
|
background_color='#424042',
|
|
tooltip="Horizontally flip camera feed.",
|
|
),
|
|
],
|
|
[sg.Image(filename="", key=self.gui_tracking_image)],
|
|
[
|
|
sg.Text("Please set a Crop.", key=self.gui_roi_message, background_color='#424042', visible=False),
|
|
],
|
|
]
|
|
|
|
self.widget_layout = [
|
|
[
|
|
sg.Text("Camera Address", background_color='#424042'),
|
|
sg.InputCombo(self.camera_list, default_value=self.config.capture_source,
|
|
key=self.gui_camera_addr,
|
|
tooltip="Enter the IP address or UVC port of your camera. (Include the 'http://')",
|
|
enable_events=True)
|
|
],
|
|
[
|
|
sg.Button("Save and Restart Tracking", key=self.gui_save_tracking_button, button_color='#539e8a'),
|
|
],
|
|
[
|
|
sg.Button("Tracking Mode", key=self.gui_tracking_button, button_color='#539e8a',
|
|
tooltip="Go here to track your mouth.", ),
|
|
sg.Button("Cropping Mode", key=self.gui_roi_button, button_color='#539e8a',
|
|
tooltip="Go here to crop out your mouth.", ),
|
|
],
|
|
[
|
|
sg.Column(self.tracking_layout, key=self.gui_tracking_layout, background_color='#424042'),
|
|
sg.Column(self.roi_layout, key=self.gui_roi_layout, background_color='#424042', visible=False),
|
|
],
|
|
]
|
|
|
|
self.x0, self.y0 = None, None
|
|
self.x1, self.y1 = None, None
|
|
self.figure = None
|
|
self.is_mouse_up = True
|
|
self.in_roi_mode = False
|
|
self.movavg_fps_queue = deque(maxlen=120)
|
|
self.movavg_bps_queue = deque(maxlen=120)
|
|
|
|
def _movavg_fps(self, next_fps):
|
|
self.movavg_fps_queue.append(next_fps)
|
|
fps = round(sum(self.movavg_fps_queue) / len(self.movavg_fps_queue))
|
|
millisec = round((1 / fps if fps else 0) * 1000)
|
|
return f"{fps} Fps {millisec} ms"
|
|
|
|
def _movavg_bps(self, next_bps):
|
|
self.movavg_bps_queue.append(next_bps)
|
|
return f"{sum(self.movavg_bps_queue) / len(self.movavg_bps_queue) * 0.001 * 0.001 * 8:.3f} Mbps"
|
|
|
|
def started(self):
|
|
return not self.cancellation_event.is_set()
|
|
|
|
def start(self):
|
|
# If we're already running, bail
|
|
if not self.cancellation_event.is_set():
|
|
return
|
|
self.cancellation_event.clear()
|
|
self.babble_cnn_thread = Thread(target=self.babble_cnn.run)
|
|
self.babble_cnn_thread.start()
|
|
self.babble_landmark_thread = Thread(target=self.babble_landmark.run)
|
|
self.babble_landmark_thread.start()
|
|
self.camera_thread = Thread(target=self.camera.run)
|
|
self.camera_thread.start()
|
|
|
|
def stop(self):
|
|
# If we're not running yet, bail
|
|
if self.cancellation_event.is_set():
|
|
return
|
|
self.cancellation_event.set()
|
|
self.babble_cnn_thread.join()
|
|
self.babble_landmark_thread.join()
|
|
self.camera_thread.join()
|
|
|
|
def render(self, window, event, values):
|
|
if self.image_queue.qsize() > 2:
|
|
with self.image_queue.mutex:
|
|
self.image_queue.queue.clear()
|
|
else:
|
|
pass
|
|
changed = False
|
|
# If anything has changed in our configuration settings, change/update those.
|
|
if (
|
|
event == self.gui_save_tracking_button
|
|
and values[self.gui_camera_addr] != self.config.capture_source
|
|
):
|
|
value = values[self.gui_camera_addr]
|
|
print("\033[94m[INFO] New value: {}\033[0m".format(value))
|
|
try:
|
|
self.config.use_ffmpeg = False
|
|
# Try storing ints as ints, for those using wired cameras.
|
|
if value not in self.camera_list:
|
|
self.config.capture_source = int(value)
|
|
else:
|
|
self.config.capture_source = value
|
|
except ValueError:
|
|
if value == "":
|
|
self.config.capture_source = None
|
|
else:
|
|
# If http is not in camera address, add it.
|
|
self.config.capture_source = value
|
|
|
|
if "udp" in value:
|
|
self.config.use_ffmpeg = True
|
|
elif (
|
|
"http" not in value
|
|
and ".mp4" not in value
|
|
and "udp" not in value
|
|
and "COM" not in value
|
|
and "/dev/tty" not in value
|
|
and value not in self.camera_list
|
|
): # If http is not in camera address, add it.
|
|
self.config.capture_source = f"http://{values[self.gui_camera_addr]}/"
|
|
|
|
changed = True
|
|
|
|
if self.config.rotation_angle != values[self.gui_rotation_slider]:
|
|
self.config.rotation_angle = int(values[self.gui_rotation_slider])
|
|
changed = True
|
|
|
|
#print(self.config.gui_vertical_flip)
|
|
if self.config.gui_vertical_flip != values[self.gui_vertical_flip]:
|
|
self.config.gui_vertical_flip = values[self.gui_vertical_flip]
|
|
changed = True
|
|
|
|
if self.config.gui_horizontal_flip != values[self.gui_horizontal_flip]:
|
|
self.config.gui_horizontal_flip = values[self.gui_horizontal_flip]
|
|
changed = True
|
|
|
|
if self.config.use_calibration != values[self.use_calibration]:
|
|
self.config.use_calibration = values[self.use_calibration]
|
|
changed = True
|
|
|
|
if self.config.use_n_calibration != values[self.use_n_calibration]:
|
|
self.config.use_n_calibration = values[self.use_n_calibration]
|
|
changed = True
|
|
|
|
if changed:
|
|
self.main_config.save()
|
|
|
|
if event == self.gui_tracking_button:
|
|
print("\033[94m[INFO] Moving to tracking mode\033[0m")
|
|
self.in_roi_mode = False
|
|
self.camera.set_output_queue(self.capture_queue)
|
|
window[self.gui_roi_layout].update(visible=False)
|
|
window[self.gui_tracking_layout].update(visible=True)
|
|
|
|
if event == self.gui_roi_button:
|
|
print("\033[94m[INFO] Move to roi mode\033[0m")
|
|
self.in_roi_mode = True
|
|
self.camera.set_output_queue(self.roi_queue)
|
|
window[self.gui_roi_layout].update(visible=True)
|
|
window[self.gui_tracking_layout].update(visible=False)
|
|
|
|
if event == "{}+UP".format(self.gui_roi_selection):
|
|
# Event for mouse button up in ROI mode
|
|
self.is_mouse_up = True
|
|
if self.x1 < 0:
|
|
self.x1 = 0
|
|
if self.y1 < 0:
|
|
self.y1 = 0
|
|
if abs(self.x0 - self.x1) != 0 and abs(self.y0 - self.y1) != 0:
|
|
self.config.roi_window_x = min([self.x0, self.x1])
|
|
self.config.roi_window_y = min([self.y0, self.y1])
|
|
self.config.roi_window_w = abs(self.x0 - self.x1)
|
|
self.config.roi_window_h = abs(self.y0 - self.y1)
|
|
self.main_config.save()
|
|
|
|
if event == self.gui_autoroi:
|
|
print("Auto ROI")
|
|
#image = self.image_queue.get()
|
|
#image = self.babble_landmark.get_frame() # Get image for pfld
|
|
#print(image)
|
|
#print(len(image))
|
|
#print(image)
|
|
#cv2.imwrite("yeah.png", image)
|
|
self.babble_landmark.infer_frame()
|
|
output = self.babble_landmark.output
|
|
print(f"Output: {output}")
|
|
self.x1 = output[2]
|
|
self.y1 = output[3]
|
|
self.x0 = output[0]
|
|
self.y0 = output[1]
|
|
self.config.roi_window_x = min([self.x0, self.x1])
|
|
self.config.roi_window_y = min([self.y0, self.y1])
|
|
self.config.roi_window_w = abs(self.x0 - self.x1)
|
|
self.config.roi_window_h = abs(self.y0 - self.y1)
|
|
self.main_config.save()
|
|
|
|
if event == self.gui_roi_selection:
|
|
# Event for mouse button down or mouse drag in ROI mode
|
|
if self.is_mouse_up:
|
|
self.is_mouse_up = False
|
|
self.x0, self.y0 = values[self.gui_roi_selection]
|
|
self.x1, self.y1 = values[self.gui_roi_selection]
|
|
|
|
if event == self.gui_restart_calibration:
|
|
self.babble_cnn.calibration_frame_counter = 1500
|
|
PlaySound('Audio/start.wav', SND_FILENAME | SND_ASYNC)
|
|
|
|
if event == self.gui_stop_calibration:
|
|
self.babble_cnn.calibration_frame_counter = 0
|
|
|
|
needs_roi_set = self.config.roi_window_h <= 0 or self.config.roi_window_w <= 0
|
|
|
|
# TODO: Refactor if statements below...
|
|
window[self.gui_tracking_fps].update('')
|
|
window[self.gui_tracking_bps].update('')
|
|
|
|
if self.config.capture_source is None or self.config.capture_source == "":
|
|
window[self.gui_mode_readout].update("Waiting for camera address")
|
|
window[self.gui_roi_message].update(visible=False)
|
|
elif self.camera.camera_status == CameraState.CONNECTING:
|
|
window[self.gui_mode_readout].update("Camera Connecting")
|
|
elif self.camera.camera_status == CameraState.DISCONNECTED:
|
|
window[self.gui_mode_readout].update("Camera Reconnecting...")
|
|
elif needs_roi_set:
|
|
window[self.gui_mode_readout].update("Awaiting Mouth Crop")
|
|
elif self.babble_cnn.calibration_frame_counter != None:
|
|
window[self.gui_mode_readout].update("Calibration")
|
|
else:
|
|
window[self.gui_mode_readout].update("Tracking")
|
|
window[self.gui_tracking_fps].update(self._movavg_fps(self.camera.fps))
|
|
window[self.gui_tracking_bps].update(self._movavg_bps(self.camera.bps))
|
|
|
|
if self.in_roi_mode:
|
|
try:
|
|
if self.roi_queue.empty():
|
|
self.capture_event.set()
|
|
maybe_image = self.roi_queue.get(block=False)
|
|
imgbytes = cv2.imencode(".ppm", maybe_image[0])[1].tobytes()
|
|
graph = window[self.gui_roi_selection]
|
|
if self.figure:
|
|
graph.delete_figure(self.figure)
|
|
# INCREDIBLY IMPORTANT ERASE. Drawing images does NOT overwrite the buffer, the fucking
|
|
# graph keeps every image fed in until you call this. Therefore we have to make sure we
|
|
# erase before we redraw, otherwise we'll leak memory *very* quickly.
|
|
graph.erase()
|
|
graph.draw_image(data=imgbytes, location=(0, 0))
|
|
if None not in (self.x0, self.y0, self.x1, self.y1):
|
|
self.figure = graph.draw_rectangle(
|
|
(self.x0, self.y0), (self.x1, self.y1), line_color="#6f4ca1"
|
|
)
|
|
except Empty:
|
|
pass
|
|
else:
|
|
if needs_roi_set:
|
|
window[self.gui_roi_message].update(visible=True)
|
|
return
|
|
try:
|
|
window[self.gui_roi_message].update(visible=False)
|
|
(maybe_image, cam_info) = self.image_queue.get(block=False)
|
|
imgbytes = cv2.imencode(".ppm", maybe_image)[1].tobytes()
|
|
window[self.gui_tracking_image].update(data=imgbytes)
|
|
|
|
# Relay information to OSC
|
|
if cam_info.info_type != CamInfoOrigin.FAILURE:
|
|
self.osc_queue.put((self.cam_id, cam_info))
|
|
except Empty:
|
|
pass
|