Files
ProjectBabble/BabbleApp/camera_widget.py

518 lines
21 KiB
Python

from collections import deque
from queue import Queue, Empty
from threading import Event, Thread
import PySimpleGUI as sg
import cv2
from babble_processor import BabbleProcessor, CamInfoOrigin
from camera import Camera, CameraState, MAX_RESOLUTION
from config import BabbleConfig
from osc import Tab
from utils.misc_utils import (
PlaySound,
SND_FILENAME,
SND_ASYNC,
list_camera_names,
get_camera_index_by_name,
bg_color_highlight,
bg_color_clear,
is_valid_int_input
)
from lang_manager import LocaleStringManager as lang
class CameraWidget:
def __init__(self, widget_id: Tab, main_config: BabbleConfig, osc_queue: Queue):
self.gui_camera_addr = f"-CAMERAADDR{widget_id}-"
self.gui_rotation_slider = f"-ROTATIONSLIDER{widget_id}-"
self.gui_roi_button = f"-ROIMODE{widget_id}-"
self.gui_roi_layout = f"-ROILAYOUT{widget_id}-"
self.gui_roi_selection = f"-GRAPH{widget_id}-"
self.gui_tracking_button = f"-TRACKINGMODE{widget_id}-"
self.gui_autoroi = f"-AUTOROI{widget_id}-"
self.gui_save_tracking_button = f"-SAVETRACKINGBUTTON{widget_id}-"
self.gui_tracking_layout = f"-TRACKINGLAYOUT{widget_id}-"
self.gui_tracking_image = f"-IMAGE{widget_id}-"
self.gui_tracking_fps = f"-TRACKINGFPS{widget_id}-"
self.gui_tracking_bps = f"-TRACKINGBPS{widget_id}-"
self.gui_output_graph = f"-OUTPUTGRAPH{widget_id}-"
self.gui_restart_calibration = f"-RESTARTCALIBRATION{widget_id}-"
self.gui_stop_calibration = f"-STOPCALIBRATION{widget_id}-"
self.gui_mode_readout = f"-APPMODE{widget_id}-"
self.gui_roi_message = f"-ROIMESSAGE{widget_id}-"
self.gui_vertical_flip = f"-VERTICALFLIP{widget_id}-"
self.gui_horizontal_flip = f"-HORIZONTALFLIP{widget_id}-"
self.use_calibration = f"-USECALIBRATION{widget_id}-"
self.gui_refresh_button = f"-REFRESHCAMLIST{widget_id}-"
self.osc_queue = osc_queue
self.main_config = main_config
self.cam_id = widget_id
self.settings_config = main_config.settings
self.config = main_config.cam
self.settings = main_config.settings
self.camera_list = list_camera_names()
self.maybe_image = None
if self.cam_id == Tab.CAM:
self.config = main_config.cam
else:
raise RuntimeError(
f'\033[91m[{lang._instance.get_string("log.warn")}] {lang._instance.get_string("error.improperTabValue")}\033[0m'
)
self.cancellation_event = Event()
# Set the event until start is called, otherwise we can block if shutdown is called.
self.cancellation_event.set()
self.capture_event = Event()
self.capture_queue = Queue(maxsize=2)
self.roi_queue = Queue(maxsize=2)
self.image_queue = Queue(maxsize=500) # This is needed to prevent the UI from freezing during widget changes.
self.babble_cnn = BabbleProcessor(
self.config,
self.settings_config,
self.main_config,
self.cancellation_event,
self.capture_event,
self.capture_queue,
self.image_queue,
self.cam_id,
self.osc_queue,
)
self.camera_status_queue = Queue(maxsize=2)
self.camera = Camera(
self.config,
0,
self.cancellation_event,
self.capture_event,
self.camera_status_queue,
self.capture_queue,
self.settings,
)
button_color = "#539e8a"
self.roi_layout = [
[
sg.Button(
lang._instance.get_string("camera.selectEntireFrame"),
key=self.gui_autoroi,
button_color=button_color,
tooltip=lang._instance.get_string(
"camera.selectEntireFrameTooltip"
),
),
],
[
sg.Graph(
(MAX_RESOLUTION, MAX_RESOLUTION),
(0, MAX_RESOLUTION),
(MAX_RESOLUTION, 0),
key=self.gui_roi_selection,
drag_submits=True,
enable_events=True,
background_color=bg_color_highlight,
)
],
]
# Define the window's contents
self.tracking_layout = [
[
sg.Text(
lang._instance.get_string("camera.rotation"),
background_color=bg_color_highlight,
),
sg.Slider(
range=(0, 360),
default_value=self.config.rotation_angle,
orientation="h",
key=self.gui_rotation_slider,
background_color=bg_color_highlight,
tooltip=lang._instance.get_string("camera.rotationTooltip"),
),
],
[
sg.Button(
lang._instance.get_string("camera.startCalibration"),
key=self.gui_restart_calibration,
button_color=button_color,
tooltip=lang._instance.get_string("camera.startCalibrationTooltip"),
disabled=not self.settings_config.use_calibration,
),
sg.Button(
lang._instance.get_string("camera.stopCalibration"),
key=self.gui_stop_calibration,
button_color=button_color,
tooltip=lang._instance.get_string("camera.startCalibrationTooltip"),
disabled=not self.settings_config.use_calibration,
),
],
[
sg.Checkbox(
f'{lang._instance.get_string("camera.enableCalibration")}:',
default=self.settings_config.use_calibration,
key=self.use_calibration,
background_color=bg_color_highlight,
tooltip=lang._instance.get_string(
"camera.enableCalibrationTooltip"
),
enable_events=True,
),
],
[
sg.Text(
f'{lang._instance.get_string("camera.mode")}:',
background_color=bg_color_highlight,
),
sg.Text(
lang._instance.get_string("camera.calibrating"),
key=self.gui_mode_readout,
background_color=button_color,
),
sg.Text(
"", key=self.gui_tracking_fps, background_color=bg_color_highlight
),
sg.Text(
"", key=self.gui_tracking_bps, background_color=bg_color_highlight
),
],
[
sg.Checkbox(
f'{lang._instance.get_string("camera.verticalFlip")}:',
default=self.config.gui_vertical_flip,
key=self.gui_vertical_flip,
background_color=bg_color_highlight,
tooltip=f'{lang._instance.get_string("camera.verticalFlipTooltip")}.',
),
sg.Checkbox(
f'{lang._instance.get_string("camera.horizontalFlip")}:',
default=self.config.gui_horizontal_flip,
key=self.gui_horizontal_flip,
background_color=bg_color_highlight,
tooltip=f'{lang._instance.get_string("camera.horizontalFlipTooltip")}:',
),
],
[sg.Image(filename="", key=self.gui_tracking_image)],
[
sg.Text(
f'{lang._instance.get_string("camera.crop")}:',
key=self.gui_roi_message,
background_color=bg_color_highlight,
visible=False,
),
],
]
self.widget_layout = [
[
sg.Text(
lang._instance.get_string("camera.cameraAddress"),
background_color=bg_color_highlight,
),
sg.InputCombo(
values=self.camera_list,
default_value=self.config.capture_source,
key=self.gui_camera_addr,
tooltip=lang._instance.get_string("camera.cameraAddressTooltip"),
enable_events=True,
size=(20,0),
),
sg.Button(
lang._instance.get_string("camera.refreshCameraList"),
key=self.gui_refresh_button,
button_color=button_color,
),
],
[
sg.Button(
lang._instance.get_string("camera.saveAndRestartTracking"),
key=self.gui_save_tracking_button,
button_color=button_color,
),
],
[
sg.Button(
lang._instance.get_string("camera.trackingMode"),
key=self.gui_tracking_button,
button_color=button_color,
tooltip=f'{lang._instance.get_string("camera.trackingModeTooltip")}.',
),
sg.Button(
lang._instance.get_string("camera.croppingMode"),
key=self.gui_roi_button,
button_color=button_color,
tooltip=f'{lang._instance.get_string("camera.croppingModeToolTip")}.',
),
],
[
sg.Column(
self.tracking_layout,
key=self.gui_tracking_layout,
background_color=bg_color_highlight,
),
sg.Column(
self.roi_layout,
key=self.gui_roi_layout,
background_color=bg_color_highlight,
visible=False,
),
],
]
self.x0, self.y0 = None, None
self.x1, self.y1 = None, None
self.figure = None
self.is_mouse_up = True
self.in_roi_mode = False
self.bps = 0
def _movavg_fps(self, next_fps):
# next_fps is already averaged
return f"{round(next_fps)} FPS {round((1 / next_fps if next_fps else 0) * 1000)} ms"
def _movavg_bps(self, next_bps):
self.bps = round(0.02 * next_bps + 0.98 * self.bps)
return f"{self.bps * 0.001 * 0.001 * 8:.3f} Mbps"
def started(self):
return not self.cancellation_event.is_set()
def start(self):
# If we're already running, bail
if not self.cancellation_event.is_set():
return
self.cancellation_event.clear()
self.babble_cnn_thread = Thread(target=self.babble_cnn.run)
self.babble_cnn_thread.start()
# self.babble_landmark_thread = Thread(target=self.babble_landmark.run)
# self.babble_landmark_thread.start()
self.camera_thread = Thread(target=self.camera.run)
self.camera_thread.start()
def stop(self):
# If we're not running yet, bail
if self.cancellation_event.is_set():
return
self.cancellation_event.set()
self.babble_cnn_thread.join()
# self.babble_landmark_thread.join()
self.camera_thread.join()
def render(self, window, event, values):
if self.image_queue.qsize() > 2:
with self.image_queue.mutex:
self.image_queue.queue.clear()
else:
pass
changed = False
# If anything has changed in our configuration settings, change/update those.
if (
event == self.gui_save_tracking_button
and values[self.gui_camera_addr] != self.config.capture_source
):
value = values[self.gui_camera_addr]
print(
f'\033[94m[{lang._instance.get_string("log.info")}] {lang._instance.get_string("info.newValue")}: {value}\033[0m'
)
try:
self.config.use_ffmpeg = False
# Try storing ints as ints, for those using wired cameras.
# if value not in self.camera_list:
# self.config.capture_source = value
# if "COM" not in value:
ports = ("COM", "/dev/ttyACM")
if any(x in str(value) for x in ports):
self.config.capture_source = value
else:
if is_valid_int_input(value):
self.config.capture_source = int(value)
else:
self.config.capture_source = value
except ValueError:
if value == "":
self.config.capture_source = None
else:
# If http is not in camera address, add it.
self.config.capture_source = value
if "udp" in value:
self.config.use_ffmpeg = True
elif (
"http" not in value
and ".mp4" not in value
and "udp" not in value
and "COM" not in value
and "/dev/ttyACM" not in value
and value not in self.camera_list
): # If http is not in camera address, add it.
self.config.capture_source = (
f"http://{values[self.gui_camera_addr]}/"
)
changed = True
if self.config.rotation_angle != values[self.gui_rotation_slider]:
self.config.rotation_angle = int(values[self.gui_rotation_slider])
changed = True
if self.config.gui_vertical_flip != values[self.gui_vertical_flip]:
self.config.gui_vertical_flip = values[self.gui_vertical_flip]
changed = True
if self.config.gui_horizontal_flip != values[self.gui_horizontal_flip]:
self.config.gui_horizontal_flip = values[self.gui_horizontal_flip]
changed = True
if self.settings_config.use_calibration != values[self.use_calibration]:
self.settings_config.use_calibration = values[self.use_calibration]
changed = True
if changed:
self.main_config.save()
if event == self.gui_tracking_button:
print(
f'\033[94m[{lang._instance.get_string("log.info")}] {lang._instance.get_string("info.moveToTrackingMode")}\033[0m'
)
self.in_roi_mode = False
self.camera.set_output_queue(self.capture_queue)
window[self.gui_roi_layout].update(visible=False)
window[self.gui_tracking_layout].update(visible=True)
if event == self.gui_roi_button:
print(
f'\033[94m[{lang._instance.get_string("log.info")}] {lang._instance.get_string("info.moveToROIMode")}\033[0m'
)
self.in_roi_mode = True
self.camera.set_output_queue(self.roi_queue)
window[self.gui_roi_layout].update(visible=True)
window[self.gui_tracking_layout].update(visible=False)
if event == self.use_calibration:
if self.settings_config.use_calibration == True:
window[self.gui_restart_calibration].update(disabled=False)
window[self.gui_stop_calibration].update(disabled=False)
print(f'[{lang._instance.get_string("log.info")}] {lang._instance.get_string("info.enabled")}')
else:
window[self.gui_restart_calibration].update(disabled=True)
window[self.gui_stop_calibration].update(disabled=True)
print(f'[{lang._instance.get_string("log.info")}] {lang._instance.get_string("info.disabled")}')
if event == "{}+UP".format(self.gui_roi_selection):
# Event for mouse button up in ROI mode
self.is_mouse_up = True
if self.x1 < 0:
self.x1 = 0
if self.y1 < 0:
self.y1 = 0
if abs(self.x0 - self.x1) != 0 and abs(self.y0 - self.y1) != 0:
self.config.roi_window_x = min([self.x0, self.x1])
self.config.roi_window_y = min([self.y0, self.y1])
self.config.roi_window_w = abs(self.x0 - self.x1)
self.config.roi_window_h = abs(self.y0 - self.y1)
self.main_config.save()
if event == self.gui_roi_selection:
# Event for mouse button down or mouse drag in ROI mode
if self.is_mouse_up:
self.is_mouse_up = False
self.x0, self.y0 = values[self.gui_roi_selection]
self.x1, self.y1 = values[self.gui_roi_selection]
if event == self.gui_autoroi:
print(lang._instance.get_string("info.setROI"))
output = self.maybe_image[0].shape
self.config.roi_window_x = 0
self.config.roi_window_y = 0
self.config.roi_window_w = output[1]
self.config.roi_window_h = output[0]
self.main_config.save()
if event == self.gui_refresh_button:
print(
f'\033[94m[{lang._instance.get_string("log.info")}] {lang._instance.get_string("info.refreshedCameraList")}\033[0m'
)
self.camera_list = list_camera_names()
#print(self.camera_list)
window[self.gui_camera_addr].update(values=self.camera_list,size=(20,0))
if event == self.gui_restart_calibration:
if (
values[self.use_calibration] == True
): # Don't start recording if the calibration filter is disabled.
self.babble_cnn.calibration_frame_counter = 1500
PlaySound("Audio/start.wav", SND_FILENAME | SND_ASYNC)
if event == self.gui_stop_calibration:
if (
self.babble_cnn.calibration_frame_counter != None
): # Only assign the variable if we are in calibration mode.
self.babble_cnn.calibration_frame_counter = 0
needs_roi_set = self.config.roi_window_h <= 0 or self.config.roi_window_w <= 0
# TODO: Refactor if statements below...
window[self.gui_tracking_fps].update("")
window[self.gui_tracking_bps].update("")
if self.config.capture_source is None or self.config.capture_source == "":
window[self.gui_mode_readout].update(
lang._instance.get_string("camera.waiting")
)
window[self.gui_roi_message].update(visible=False)
elif self.camera.camera_status == CameraState.CONNECTING:
window[self.gui_mode_readout].update(
lang._instance.get_string("camera.connecting")
)
elif self.camera.camera_status == CameraState.DISCONNECTED:
window[self.gui_mode_readout].update(
lang._instance.get_string("camera.reconnecting")
)
elif needs_roi_set:
window[self.gui_mode_readout].update(
lang._instance.get_string("camera.waitingOnMouthCrop")
)
elif self.babble_cnn.calibration_frame_counter != None:
window[self.gui_mode_readout].update(
lang._instance.get_string("camera.calibration")
)
else:
window[self.gui_mode_readout].update(
lang._instance.get_string("camera.tracking")
)
window[self.gui_tracking_fps].update(self._movavg_fps(self.camera.fps))
window[self.gui_tracking_bps].update(self._movavg_bps(self.camera.bps))
if self.in_roi_mode:
try:
if self.roi_queue.empty():
self.capture_event.set()
maybe_image = self.roi_queue.get(block=False)
self.maybe_image = maybe_image
imgbytes = cv2.imencode(".ppm", maybe_image[0])[1].tobytes()
graph = window[self.gui_roi_selection]
if self.figure:
graph.delete_figure(self.figure)
# INCREDIBLY IMPORTANT ERASE. Drawing images does NOT overwrite the buffer, the fucking
# graph keeps every image fed in until you call this. Therefore we have to make sure we
# erase before we redraw, otherwise we'll leak memory *very* quickly.
graph.erase()
graph.draw_image(data=imgbytes, location=(0, 0))
if None not in (self.x0, self.y0, self.x1, self.y1):
self.figure = graph.draw_rectangle(
(self.x0, self.y0), (self.x1, self.y1), line_color="#6f4ca1"
)
except Empty:
pass
else:
if needs_roi_set:
window[self.gui_roi_message].update(visible=True)
return
try:
window[self.gui_roi_message].update(visible=False)
(maybe_image, cam_info) = self.image_queue.get(block=False)
imgbytes = cv2.imencode(".ppm", maybe_image)[1].tobytes()
window[self.gui_tracking_image].update(data=imgbytes)
except Empty:
pass