mirror of
https://github.com/WiVRn/WiVRn.git
synced 2026-11-20 00:51:32 -08:00
388 lines
12 KiB
C++
388 lines
12 KiB
C++
/*
|
|
* WiVRn VR streaming
|
|
* Copyright (C) 2022 Guillaume Meunier <[email protected]>
|
|
* Copyright (C) 2022 Patrick Nicolas <[email protected]>
|
|
*
|
|
* This program is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation, either version 3 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
#pragma once
|
|
|
|
#include "app_launcher.h"
|
|
#include "application.h"
|
|
#include "audio/audio.h"
|
|
#include "configuration.h"
|
|
#include "decoder/shard_accumulator.h"
|
|
#include "foveation.h"
|
|
#include "render/imgui_impl.h"
|
|
#include "scene.h"
|
|
#include "scenes/input_profile.h"
|
|
#include "stream_defoveator.h"
|
|
#include "utils/thread_safe.h"
|
|
#include "wifi_lock.h"
|
|
#include "wivrn_client.h"
|
|
#include "wivrn_packets.h"
|
|
#include "xr/space.h"
|
|
#include <mutex>
|
|
#include <optional>
|
|
#include <queue>
|
|
#include <shared_mutex>
|
|
#include <thread>
|
|
#include <vulkan/vulkan_core.h>
|
|
|
|
namespace scenes
|
|
{
|
|
class stream : public scene_impl<stream>, public std::enable_shared_from_this<stream>
|
|
{
|
|
public:
|
|
enum class state
|
|
{
|
|
initializing,
|
|
streaming,
|
|
stalled,
|
|
shutdown,
|
|
};
|
|
static const size_t image_buffer_size = 3;
|
|
|
|
app_launcher apps;
|
|
|
|
private:
|
|
static const size_t view_count = 2;
|
|
static const size_t decoder_count = view_count + 1;
|
|
|
|
struct accumulator_images
|
|
{
|
|
std::unique_ptr<wivrn::shard_accumulator> decoder;
|
|
// latest frames, rolling buffer
|
|
std::array<std::shared_ptr<wivrn::shard_accumulator::blit_handle>, image_buffer_size> latest_frames;
|
|
|
|
std::shared_ptr<wivrn::shard_accumulator::blit_handle> frame(uint64_t id) const;
|
|
bool empty() const;
|
|
};
|
|
|
|
wifi_lock::wifi wifi;
|
|
|
|
// for frames inside accumulator images
|
|
std::mutex frames_mutex;
|
|
std::array<std::shared_ptr<wivrn::shard_accumulator::blit_handle>, decoder_count> common_frame(XrTime display_time);
|
|
|
|
std::unique_ptr<wivrn_session> network_session;
|
|
std::thread network_thread;
|
|
thread_safe<to_headset::tracking_control> tracking_control{};
|
|
std::array<std::atomic<interaction_profile>, 3> interaction_profiles; // left hand, right hand, gamepad
|
|
std::atomic<bool> gamepad_mode = false;
|
|
|
|
struct chord_state
|
|
{
|
|
XrAction a = XR_NULL_HANDLE;
|
|
XrAction b = XR_NULL_HANDLE;
|
|
uint32_t clicks_needed = 1;
|
|
|
|
bool both_released = false;
|
|
uint32_t clicks_done = 0;
|
|
XrTime first_click_time = 0;
|
|
|
|
void reset()
|
|
{
|
|
both_released = false;
|
|
clicks_done = 0;
|
|
first_click_time = 0;
|
|
}
|
|
|
|
// Read actions a and b and determine if they were activated together clicks_needed times in a row
|
|
bool check(XrTime now);
|
|
};
|
|
chord_state menu_chord = {};
|
|
chord_state gamepad_chord = {.clicks_needed = 2};
|
|
|
|
bool using_frame_gamepad() const
|
|
{
|
|
return gamepad_mode.load() and
|
|
interaction_profiles[0].load() == interaction_profile::valve_frame_controller_valve and
|
|
interaction_profiles[1].load() == interaction_profile::valve_frame_controller_valve;
|
|
}
|
|
std::atomic<bool> interaction_profile_changed = false;
|
|
std::atomic<XrTime> scheduled_derived_pose = 0; // Tracking thread will compute derived pose when time is reached
|
|
std::atomic<bool> recenter_requested = false;
|
|
std::atomic<bool> hid_forwarding = false;
|
|
std::atomic<XrDuration> display_time_phase = 0;
|
|
std::atomic<XrDuration> display_time_period = 0;
|
|
XrTime last_display_time = 0;
|
|
std::atomic<XrDuration> real_display_period = 0;
|
|
std::optional<std::thread> tracking_thread;
|
|
|
|
std::shared_mutex decoder_mutex;
|
|
std::optional<to_headset::video_stream_description> video_stream_description;
|
|
std::array<accumulator_images, decoder_count> decoders; // Locked by decoder_mutex
|
|
|
|
std::optional<stream_defoveator> defoveator;
|
|
|
|
vk::raii::Fence fence = nullptr;
|
|
vk::raii::CommandBuffer command_buffer = nullptr;
|
|
|
|
struct haptics_action
|
|
{
|
|
XrAction action;
|
|
XrPath path;
|
|
float amplitude;
|
|
};
|
|
std::unordered_multimap<device_id, haptics_action> haptics_actions;
|
|
std::vector<std::tuple<device_id, XrAction, XrActionType>> input_actions;
|
|
|
|
std::atomic<state> state_ = state::initializing;
|
|
|
|
void set_state(state new_state)
|
|
{
|
|
state prev = state_;
|
|
if (prev == state::shutdown)
|
|
return;
|
|
|
|
state_.compare_exchange_strong(prev, new_state);
|
|
}
|
|
|
|
client_foveation::center foveation_center;
|
|
xr::swapchain swapchain;
|
|
|
|
std::optional<audio> audio_handle;
|
|
|
|
std::optional<xr::hand_tracker> left_hand;
|
|
std::optional<xr::hand_tracker> right_hand;
|
|
std::optional<input_profile> input;
|
|
static inline const uint32_t layer_controllers = 1 << 0;
|
|
static inline const uint32_t layer_rays = 1 << 1;
|
|
|
|
// Size of the composition layer used for the controllers
|
|
uint32_t width;
|
|
uint32_t height;
|
|
|
|
std::optional<imgui_context> imgui_ctx;
|
|
ImTextureID wivrn_logo = 0; // wordmark logo shown in the top bar, like the lobby
|
|
struct gui_toast
|
|
{
|
|
std::string content;
|
|
bool is_urgent = false;
|
|
};
|
|
|
|
static bool is_interactable(stream_tab);
|
|
bool is_gui_interactable() const;
|
|
|
|
// settings sub-page, client-only: the wire stream_tab stays settings
|
|
enum class settings_page
|
|
{
|
|
video,
|
|
audio,
|
|
streaming,
|
|
post_processing,
|
|
devices,
|
|
tracking,
|
|
theme,
|
|
system,
|
|
};
|
|
settings_page current_settings_page = settings_page::video;
|
|
|
|
// Tab currently being displayed
|
|
stream_tab gui_status = stream_tab::hidden;
|
|
// Tab that we will switch to if button is pressed
|
|
stream_tab stored_gui_status = stream_tab::applications;
|
|
// Tab that will be displayed on next render()
|
|
std::atomic<stream_tab> next_gui_status = stream_tab::hidden;
|
|
float dimming = 0;
|
|
|
|
thread_safe<std::optional<gui_toast>> gui_toast;
|
|
std::atomic<XrTime> gui_status_last_change;
|
|
|
|
thread_safe<std::queue<std::string>> stream_error_queue;
|
|
|
|
XrAction recenter_left = XR_NULL_HANDLE;
|
|
XrAction recenter_right = XR_NULL_HANDLE;
|
|
XrAction gui_distance_left = XR_NULL_HANDLE;
|
|
XrAction gui_distance_right = XR_NULL_HANDLE;
|
|
XrAction settings_adjust = XR_NULL_HANDLE;
|
|
XrAction foveation_distance = XR_NULL_HANDLE;
|
|
XrAction foveation_ok = XR_NULL_HANDLE;
|
|
XrAction foveation_cancel = XR_NULL_HANDLE;
|
|
|
|
// Position of the GUI relative to the view space, in view space axes, used when the GUI is not interactable
|
|
glm::vec3 head_gui_position{-0.1, -0.3, -1.2}; // Shift 10cm left by default so that the stats are centered accounting for the tab list
|
|
glm::quat head_gui_orientation{1, 0, 0, 0};
|
|
|
|
// Position of the GUI relative to the world space, in world space axes, used when the GUI is interactable
|
|
glm::vec3 world_gui_position;
|
|
glm::quat world_gui_orientation;
|
|
|
|
// Which controller is used for recentering and position of the GUI relative to the controller, in controller axes, during recentering
|
|
std::optional<std::tuple<xr::spaces, glm::vec3, glm::quat>> recentering_context;
|
|
void update_gui_position(xr::spaces controller, float predicted_display_period);
|
|
|
|
// Keep a reference to the resources needed to blit the images until vkWaitForFences
|
|
std::array<std::shared_ptr<wivrn::shard_accumulator::blit_handle>, decoder_count> current_blit_handles;
|
|
|
|
XrTime running_application_req = 0;
|
|
thread_safe<to_headset::running_applications> running_applications;
|
|
|
|
stream(std::string server_name, scene & parent_scene);
|
|
|
|
bool forward_hid_input(from_headset::hid::input_t, bool device_enabled);
|
|
|
|
public:
|
|
~stream();
|
|
|
|
static std::shared_ptr<stream> create(
|
|
std::unique_ptr<wivrn_session> session,
|
|
float guessed_fps,
|
|
std::string server_name,
|
|
scene & parent_scene);
|
|
|
|
void render(const XrFrameState &) override;
|
|
void on_focused() override;
|
|
void on_unfocused() override;
|
|
void on_xr_event(const xr::event &) override;
|
|
void on_battery_update(battery::device device, const battery::battery_status &) override;
|
|
|
|
bool on_input_key_down(uint8_t key_code) override;
|
|
bool on_input_key_up(uint8_t key_code) override;
|
|
bool on_input_mouse_move(float x, float y) override;
|
|
bool on_input_button_down(uint8_t button) override;
|
|
bool on_input_button_up(uint8_t button) override;
|
|
bool on_input_scroll(float h, float v) override;
|
|
|
|
void operator()(to_headset::crypto_handshake &&) {};
|
|
void operator()(to_headset::pin_check_2 &&) {};
|
|
void operator()(to_headset::pin_check_4 &&) {};
|
|
void operator()(to_headset::handshake &&) {};
|
|
void operator()(to_headset::server_message &&);
|
|
void operator()(to_headset::video_stream_data_shard &&);
|
|
void operator()(to_headset::haptics &&);
|
|
void operator()(to_headset::timesync_query &&);
|
|
void operator()(to_headset::tracking_control &&);
|
|
void operator()(to_headset::feature_control &&);
|
|
void operator()(to_headset::audio_stream_description &&);
|
|
void operator()(to_headset::video_stream_description &&);
|
|
void operator()(to_headset::refresh_rate_change &&);
|
|
void operator()(to_headset::stream_tab_change &&);
|
|
void operator()(to_headset::application_list &&);
|
|
void operator()(to_headset::application_icon &&);
|
|
void operator()(to_headset::running_applications &&);
|
|
void operator()(audio_data &&);
|
|
|
|
void push_blit_handle(wivrn::shard_accumulator * decoder, std::shared_ptr<wivrn::shard_accumulator::blit_handle> handle);
|
|
|
|
void send_feedback(const wivrn::from_headset::feedback & feedback);
|
|
|
|
state current_state() const
|
|
{
|
|
return state_;
|
|
}
|
|
|
|
// Whether the server mirrors forwarded input devices to uinput. The gamepad is also exposed
|
|
// through OpenXR regardless, so this only affects forwarded keyboard and mouse.
|
|
bool hid_forwarding_enabled() const
|
|
{
|
|
return hid_forwarding;
|
|
}
|
|
|
|
void exit();
|
|
void start_application(std::string appid);
|
|
|
|
static meta & get_meta_scene();
|
|
std::optional<std::string> pop_stream_error();
|
|
|
|
private:
|
|
void process_packets();
|
|
void tracking();
|
|
void read_actions();
|
|
|
|
void on_interaction_profile_changed(const XrEventDataInteractionProfileChanged &);
|
|
void send_derived_pose();
|
|
|
|
void setup(const to_headset::video_stream_description &);
|
|
void setup_reprojection_swapchain(uint32_t width, uint32_t height, bool foveation);
|
|
void ensure_swapchain(bool cropped, uint32_t width, uint32_t height);
|
|
|
|
vk::raii::QueryPool query_pool = nullptr;
|
|
bool query_pool_filled = false;
|
|
|
|
// Used for plots
|
|
uint64_t bytes_received = 0;
|
|
uint64_t bytes_sent = 0;
|
|
float bandwidth_rx = 0;
|
|
float bandwidth_tx = 0;
|
|
|
|
struct gpu_timestamps
|
|
{
|
|
float gpu_time = 0;
|
|
};
|
|
|
|
struct global_metric
|
|
{
|
|
float gpu_time;
|
|
float cpu_time = 0;
|
|
float bandwidth_rx = 0;
|
|
float bandwidth_tx = 0;
|
|
};
|
|
|
|
struct plot
|
|
{
|
|
std::string title;
|
|
struct subplot
|
|
{
|
|
std::string title;
|
|
float scenes::stream::global_metric::* data;
|
|
};
|
|
std::vector<subplot> subplots;
|
|
const char * unit;
|
|
};
|
|
|
|
static const inline int size_gpu_timestamps = 1 + sizeof(gpu_timestamps) / sizeof(float);
|
|
|
|
struct decoder_metric
|
|
{
|
|
// All times are in seconds relative to encode_begin
|
|
float encode_begin;
|
|
float encode_end;
|
|
float send_begin;
|
|
float send_end;
|
|
float received_first_packet;
|
|
float received_last_packet;
|
|
float sent_to_decoder;
|
|
float received_from_decoder;
|
|
float blitted;
|
|
float displayed;
|
|
float predicted_display;
|
|
};
|
|
|
|
std::vector<global_metric> global_metrics{300};
|
|
std::vector<std::vector<decoder_metric>> decoder_metrics;
|
|
std::vector<float> axis_scale;
|
|
XrTime last_metric_time = 0;
|
|
int metrics_offset = 0;
|
|
|
|
// Used for compact view
|
|
float compact_bandwidth_rx = 0;
|
|
float compact_bandwidth_tx = 0;
|
|
float compact_cpu_time = 0;
|
|
float compact_gpu_time = 0;
|
|
|
|
void accumulate_metrics(XrTime predicted_display_time, const std::array<std::shared_ptr<wivrn::shard_accumulator::blit_handle>, decoder_count> & blit_handles, const gpu_timestamps & timestamps);
|
|
void gui_performance_metrics();
|
|
void gui_compact_view();
|
|
void gui_settings(float predicted_display_period);
|
|
void gui_bitrate_settings(float predicted_display_period);
|
|
void gui_foveation_settings(float predicted_display_period);
|
|
void gui_applications();
|
|
void gui_toasts();
|
|
void draw_gui(XrTime predicted_display_time, XrDuration predicted_display_period);
|
|
};
|
|
} // namespace scenes
|