Files

388 lines
12 KiB
C++

/*
* WiVRn VR streaming
* Copyright (C) 2022 Guillaume Meunier <[email protected]>
* Copyright (C) 2022 Patrick Nicolas <[email protected]>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
#pragma once
#include "app_launcher.h"
#include "application.h"
#include "audio/audio.h"
#include "configuration.h"
#include "decoder/shard_accumulator.h"
#include "foveation.h"
#include "render/imgui_impl.h"
#include "scene.h"
#include "scenes/input_profile.h"
#include "stream_defoveator.h"
#include "utils/thread_safe.h"
#include "wifi_lock.h"
#include "wivrn_client.h"
#include "wivrn_packets.h"
#include "xr/space.h"
#include <mutex>
#include <optional>
#include <queue>
#include <shared_mutex>
#include <thread>
#include <vulkan/vulkan_core.h>
namespace scenes
{
class stream : public scene_impl<stream>, public std::enable_shared_from_this<stream>
{
public:
enum class state
{
initializing,
streaming,
stalled,
shutdown,
};
static const size_t image_buffer_size = 3;
app_launcher apps;
private:
static const size_t view_count = 2;
static const size_t decoder_count = view_count + 1;
struct accumulator_images
{
std::unique_ptr<wivrn::shard_accumulator> decoder;
// latest frames, rolling buffer
std::array<std::shared_ptr<wivrn::shard_accumulator::blit_handle>, image_buffer_size> latest_frames;
std::shared_ptr<wivrn::shard_accumulator::blit_handle> frame(uint64_t id) const;
bool empty() const;
};
wifi_lock::wifi wifi;
// for frames inside accumulator images
std::mutex frames_mutex;
std::array<std::shared_ptr<wivrn::shard_accumulator::blit_handle>, decoder_count> common_frame(XrTime display_time);
std::unique_ptr<wivrn_session> network_session;
std::thread network_thread;
thread_safe<to_headset::tracking_control> tracking_control{};
std::array<std::atomic<interaction_profile>, 3> interaction_profiles; // left hand, right hand, gamepad
std::atomic<bool> gamepad_mode = false;
struct chord_state
{
XrAction a = XR_NULL_HANDLE;
XrAction b = XR_NULL_HANDLE;
uint32_t clicks_needed = 1;
bool both_released = false;
uint32_t clicks_done = 0;
XrTime first_click_time = 0;
void reset()
{
both_released = false;
clicks_done = 0;
first_click_time = 0;
}
// Read actions a and b and determine if they were activated together clicks_needed times in a row
bool check(XrTime now);
};
chord_state menu_chord = {};
chord_state gamepad_chord = {.clicks_needed = 2};
bool using_frame_gamepad() const
{
return gamepad_mode.load() and
interaction_profiles[0].load() == interaction_profile::valve_frame_controller_valve and
interaction_profiles[1].load() == interaction_profile::valve_frame_controller_valve;
}
std::atomic<bool> interaction_profile_changed = false;
std::atomic<XrTime> scheduled_derived_pose = 0; // Tracking thread will compute derived pose when time is reached
std::atomic<bool> recenter_requested = false;
std::atomic<bool> hid_forwarding = false;
std::atomic<XrDuration> display_time_phase = 0;
std::atomic<XrDuration> display_time_period = 0;
XrTime last_display_time = 0;
std::atomic<XrDuration> real_display_period = 0;
std::optional<std::thread> tracking_thread;
std::shared_mutex decoder_mutex;
std::optional<to_headset::video_stream_description> video_stream_description;
std::array<accumulator_images, decoder_count> decoders; // Locked by decoder_mutex
std::optional<stream_defoveator> defoveator;
vk::raii::Fence fence = nullptr;
vk::raii::CommandBuffer command_buffer = nullptr;
struct haptics_action
{
XrAction action;
XrPath path;
float amplitude;
};
std::unordered_multimap<device_id, haptics_action> haptics_actions;
std::vector<std::tuple<device_id, XrAction, XrActionType>> input_actions;
std::atomic<state> state_ = state::initializing;
void set_state(state new_state)
{
state prev = state_;
if (prev == state::shutdown)
return;
state_.compare_exchange_strong(prev, new_state);
}
client_foveation::center foveation_center;
xr::swapchain swapchain;
std::optional<audio> audio_handle;
std::optional<xr::hand_tracker> left_hand;
std::optional<xr::hand_tracker> right_hand;
std::optional<input_profile> input;
static inline const uint32_t layer_controllers = 1 << 0;
static inline const uint32_t layer_rays = 1 << 1;
// Size of the composition layer used for the controllers
uint32_t width;
uint32_t height;
std::optional<imgui_context> imgui_ctx;
ImTextureID wivrn_logo = 0; // wordmark logo shown in the top bar, like the lobby
struct gui_toast
{
std::string content;
bool is_urgent = false;
};
static bool is_interactable(stream_tab);
bool is_gui_interactable() const;
// settings sub-page, client-only: the wire stream_tab stays settings
enum class settings_page
{
video,
audio,
streaming,
post_processing,
devices,
tracking,
theme,
system,
};
settings_page current_settings_page = settings_page::video;
// Tab currently being displayed
stream_tab gui_status = stream_tab::hidden;
// Tab that we will switch to if button is pressed
stream_tab stored_gui_status = stream_tab::applications;
// Tab that will be displayed on next render()
std::atomic<stream_tab> next_gui_status = stream_tab::hidden;
float dimming = 0;
thread_safe<std::optional<gui_toast>> gui_toast;
std::atomic<XrTime> gui_status_last_change;
thread_safe<std::queue<std::string>> stream_error_queue;
XrAction recenter_left = XR_NULL_HANDLE;
XrAction recenter_right = XR_NULL_HANDLE;
XrAction gui_distance_left = XR_NULL_HANDLE;
XrAction gui_distance_right = XR_NULL_HANDLE;
XrAction settings_adjust = XR_NULL_HANDLE;
XrAction foveation_distance = XR_NULL_HANDLE;
XrAction foveation_ok = XR_NULL_HANDLE;
XrAction foveation_cancel = XR_NULL_HANDLE;
// Position of the GUI relative to the view space, in view space axes, used when the GUI is not interactable
glm::vec3 head_gui_position{-0.1, -0.3, -1.2}; // Shift 10cm left by default so that the stats are centered accounting for the tab list
glm::quat head_gui_orientation{1, 0, 0, 0};
// Position of the GUI relative to the world space, in world space axes, used when the GUI is interactable
glm::vec3 world_gui_position;
glm::quat world_gui_orientation;
// Which controller is used for recentering and position of the GUI relative to the controller, in controller axes, during recentering
std::optional<std::tuple<xr::spaces, glm::vec3, glm::quat>> recentering_context;
void update_gui_position(xr::spaces controller, float predicted_display_period);
// Keep a reference to the resources needed to blit the images until vkWaitForFences
std::array<std::shared_ptr<wivrn::shard_accumulator::blit_handle>, decoder_count> current_blit_handles;
XrTime running_application_req = 0;
thread_safe<to_headset::running_applications> running_applications;
stream(std::string server_name, scene & parent_scene);
bool forward_hid_input(from_headset::hid::input_t, bool device_enabled);
public:
~stream();
static std::shared_ptr<stream> create(
std::unique_ptr<wivrn_session> session,
float guessed_fps,
std::string server_name,
scene & parent_scene);
void render(const XrFrameState &) override;
void on_focused() override;
void on_unfocused() override;
void on_xr_event(const xr::event &) override;
void on_battery_update(battery::device device, const battery::battery_status &) override;
bool on_input_key_down(uint8_t key_code) override;
bool on_input_key_up(uint8_t key_code) override;
bool on_input_mouse_move(float x, float y) override;
bool on_input_button_down(uint8_t button) override;
bool on_input_button_up(uint8_t button) override;
bool on_input_scroll(float h, float v) override;
void operator()(to_headset::crypto_handshake &&) {};
void operator()(to_headset::pin_check_2 &&) {};
void operator()(to_headset::pin_check_4 &&) {};
void operator()(to_headset::handshake &&) {};
void operator()(to_headset::server_message &&);
void operator()(to_headset::video_stream_data_shard &&);
void operator()(to_headset::haptics &&);
void operator()(to_headset::timesync_query &&);
void operator()(to_headset::tracking_control &&);
void operator()(to_headset::feature_control &&);
void operator()(to_headset::audio_stream_description &&);
void operator()(to_headset::video_stream_description &&);
void operator()(to_headset::refresh_rate_change &&);
void operator()(to_headset::stream_tab_change &&);
void operator()(to_headset::application_list &&);
void operator()(to_headset::application_icon &&);
void operator()(to_headset::running_applications &&);
void operator()(audio_data &&);
void push_blit_handle(wivrn::shard_accumulator * decoder, std::shared_ptr<wivrn::shard_accumulator::blit_handle> handle);
void send_feedback(const wivrn::from_headset::feedback & feedback);
state current_state() const
{
return state_;
}
// Whether the server mirrors forwarded input devices to uinput. The gamepad is also exposed
// through OpenXR regardless, so this only affects forwarded keyboard and mouse.
bool hid_forwarding_enabled() const
{
return hid_forwarding;
}
void exit();
void start_application(std::string appid);
static meta & get_meta_scene();
std::optional<std::string> pop_stream_error();
private:
void process_packets();
void tracking();
void read_actions();
void on_interaction_profile_changed(const XrEventDataInteractionProfileChanged &);
void send_derived_pose();
void setup(const to_headset::video_stream_description &);
void setup_reprojection_swapchain(uint32_t width, uint32_t height, bool foveation);
void ensure_swapchain(bool cropped, uint32_t width, uint32_t height);
vk::raii::QueryPool query_pool = nullptr;
bool query_pool_filled = false;
// Used for plots
uint64_t bytes_received = 0;
uint64_t bytes_sent = 0;
float bandwidth_rx = 0;
float bandwidth_tx = 0;
struct gpu_timestamps
{
float gpu_time = 0;
};
struct global_metric
{
float gpu_time;
float cpu_time = 0;
float bandwidth_rx = 0;
float bandwidth_tx = 0;
};
struct plot
{
std::string title;
struct subplot
{
std::string title;
float scenes::stream::global_metric::* data;
};
std::vector<subplot> subplots;
const char * unit;
};
static const inline int size_gpu_timestamps = 1 + sizeof(gpu_timestamps) / sizeof(float);
struct decoder_metric
{
// All times are in seconds relative to encode_begin
float encode_begin;
float encode_end;
float send_begin;
float send_end;
float received_first_packet;
float received_last_packet;
float sent_to_decoder;
float received_from_decoder;
float blitted;
float displayed;
float predicted_display;
};
std::vector<global_metric> global_metrics{300};
std::vector<std::vector<decoder_metric>> decoder_metrics;
std::vector<float> axis_scale;
XrTime last_metric_time = 0;
int metrics_offset = 0;
// Used for compact view
float compact_bandwidth_rx = 0;
float compact_bandwidth_tx = 0;
float compact_cpu_time = 0;
float compact_gpu_time = 0;
void accumulate_metrics(XrTime predicted_display_time, const std::array<std::shared_ptr<wivrn::shard_accumulator::blit_handle>, decoder_count> & blit_handles, const gpu_timestamps & timestamps);
void gui_performance_metrics();
void gui_compact_view();
void gui_settings(float predicted_display_period);
void gui_bitrate_settings(float predicted_display_period);
void gui_foveation_settings(float predicted_display_period);
void gui_applications();
void gui_toasts();
void draw_gui(XrTime predicted_display_time, XrDuration predicted_display_period);
};
} // namespace scenes