/* * WiVRn VR streaming * Copyright (C) 2022 Guillaume Meunier * Copyright (C) 2022 Patrick Nicolas * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #include "constants.h" #include "xr/body_tracker.h" #include "xr/face_tracker.h" #include "xr/fb_face_tracker2.h" #include "xr/space.h" #include "xr/system.h" #include #include #include #include #define GLM_FORCE_RADIANS #include "stream.h" #include "application.h" #include "audio/audio.h" #include "boost/pfr/core.hpp" #include "decoder/shard_accumulator.h" #include "inplace_vector.hpp" #include "spdlog/spdlog.h" #include "utils/i18n.h" #include "utils/named_thread.h" #include "utils/ranges.h" #include "wivrn_packets.h" #include #include #include #include #include #include #include "wivrn_config.h" #if WIVRN_FEATURE_RENDERDOC #include "vk/renderdoc.h" #endif using namespace wivrn; using namespace beman::inplace_vector; // clang-format off static const std::unordered_map device_ids = { {"/user/hand/left/input/dpad_up/click", device_id::DPAD_UP_CLICK}, {"/user/hand/left/input/dpad_up/touch", device_id::DPAD_UP_TOUCH}, {"/user/hand/left/input/dpad_left/click", device_id::DPAD_LEFT_CLICK}, {"/user/hand/left/input/dpad_left/touch", device_id::DPAD_LEFT_TOUCH}, {"/user/hand/left/input/dpad_down/click", device_id::LEFT_X_CLICK}, {"/user/hand/left/input/dpad_down/touch", device_id::LEFT_X_TOUCH}, {"/user/hand/left/input/dpad_right/click", device_id::LEFT_Y_CLICK}, {"/user/hand/left/input/dpad_right/touch", device_id::LEFT_Y_TOUCH}, {"/user/hand/left/input/x/click", device_id::LEFT_X_CLICK}, {"/user/hand/left/input/x/touch", device_id::LEFT_X_TOUCH}, {"/user/hand/left/input/y/click", device_id::LEFT_Y_CLICK}, {"/user/hand/left/input/y/touch", device_id::LEFT_Y_TOUCH}, {"/user/hand/left/input/view/click", device_id::LEFT_MENU_CLICK}, {"/user/hand/left/input/view/touch", device_id::LEFT_MENU_TOUCH}, {"/user/hand/left/input/menu/click", device_id::LEFT_MENU_CLICK}, {"/user/hand/left/input/menu/touch", device_id::LEFT_MENU_TOUCH}, {"/user/hand/left/input/system/click", device_id::LEFT_SYSTEM_CLICK}, {"/user/hand/left/input/system/touch", device_id::LEFT_SYSTEM_TOUCH}, {"/user/hand/left/input/bumper/click", device_id::LEFT_BUMPER_CLICK}, {"/user/hand/left/input/bumper/touch", device_id::LEFT_BUMPER_TOUCH}, {"/user/hand/left/input/squeeze/click", device_id::LEFT_SQUEEZE_CLICK}, {"/user/hand/left/input/squeeze/force", device_id::LEFT_SQUEEZE_FORCE}, {"/user/hand/left/input/squeeze/touch", device_id::LEFT_SQUEEZE_TOUCH}, {"/user/hand/left/input/squeeze/value", device_id::LEFT_SQUEEZE_VALUE}, {"/user/hand/left/input/trigger/value", device_id::LEFT_TRIGGER_VALUE}, {"/user/hand/left/input/trigger/click", device_id::LEFT_TRIGGER_CLICK}, {"/user/hand/left/input/trigger/touch", device_id::LEFT_TRIGGER_TOUCH}, {"/user/hand/left/input/trigger/proximity", device_id::LEFT_TRIGGER_PROXIMITY}, {"/user/hand/left/input/trigger/proximity_fb",device_id::LEFT_TRIGGER_PROXIMITY}, {"/user/hand/left/input/trigger/proximity_meta",device_id::LEFT_TRIGGER_PROXIMITY}, {"/user/hand/left/input/trigger/curl_fb", device_id::LEFT_TRIGGER_CURL}, {"/user/hand/left/input/trigger/curl_meta", device_id::LEFT_TRIGGER_CURL}, {"/user/hand/left/input/trigger_curl/value", device_id::LEFT_TRIGGER_CURL}, {"/user/hand/left/input/trigger/slide_fb", device_id::LEFT_TRIGGER_SLIDE}, {"/user/hand/left/input/trigger/slide_meta", device_id::LEFT_TRIGGER_SLIDE}, {"/user/hand/left/input/trigger_slide/value", device_id::LEFT_TRIGGER_SLIDE}, {"/user/hand/left/input/trigger/force", device_id::LEFT_TRIGGER_FORCE}, {"/user/hand/left/input/thumbstick", device_id::LEFT_THUMBSTICK_X}, {"/user/hand/left/input/thumbstick/click", device_id::LEFT_THUMBSTICK_CLICK}, {"/user/hand/left/input/thumbstick/touch", device_id::LEFT_THUMBSTICK_TOUCH}, {"/user/hand/left/input/thumbrest/touch", device_id::LEFT_THUMBREST_TOUCH}, {"/user/hand/left/input/thumbrest/force", device_id::LEFT_THUMBREST_FORCE}, {"/user/hand/left/input/thumb_resting_surfaces/proximity",device_id::LEFT_THUMB_PROXIMITY}, {"/user/hand/left/input/thumb_meta/proximity_meta",device_id::LEFT_THUMB_PROXIMITY}, {"/user/hand/left/input/trackpad", device_id::LEFT_TRACKPAD_X}, {"/user/hand/left/input/trackpad/click", device_id::LEFT_TRACKPAD_CLICK}, {"/user/hand/left/input/trackpad/touch", device_id::LEFT_TRACKPAD_TOUCH}, {"/user/hand/left/input/trackpad/force", device_id::LEFT_TRACKPAD_FORCE}, {"/user/hand/left/input/stylus/force", device_id::LEFT_STYLUS_FORCE}, {"/user/hand/left/input/stylus_fb/force", device_id::LEFT_STYLUS_FORCE}, {"/user/hand/right/input/a/click", device_id::A_CLICK}, {"/user/hand/right/input/a/touch", device_id::A_TOUCH}, {"/user/hand/right/input/b/click", device_id::B_CLICK}, {"/user/hand/right/input/b/touch", device_id::B_TOUCH}, {"/user/hand/right/input/x/click", device_id::RIGHT_X_CLICK}, {"/user/hand/right/input/x/touch", device_id::RIGHT_X_TOUCH}, {"/user/hand/right/input/y/click", device_id::RIGHT_Y_CLICK}, {"/user/hand/right/input/y/touch", device_id::RIGHT_Y_TOUCH}, {"/user/hand/right/input/menu/click", device_id::RIGHT_MENU_CLICK}, {"/user/hand/right/input/menu/touch", device_id::RIGHT_MENU_TOUCH}, {"/user/hand/right/input/system/click", device_id::RIGHT_SYSTEM_CLICK}, {"/user/hand/right/input/system/touch", device_id::RIGHT_SYSTEM_TOUCH}, {"/user/hand/right/input/bumper/click", device_id::RIGHT_BUMPER_CLICK}, {"/user/hand/right/input/bumper/touch", device_id::RIGHT_BUMPER_TOUCH}, {"/user/hand/right/input/squeeze/click", device_id::RIGHT_SQUEEZE_CLICK}, {"/user/hand/right/input/squeeze/force", device_id::RIGHT_SQUEEZE_FORCE}, {"/user/hand/right/input/squeeze/touch", device_id::RIGHT_SQUEEZE_TOUCH}, {"/user/hand/right/input/squeeze/value", device_id::RIGHT_SQUEEZE_VALUE}, {"/user/hand/right/input/trigger/value", device_id::RIGHT_TRIGGER_VALUE}, {"/user/hand/right/input/trigger/click", device_id::RIGHT_TRIGGER_CLICK}, {"/user/hand/right/input/trigger/touch", device_id::RIGHT_TRIGGER_TOUCH}, {"/user/hand/right/input/trigger/proximity", device_id::RIGHT_TRIGGER_PROXIMITY}, {"/user/hand/right/input/trigger/proximity_fb",device_id::RIGHT_TRIGGER_PROXIMITY}, {"/user/hand/right/input/trigger/proximity_meta",device_id::RIGHT_TRIGGER_PROXIMITY}, {"/user/hand/right/input/trigger/curl_fb", device_id::RIGHT_TRIGGER_CURL}, {"/user/hand/right/input/trigger/curl_meta", device_id::RIGHT_TRIGGER_CURL}, {"/user/hand/right/input/trigger_curl/value", device_id::RIGHT_TRIGGER_CURL}, {"/user/hand/right/input/trigger/slide_fb", device_id::RIGHT_TRIGGER_SLIDE}, {"/user/hand/right/input/trigger/slide_meta", device_id::RIGHT_TRIGGER_SLIDE}, {"/user/hand/right/input/trigger_slide/value", device_id::RIGHT_TRIGGER_SLIDE}, {"/user/hand/right/input/trigger/force", device_id::RIGHT_TRIGGER_FORCE}, {"/user/hand/right/input/thumbstick", device_id::RIGHT_THUMBSTICK_X}, {"/user/hand/right/input/thumbstick/click", device_id::RIGHT_THUMBSTICK_CLICK}, {"/user/hand/right/input/thumbstick/touch", device_id::RIGHT_THUMBSTICK_TOUCH}, {"/user/hand/right/input/thumbrest/touch", device_id::RIGHT_THUMBREST_TOUCH}, {"/user/hand/right/input/thumbrest/force", device_id::RIGHT_THUMBREST_FORCE}, {"/user/hand/right/input/thumb_resting_surfaces/proximity",device_id::RIGHT_THUMB_PROXIMITY}, {"/user/hand/right/input/thumb_meta/proximity_meta",device_id::RIGHT_THUMB_PROXIMITY}, {"/user/hand/right/input/trackpad", device_id::RIGHT_TRACKPAD_X}, {"/user/hand/right/input/trackpad/click", device_id::RIGHT_TRACKPAD_CLICK}, {"/user/hand/right/input/trackpad/touch", device_id::RIGHT_TRACKPAD_TOUCH}, {"/user/hand/right/input/trackpad/force", device_id::RIGHT_TRACKPAD_FORCE}, {"/user/hand/right/input/stylus/force", device_id::RIGHT_STYLUS_FORCE}, {"/user/hand/right/input/stylus_fb/force", device_id::RIGHT_STYLUS_FORCE}, // XR_EXT_hand_interaction {"/user/hand/left/input/pinch_ext/value", device_id::LEFT_PINCH_VALUE}, {"/user/hand/left/input/pinch_ext/ready_ext", device_id::LEFT_PINCH_READY}, {"/user/hand/left/input/aim_activate_ext/value",device_id::LEFT_AIM_ACTIVATE_VALUE}, {"/user/hand/left/input/aim_activate_ext/ready_ext",device_id::LEFT_AIM_ACTIVATE_READY}, {"/user/hand/left/input/grasp_ext/value", device_id::LEFT_GRASP_VALUE}, {"/user/hand/left/input/grasp_ext/ready_ext", device_id::LEFT_GRASP_READY}, {"/user/hand/right/input/pinch_ext/value", device_id::RIGHT_PINCH_VALUE}, {"/user/hand/right/input/pinch_ext/ready_ext", device_id::RIGHT_PINCH_READY}, {"/user/hand/right/input/aim_activate_ext/value",device_id::RIGHT_AIM_ACTIVATE_VALUE}, {"/user/hand/right/input/aim_activate_ext/ready_ext",device_id::RIGHT_AIM_ACTIVATE_READY}, {"/user/hand/right/input/grasp_ext/value", device_id::RIGHT_GRASP_VALUE}, {"/user/hand/right/input/grasp_ext/ready_ext", device_id::RIGHT_GRASP_READY}, {"/user/gamepad/input/menu/click", device_id::GAMEPAD_MENU_CLICK}, {"/user/gamepad/input/view/click", device_id::GAMEPAD_VIEW_CLICK}, {"/user/gamepad/input/a/click", device_id::GAMEPAD_A_CLICK}, {"/user/gamepad/input/b/click", device_id::GAMEPAD_B_CLICK}, {"/user/gamepad/input/x/click", device_id::GAMEPAD_X_CLICK}, {"/user/gamepad/input/y/click", device_id::GAMEPAD_Y_CLICK}, {"/user/gamepad/input/dpad_down/click", device_id::GAMEPAD_DPAD_DOWN_CLICK}, {"/user/gamepad/input/dpad_right/click", device_id::GAMEPAD_DPAD_RIGHT_CLICK}, {"/user/gamepad/input/dpad_up/click", device_id::GAMEPAD_DPAD_UP_CLICK}, {"/user/gamepad/input/dpad_left/click", device_id::GAMEPAD_DPAD_LEFT_CLICK}, {"/user/gamepad/input/shoulder_left/click", device_id::GAMEPAD_SHOULDER_LEFT_CLICK}, {"/user/gamepad/input/shoulder_right/click", device_id::GAMEPAD_SHOULDER_RIGHT_CLICK}, {"/user/gamepad/input/thumbstick_left/click", device_id::GAMEPAD_THUMBSTICK_LEFT_CLICK}, {"/user/gamepad/input/thumbstick_right/click", device_id::GAMEPAD_THUMBSTICK_RIGHT_CLICK}, {"/user/gamepad/input/trigger_left/value", device_id::GAMEPAD_TRIGGER_LEFT_VALUE}, {"/user/gamepad/input/trigger_right/value", device_id::GAMEPAD_TRIGGER_RIGHT_VALUE}, {"/user/gamepad/input/thumbstick_left/x", device_id::GAMEPAD_THUMBSTICK_LEFT_X}, {"/user/gamepad/input/thumbstick_left/y", device_id::GAMEPAD_THUMBSTICK_LEFT_Y}, {"/user/gamepad/input/thumbstick_right/x", device_id::GAMEPAD_THUMBSTICK_RIGHT_X}, {"/user/gamepad/input/thumbstick_right/y", device_id::GAMEPAD_THUMBSTICK_RIGHT_Y}, }; // clang-format on static const std::array supported_color_formats = { vk::Format::eR8G8B8A8Srgb, vk::Format::eB8G8R8A8Srgb, }; static const std::array supported_depth_formats{ vk::Format::eD32Sfloat, vk::Format::eX8D24UnormPack32, }; scenes::stream::stream(std::string server_name, scene & parent_scene) : scene_impl(supported_color_formats, supported_depth_formats, parent_scene), apps{*this, std::move(server_name)}, foveation_center{instance, system, session, device, commandpool, queue, swapchain_format} { const auto & view = system.stereo_view(); width = view.recommendedImageRectWidth; height = view.recommendedImageRectHeight; } static from_headset::visibility_mask_changed::masks get_visibility_mask(xr::instance & inst, xr::session & session, int view) { assert(inst.has_extension(XR_KHR_VISIBILITY_MASK_EXTENSION_NAME)); static auto xrGetVisibilityMaskKHR = inst.get_proc("xrGetVisibilityMaskKHR"); from_headset::visibility_mask_changed::masks res{}; for (auto [type, mask]: utils::enumerate(res)) { XrVisibilityMaskKHR xr_mask{ .type = XR_TYPE_VISIBILITY_MASK_KHR, }; CHECK_XR(xrGetVisibilityMaskKHR(session, XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO, view, XrVisibilityMaskTypeKHR(type + 1), &xr_mask)); mask.vertices.resize(xr_mask.vertexCountOutput); mask.indices.resize(xr_mask.indexCountOutput); xr_mask = { .type = XR_TYPE_VISIBILITY_MASK_KHR, .vertexCapacityInput = uint32_t(mask.vertices.size()), .vertices = mask.vertices.data(), .indexCapacityInput = uint32_t(mask.indices.size()), .indices = mask.indices.data(), }; CHECK_XR(xrGetVisibilityMaskKHR(session, XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO, view, XrVisibilityMaskTypeKHR(type + 1), &xr_mask)); mask.vertices.resize(xr_mask.vertexCountOutput); mask.indices.resize(xr_mask.indexCountOutput); } return res; } std::shared_ptr scenes::stream::create(std::unique_ptr network_session, float guessed_fps, std::string server_name, scene & parent_scene) { std::shared_ptr self{new stream{std::move(server_name), parent_scene}}; self->network_session = std::move(network_session); self->network_session->send_control([&]() { from_headset::headset_info_packet info{ .language = application::get_messages_info().language, .country = application::get_messages_info().country, .variant = application::get_messages_info().variant, }; const auto & config = application::get_config(); { auto view = self->system.stereo_view(); view = application::get_hmd_traits().override_view(view); info.render_eye_width = view.recommendedImageRectWidth * config.resolution_scale; info.render_eye_height = view.recommendedImageRectHeight * config.resolution_scale; } { auto scale = config.get_stream_scale(); info.stream_eye_width = info.render_eye_width * scale; info.stream_eye_height = info.render_eye_height * scale; } if (self->instance.has_extension(XR_FB_DISPLAY_REFRESH_RATE_EXTENSION_NAME)) { info.available_refresh_rates = self->session.get_refresh_rates(); info.settings.preferred_refresh_rate = config.preferred_refresh_rate; info.settings.minimum_refresh_rate = config.minimum_refresh_rate.value_or(0); } info.settings.fps_divider = config.fps_divider; if (info.available_refresh_rates.empty()) { spdlog::warn("Unable to detect refresh rates"); info.available_refresh_rates = {guessed_fps}; info.settings.preferred_refresh_rate = guessed_fps; } info.settings.bitrate_bps = config.bitrate_bps; info.settings.mirror_gamepad = config.forward_gamepad; info.settings.enabled_body_parts = config.body_part_mask; info.battery_status = self->instance.has_extension(XR_EXT_INTERACTION_PROFILE_BATTERY_STATE_DISPLAY_EXTENSION_NAME); info.hand_tracking = config.check_feature(feature::hand_tracking); info.eye_gaze = config.check_feature(feature::eye_gaze); if (self->instance.has_extension(XR_EXT_USER_PRESENCE_EXTENSION_NAME)) { info.user_presence = self->system.user_presence_properties().supportsUserPresence; } if (config.check_feature(feature::face_tracking)) { switch (self->system.face_tracker_supported()) { case xr::face_tracker_type::none: info.face_tracking = from_headset::face_type::none; break; case xr::face_tracker_type::android: info.face_tracking = from_headset::face_type::android; break; case xr::face_tracker_type::fb: case xr::face_tracker_type::pico: info.face_tracking = from_headset::face_type::fb2; break; case xr::face_tracker_type::htc: info.face_tracking = from_headset::face_type::htc; break; } } if (config.check_feature(feature::body_tracking)) { switch (self->system.body_tracker_supported()) { case xr::body_tracker_type::none: info.body_tracking = from_headset::body_type::none; break; case xr::body_tracker_type::fb: info.body_tracking = from_headset::body_type::fb; break; case xr::body_tracker_type::meta: info.body_tracking = from_headset::body_type::meta; break; case xr::body_tracker_type::pico: info.body_tracking = from_headset::body_type::bd; break; case xr::body_tracker_type::htc: info.body_tracking = from_headset::body_type::htc; info.num_generic_trackers = application::get_generic_trackers().size(); break; } } info.palm_pose = application::space(xr::spaces::palm_left) or application::space(xr::spaces::palm_right); info.passthrough = self->system.passthrough_supported() != xr::passthrough_type::none; info.system_name = std::string(self->system.properties().systemName); audio::get_audio_description(info); if (not(config.check_feature(feature::microphone))) info.microphone = {}; if (config.codec) { const bool supports_10bit = decoder::supports_10bit(*config.codec); info.supported_codecs = {{.codec = *config.codec, .supports_10bit = supports_10bit}}; switch (*config.codec) { case h264: case raw: break; case h265: case av1: info.bit_depth = supports_10bit ? config.bit_depth : 8; } } else { info.supported_codecs = decoder::supported_codecs(); } return info; }()); self->network_session->send_control(from_headset::session_state_changed{ .state = application::get_session_state(), }); self->network_session->send_control(from_headset::stream_tab_changed{ .tab = self->gui_status, }); if (self->instance.has_extension(XR_KHR_VISIBILITY_MASK_EXTENSION_NAME)) { for (uint8_t view = 0; view < view_count; ++view) { try { self->network_session->send_control(from_headset::visibility_mask_changed{ .data = get_visibility_mask(self->instance, self->session, view), .view_index = view}); } catch (std::exception & e) { spdlog::warn("Failed to get visibility mask: ", e.what()); } } } self->network_thread = utils::named_thread("network_thread", &stream::process_packets, self.get()); self->command_buffer = std::move(self->device.allocateCommandBuffers({ .commandPool = *self->commandpool, .level = vk::CommandBufferLevel::ePrimary, .commandBufferCount = 1, })[0]); self->fence = self->device.createFence({.flags = vk::FenceCreateFlagBits::eSignaled}); // Look up the XrActions for haptics for (auto [id, path, output]: { std::tuple(device_id::LEFT_CONTROLLER_HAPTIC, "/user/hand/left", "/output/haptic"), std::tuple(device_id::RIGHT_CONTROLLER_HAPTIC, "/user/hand/right", "/output/haptic"), std::tuple(device_id::LEFT_TRIGGER_HAPTIC, "/user/hand/left", "/output/haptic_trigger"), std::tuple(device_id::RIGHT_TRIGGER_HAPTIC, "/user/hand/right", "/output/haptic_trigger"), std::tuple(device_id::LEFT_TRIGGER_HAPTIC, "/user/hand/left", "/output/haptic_trigger_fb"), std::tuple(device_id::RIGHT_TRIGGER_HAPTIC, "/user/hand/right", "/output/haptic_trigger_fb"), std::tuple(device_id::LEFT_THUMB_HAPTIC, "/user/hand/left", "/output/haptic_thumb"), std::tuple(device_id::RIGHT_THUMB_HAPTIC, "/user/hand/right", "/output/haptic_thumb"), std::tuple(device_id::LEFT_THUMB_HAPTIC, "/user/hand/left", "/output/haptic_thumb_fb"), std::tuple(device_id::RIGHT_THUMB_HAPTIC, "/user/hand/right", "/output/haptic_thumb_fb"), std::tuple(device_id::GAMEPAD_HAPTIC_LEFT, "/user/gamepad", "/output/haptic_left"), std::tuple(device_id::GAMEPAD_HAPTIC_RIGHT, "/user/gamepad", "/output/haptic_right"), std::tuple(device_id::GAMEPAD_HAPTIC_LEFT_TRIGGER, "/user/gamepad", "/output/haptic_left_trigger"), std::tuple(device_id::GAMEPAD_HAPTIC_RIGHT_TRIGGER, "/user/gamepad", "/output/haptic_right_trigger")}) { if (auto action = application::get_action(std::string(path) + output); action.first) { self->haptics_actions.emplace(id, haptics_action{ .action = action.first, .path = self->instance.string_to_path(path), }); } } // Look up the XrActions for input for (const auto & [action, action_type, name]: application::inputs()) { auto it = device_ids.find(name); if (it == device_ids.end()) continue; self->input_actions.emplace_back(it->second, action, action_type); } spdlog::info("Using format {}", vk::to_string(self->swapchain_format)); self->query_pool = vk::raii::QueryPool( self->device, vk::QueryPoolCreateInfo{ .queryType = vk::QueryType::eTimestamp, .queryCount = size_gpu_timestamps, }); self->wifi = application::get_wifi_lock().get_wifi_lock(); return self; } void scenes::stream::on_focused() { gui_status_last_change = instance.now(); const auto & profile = application::get_hmd_traits().controller_profile; input.emplace( *this, "assets://controllers/" + profile + "/profile.json", layer_controllers, layer_rays, get_action("left_trigger").first, get_action("right_trigger").first); spdlog::info("Loaded input profile {}", input->id); for (auto i: {xr::spaces::aim_left, xr::spaces::aim_right, xr::spaces::grip_left, xr::spaces::grip_right}) { auto [p, q] = input->offset[i] = application::get_hmd_traits().controller_offset(i); auto rot = glm::degrees(glm::eulerAngles(q)); spdlog::info("Initializing offset of space {} to ({}, {}, {}) mm, ({}, {}, {})°", magic_enum::enum_name(i), 1000 * p.x, 1000 * p.y, 1000 * p.z, rot.x, rot.y, rot.z); } std::array imgui_inputs{ imgui_context::controller{ .aim = get_action_space("left_aim"), .offset = input->offset[xr::spaces::aim_left], .trigger = get_action("left_trigger").first, .squeeze = get_action("left_squeeze").first, .scroll = get_action("left_scroll").first, .haptic_output = get_action("left_haptic").first, }, imgui_context::controller{ .aim = get_action_space("right_aim"), .offset = input->offset[xr::spaces::aim_right], .trigger = get_action("right_trigger").first, .squeeze = get_action("right_squeeze").first, .scroll = get_action("right_scroll").first, .haptic_output = get_action("right_haptic").first, }, }; xr::swapchain swapchain_imgui( instance, session, device, swapchain_format, 3600, 1200); std::vector vps{ { // Main window .space = xr::spaces::world, // Position and orientation are set at each frame .size = {1.2, 0.6666}, .vp_origin = {0, 0}, .vp_size = {1800, 1000}, }, { .space = xr::spaces::world, .size = {1.2, 0.1333}, .vp_origin = {0, 1000}, .vp_size = {1800, 200}, .tooltip_viewport = true, }, { // popup window for combos and modals, tracks the main panel each frame at the same pixel density .space = xr::spaces::world, .size = {1.2, 0.6666}, .vp_origin = {1800, 0}, .vp_size = {1800, 1000}, }, }; imgui_ctx.emplace(physical_device, device, queue_family_index, queue, imgui_inputs, std::move(swapchain_imgui), std::move(vps), image_cache); // match the lobby's seasonal wordmark logo { auto t = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); auto tm = std::localtime(&t); switch (tm->tm_mon) { case 5: wivrn_logo = imgui_ctx->load_texture("assets://wivrn-pride.ktx2"); break; case 11: wivrn_logo = imgui_ctx->load_texture("assets://wivrn-christmas.ktx2"); break; default: wivrn_logo = imgui_ctx->load_texture("assets://wivrn.ktx2"); break; } } menu_chord.a = get_action("menu_toggle_1").first; menu_chord.b = get_action("menu_toggle_2").first; gamepad_chord.a = get_action("gamepad_mode_1").first; gamepad_chord.b = get_action("gamepad_mode_2").first; recenter_left = get_action("recenter_left").first; recenter_right = get_action("recenter_right").first; gui_distance_left = get_action("gui_distance_left").first; gui_distance_right = get_action("gui_distance_right").first; settings_adjust = get_action("settings_adjust").first; foveation_distance = get_action("foveation_distance").first; foveation_ok = get_action("foveation_ok").first; foveation_cancel = get_action("foveation_cancel").first; if (application::get_config().high_power_mode) { session.set_performance_level(XR_PERF_SETTINGS_DOMAIN_CPU_EXT, XR_PERF_SETTINGS_LEVEL_SUSTAINED_HIGH_EXT); session.set_performance_level(XR_PERF_SETTINGS_DOMAIN_GPU_EXT, XR_PERF_SETTINGS_LEVEL_SUSTAINED_HIGH_EXT); } else { session.set_performance_level(XR_PERF_SETTINGS_DOMAIN_CPU_EXT, XR_PERF_SETTINGS_LEVEL_SUSTAINED_LOW_EXT); session.set_performance_level(XR_PERF_SETTINGS_DOMAIN_GPU_EXT, XR_PERF_SETTINGS_LEVEL_SUSTAINED_LOW_EXT); } } void scenes::stream::on_unfocused() { renderer->wait_idle(); // Must be before the scene data because the renderer uses its descriptor sets; world.clear(); input.reset(); left_hand.reset(); right_hand.reset(); apps.reset(); imgui_ctx.reset(); } scenes::stream::~stream() { exit(); if (tracking_thread && tracking_thread->joinable()) tracking_thread->join(); if (network_thread.joinable()) network_thread.join(); } void scenes::stream::push_blit_handle(shard_accumulator * decoder, std::shared_ptr handle) { assert(handle); if (!application::is_visible()) return; { std::shared_lock lock(decoder_mutex); std::unique_lock frame_lock(frames_mutex); auto stream = handle->feedback.stream_index; if (stream < decoders.size()) { if (decoder != decoders[stream].decoder.get()) return; handle->feedback.received_from_decoder = instance.now(); std::swap(handle, decoders[stream].latest_frames[handle->feedback.frame_index % decoders[stream].latest_frames.size()]); } if (state_ != state::streaming and not(decoders[0].empty() or decoders[1].empty())) { set_state(state::streaming); spdlog::info("Stream scene ready at t={}", instance.now()); } } if (handle and not handle->feedback.blitted) { send_feedback(handle->feedback); } } bool scenes::stream::accumulator_images::empty() const { for (const auto & frame: latest_frames) { if (frame) return false; } return true; } std::array, 3> scenes::stream::common_frame(XrTime display_time) { if (decoders.empty()) return {}; std::unique_lock lock(frames_mutex); inplace_vector common_frames; const bool alpha = decoders[0].latest_frames[0] and decoders[0].latest_frames[0]->view_info.alpha; for (size_t i = 0; i < view_count + alpha; ++i) { if (i == 0) { for (const auto & h: decoders[i].latest_frames) if (h) common_frames.push_back(h.get()); } else { // clang-format off erase_if(common_frames, [this, i](auto & left) { return std::ranges::none_of( decoders[i].latest_frames, [&left](auto & right) { return right and left->feedback.frame_index == right->feedback.frame_index; }); }); // clang-format on } } std::array, decoder_count> result; if (not common_frames.empty()) { auto min = std::ranges::min_element(common_frames, std::ranges::less{}, [display_time](auto frame) { if (not frame) return std::numeric_limits::max(); return std::abs(frame->view_info.display_time - display_time); }); assert(*min); auto frame_index = (*min)->feedback.frame_index; for (auto [i, decoder]: utils::enumerate(decoders)) { if (alpha or i < view_count) result[i] = decoder.frame(frame_index); } } else { spdlog::warn("Failed to find a common frame for all decoders, dumping available frames per decoder"); for (const auto & decoder: decoders) { std::string frames; for (const auto & frame: decoder.latest_frames) { if (frame) frames += " " + std::to_string(frame->feedback.frame_index); else frames += " -"; } spdlog::warn(frames); } for (auto [i, decoder]: utils::enumerate(decoders)) { if (alpha or i < view_count) { auto min = std::ranges::min_element(decoder.latest_frames, std::ranges::less{}, [display_time](auto frame) { if (not frame) return std::numeric_limits::max(); return std::abs(frame->view_info.display_time - display_time); }); result[i] = *min; } } } return result; } std::shared_ptr scenes::stream::accumulator_images::frame(uint64_t id) const { auto & frame = latest_frames[id % latest_frames.size()]; if (frame and frame->feedback.frame_index != id) return nullptr; return frame; } void scenes::stream::update_gui_position(xr::spaces controller, float predicted_display_period) { std::optional> aim = application::locate_controller( application::space(controller), application::space(xr::spaces::world), predicted_display_time); if (not aim) return; auto [offset_position, offset_orientation] = input->offset[controller]; auto world_controller_position = aim->first + glm::mat3_cast(aim->second * offset_orientation) * offset_position; auto world_controller_orientation = aim->second * offset_orientation; auto world_controller_direction = -glm::column(glm::mat3_cast(world_controller_orientation), 2); if (not recentering_context) { // First frame of recentering: get the GUI position relative to the controller // Compute the intersection of the ray with the GUI auto gui_controller_direction = glm::conjugate(world_gui_orientation) * world_controller_direction; auto gui_controller_position = glm::conjugate(world_gui_orientation) * (world_controller_position - world_gui_position); float lambda = -gui_controller_position.z / gui_controller_direction.z; auto gui_intersection = gui_controller_position + lambda * gui_controller_direction; auto viewport_size = imgui_ctx->layers()[0].size; if (std::isnan(lambda) or lambda < 0 or std::abs(gui_intersection.x) > viewport_size.x / 2 or std::abs(gui_intersection.y) > viewport_size.y / 2) { // Reset the relative GUI position if the ray does not intersect recentering_context.emplace(controller, glm::vec3{0, 0, -1}, glm::quat{1, 0, 0, 0}); } else { glm::vec3 controller_gui_position = glm::conjugate(world_controller_orientation) * (world_gui_position - world_controller_position); glm::quat controller_gui_orientation = glm::conjugate(world_controller_orientation) * world_gui_orientation; recentering_context.emplace(controller, controller_gui_position, controller_gui_orientation); } } else { // Subsequent frames of recentering: keep the GUI locked to the controller auto & [_, controller_gui_position, controller_gui_orientation] = *recentering_context; if (auto gui_distance = application::read_action_float(controller == xr::spaces::aim_left ? gui_distance_left : gui_distance_right)) { controller_gui_position.z *= std::pow(constants::stream::gui_max_layer_speed, gui_distance->second * predicted_display_period); controller_gui_position.z = -std::clamp(-controller_gui_position.z, constants::stream::gui_min_layer_distance, constants::stream::gui_max_layer_distance); } world_gui_position = world_controller_position + world_controller_orientation * controller_gui_position; world_gui_orientation = world_controller_orientation * controller_gui_orientation; } } bool scenes::stream::is_interactable(stream_tab tab) { switch (tab) { case stream_tab::stats: case stream_tab::settings: case stream_tab::foveation_settings: case stream_tab::applications: case stream_tab::application_launcher: return true; case stream_tab::hidden: case stream_tab::overlay_only: case stream_tab::compact: return false; } spdlog::warn("Invalid tab value {}", int(tab)); return false; } bool scenes::stream::is_gui_interactable() const { return is_interactable(gui_status); } void scenes::stream::render(const XrFrameState & frame_state) { if (state_ == state::shutdown) application::pop_scene(); display_time_phase = frame_state.predictedDisplayPeriod ? frame_state.predictedDisplayTime % frame_state.predictedDisplayPeriod : 0; display_time_period = frame_state.predictedDisplayPeriod; real_display_period = last_display_time ? frame_state.predictedDisplayTime - last_display_time : frame_state.predictedDisplayPeriod; last_display_time = frame_state.predictedDisplayTime; std::shared_lock lock(decoder_mutex); if (not frame_state.shouldRender or decoders[0].empty() or decoders[1].empty() or state_ == state::shutdown) { // TODO: stop/restart video stream session.begin_frame(); session.end_frame(frame_state.predictedDisplayTime, {}); return; } if (state_ == state::stalled) { network_session->send_control(from_headset::get_application_list{ .language = application::get_messages_info().language, .country = application::get_messages_info().country, .variant = application::get_messages_info().variant, }); next_gui_status = stream_tab::hidden; application::pop_scene(); } if (device.waitForFences(*fence, VK_TRUE, UINT64_MAX) == vk::Result::eTimeout) throw std::runtime_error("Vulkan fence timeout"); // We don't need those after vkWaitForFences current_blit_handles.fill(nullptr); gpu_timestamps timestamps; if (query_pool_filled) { auto [res, timestamps2] = query_pool.getResults( 0, size_gpu_timestamps, size_gpu_timestamps * sizeof(uint64_t), sizeof(uint64_t), vk::QueryResultFlagBits::e64 | vk::QueryResultFlagBits::eWait); if (res == vk::Result::eSuccess) { boost::pfr::for_each_field(timestamps, [n = 1, ×tamps2](float & t) mutable { t = (timestamps2[n++] - timestamps2[0]) * application::get_physical_device_properties().limits.timestampPeriod / 1e9; }); } } // Search for frame with desired display time on all decoders // If no such frame exists, use the latest frame for each decoder current_blit_handles = common_frame(frame_state.predictedDisplayTime); std::array pose; std::array fov{}; std::array valid_fov{}; std::array foveation; std::array extents; int32_t max_width = 0; int32_t max_height = 0; for (size_t i = 0; i < view_count; ++i) { auto & blit_handle = current_blit_handles[i]; if (!blit_handle) continue; fov[i] = blit_handle->view_info.fov[i]; valid_fov[i] = true; foveation[i] = blit_handle->view_info.foveation[i]; extents[i] = stream_defoveator::defoveated_size(foveation[i]); max_width = std::max(max_width, extents[i].width); max_height = std::max(max_height, extents[i].height); } if (max_width > 0 and max_height > 0) ensure_swapchain(foveation_center.is_cropped(fov, valid_fov), max_width, max_height); session.begin_frame(); #if WIVRN_FEATURE_RENDERDOC renderdoc_begin(*vk_instance); #endif command_buffer.reset(); command_buffer.begin(vk::CommandBufferBeginInfo{ .flags = vk::CommandBufferUsageFlagBits::eOneTimeSubmit, }); // Keep a reference to the resources needed to blit the images until vkWaitForFences command_buffer.resetQueryPool(*query_pool, 0, size_gpu_timestamps); command_buffer.writeTimestamp(vk::PipelineStageFlagBits::eTopOfPipe, *query_pool, 0); bool use_alpha = false; auto now = instance.now(); std::array images; for (size_t i = 0; i < view_count + use_alpha; ++i) { auto & blit_handle = current_blit_handles[i]; if (not blit_handle) { if (i == view_count) use_alpha = false; continue; } blit_handle->feedback.blitted = now; if (blit_handle->feedback.blitted - blit_handle->feedback.received_from_decoder > 1'000'000'000) set_state(state::stalled); ++blit_handle->feedback.times_displayed; blit_handle->feedback.displayed = frame_state.predictedDisplayTime; use_alpha = blit_handle->view_info.alpha; #if WIVRN_USE_V4L2 if (blit_handle->foreign_queue_family != vk::QueueFamilyIgnored) { // decoder completed its write before handing us this image // acquire ownership from the external media engine before sampling it vk::ImageMemoryBarrier barrier{ .srcAccessMask = vk::AccessFlagBits::eNone, .dstAccessMask = vk::AccessFlagBits::eShaderRead, .oldLayout = blit_handle->current_layout, .newLayout = vk::ImageLayout::eGeneral, .srcQueueFamilyIndex = blit_handle->foreign_queue_family, .dstQueueFamilyIndex = queue_family_index, .image = blit_handle->image, .subresourceRange = { .aspectMask = vk::ImageAspectFlagBits::eColor, .levelCount = 1, .layerCount = 1, }, }; command_buffer.pipelineBarrier(vk::PipelineStageFlagBits::eTopOfPipe, vk::PipelineStageFlagBits::eFragmentShader, {}, {}, {}, barrier); blit_handle->current_layout = vk::ImageLayout::eGeneral; } else #endif if (blit_handle->current_layout == vk::ImageLayout::eUndefined) { vk::ImageMemoryBarrier barrier{ .srcAccessMask = vk::AccessFlagBits::eNone, .dstAccessMask = vk::AccessFlagBits::eMemoryRead, .oldLayout = vk::ImageLayout::eUndefined, .newLayout = vk::ImageLayout::eGeneral, .image = blit_handle->image, .subresourceRange = { .aspectMask = vk::ImageAspectFlagBits::eColor, .levelCount = 1, .layerCount = 1, }, }; command_buffer.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands, vk::PipelineStageFlagBits::eAllCommands, {}, {}, {}, barrier); blit_handle->current_layout = vk::ImageLayout::eGeneral; } if (i < view_count) { foveation[i] = blit_handle->view_info.foveation[i]; pose[i] = blit_handle->view_info.pose[i]; fov[i] = blit_handle->view_info.fov[i]; // colour image images[i] = { .rgb = blit_handle->image_view, .sampler_rgb = decoders[i].decoder->sampler(), .rect_rgb = { .extent = blit_handle->extent, }, .layout_rgb = blit_handle->current_layout, }; } else { // alpha image, must set for each view for (int j = 0; j < view_count; ++j) { images[j].a = blit_handle->image_view; images[j].sampler_a = decoders[i].decoder->sampler(); images[j].rect_a = vk::Rect2D{ // in full size pixels .offset = { .x = j * video_stream_description->width, }, .extent = { .width = blit_handle->extent.width * 2, .height = blit_handle->extent.height * 2, }, }; images[j].layout_a = blit_handle->current_layout; } } } // Allow the headset to time warp if we are redisplaying a frame if ((not application::get_hmd_traits().discard_frame) or std::ranges::any_of(current_blit_handles, [](const auto & h) { return h and h->feedback.times_displayed < 2; }) or is_gui_interactable()) { assert(swapchain); // Swapchains only recreated when switching between cropped/full // Otherwise we clamp and resize the existing one for (size_t i = 0; i < view_count; ++i) { extents[i].width = std::min(extents[i].width, swapchain.width()); extents[i].height = std::min(extents[i].height, swapchain.height()); } foveation_center.update(fov, valid_fov, swapchain, frame_state.predictedDisplayTime); // defoveate the image, apply scale/bias int image_index = swapchain.acquire(); swapchain.wait(); switch (gui_status) { case stream_tab::hidden: case stream_tab::foveation_settings: case stream_tab::compact: case stream_tab::overlay_only: dimming = dimming - frame_state.predictedDisplayPeriod / (1e9 * constants::stream::fade_duration); break; case stream_tab::stats: case stream_tab::settings: case stream_tab::applications: case stream_tab::application_launcher: dimming = dimming + frame_state.predictedDisplayPeriod / (1e9 * constants::stream::fade_duration); break; } dimming = std::clamp(dimming, 0, 1); float x = dimming * dimming * (3 - 2 * dimming); // Easing function const float scale = std::lerp(1, constants::stream::dimming_scale, x); const float bias = std::lerp(0, constants::stream::dimming_bias, x); defoveator->defoveate(command_buffer, foveation, images, {scale, scale, scale, 1.}, {bias, bias, bias, 0.}, image_index); #if WIVRN_USE_V4L2 // the blit handles keep CAPTURE buffers alive until this cmd buffer's fence has completed inplace_vector foreign_release_barriers; for (const auto & handle: current_blit_handles) { if (handle && handle->foreign_queue_family != vk::QueueFamilyIgnored) { foreign_release_barriers.push_back({ .srcAccessMask = vk::AccessFlagBits::eShaderRead, .dstAccessMask = vk::AccessFlagBits::eNone, .oldLayout = handle->current_layout, .newLayout = vk::ImageLayout::eGeneral, .srcQueueFamilyIndex = queue_family_index, .dstQueueFamilyIndex = handle->foreign_queue_family, .image = handle->image, .subresourceRange = { .aspectMask = vk::ImageAspectFlagBits::eColor, .levelCount = 1, .layerCount = 1, }, }); handle->current_layout = vk::ImageLayout::eGeneral; } } if (!foreign_release_barriers.empty()) { command_buffer.pipelineBarrier(vk::PipelineStageFlagBits::eFragmentShader, vk::PipelineStageFlagBits::eBottomOfPipe, {}, {}, {}, foreign_release_barriers); } #endif command_buffer.writeTimestamp(vk::PipelineStageFlagBits::eBottomOfPipe, *query_pool, 1); command_buffer.end(); vk::SubmitInfo submit_info; submit_info.setCommandBuffers(*command_buffer); inplace_vector semaphores; inplace_vector semaphore_vals; inplace_vector wait_stages; for (auto b: current_blit_handles) { if (b and b->semaphore) { assert(b->semaphore_val); semaphores.push_back(b->semaphore); semaphore_vals.push_back(*b->semaphore_val); wait_stages.push_back(vk::PipelineStageFlagBits::eFragmentShader); } } submit_info.setWaitDstStageMask(wait_stages); submit_info.setWaitSemaphores(semaphores); vk::TimelineSemaphoreSubmitInfo sem_info{ .waitSemaphoreValueCount = uint32_t(semaphore_vals.size()), .pWaitSemaphoreValues = semaphore_vals.data(), }; submit_info.pNext = &sem_info; device.resetFences(*fence); queue.lock()->submit(submit_info, *fence); #if WIVRN_FEATURE_RENDERDOC renderdoc_end(*vk_instance); #endif swapchain.release(); if (use_alpha) session.enable_passthrough(system); else session.disable_passthrough(); render_start(use_alpha, frame_state.predictedDisplayTime); // full-FOV dummy layer for Meta foveation, used when cropping FOV if (auto layer = foveation_center.get_layer()) add_projection_layer(layer->flags, layer->space, layer->views); // Add the layer with the streamed content std::array layer_view; for (uint32_t view = 0; view < view_count; view++) { layer_view[view] = { .type = XR_TYPE_COMPOSITION_LAYER_PROJECTION_VIEW, .pose = pose[view], .fov = fov[view], .subImage = { .swapchain = swapchain, .imageRect = { .offset = {0, 0}, .extent = extents[view], }, .imageArrayIndex = view, }, }; } add_projection_layer( use_alpha ? XR_COMPOSITION_LAYER_BLEND_TEXTURE_SOURCE_ALPHA_BIT : 0, application::space(xr::spaces::world), layer_view); if (const configuration::openxr_post_processing_settings openxr_post_processing = application::get_config().openxr_post_processing; (openxr_post_processing.sharpening | openxr_post_processing.super_sampling) > 0) set_layer_settings(openxr_post_processing.sharpening | openxr_post_processing.super_sampling); accumulate_metrics(frame_state.predictedDisplayTime, current_blit_handles, timestamps); draw_gui(frame_state.predictedDisplayTime, frame_state.predictedDisplayPeriod); try { render_end(); } catch (std::system_error & e) { if (e.code().category() == xr::error_category() and e.code().value() == XR_ERROR_POSE_INVALID) spdlog::info("Invalid pose submitted"); else throw; } } // Network operations may be blocking, do them once everything was submitted { // Keep a copy of the feedback packets as they can be modified if they're encrypted inplace_vector feedbacks; // +1 for optional foveation packet inplace_vector packets; feedbacks.reserve(current_blit_handles.size()); packets.reserve(current_blit_handles.size()); for (const auto & handle: current_blit_handles) { if (handle) { auto & packet = packets.emplace_back(); wivrn_session::control_socket_t::serialize(packet, feedbacks.emplace_back(handle->feedback)); } } if (auto foveation_packet = foveation_center.take_packet()) { auto & packet = packets.emplace_back(); wivrn_session::control_socket_t::serialize(packet, *foveation_packet); } if (not packets.empty()) { try { network_session->send_control(std::span(packets)); } catch (std::exception & e) { spdlog::warn("Exception while sending feedback packet: {}", e.what()); } } } auto show_gamepad_toast = [this, now](bool enabled) { { auto toast = gui_toast.lock(); toast->emplace(enabled ? _("Now emulating a gamepad. Double-click both grips to go back to using VR controllers.") : _("No longer emulating a gamepad."), false); } gui_status_last_change = now; }; if (not application::get_config().emulated_gamepad) { if (gamepad_mode.exchange(false)) { scheduled_derived_pose = now; show_gamepad_toast(false); spdlog::info("Controller gamepad emulation disabled due to setting change"); } gamepad_chord.reset(); } else if (gamepad_chord.check(now)) { const bool enabled = not gamepad_mode.load(); gamepad_mode = enabled; if (not enabled) scheduled_derived_pose = now; show_gamepad_toast(enabled); spdlog::info("Controller gamepad mode {}", enabled ? "enabled" : "disabled"); } read_actions(); if (application::get_config().enable_stream_gui and menu_chord.check(now)) { // Arbitrary transitions can happen from network commands // Ensure we can't have a set of 2 non interactable states if (is_gui_interactable()) next_gui_status = stream_tab::hidden; else if (is_interactable(stored_gui_status)) next_gui_status = stored_gui_status; else next_gui_status = stream_tab::applications; } query_pool_filled = true; } void scenes::stream::exit() { state_ = state::shutdown; } void scenes::stream::setup(const to_headset::video_stream_description & description) { spdlog::info("setup, refresh rate {}", description.refresh_rate); session.set_refresh_rate(description.refresh_rate); battery & battery_monitor = application::get_battery_monitor(); for (auto device: magic_enum::enum_values()) { on_battery_update(device, battery_monitor.get_battery_status(device)); } std::unique_lock lock(decoder_mutex); if (video_stream_description == description) return; spdlog::info("Creating decoders, size {}x{}", description.width, description.height); video_stream_description = description; for (const auto & [stream_index, item]: utils::enumerate(decoders)) { item = accumulator_images{ .decoder = std::make_unique(device, physical_device, instance, queue_family_index, description, weak_from_this(), stream_index), }; } if (defoveator) defoveator->reset_pipelines(); } void scenes::stream::setup_reprojection_swapchain(uint32_t swapchain_width, uint32_t swapchain_height, bool foveation) { assert(swapchain_width); assert(swapchain_height); device.waitIdle(); spdlog::info("swapchain setup, refresh rate {}", video_stream_description->refresh_rate); session.set_refresh_rate(video_stream_description->refresh_rate); auto views = system.view_configuration_views(viewconfig); swapchain = xr::swapchain( instance, session, device, swapchain_format, swapchain_width, swapchain_height, 1, views.size(), foveation); spdlog::info( "Created stream swapchain: {}x{}{}", swapchain.width(), swapchain.height(), foveation ? " with eye-tracked foveation" : ""); for (auto view: views) { if (swapchain.width() > view.maxImageRectWidth or swapchain.height() > view.maxImageRectHeight) spdlog::warn("Swapchain size larger than maximum {}x{}", view.maxImageRectWidth, view.maxImageRectHeight); } spdlog::info("Initializing reprojector"); vk::Extent2D extent = {(uint32_t)swapchain.width(), (uint32_t)swapchain.height()}; defoveator.emplace( device, physical_device, swapchain.images(), extent, swapchain.format()); } void scenes::stream::ensure_swapchain(bool cropped, uint32_t width, uint32_t height) { if (!swapchain or swapchain.width() < width or swapchain.height() < height or foveation_center.needs_swapchain_recreation(cropped)) { try { setup_reprojection_swapchain(width, height, foveation_center.use_meta_swapchain(cropped)); foveation_center.on_swapchain_recreated(cropped); } catch (std::exception & e) { spdlog::warn("failed to create swapchain"); } } else { foveation_center.set_cropped(cropped); } } scene::meta & scenes::stream::get_meta_scene() { static meta m{ .name = "Stream", .actions = { {"left_aim", XR_ACTION_TYPE_POSE_INPUT}, {"left_trigger", XR_ACTION_TYPE_FLOAT_INPUT}, {"left_squeeze", XR_ACTION_TYPE_FLOAT_INPUT}, {"left_scroll", XR_ACTION_TYPE_VECTOR2F_INPUT}, {"left_haptic", XR_ACTION_TYPE_VIBRATION_OUTPUT}, {"right_aim", XR_ACTION_TYPE_POSE_INPUT}, {"right_trigger", XR_ACTION_TYPE_FLOAT_INPUT}, {"right_squeeze", XR_ACTION_TYPE_FLOAT_INPUT}, {"right_scroll", XR_ACTION_TYPE_VECTOR2F_INPUT}, {"right_haptic", XR_ACTION_TYPE_VIBRATION_OUTPUT}, {"menu_toggle_1", XR_ACTION_TYPE_BOOLEAN_INPUT}, {"menu_toggle_2", XR_ACTION_TYPE_BOOLEAN_INPUT}, {"gamepad_mode_1", XR_ACTION_TYPE_BOOLEAN_INPUT}, {"gamepad_mode_2", XR_ACTION_TYPE_BOOLEAN_INPUT}, {"recenter_left", XR_ACTION_TYPE_BOOLEAN_INPUT}, {"recenter_right", XR_ACTION_TYPE_BOOLEAN_INPUT}, {"gui_distance_left", XR_ACTION_TYPE_FLOAT_INPUT}, {"gui_distance_right", XR_ACTION_TYPE_FLOAT_INPUT}, {"settings_adjust", XR_ACTION_TYPE_FLOAT_INPUT}, {"foveation_distance", XR_ACTION_TYPE_FLOAT_INPUT}, {"foveation_ok", XR_ACTION_TYPE_BOOLEAN_INPUT}, {"foveation_cancel", XR_ACTION_TYPE_BOOLEAN_INPUT}, }, .bindings = { suggested_binding{ { "/interaction_profiles/oculus/touch_controller", "/interaction_profiles/facebook/touch_controller_pro", "/interaction_profiles/meta/touch_pro_controller", "/interaction_profiles/meta/touch_controller_plus", "/interaction_profiles/meta/touch_plus_controller", "/interaction_profiles/bytedance/pico_neo3_controller", "/interaction_profiles/bytedance/pico4_controller", "/interaction_profiles/bytedance/pico4s_controller", "/interaction_profiles/yvr/touch_controller_yvr", "/interaction_profiles/htc/vive_focus3_controller", "/interaction_profiles/valve/frame_controller_valve", }, { {"left_aim", "/user/hand/left/input/aim/pose"}, {"left_trigger", "/user/hand/left/input/trigger/value"}, {"left_squeeze", "/user/hand/left/input/squeeze/value"}, {"left_scroll", "/user/hand/left/input/thumbstick"}, {"left_haptic", "/user/hand/left/output/haptic"}, {"right_aim", "/user/hand/right/input/aim/pose"}, {"right_trigger", "/user/hand/right/input/trigger/value"}, {"right_squeeze", "/user/hand/right/input/squeeze/value"}, {"right_scroll", "/user/hand/right/input/thumbstick"}, {"right_haptic", "/user/hand/right/output/haptic"}, {"recenter_left", "/user/hand/left/input/squeeze/value"}, {"recenter_right", "/user/hand/right/input/squeeze/value"}, {"gui_distance_left", "/user/hand/left/input/thumbstick/y"}, {"gui_distance_right", "/user/hand/right/input/thumbstick/y"}, {"settings_adjust", "/user/hand/right/input/thumbstick/y"}, {"foveation_distance", "/user/hand/left/input/thumbstick/y"}, {"foveation_ok", "/user/hand/right/input/a/click"}, {"foveation_cancel", "/user/hand/right/input/b/click"}, {"menu_toggle_1", "/user/hand/left/input/thumbstick/click"}, {"menu_toggle_2", "/user/hand/right/input/thumbstick/click"}, }, }, suggested_binding{ { "/interaction_profiles/valve/frame_controller_valve", }, { {"left_trigger", "/user/hand/left/input/bumper/click"}, {"right_trigger", "/user/hand/right/input/bumper/click"}, {"gamepad_mode_1", "/user/hand/left/input/squeeze/click"}, {"gamepad_mode_2", "/user/hand/right/input/squeeze/click"}, }, }, suggested_binding{ { "/interaction_profiles/khr/simple_controller", }, {}, }, }, }; return m; } std::optional scenes::stream::pop_stream_error() { auto queue = stream_error_queue.lock(); if (queue->empty()) return std::nullopt; auto err = std::make_optional(queue->front()); queue->pop(); return err; } void scenes::stream::on_xr_event(const xr::event & event) { switch (event.header.type) { case XR_TYPE_EVENT_DATA_REFERENCE_SPACE_CHANGE_PENDING: if (event.space_changed_pending.referenceSpaceType == XrReferenceSpaceType::XR_REFERENCE_SPACE_TYPE_LOCAL) recenter_requested = true; break; case XR_TYPE_EVENT_DATA_DISPLAY_REFRESH_RATE_CHANGED_FB: network_session->send_control(from_headset::refresh_rate_changed{ .from = event.refresh_rate_changed.fromDisplayRefreshRate, .to = event.refresh_rate_changed.toDisplayRefreshRate, }); break; case XR_TYPE_EVENT_DATA_VISIBILITY_MASK_CHANGED_KHR: network_session->send_control(from_headset::visibility_mask_changed{ .data = get_visibility_mask(instance, session, event.visibility_mask_changed.viewIndex), .view_index = uint8_t(event.visibility_mask_changed.viewIndex), }); break; case XR_TYPE_EVENT_DATA_SESSION_STATE_CHANGED: // Override session state if the GUI is interactable if (event.state_changed.state == XR_SESSION_STATE_FOCUSED and is_gui_interactable()) network_session->send_control(from_headset::session_state_changed{ .state = XR_SESSION_STATE_VISIBLE, }); else network_session->send_control(from_headset::session_state_changed{ .state = event.state_changed.state, }); break; case XR_TYPE_EVENT_DATA_USER_PRESENCE_CHANGED_EXT: network_session->send_control(from_headset::user_presence_changed{ .present = (bool)event.user_presence_changed.isUserPresent, .change_time = instance.now(), }); break; case XR_TYPE_EVENT_DATA_INTERACTION_PROFILE_CHANGED: on_interaction_profile_changed(event.interaction_profile_changed); break; default: break; } } void scenes::stream::on_battery_update(battery::device device, const battery::battery_status & battery) { device_id id = [&] { switch (device) { case battery::device::head: return device_id::HEAD; case battery::device::left_controller: return device_id::LEFT_CONTROLLER; case battery::device::right_controller: return device_id::RIGHT_CONTROLLER; case battery::device::gamepad: return device_id::GAMEPAD; default: assert(false); __builtin_unreachable(); } }(); network_session->send_control(from_headset::battery{ .device = id, .charge = battery.charge, .charging = battery.charging, }); } bool scenes::stream::forward_hid_input(from_headset::hid::input_t packet, bool device_enabled) { // hid_forwarding is whether the server permits it; device_enabled is the headset toggle. if (not hid_forwarding or not device_enabled) return false; network_session->send_control(from_headset::hid::input{packet}); return true; } bool scenes::stream::on_input_key_down(uint8_t key_code) { return forward_hid_input(from_headset::hid::key_down{key_code}, application::get_config().forward_keyboard); } bool scenes::stream::on_input_key_up(uint8_t key_code) { return forward_hid_input(from_headset::hid::key_up{key_code}, application::get_config().forward_keyboard); } bool scenes::stream::on_input_mouse_move(float x, float y) { return forward_hid_input(from_headset::hid::mouse_move{x, y}, application::get_config().forward_mouse); } bool scenes::stream::on_input_button_down(uint8_t button) { return forward_hid_input(from_headset::hid::button_down{button}, application::get_config().forward_mouse); } bool scenes::stream::on_input_button_up(uint8_t button) { return forward_hid_input(from_headset::hid::button_up{button}, application::get_config().forward_mouse); } bool scenes::stream::on_input_scroll(float h, float v) { return forward_hid_input(from_headset::hid::mouse_scroll{h, v}, application::get_config().forward_mouse); }