/* * WiVRn VR streaming * Copyright (C) 2022-2024 Guillaume Meunier * Copyright (C) 2022-2024 Patrick Nicolas * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #include "lobby.h" #include "version.h" #include "application.h" #include "glm/geometric.hpp" #include "imgui.h" #include "input_profile.h" #include "openxr/openxr.h" #include "render/scene_data.h" #include "render/scene_renderer.h" #include "stream.h" #include "hardware.h" #include "utils/contains.h" #include "wivrn_client.h" #include "xr/passthrough.h" #include #include "wivrn_discover.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static const std::string discover_service = "_wivrn._tcp.local."; static bool force_autoconnect = false; scenes::lobby::~lobby() { if (renderer) renderer->wait_idle(); } static std::string choose_webxr_profile() { #ifndef __ANDROID__ const char * controller = std::getenv("WIVRN_CONTROLLER"); if (controller && strcmp(controller, "")) return controller; #endif switch(guess_model()) { case model::oculus_quest: return "oculus-touch-v2"; case model::oculus_quest_2: return "oculus-touch-v3"; case model::meta_quest_pro: return "meta-quest-touch-pro"; case model::meta_quest_3: return "meta-quest-touch-plus"; case model::pico_neo_3: return "pico-neo3"; case model::pico_4: return "pico-4"; case model::htc_vive_focus_3: case model::htc_vive_xr_elite: return "htc-vive-focus-3"; case model::unknown: return "generic-trigger-squeeze"; } __builtin_unreachable(); } static std::string json_string(const std::string& in) { std::string out; out.reserve(in.size() + 2); out += '"'; for(char c: in) { switch(c) { case '\b': out += "\\b"; break; case '\f': out += "\\f"; break; case '\n': out += "\\n"; break; case '\r': out += "\\r"; break; case '\t': out += "\\t"; break; case '"': out += "\\\""; break; case '\\': out += "\\\\"; break; default: out += c; break; } } out += '"'; return out; } static const std::array supported_formats = { vk::Format::eR8G8B8A8Srgb, vk::Format::eB8G8R8A8Srgb }; void scenes::lobby::move_gui(glm::vec3 position, glm::quat orientation, XrTime predicted_display_time) { const float gui_target_distance = 0.75; const float gui_target_altitude = 1.; const float gui_target_pitch = -0.5; glm::vec3 gui_direction = glm::column(glm::mat3_cast(imgui_ctx->orientation()), 2); float gui_yaw = atan2(gui_direction.x, gui_direction.z); glm::vec3 head_direction = -glm::column(glm::mat3_cast(orientation), 2); float head_yaw = atan2(head_direction.x, head_direction.z) + M_PI; head_direction.y = 0; head_direction = glm::normalize(head_direction); position.y = gui_target_altitude; glm::vec3 gui_target_position = position + gui_target_distance * head_direction; glm::vec3 gui_position_error = gui_target_position - imgui_ctx->position(); float gui_yaw_error = remainderf(head_yaw - gui_yaw, 2 * M_PI); if (move_gui_first_time) { move_gui_first_time = false; gui_yaw += gui_yaw_error; imgui_ctx->position() += gui_position_error; imgui_ctx->orientation() = glm::quat(cos(gui_yaw/2), 0, sin(gui_yaw/2), 0) * glm::quat(cos(gui_target_pitch/2), sin(gui_target_pitch/2), 0, 0); } } scenes::lobby::lobby() { if (std::getenv("WIVRN_AUTOCONNECT")) force_autoconnect = true; passthrough_supported = system.passthrough_supported(); passthrough_enabled = passthrough_supported == xr::system::passthrough_type::color; try { simdjson::dom::parser parser; simdjson::dom::element root = parser.load(application::get_config_path() / "client.json"); for(simdjson::dom::object i: simdjson::dom::array(root["servers"])) { server_data data{ .autoconnect = i["autoconnect"].get_bool(), .manual = i["manual"].get_bool(), .visible = false, .compatible = true, .service = { .name = (std::string)i["pretty_name"], .hostname = (std::string)i["hostname"], .port = (int)i["port"].get_int64(), .txt = {{"cookie", (std::string)i["cookie"]}} } }; servers.emplace(data.service.txt["cookie"], data); } auto json_show_plots = root["show_performance_metrics"]; if (json_show_plots.is_bool()) show_performance_metrics = json_show_plots.get_bool(); auto rate = root["preferred_refresh_rate"]; if (rate.is_double()) { preferred_refresh_rate = rate.get_double(); session.set_refresh_rate(preferred_refresh_rate); } auto mic = root["microphone"]; if (mic.is_bool()) microphone = mic.get_bool(); auto passthrough = root["passthrough_enabled"]; if (passthrough.is_bool()) passthrough_enabled = passthrough.get_bool(); } catch(std::exception& e) { spdlog::warn("Cannot read configuration: {}", e.what()); servers.clear(); show_performance_metrics = false; } spdlog::info("{} known server(s):", servers.size()); for(auto& i: servers) { spdlog::info(" {}", i.second.service.name); } haptic_output[0] = get_action("left_haptic").first; haptic_output[1] = get_action("right_haptic").first; swapchain_format = vk::Format::eUndefined; spdlog::info("Supported swapchain formats:"); for (auto format: session.get_swapchain_formats()) { spdlog::info(" {}", vk::to_string(format)); } for (auto format: session.get_swapchain_formats()) { if (std::find(supported_formats.begin(), supported_formats.end(), format) != supported_formats.end()) { swapchain_format = format; break; } } if (swapchain_format == vk::Format::eUndefined) throw std::runtime_error("No supported swapchain format"); spdlog::info("Using format {}", vk::to_string(swapchain_format)); } void scenes::lobby::save_config() { std::stringstream ss; for (auto& [cookie, server_data]: servers) { if (server_data.autoconnect || server_data.manual) { ss << "{"; ss << "\"autoconnect\":" << std::boolalpha << server_data.autoconnect << ","; ss << "\"manual\":" << std::boolalpha << server_data.manual << ","; ss << "\"pretty_name\":" << json_string(server_data.service.name) << ","; ss << "\"hostname\":" << json_string(server_data.service.hostname) << ","; ss << "\"port\":" << server_data.service.port << ","; ss << "\"cookie\":" << json_string(cookie); ss << "},"; } } std::string servers_str = ss.str(); if (servers_str != "") servers_str.pop_back(); // Remove last comma std::ofstream json(application::get_config_path() / "client.json"); json << "{\"servers\":[" << servers_str << "]," << "\"show_performance_metrics\":" << std::boolalpha << show_performance_metrics; if (preferred_refresh_rate != 0.) json << ",\"preferred_refresh_rate\":" << preferred_refresh_rate ; json << ",\"microphone\":" << std::boolalpha << microphone; json << ",\"passthrough_enabled\":" << std::boolalpha << passthrough_enabled; json << "}"; } static std::string ip_address_to_string(const in_addr& addr) { char buf[100]; inet_ntop(AF_INET, &addr, buf, sizeof(buf)); return buf; } static std::string ip_address_to_string(const in6_addr& addr) { char buf[100]; inet_ntop(AF_INET6, &addr, buf, sizeof(buf)); return buf; } std::unique_ptr connect_to_session(wivrn_discover::service service, bool manual_connection) { if (!manual_connection) { char protocol_string[17]; sprintf(protocol_string, "%016lx", xrt::drivers::wivrn::protocol_version); spdlog::debug("Client protocol version: {}", protocol_string); spdlog::debug("Server TXT:"); for(auto& [i,j]: service.txt) { spdlog::debug(" {}=\"{}\"", i, j); } auto protocol = service.txt.find("protocol"); if (protocol == service.txt.end()) throw std::runtime_error("Incompatible WiVRn server: no protocol field in TXT"); if (protocol->second != protocol_string) throw std::runtime_error(fmt::format("Incompatible WiVRn server protocol (client: {}, server: {})", protocol_string, protocol->second)); } // Only the automatically discovered servers already have their IP addresses available if (manual_connection) { addrinfo hint { .ai_flags = AI_ADDRCONFIG, .ai_family = AF_UNSPEC, .ai_socktype = SOCK_STREAM, }; addrinfo * addresses; if (int err = getaddrinfo(service.hostname.c_str(), nullptr, &hint, &addresses)) { spdlog::error("Cannot resolve hostname {}: {}", service.hostname, gai_strerror(err)); throw std::runtime_error("Cannot resolve hostname: " + std::string(gai_strerror(err))); } for(auto i = addresses; i; i = i->ai_next) { switch (i->ai_family) { case AF_INET: service.addresses.push_back(((sockaddr_in*)i->ai_addr)->sin_addr); break; case AF_INET6: service.addresses.push_back(((sockaddr_in6*)i->ai_addr)->sin6_addr); break; } } freeaddrinfo(addresses); } for (const auto & address: service.addresses) { std::string address_string = std::visit([](auto & address) { return ip_address_to_string(address); }, address); try { spdlog::debug("Trying address {}", address_string); return std::visit([port = service.port](auto & address) { return std::make_unique(address, port); }, address); } catch (std::exception & e) { spdlog::warn("Cannot connect to {} ({}): {}", service.hostname, address_string, e.what()); } } throw std::runtime_error("No usable address"); } static glm::mat4 projection_matrix(XrFovf fov, float zn = 0.02) { float r = tan(fov.angleRight); float l = tan(fov.angleLeft); float t = tan(fov.angleUp); float b = tan(fov.angleDown); // clang-format off return glm::mat4{ { 2/(r-l), 0, 0, 0 }, { 0, 2/(b-t), 0, 0 }, { (l+r)/(r-l), (t+b)/(b-t), -1, -1 }, { 0, 0, -2*zn, 0 } }; // clang-format on } static glm::mat4 view_matrix(XrPosef pose) { XrQuaternionf q = pose.orientation; XrVector3f pos = pose.position; glm::mat4 inv_view_matrix = glm::mat4_cast(glm::quat(q.w, q.x, q.y, q.z)); inv_view_matrix = glm::translate(glm::mat4(1), glm::vec3(pos.x, pos.y, pos.z)) * inv_view_matrix; return glm::inverse(inv_view_matrix); } void scenes::lobby::update_server_list() { if (application::is_focused() && !discover) discover.emplace(); else if (!application::is_focused() && discover) discover.reset(); if (!discover) return; std::vector discovered_services = discover->get_services(); // TODO: only if discovered_services changed for(auto& [cookie, data]: servers) { data.visible = false; } char protocol_string[17]; sprintf(protocol_string, "%016lx", xrt::drivers::wivrn::protocol_version); for(auto& service: discovered_services) { std::string cookie; bool compatible = true; auto it = service.txt.find("cookie"); if (it == service.txt.end()) { cookie = service.hostname; compatible = false; } else cookie = it->second; auto protocol = service.txt.find("protocol"); if (protocol == service.txt.end()) compatible = false; if (protocol->second != protocol_string) compatible = false; auto server = servers.find(cookie); if (server == servers.end()) { // Newly discovered server: add it to the list servers.emplace(cookie, server_data{ .autoconnect = false, .manual = false, .visible = true, .compatible = compatible, .service = service }); } else { server->second.visible = true; server->second.service = service; server->second.compatible = compatible; } } } void scenes::lobby::connect(server_data& data) { server_name = data.service.name; async_error.reset(); async_session = utils::async, std::string>([](auto token, wivrn_discover::service service, bool manual) { token.set_progress("Waiting for connection"); return connect_to_session(service, manual); }, data.service, data.manual); } static std::vector render_layer(std::vector& views, std::vector& swapchains, scene_renderer& renderer, scene_data& data, const std::array& clear_color) { std::vector frames; frames.reserve(views.size()); std::vector layer_views; layer_views.reserve(views.size()); for (auto && [view, swapchain]: utils::zip(views, swapchains)) { int image_index = swapchain.acquire(); swapchain.wait(); frames.push_back({.destination = swapchain.images()[image_index].image, .projection = projection_matrix(view.fov), .view = view_matrix(view.pose)}); layer_views.push_back({ .type = XR_TYPE_COMPOSITION_LAYER_PROJECTION_VIEW, .pose = view.pose, .fov = view.fov, .subImage = { .swapchain = swapchain, .imageRect = { .offset = {0, 0}, .extent = swapchain.extent()}}, }); } renderer.render(data, clear_color, frames); return layer_views; } void scenes::lobby::render(XrTime predicted_display_time, bool should_render) { if (async_session.valid() && async_session.poll() == utils::future_status::ready) { try { auto session = async_session.get(); if (session) next_scene = stream::create(std::move(session), show_performance_metrics, microphone); async_session.reset(); } catch(std::exception& e) { spdlog::error("Error connecting to server: {}", e.what()); async_session.cancel(); async_error = e.what(); } } if (next_scene) { if (!next_scene->alive()) next_scene.reset(); else if (next_scene->current_state() == scenes::stream::state::streaming) application::push_scene(next_scene); } update_server_list(); imgui_ctx->set_current(); if (!async_session.valid() && !next_scene && !ImGui::IsPopupOpen("", ImGuiPopupFlags_AnyPopup)) { for(auto&& [cookie, data]: servers) { if (data.visible && (data.autoconnect || force_autoconnect) && data.compatible) { connect(data); } } } if (!should_render) { session.begin_frame(); session.end_frame(predicted_display_time, {}); return; } session.begin_frame(); auto [flags, views] = session.locate_views(viewconfig, predicted_display_time, world_space); assert(views.size() == swapchains_lobby.size()); bool hide_left_controller = false; bool hide_right_controller = false; if (application::get_hand_tracking_supported()) { if (left_hand) { auto& hand = application::get_left_hand(); auto joints = hand.locate(world_space, predicted_display_time); left_hand->apply(joints); if (joints) hide_left_controller = true; } if (right_hand) { auto& hand = application::get_right_hand(); auto joints = hand.locate(world_space, predicted_display_time); right_hand->apply(joints); if (joints) hide_right_controller = true; } } input->apply(world_space, predicted_display_time, hide_left_controller, hide_right_controller); XrCompositionLayerQuad imgui_layer = draw_gui(predicted_display_time); assert(renderer); renderer->start_frame(); std::vector lobby_layer_views; if (!passthrough_enabled) lobby_layer_views = render_layer(views, swapchains_lobby, *renderer, *lobby_scene, {0, 0.25, 0.5, 1}); auto controllers_layer_views = render_layer(views, swapchains_controllers, *renderer, *controllers_scene, {0, 0, 0, 0}); renderer->end_frame(); // After end_frame because the command buffers are submitted in end_frame if (!passthrough_enabled) { for (auto & swapchain: swapchains_lobby) swapchain.release(); } for (auto & swapchain: swapchains_controllers) swapchain.release(); XrCompositionLayerProjection lobby_layer{ .type = XR_TYPE_COMPOSITION_LAYER_PROJECTION, .layerFlags = 0, .space = world_space, .viewCount = (uint32_t)lobby_layer_views.size(), .views = lobby_layer_views.data(), }; XrCompositionLayerProjection controllers_layer{ .type = XR_TYPE_COMPOSITION_LAYER_PROJECTION, .layerFlags = XR_COMPOSITION_LAYER_BLEND_TEXTURE_SOURCE_ALPHA_BIT, .space = world_space, .viewCount = (uint32_t)controllers_layer_views.size(), .views = controllers_layer_views.data(), }; std::vector layers_base; if (passthrough_enabled) { std::visit([&](auto& p){ layers_base.push_back(p.layer()); }, passthrough); } else { layers_base.push_back(reinterpret_cast(&lobby_layer)); } layers_base.push_back(reinterpret_cast(&imgui_layer)); layers_base.push_back(reinterpret_cast(&controllers_layer)); session.end_frame(predicted_display_time, layers_base); } void scenes::lobby::on_focused() { move_gui_first_time = true; auto views = system.view_configuration_views(viewconfig); uint32_t width = views[0].recommendedImageRectWidth; uint32_t height = views[0].recommendedImageRectHeight; swapchains_lobby.reserve(views.size()); swapchains_controllers.reserve(views.size()); for ([[maybe_unused]] auto view: views) { assert(view.recommendedImageRectWidth == width); assert(view.recommendedImageRectHeight == height); swapchains_lobby.emplace_back(session, device, swapchain_format, width, height); swapchains_controllers.emplace_back(session, device, swapchain_format, width, height); } spdlog::info("Created lobby swapchains: {}x{}", width, height); vk::Extent2D output_size{width, height}; std::array depth_formats{ vk::Format::eX8D24UnormPack32, vk::Format::eD32Sfloat, }; renderer.emplace(device, physical_device, queue, commandpool, output_size, swapchain_format, depth_formats); scene_loader loader(device, physical_device, queue, application::queue_family_index(), renderer->get_default_material()); lobby_scene.emplace(); lobby_scene->import(loader("ground.gltf")); controllers_scene.emplace(); input = input_profile("controllers/" + choose_webxr_profile() + "/profile.json", loader, *controllers_scene); spdlog::info("Loaded input profile {}", input->id); if (application::get_hand_tracking_supported()) { left_hand.emplace( "left-hand.glb", loader, *controllers_scene); right_hand.emplace("right-hand.glb", loader, *controllers_scene); } std::array imgui_inputs{ imgui_context::controller{ .aim = get_action_space("left_aim"), .trigger = get_action("left_trigger").first, .squeeze = get_action("left_squeeze").first, .scroll = get_action("left_scroll").first, .hand = application::get_left_hand(), }, imgui_context::controller{ .aim = get_action_space("right_aim"), .trigger = get_action("right_trigger").first, .squeeze = get_action("right_squeeze").first, .scroll = get_action("right_scroll").first, .hand = application::get_right_hand(), }, }; swapchain_imgui = xr::swapchain(session, device, swapchain_format, 1500, 1000); imgui_ctx.emplace(physical_device, device, queue_family_index, queue, world_space, imgui_inputs, swapchain_imgui, glm::vec2{1.0, 0.6666}); try { about_picture = imgui_ctx->load_texture("wivrn.ktx2"); } catch(...) { about_picture = imgui_ctx->load_texture("wivrn.png"); } if (passthrough_supported != xr::system::passthrough_type::no_passthrough) { if (utils::contains(application::get_xr_extensions(), XR_FB_PASSTHROUGH_EXTENSION_NAME)) { passthrough.emplace(instance, session); } else if (utils::contains(application::get_xr_extensions(), XR_HTC_PASSTHROUGH_EXTENSION_NAME)) { passthrough.emplace(instance, session); } std::visit([](auto& p) {p.start();}, passthrough); } } void scenes::lobby::on_unfocused() { discover.reset(); renderer->wait_idle(); // Must be before the scene data because the renderer uses its descriptor sets about_picture = nullptr; imgui_ctx.reset(); lobby_scene.reset(); // Must be reset before the renderer so that the descriptor sets are freed before their pools controllers_scene.reset(); input.reset(); left_hand.reset(); right_hand.reset(); renderer.reset(); swapchains_lobby.clear(); swapchains_controllers.clear(); swapchain_imgui = xr::swapchain(); passthrough.emplace(); // default constructor does no passthrough } void scenes::lobby::on_session_state_changed(XrSessionState state) { if (state == XR_SESSION_STATE_STOPPING) discover.reset(); } scene::meta& scenes::lobby::get_meta_scene() { static meta m{ .name = "Lobby", .actions = { {"left_aim", XR_ACTION_TYPE_POSE_INPUT}, {"left_trigger", XR_ACTION_TYPE_FLOAT_INPUT}, {"left_squeeze", XR_ACTION_TYPE_FLOAT_INPUT}, {"left_scroll", XR_ACTION_TYPE_VECTOR2F_INPUT}, {"left_haptic", XR_ACTION_TYPE_VIBRATION_OUTPUT}, {"right_aim", XR_ACTION_TYPE_POSE_INPUT}, {"right_trigger", XR_ACTION_TYPE_FLOAT_INPUT}, {"right_squeeze", XR_ACTION_TYPE_FLOAT_INPUT}, {"right_scroll", XR_ACTION_TYPE_VECTOR2F_INPUT}, {"right_haptic", XR_ACTION_TYPE_VIBRATION_OUTPUT}, }, .bindings = { suggested_binding{ "/interaction_profiles/oculus/touch_controller", { {"left_aim", "/user/hand/left/input/aim/pose"}, {"left_trigger", "/user/hand/left/input/trigger/value"}, {"left_squeeze", "/user/hand/left/input/squeeze/value"}, {"left_scroll", "/user/hand/left/input/thumbstick"}, {"left_haptic", "/user/hand/left/output/haptic"}, {"right_aim", "/user/hand/right/input/aim/pose"}, {"right_trigger", "/user/hand/right/input/trigger/value"}, {"right_squeeze", "/user/hand/right/input/squeeze/value"}, {"right_scroll", "/user/hand/right/input/thumbstick"}, {"right_haptic", "/user/hand/right/output/haptic"}, } }, suggested_binding{ "/interaction_profiles/bytedance/pico_neo3_controller", { {"left_aim", "/user/hand/left/input/aim/pose"}, {"left_trigger", "/user/hand/left/input/trigger/value"}, {"left_squeeze", "/user/hand/left/input/squeeze/value"}, {"left_scroll", "/user/hand/left/input/thumbstick"}, {"left_haptic", "/user/hand/left/output/haptic"}, {"right_aim", "/user/hand/right/input/aim/pose"}, {"right_trigger", "/user/hand/right/input/trigger/value"}, {"right_squeeze", "/user/hand/right/input/squeeze/value"}, {"right_scroll", "/user/hand/right/input/thumbstick"}, {"right_haptic", "/user/hand/right/output/haptic"}, } }, suggested_binding{ "/interaction_profiles/bytedance/pico4_controller", { {"left_aim", "/user/hand/left/input/aim/pose"}, {"left_trigger", "/user/hand/left/input/trigger/value"}, {"left_squeeze", "/user/hand/left/input/squeeze/value"}, {"left_scroll", "/user/hand/left/input/thumbstick"}, {"left_haptic", "/user/hand/left/output/haptic"}, {"right_aim", "/user/hand/right/input/aim/pose"}, {"right_trigger", "/user/hand/right/input/trigger/value"}, {"right_squeeze", "/user/hand/right/input/squeeze/value"}, {"right_scroll", "/user/hand/right/input/thumbstick"}, {"right_haptic", "/user/hand/right/output/haptic"}, } }, suggested_binding{ "/interaction_profiles/htc/vive_focus3_controller", { {"left_aim", "/user/hand/left/input/aim/pose"}, {"left_trigger", "/user/hand/left/input/trigger/value"}, {"left_squeeze", "/user/hand/left/input/squeeze/value"}, {"left_scroll", "/user/hand/left/input/thumbstick"}, {"left_haptic", "/user/hand/left/output/haptic"}, {"right_aim", "/user/hand/right/input/aim/pose"}, {"right_trigger", "/user/hand/right/input/trigger/value"}, {"right_squeeze", "/user/hand/right/input/squeeze/value"}, {"right_scroll", "/user/hand/right/input/thumbstick"}, {"right_haptic", "/user/hand/right/output/haptic"}, } }, suggested_binding{ "/interaction_profiles/khr/simple_controller", { {"left_aim", "/user/hand/left/input/aim/pose"}, {"left_trigger", "/user/hand/left/input/select/click"}, {"left_squeeze", "/user/hand/left/input/menu/click"}, {"right_aim", "/user/hand/right/input/aim/pose"}, {"right_trigger", "/user/hand/right/input/select/click"}, {"right_squeeze", "/user/hand/right/input/menu/click"}, } }, } }; return m; }