/* * WiVRn VR streaming * Copyright (C) 2026 galister * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #include "foveation.h" #include "application.h" #include "xr/instance.h" #include "xr/session.h" #include "xr/system.h" #include #include #include #include #include namespace { float convergence_angle(float distance, float eye_x, float yaw) { const float target_x = distance * std::sin(yaw); const float target_z = distance * std::cos(yaw); return std::atan2(target_x - eye_x, target_z); } XrVector2f yaw_pitch(const XrQuaternionf & q) { const float sine_theta = std::clamp(-2.0f * (q.y * q.z - q.w * q.x), -1.0f, 1.0f); const float pitch = std::asin(sine_theta); if (std::abs(sine_theta) > 0.99999f) { const float scale = std::copysign(2.0f, sine_theta); return {scale * std::atan2(-q.z, q.w), pitch}; } return { std::atan2(2.0f * (q.x * q.z + q.w * q.y), q.w * q.w - q.x * q.x - q.y * q.y + q.z * q.z), pitch}; } float ndc_to_angle(float center, float a0, float a1) { const float u = std::clamp(center * 0.5f + 0.5f, 0.0f, 1.0f); const float tangent = std::lerp(std::tan(a0), std::tan(a1), u); return std::atan(tangent); } client_foveation::angles center_to_angles(const std::array & center, std::span fovs) { assert(fovs.size() == 2); client_foveation::angles result{}; for (size_t i = 0; i < result.size(); ++i) { result[i].x = ndc_to_angle(center[i].x, fovs[i].angleLeft, fovs[i].angleRight); result[i].y = ndc_to_angle(center[i].y, fovs[i].angleUp, fovs[i].angleDown); } return result; } bool fov_equal(const XrFovf & a, const XrFovf & b, const float epsilon) { return std::abs(a.angleLeft - b.angleLeft) <= epsilon and std::abs(a.angleRight - b.angleRight) <= epsilon and std::abs(a.angleUp - b.angleUp) <= epsilon and std::abs(a.angleDown - b.angleDown) <= epsilon; } client_foveation::angles target_angles( float yaw, float pitch, float convergence_distance, std::span views) { assert(views.size() == 2); client_foveation::angles result{}; for (size_t i = 0; i < result.size(); ++i) { result[i].x = convergence_angle(convergence_distance, views[i].pose.position.x, yaw); result[i].y = pitch; } return result; } std::optional eye_gaze_angles( XrTime time, XrViewStateFlags view_flags, std::span views) { if (not(view_flags & XR_VIEW_STATE_POSITION_VALID_BIT)) return std::nullopt; auto gaze_quat = application::instance().locate_gaze(time); if (not gaze_quat) return std::nullopt; const XrVector2f gaze = yaw_pitch(*gaze_quat); return target_angles(-gaze.x, gaze.y, client_foveation::default_convergence_distance, views); } } // namespace namespace client_foveation { center::center(xr::instance & inst, xr::system & sys, xr::session & xr_session, vk::raii::Device & vk_device, vk::raii::CommandPool & vk_commandpool, thread_safe & vk_queue, vk::Format format) : instance(inst), session(xr_session), device(vk_device), commandpool(vk_commandpool), queue(vk_queue), swapchain_format(format) { if (application::get_foveation_center_supported() and application::get_foveation_vulkan_supported() and inst.has_extension(XR_META_FOVEATION_EYE_TRACKED_EXTENSION_NAME) and inst.has_extension(XR_FB_FOVEATION_EXTENSION_NAME) and inst.has_extension(XR_FB_FOVEATION_CONFIGURATION_EXTENSION_NAME) and inst.has_extension(XR_FB_FOVEATION_VULKAN_EXTENSION_NAME) and inst.has_extension(XR_FB_SWAPCHAIN_UPDATE_STATE_EXTENSION_NAME) and inst.has_extension(XR_META_VULKAN_SWAPCHAIN_CREATE_INFO_EXTENSION_NAME) and sys.foveation_eye_tracked_properties().supportsFoveationEyeTracked) { xrCreateFoveationProfileFB = inst.get_proc("xrCreateFoveationProfileFB"); xrDestroyFoveationProfileFB = inst.get_proc("xrDestroyFoveationProfileFB"); xrUpdateSwapchainFB = inst.get_proc("xrUpdateSwapchainFB"); xrGetFoveationEyeTrackedStateMETA = inst.get_proc("xrGetFoveationEyeTrackedStateMETA"); if (xrCreateFoveationProfileFB and xrDestroyFoveationProfileFB and xrUpdateSwapchainFB and xrGetFoveationEyeTrackedStateMETA) { XrFoveationEyeTrackedProfileCreateInfoMETA eye_tracked_info{ .type = XR_TYPE_FOVEATION_EYE_TRACKED_PROFILE_CREATE_INFO_META, .flags = 0, }; XrFoveationLevelProfileCreateInfoFB level_info{ .type = XR_TYPE_FOVEATION_LEVEL_PROFILE_CREATE_INFO_FB, .next = &eye_tracked_info, .level = XR_FOVEATION_LEVEL_LOW_FB, .verticalOffset = 0, .dynamic = XR_FOVEATION_DYNAMIC_DISABLED_FB, }; XrFoveationProfileCreateInfoFB profile_info{ .type = XR_TYPE_FOVEATION_PROFILE_CREATE_INFO_FB, .next = &level_info, }; XrFoveationProfileFB raw_profile = XR_NULL_HANDLE; XrResult profile_result = xrCreateFoveationProfileFB(session, &profile_info, &raw_profile); if (XR_SUCCEEDED(profile_result)) foveation_profile.emplace(raw_profile, xrDestroyFoveationProfileFB); } } if (foveation_profile.has_value()) spdlog::info("XR_META_foveation_eye_tracked will be used for this stream"); else if (application::get_eye_gaze_supported()) spdlog::info("Eye gaze tracking will be used for this stream"); else spdlog::info("Static foveation will be used for this stream"); auto [flags, views] = session.locate_views( XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO, instance.now(), application::space(xr::spaces::view)); (void)flags; assert(views.size() == headset_fov.size()); for (size_t i = 0; i < headset_fov.size(); ++i) headset_fov[i] = views[i].fov; const auto & config = application::get_config(); set_manual_mode({ .enabled = config.override_foveation_enable, .pitch = config.override_foveation_pitch, .distance = config.override_foveation_distance, }); } // member order ensures swapchain is dropped before foveation profile center::~center() = default; bool center::supports_meta() const { return foveation_profile.has_value(); } manual_override center::get_manual_mode() const { return *manual_mode_.lock(); } void center::set_manual_mode(manual_override mode) { *manual_mode_.lock() = mode; } bool center::is_cropped(std::span submitted_fov, std::span valid) const { assert(submitted_fov.size() == view_count and valid.size() == view_count); for (size_t i = 0; i < view_count; ++i) { // generous epsilon of ~6° per-side to filter out false positives if (valid[i] and not fov_equal(submitted_fov[i], headset_fov[i], 0.1f)) return true; } return false; } bool center::needs_swapchain_recreation(bool cropped) const { return supports_meta() and (not cropped_mode or *cropped_mode != cropped); } bool center::use_meta_swapchain(bool cropped) const { return supports_meta() and not cropped; } void center::set_cropped(bool cropped) { if (not cropped_mode or *cropped_mode != cropped) spdlog::info("Stream FOV mode: {}", cropped ? "cropped" : "full"); cropped_mode = cropped; } void center::on_swapchain_recreated(bool cropped) { dummy_swapchain = {}; if (supports_meta() and cropped) create_dummy_swapchain(); set_cropped(cropped); } bool center::update_meta_swapchain(XrSwapchain swapchain) const { if (not supports_meta() or not swapchain) return false; XrSwapchainStateFoveationFB swapchain_state{ .type = XR_TYPE_SWAPCHAIN_STATE_FOVEATION_FB, .flags = 0, .profile = *foveation_profile, }; XrResult update_result = xrUpdateSwapchainFB( swapchain, reinterpret_cast(&swapchain_state)); return XR_SUCCEEDED(update_result); } std::optional center::meta_angles(XrSwapchain swapchain, std::span fov) const { if (not update_meta_swapchain(swapchain)) return {}; XrFoveationEyeTrackedStateMETA state{ .type = XR_TYPE_FOVEATION_EYE_TRACKED_STATE_META, }; XrResult result = xrGetFoveationEyeTrackedStateMETA(session, &state); if (XR_FAILED(result) or not(state.flags & XR_FOVEATION_EYE_TRACKED_STATE_VALID_BIT_META)) return {}; std::array ndc_center{state.foveationCenter[0], state.foveationCenter[1]}; return center_to_angles(ndc_center, fov); } void center::update(std::span submitted_fov, std::span valid, XrSwapchain stream_swapchain, XrTime display_time) { assert(submitted_fov.size() == view_count and valid.size() == view_count); layer_.reset(); packet_.reset(); XrSpace view_space = application::space(xr::spaces::view); auto [view_flags, views] = session.locate_views( XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO, display_time, view_space); constexpr XrViewStateFlags layer_flags = XR_VIEW_STATE_POSITION_VALID_BIT | XR_VIEW_STATE_ORIENTATION_VALID_BIT; if (supports_meta() and cropped_mode.value_or(false) and dummy_swapchain and views.size() == view_count and (view_flags & layer_flags) == layer_flags) { projection_layer layer{ .flags = XR_COMPOSITION_LAYER_BLEND_TEXTURE_SOURCE_ALPHA_BIT, .space = view_space, }; for (size_t i = 0; i < view_count; ++i) { layer.views[i] = { .type = XR_TYPE_COMPOSITION_LAYER_PROJECTION_VIEW, .pose = views[i].pose, .fov = headset_fov[i], .subImage = { .swapchain = dummy_swapchain, .imageRect = { .offset = {0, 0}, .extent = {dummy_swapchain_size, dummy_swapchain_size}, }, .imageArrayIndex = uint32_t(i), }, }; } layer_ = layer; } if (views.size() != view_count or not(view_flags & XR_VIEW_STATE_POSITION_VALID_BIT)) return; // dummy layer is always full FOV std::array effective_fov{}; for (size_t i = 0; i < view_count; ++i) effective_fov[i] = valid[i] ? submitted_fov[i] : headset_fov[i]; const XrSwapchain meta_swapchain = cropped_mode.value_or(false) ? static_cast(dummy_swapchain) : stream_swapchain; const std::span meta_fov = cropped_mode.value_or(false) ? std::span(headset_fov) : std::span(effective_fov); const auto manual = get_manual_mode(); std::optional angles; if (manual.enabled) { if (supports_meta()) update_meta_swapchain(meta_swapchain); angles = target_angles(0, manual.pitch, manual.distance, views); } else { if (supports_meta()) { if (auto a = meta_angles(meta_swapchain, meta_fov)) angles = *a; } if (not angles and application::get_eye_gaze_supported()) { if (auto a = eye_gaze_angles(display_time, view_flags, views)) angles = *a; } if (not angles) angles = target_angles(0, default_foveation_pitch, default_convergence_distance, views); } // angles guaranteed to have a value at this point packet_ = wivrn::from_headset::foveation{.angles = *angles}; } std::optional center::get_layer() const { return layer_; } std::optional center::take_packet() { // A skipped render must not resend the previous frame's center. auto result = packet_; packet_.reset(); return result; } void center::create_dummy_swapchain() { device.waitIdle(); dummy_swapchain = xr::swapchain( instance, session, device, swapchain_format, dummy_swapchain_size, dummy_swapchain_size, 1, view_count, true, true); if (not update_meta_swapchain(dummy_swapchain)) throw std::runtime_error("Failed to apply eye-tracked foveation profile to dummy swapchain"); const int image_index = dummy_swapchain.acquire(); dummy_swapchain.wait(); auto clear_command_buffer = std::move(device.allocateCommandBuffers({ .commandPool = *commandpool, .level = vk::CommandBufferLevel::ePrimary, .commandBufferCount = 1, })[0]); auto clear_fence = device.createFence({}); clear_command_buffer.begin(vk::CommandBufferBeginInfo{ .flags = vk::CommandBufferUsageFlagBits::eOneTimeSubmit, }); vk::ImageMemoryBarrier to_transfer{ .srcAccessMask = vk::AccessFlagBits::eNone, .dstAccessMask = vk::AccessFlagBits::eTransferWrite, .oldLayout = vk::ImageLayout::eUndefined, .newLayout = vk::ImageLayout::eTransferDstOptimal, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = dummy_swapchain.image(image_index), .subresourceRange = { .aspectMask = vk::ImageAspectFlagBits::eColor, .baseMipLevel = 0, .levelCount = 1, .baseArrayLayer = 0, .layerCount = view_count, }, }; clear_command_buffer.pipelineBarrier( vk::PipelineStageFlagBits::eTopOfPipe, vk::PipelineStageFlagBits::eTransfer, {}, {}, {}, to_transfer); clear_command_buffer.clearColorImage( dummy_swapchain.image(image_index), vk::ImageLayout::eTransferDstOptimal, vk::ClearColorValue{std::array{0, 0, 0, 0}}, to_transfer.subresourceRange); vk::ImageMemoryBarrier to_runtime{ .srcAccessMask = vk::AccessFlagBits::eTransferWrite, .dstAccessMask = vk::AccessFlagBits::eMemoryRead, .oldLayout = vk::ImageLayout::eTransferDstOptimal, .newLayout = vk::ImageLayout::eColorAttachmentOptimal, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = dummy_swapchain.image(image_index), .subresourceRange = to_transfer.subresourceRange, }; clear_command_buffer.pipelineBarrier( vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eAllCommands, {}, {}, {}, to_runtime); clear_command_buffer.end(); vk::SubmitInfo submit_info; submit_info.setCommandBuffers(*clear_command_buffer); queue.lock()->submit(submit_info, *clear_fence); if (device.waitForFences(*clear_fence, VK_TRUE, UINT64_MAX) == vk::Result::eTimeout) throw std::runtime_error("Vulkan fence timeout while clearing foveation dummy swapchain"); dummy_swapchain.release(); spdlog::info( "Created {}x{} eye-tracked foveation dummy swapchain", dummy_swapchain_size, dummy_swapchain_size); } } // namespace client_foveation