mirror of
https://github.com/WiVRn/WiVRn.git
synced 2026-11-20 00:51:32 -08:00
488 lines
16 KiB
C++
488 lines
16 KiB
C++
/*
|
|
* WiVRn VR streaming
|
|
* Copyright (C) 2026 galister <[email protected]>
|
|
*
|
|
* This program is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation, either version 3 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
#include "foveation.h"
|
|
|
|
#include "application.h"
|
|
#include "xr/instance.h"
|
|
#include "xr/session.h"
|
|
#include "xr/system.h"
|
|
|
|
#include <algorithm>
|
|
#include <cassert>
|
|
#include <cmath>
|
|
#include <spdlog/spdlog.h>
|
|
#include <stdexcept>
|
|
|
|
namespace
|
|
{
|
|
float convergence_angle(float distance, float eye_x, float yaw)
|
|
{
|
|
const float target_x = distance * std::sin(yaw);
|
|
const float target_z = distance * std::cos(yaw);
|
|
|
|
return std::atan2(target_x - eye_x, target_z);
|
|
}
|
|
|
|
XrVector2f yaw_pitch(const XrQuaternionf & q)
|
|
{
|
|
const float sine_theta = std::clamp(-2.0f * (q.y * q.z - q.w * q.x), -1.0f, 1.0f);
|
|
const float pitch = std::asin(sine_theta);
|
|
|
|
if (std::abs(sine_theta) > 0.99999f)
|
|
{
|
|
const float scale = std::copysign(2.0f, sine_theta);
|
|
return {scale * std::atan2(-q.z, q.w), pitch};
|
|
}
|
|
|
|
return {
|
|
std::atan2(2.0f * (q.x * q.z + q.w * q.y),
|
|
q.w * q.w - q.x * q.x - q.y * q.y + q.z * q.z),
|
|
pitch};
|
|
}
|
|
|
|
float ndc_to_angle(float center, float a0, float a1)
|
|
{
|
|
const float u = std::clamp(center * 0.5f + 0.5f, 0.0f, 1.0f);
|
|
const float tangent = std::lerp(std::tan(a0), std::tan(a1), u);
|
|
return std::atan(tangent);
|
|
}
|
|
|
|
client_foveation::angles center_to_angles(const std::array<XrVector2f, 2> & center, std::span<const XrFovf> fovs)
|
|
{
|
|
assert(fovs.size() == 2);
|
|
client_foveation::angles result{};
|
|
|
|
for (size_t i = 0; i < result.size(); ++i)
|
|
{
|
|
result[i].x = ndc_to_angle(center[i].x, fovs[i].angleLeft, fovs[i].angleRight);
|
|
result[i].y = ndc_to_angle(center[i].y, fovs[i].angleUp, fovs[i].angleDown);
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
bool fov_equal(const XrFovf & a, const XrFovf & b, const float epsilon)
|
|
{
|
|
return std::abs(a.angleLeft - b.angleLeft) <= epsilon and
|
|
std::abs(a.angleRight - b.angleRight) <= epsilon and
|
|
std::abs(a.angleUp - b.angleUp) <= epsilon and
|
|
std::abs(a.angleDown - b.angleDown) <= epsilon;
|
|
}
|
|
|
|
client_foveation::angles target_angles(
|
|
float yaw,
|
|
float pitch,
|
|
float convergence_distance,
|
|
std::span<const XrView> views)
|
|
{
|
|
assert(views.size() == 2);
|
|
client_foveation::angles result{};
|
|
|
|
for (size_t i = 0; i < result.size(); ++i)
|
|
{
|
|
result[i].x = convergence_angle(convergence_distance, views[i].pose.position.x, yaw);
|
|
result[i].y = pitch;
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
std::optional<client_foveation::angles> eye_gaze_angles(
|
|
XrSpace eye_gaze_space,
|
|
XrSpace view_space,
|
|
XrTime time,
|
|
XrViewStateFlags view_flags,
|
|
std::span<const XrView> views)
|
|
{
|
|
if (not eye_gaze_space or not view_space or views.size() != 2)
|
|
return {};
|
|
|
|
XrSpaceLocation location{
|
|
.type = XR_TYPE_SPACE_LOCATION,
|
|
};
|
|
|
|
XrResult res = xrLocateSpace(eye_gaze_space, view_space, time, &location);
|
|
if (XR_FAILED(res) or
|
|
not(location.locationFlags & XR_SPACE_LOCATION_ORIENTATION_VALID_BIT) or
|
|
not(location.locationFlags & XR_SPACE_LOCATION_ORIENTATION_TRACKED_BIT))
|
|
return {};
|
|
|
|
if (not(view_flags & XR_VIEW_STATE_POSITION_VALID_BIT))
|
|
return {};
|
|
|
|
const XrVector2f gaze = yaw_pitch(location.pose.orientation);
|
|
return target_angles(-gaze.x, gaze.y, client_foveation::default_convergence_distance, views);
|
|
}
|
|
|
|
} // namespace
|
|
|
|
namespace client_foveation
|
|
{
|
|
|
|
center::center(xr::instance & inst, xr::system & sys, xr::session & xr_session, vk::raii::Device & vk_device, vk::raii::CommandPool & vk_commandpool, thread_safe<vk::raii::Queue> & vk_queue, vk::Format format) :
|
|
instance(inst),
|
|
session(xr_session),
|
|
device(vk_device),
|
|
commandpool(vk_commandpool),
|
|
queue(vk_queue),
|
|
swapchain_format(format)
|
|
{
|
|
if (application::get_foveation_center_supported() and
|
|
application::get_foveation_vulkan_supported() and
|
|
inst.has_extension(XR_META_FOVEATION_EYE_TRACKED_EXTENSION_NAME) and
|
|
inst.has_extension(XR_FB_FOVEATION_EXTENSION_NAME) and
|
|
inst.has_extension(XR_FB_FOVEATION_CONFIGURATION_EXTENSION_NAME) and
|
|
inst.has_extension(XR_FB_FOVEATION_VULKAN_EXTENSION_NAME) and
|
|
inst.has_extension(XR_FB_SWAPCHAIN_UPDATE_STATE_EXTENSION_NAME) and
|
|
inst.has_extension(XR_META_VULKAN_SWAPCHAIN_CREATE_INFO_EXTENSION_NAME) and
|
|
sys.foveation_eye_tracked_properties().supportsFoveationEyeTracked)
|
|
{
|
|
xrCreateFoveationProfileFB = inst.get_proc<PFN_xrCreateFoveationProfileFB>("xrCreateFoveationProfileFB");
|
|
xrDestroyFoveationProfileFB = inst.get_proc<PFN_xrDestroyFoveationProfileFB>("xrDestroyFoveationProfileFB");
|
|
xrUpdateSwapchainFB = inst.get_proc<PFN_xrUpdateSwapchainFB>("xrUpdateSwapchainFB");
|
|
xrGetFoveationEyeTrackedStateMETA = inst.get_proc<PFN_xrGetFoveationEyeTrackedStateMETA>("xrGetFoveationEyeTrackedStateMETA");
|
|
|
|
if (xrCreateFoveationProfileFB and xrDestroyFoveationProfileFB and
|
|
xrUpdateSwapchainFB and xrGetFoveationEyeTrackedStateMETA)
|
|
{
|
|
XrFoveationEyeTrackedProfileCreateInfoMETA eye_tracked_info{
|
|
.type = XR_TYPE_FOVEATION_EYE_TRACKED_PROFILE_CREATE_INFO_META,
|
|
.flags = 0,
|
|
};
|
|
XrFoveationLevelProfileCreateInfoFB level_info{
|
|
.type = XR_TYPE_FOVEATION_LEVEL_PROFILE_CREATE_INFO_FB,
|
|
.next = &eye_tracked_info,
|
|
.level = XR_FOVEATION_LEVEL_LOW_FB,
|
|
.verticalOffset = 0,
|
|
.dynamic = XR_FOVEATION_DYNAMIC_DISABLED_FB,
|
|
};
|
|
XrFoveationProfileCreateInfoFB profile_info{
|
|
.type = XR_TYPE_FOVEATION_PROFILE_CREATE_INFO_FB,
|
|
.next = &level_info,
|
|
};
|
|
|
|
XrFoveationProfileFB raw_profile = XR_NULL_HANDLE;
|
|
XrResult profile_result = xrCreateFoveationProfileFB(session, &profile_info, &raw_profile);
|
|
if (XR_SUCCEEDED(profile_result))
|
|
foveation_profile.emplace(raw_profile, xrDestroyFoveationProfileFB);
|
|
}
|
|
}
|
|
|
|
if (foveation_profile.has_value())
|
|
spdlog::info("XR_META_foveation_eye_tracked will be used for this stream");
|
|
else if (application::get_eye_gaze_supported())
|
|
spdlog::info("Eye gaze tracking will be used for this stream");
|
|
else
|
|
spdlog::info("Static foveation will be used for this stream");
|
|
|
|
auto [flags, views] = session.locate_views(
|
|
XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO,
|
|
instance.now(),
|
|
application::space(xr::spaces::view));
|
|
(void)flags;
|
|
assert(views.size() == headset_fov.size());
|
|
for (size_t i = 0; i < headset_fov.size(); ++i)
|
|
headset_fov[i] = views[i].fov;
|
|
|
|
const auto & config = application::get_config();
|
|
set_manual_mode({
|
|
.enabled = config.override_foveation_enable,
|
|
.pitch = config.override_foveation_pitch,
|
|
.distance = config.override_foveation_distance,
|
|
});
|
|
}
|
|
|
|
// member order ensures swapchain is dropped before foveation profile
|
|
center::~center() = default;
|
|
|
|
bool center::supports_meta() const
|
|
{
|
|
return foveation_profile.has_value();
|
|
}
|
|
|
|
manual_override center::get_manual_mode() const
|
|
{
|
|
return *manual_mode_.lock();
|
|
}
|
|
|
|
void center::set_manual_mode(manual_override mode)
|
|
{
|
|
*manual_mode_.lock() = mode;
|
|
}
|
|
|
|
bool center::is_cropped(std::span<const XrFovf> submitted_fov, std::span<const bool> valid) const
|
|
{
|
|
assert(submitted_fov.size() == view_count and valid.size() == view_count);
|
|
for (size_t i = 0; i < view_count; ++i)
|
|
{
|
|
// generous epsilon of ~6° per-side to filter out false positives
|
|
if (valid[i] and not fov_equal(submitted_fov[i], headset_fov[i], 0.1f))
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
bool center::needs_swapchain_recreation(bool cropped) const
|
|
{
|
|
return supports_meta() and (not cropped_mode or *cropped_mode != cropped);
|
|
}
|
|
|
|
bool center::use_meta_swapchain(bool cropped) const
|
|
{
|
|
return supports_meta() and not cropped;
|
|
}
|
|
|
|
void center::set_cropped(bool cropped)
|
|
{
|
|
if (not cropped_mode or *cropped_mode != cropped)
|
|
spdlog::info("Stream FOV mode: {}", cropped ? "cropped" : "full");
|
|
cropped_mode = cropped;
|
|
}
|
|
|
|
void center::on_swapchain_recreated(bool cropped)
|
|
{
|
|
dummy_swapchain = {};
|
|
if (supports_meta() and cropped)
|
|
create_dummy_swapchain();
|
|
set_cropped(cropped);
|
|
}
|
|
|
|
bool center::update_meta_swapchain(XrSwapchain swapchain) const
|
|
{
|
|
if (not supports_meta() or not swapchain)
|
|
return false;
|
|
|
|
XrSwapchainStateFoveationFB swapchain_state{
|
|
.type = XR_TYPE_SWAPCHAIN_STATE_FOVEATION_FB,
|
|
.flags = 0,
|
|
.profile = *foveation_profile,
|
|
};
|
|
|
|
XrResult update_result = xrUpdateSwapchainFB(
|
|
swapchain,
|
|
reinterpret_cast<const XrSwapchainStateBaseHeaderFB *>(&swapchain_state));
|
|
return XR_SUCCEEDED(update_result);
|
|
}
|
|
|
|
std::optional<angles> center::meta_angles(XrSwapchain swapchain, std::span<const XrFovf> fov) const
|
|
{
|
|
if (not update_meta_swapchain(swapchain))
|
|
return {};
|
|
|
|
XrFoveationEyeTrackedStateMETA state{
|
|
.type = XR_TYPE_FOVEATION_EYE_TRACKED_STATE_META,
|
|
};
|
|
|
|
XrResult result = xrGetFoveationEyeTrackedStateMETA(session, &state);
|
|
if (XR_FAILED(result) or not(state.flags & XR_FOVEATION_EYE_TRACKED_STATE_VALID_BIT_META))
|
|
return {};
|
|
|
|
std::array<XrVector2f, view_count> ndc_center{state.foveationCenter[0], state.foveationCenter[1]};
|
|
return center_to_angles(ndc_center, fov);
|
|
}
|
|
|
|
void center::update(std::span<const XrFovf> submitted_fov, std::span<const bool> valid, XrSwapchain stream_swapchain, XrTime display_time)
|
|
{
|
|
assert(submitted_fov.size() == view_count and valid.size() == view_count);
|
|
layer_.reset();
|
|
packet_.reset();
|
|
|
|
XrSpace view_space = application::space(xr::spaces::view);
|
|
auto [view_flags, views] = session.locate_views(
|
|
XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO, display_time, view_space);
|
|
|
|
constexpr XrViewStateFlags layer_flags =
|
|
XR_VIEW_STATE_POSITION_VALID_BIT | XR_VIEW_STATE_ORIENTATION_VALID_BIT;
|
|
if (supports_meta() and cropped_mode.value_or(false) and dummy_swapchain and
|
|
views.size() == view_count and (view_flags & layer_flags) == layer_flags)
|
|
{
|
|
projection_layer layer{
|
|
.flags = XR_COMPOSITION_LAYER_BLEND_TEXTURE_SOURCE_ALPHA_BIT,
|
|
.space = view_space,
|
|
};
|
|
for (size_t i = 0; i < view_count; ++i)
|
|
{
|
|
layer.views[i] = {
|
|
.type = XR_TYPE_COMPOSITION_LAYER_PROJECTION_VIEW,
|
|
.pose = views[i].pose,
|
|
.fov = headset_fov[i],
|
|
.subImage = {
|
|
.swapchain = dummy_swapchain,
|
|
.imageRect = {
|
|
.offset = {0, 0},
|
|
.extent = {dummy_swapchain_size, dummy_swapchain_size},
|
|
},
|
|
.imageArrayIndex = uint32_t(i),
|
|
},
|
|
};
|
|
}
|
|
layer_ = layer;
|
|
}
|
|
|
|
if (views.size() != view_count or not(view_flags & XR_VIEW_STATE_POSITION_VALID_BIT))
|
|
return;
|
|
|
|
// dummy layer is always full FOV
|
|
std::array<XrFovf, view_count> effective_fov{};
|
|
for (size_t i = 0; i < view_count; ++i)
|
|
effective_fov[i] = valid[i] ? submitted_fov[i] : headset_fov[i];
|
|
const XrSwapchain meta_swapchain = cropped_mode.value_or(false) ? static_cast<XrSwapchain>(dummy_swapchain) : stream_swapchain;
|
|
const std::span<const XrFovf> meta_fov = cropped_mode.value_or(false) ? std::span<const XrFovf>(headset_fov) : std::span<const XrFovf>(effective_fov);
|
|
|
|
const auto manual = get_manual_mode();
|
|
std::optional<angles> angles;
|
|
if (manual.enabled)
|
|
{
|
|
if (supports_meta())
|
|
update_meta_swapchain(meta_swapchain);
|
|
angles = target_angles(0, manual.pitch, manual.distance, views);
|
|
}
|
|
else
|
|
{
|
|
if (supports_meta())
|
|
{
|
|
if (auto a = meta_angles(meta_swapchain, meta_fov))
|
|
angles = *a;
|
|
}
|
|
if (not angles and application::get_eye_gaze_supported())
|
|
{
|
|
if (auto a = eye_gaze_angles(application::space(xr::spaces::eye_gaze),
|
|
view_space,
|
|
display_time,
|
|
view_flags,
|
|
views))
|
|
angles = *a;
|
|
}
|
|
if (not angles)
|
|
angles = target_angles(0, default_foveation_pitch, default_convergence_distance, views);
|
|
}
|
|
// angles guaranteed to have a value at this point
|
|
packet_ = wivrn::from_headset::foveation{.angles = *angles};
|
|
}
|
|
|
|
std::optional<center::projection_layer> center::get_layer() const
|
|
{
|
|
return layer_;
|
|
}
|
|
|
|
std::optional<wivrn::from_headset::foveation> center::take_packet()
|
|
{
|
|
// A skipped render must not resend the previous frame's center.
|
|
auto result = packet_;
|
|
packet_.reset();
|
|
return result;
|
|
}
|
|
|
|
void center::create_dummy_swapchain()
|
|
{
|
|
device.waitIdle();
|
|
|
|
dummy_swapchain = xr::swapchain(
|
|
instance,
|
|
session,
|
|
device,
|
|
swapchain_format,
|
|
dummy_swapchain_size,
|
|
dummy_swapchain_size,
|
|
1,
|
|
view_count,
|
|
true,
|
|
true);
|
|
|
|
if (not update_meta_swapchain(dummy_swapchain))
|
|
throw std::runtime_error("Failed to apply eye-tracked foveation profile to dummy swapchain");
|
|
|
|
const int image_index = dummy_swapchain.acquire();
|
|
dummy_swapchain.wait();
|
|
|
|
auto clear_command_buffer = std::move(device.allocateCommandBuffers({
|
|
.commandPool = *commandpool,
|
|
.level = vk::CommandBufferLevel::ePrimary,
|
|
.commandBufferCount = 1,
|
|
})[0]);
|
|
auto clear_fence = device.createFence({});
|
|
|
|
clear_command_buffer.begin(vk::CommandBufferBeginInfo{
|
|
.flags = vk::CommandBufferUsageFlagBits::eOneTimeSubmit,
|
|
});
|
|
|
|
vk::ImageMemoryBarrier to_transfer{
|
|
.srcAccessMask = vk::AccessFlagBits::eNone,
|
|
.dstAccessMask = vk::AccessFlagBits::eTransferWrite,
|
|
.oldLayout = vk::ImageLayout::eUndefined,
|
|
.newLayout = vk::ImageLayout::eTransferDstOptimal,
|
|
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
|
|
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
|
|
.image = dummy_swapchain.image(image_index),
|
|
.subresourceRange = {
|
|
.aspectMask = vk::ImageAspectFlagBits::eColor,
|
|
.baseMipLevel = 0,
|
|
.levelCount = 1,
|
|
.baseArrayLayer = 0,
|
|
.layerCount = view_count,
|
|
},
|
|
};
|
|
clear_command_buffer.pipelineBarrier(
|
|
vk::PipelineStageFlagBits::eTopOfPipe,
|
|
vk::PipelineStageFlagBits::eTransfer,
|
|
{},
|
|
{},
|
|
{},
|
|
to_transfer);
|
|
|
|
clear_command_buffer.clearColorImage(
|
|
dummy_swapchain.image(image_index),
|
|
vk::ImageLayout::eTransferDstOptimal,
|
|
vk::ClearColorValue{std::array<float, 4>{0, 0, 0, 0}},
|
|
to_transfer.subresourceRange);
|
|
|
|
vk::ImageMemoryBarrier to_runtime{
|
|
.srcAccessMask = vk::AccessFlagBits::eTransferWrite,
|
|
.dstAccessMask = vk::AccessFlagBits::eMemoryRead,
|
|
.oldLayout = vk::ImageLayout::eTransferDstOptimal,
|
|
.newLayout = vk::ImageLayout::eColorAttachmentOptimal,
|
|
.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
|
|
.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
|
|
.image = dummy_swapchain.image(image_index),
|
|
.subresourceRange = to_transfer.subresourceRange,
|
|
};
|
|
clear_command_buffer.pipelineBarrier(
|
|
vk::PipelineStageFlagBits::eTransfer,
|
|
vk::PipelineStageFlagBits::eAllCommands,
|
|
{},
|
|
{},
|
|
{},
|
|
to_runtime);
|
|
clear_command_buffer.end();
|
|
|
|
vk::SubmitInfo submit_info;
|
|
submit_info.setCommandBuffers(*clear_command_buffer);
|
|
queue.lock()->submit(submit_info, *clear_fence);
|
|
if (device.waitForFences(*clear_fence, VK_TRUE, UINT64_MAX) == vk::Result::eTimeout)
|
|
throw std::runtime_error("Vulkan fence timeout while clearing foveation dummy swapchain");
|
|
|
|
dummy_swapchain.release();
|
|
spdlog::info(
|
|
"Created {}x{} eye-tracked foveation dummy swapchain",
|
|
dummy_swapchain_size,
|
|
dummy_swapchain_size);
|
|
}
|
|
|
|
} // namespace client_foveation
|