mirror of
https://github.com/WiVRn/WiVRn.git
synced 2026-11-20 17:11:36 -08:00
In the layer squasher, world pose is retrieved from a projection layer if possible, but the eye pose is incorrect in that case. Get the data from hmd if it is needed by any of the layers.
991 lines
35 KiB
C++
991 lines
35 KiB
C++
/*
|
|
* WiVRn VR streaming
|
|
* Copyright (C) 2026 Patrick Nicolas <[email protected]>
|
|
*
|
|
* This program is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation, either version 3 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
// uses code borrowed from Monado with the following copyright notice:
|
|
// Copyright 2019-2024, Collabora, Ltd.
|
|
// Copyright 2025, NVIDIA CORPORATION.
|
|
// SPDX-License-Identifier: BSL-1.0
|
|
/*!
|
|
* @file
|
|
* @brief Compositor (compute shader) rendering code.
|
|
* @author Jakob Bornecrantz <[email protected]>
|
|
* @author Christoph Haag <[email protected]>
|
|
* @author Fernando Velazquez Innella <[email protected]>
|
|
* @author Rylie Pavlik <[email protected]>
|
|
* @ingroup comp_util
|
|
*/
|
|
|
|
#include "render/render_interface.h"
|
|
|
|
#include "layer_squasher.h"
|
|
|
|
#include "driver/wivrn_hmd.h"
|
|
#include "utils/wivrn_vk_bundle.h"
|
|
#include "vk/specialization_constants.h"
|
|
|
|
#include "main/comp_compositor.h"
|
|
#include "math/m_space.h"
|
|
#include "shaders/layer_defines.inc.glsl"
|
|
#include "util/comp_layer_accum.h"
|
|
#include "util/comp_render_helpers.h"
|
|
|
|
#include <algorithm>
|
|
#include <array>
|
|
#include <format>
|
|
#include <ranges>
|
|
|
|
namespace
|
|
{
|
|
const uint32_t view_count = 2;
|
|
const bool do_timewarp = true;
|
|
|
|
const std::array formats{
|
|
vk::Format::eR8G8B8A8Srgb,
|
|
vk::Format::eR8G8B8A8Unorm,
|
|
};
|
|
|
|
const comp_layer *
|
|
get_projection_layer(const comp_layer_accum & layers)
|
|
{
|
|
for (const auto & layer: std::span(layers.layers, layers.layer_count))
|
|
switch (layer.data.type)
|
|
{
|
|
case XRT_LAYER_PROJECTION:
|
|
case XRT_LAYER_PROJECTION_DEPTH:
|
|
return &layer;
|
|
case XRT_LAYER_QUAD:
|
|
case XRT_LAYER_CUBE:
|
|
case XRT_LAYER_CYLINDER:
|
|
case XRT_LAYER_EQUIRECT1:
|
|
case XRT_LAYER_EQUIRECT2:
|
|
case XRT_LAYER_PASSTHROUGH:
|
|
break;
|
|
}
|
|
return NULL;
|
|
}
|
|
struct pose_data
|
|
{
|
|
std::array<xrt_fov, XRT_MAX_VIEWS> fovs;
|
|
std::array<xrt_pose, XRT_MAX_VIEWS> world_poses;
|
|
std::array<xrt_pose, XRT_MAX_VIEWS> eye_poses;
|
|
pose_data(
|
|
wivrn::wivrn_hmd & hmd,
|
|
uint64_t frame_interval_ns,
|
|
const comp_frame & frame,
|
|
const comp_layer_accum & layers)
|
|
{
|
|
// Get pose data from projection layer so we avoid one resampling step
|
|
const auto * proj = get_projection_layer(layers);
|
|
// Except if it's not for the right frame, as it would make quads stutter
|
|
int64_t predicted_display_time_ns = frame.predicted_display_time_ns;
|
|
int64_t cutoff_ns = 3 * frame_interval_ns;
|
|
const bool use_proj_view = proj and llabs(predicted_display_time_ns - proj->data.timestamp) <= cutoff_ns;
|
|
if (use_proj_view)
|
|
{
|
|
// depth has the same first member as proj
|
|
for (auto [i, data]: std::ranges::enumerate_view(proj->data.proj.v))
|
|
{
|
|
fovs[i] = data.fov;
|
|
world_poses[i] = data.pose;
|
|
eye_poses[i] = data.pose;
|
|
}
|
|
}
|
|
|
|
if (
|
|
(not use_proj_view) or
|
|
std::ranges::any_of(
|
|
std::span(layers.layers, layers.layer_count),
|
|
[](const auto & layer) { return is_layer_view_space(&layer.data); }))
|
|
{
|
|
xrt_space_relation head_rel;
|
|
hmd.get_view_poses(
|
|
nullptr, // unused
|
|
frame.predicted_display_time_ns,
|
|
XRT_VIEW_TYPE_STEREO,
|
|
view_count,
|
|
&head_rel,
|
|
fovs.data(),
|
|
eye_poses.data());
|
|
if (not use_proj_view)
|
|
{
|
|
for (uint32_t i = 0; i < view_count; ++i)
|
|
{
|
|
xrt_relation_chain xrc{};
|
|
xrt_space_relation rel = XRT_SPACE_RELATION_ZERO;
|
|
|
|
m_relation_chain_push_pose_if_not_identity(&xrc, &eye_poses[i]);
|
|
m_relation_chain_push_relation(&xrc, &head_rel);
|
|
m_relation_chain_resolve(&xrc, &rel);
|
|
world_poses[i] = rel.pose;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
};
|
|
|
|
const comp_swapchain_image & get_layer_image(const comp_layer & layer, uint32_t swapchain_index, uint32_t image_index)
|
|
{
|
|
return reinterpret_cast<struct comp_swapchain *>(comp_layer_get_swapchain(&layer, swapchain_index))->images[image_index];
|
|
}
|
|
|
|
const struct comp_swapchain_image &
|
|
get_layer_depth_image(const comp_layer & layer, uint32_t swapchain_index, uint32_t image_index)
|
|
{
|
|
return reinterpret_cast<struct comp_swapchain *>(comp_layer_get_depth_swapchain(&layer, swapchain_index))->images[image_index];
|
|
}
|
|
|
|
uint32_t
|
|
uint_divide_and_round_up(uint32_t a, uint32_t b)
|
|
{
|
|
return (a + (b - 1)) / b;
|
|
}
|
|
|
|
std::array<uint32_t, 2> calc_dispatch_dims_1_view(const render_viewport_data & views)
|
|
{
|
|
// Power of two divide and round up.
|
|
return {
|
|
uint_divide_and_round_up(views.w, 8),
|
|
uint_divide_and_round_up(views.h, 8),
|
|
};
|
|
}
|
|
|
|
buffer_allocation make_ubo(vk::raii::Device & device, int i)
|
|
{
|
|
return buffer_allocation{
|
|
device,
|
|
vk::BufferCreateInfo{
|
|
.size = sizeof(render_compute_layer_ubo_data),
|
|
.usage = vk::BufferUsageFlagBits::eUniformBuffer,
|
|
},
|
|
VmaAllocationCreateInfo{
|
|
.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT,
|
|
.usage = VMA_MEMORY_USAGE_AUTO,
|
|
},
|
|
std::format("layer squasher UBO {}", i)};
|
|
}
|
|
|
|
vk::raii::ImageView make_view(wivrn::vk_bundle & vk, image_allocation & image, vk::ImageUsageFlagBits flag, uint32_t view)
|
|
{
|
|
vk::raii::ImageView res{vk.device,
|
|
vk::StructureChain{
|
|
vk::ImageViewCreateInfo{
|
|
.image = image,
|
|
.viewType = vk::ImageViewType::e2D,
|
|
.format = flag == vk::ImageUsageFlagBits::eSampled ? formats[0] : formats[1],
|
|
.subresourceRange = {
|
|
.aspectMask = vk::ImageAspectFlagBits::eColor,
|
|
.levelCount = 1,
|
|
.baseArrayLayer = view,
|
|
.layerCount = 1,
|
|
},
|
|
},
|
|
vk::ImageViewUsageCreateInfo{
|
|
.usage = flag,
|
|
},
|
|
|
|
}
|
|
.get()};
|
|
vk.name(*res, std::format("layer squasher view {}", view).c_str());
|
|
return res;
|
|
}
|
|
|
|
vk::raii::DescriptorSetLayout make_ds_layout(wivrn::vk_bundle & vk, uint32_t image_array_size, bool partially_bound_desc)
|
|
{
|
|
std::array bindings{
|
|
vk::DescriptorSetLayoutBinding{
|
|
.binding = 0,
|
|
.descriptorType = vk::DescriptorType::eCombinedImageSampler,
|
|
.descriptorCount = image_array_size,
|
|
.stageFlags = vk::ShaderStageFlagBits::eCompute,
|
|
},
|
|
vk::DescriptorSetLayoutBinding{
|
|
.binding = 2,
|
|
.descriptorType = vk::DescriptorType::eStorageImage,
|
|
.descriptorCount = 1,
|
|
.stageFlags = vk::ShaderStageFlagBits::eCompute,
|
|
},
|
|
vk::DescriptorSetLayoutBinding{
|
|
.binding = 3,
|
|
.descriptorType = vk::DescriptorType::eUniformBuffer,
|
|
.descriptorCount = 1,
|
|
.stageFlags = vk::ShaderStageFlagBits::eCompute,
|
|
},
|
|
};
|
|
std::array<vk::DescriptorBindingFlags, std::size(bindings)> flags{
|
|
vk::DescriptorBindingFlagBits::ePartiallyBound,
|
|
vk::DescriptorBindingFlagBits::ePartiallyBound,
|
|
vk::DescriptorBindingFlags{},
|
|
};
|
|
|
|
vk::StructureChain info{
|
|
vk::DescriptorSetLayoutCreateInfo{
|
|
.bindingCount = bindings.size(),
|
|
.pBindings = bindings.data(),
|
|
},
|
|
vk::DescriptorSetLayoutBindingFlagsCreateInfo{
|
|
.bindingCount = flags.size(),
|
|
.pBindingFlags = flags.data(),
|
|
}};
|
|
|
|
if (not partially_bound_desc)
|
|
info.unlink<vk::DescriptorSetLayoutBindingFlagsCreateInfo>();
|
|
|
|
vk::raii::DescriptorSetLayout res(vk.device, info.get());
|
|
vk.name(*res, "layer squasher descriptor set layout");
|
|
return res;
|
|
}
|
|
vk::raii::PipelineLayout make_layout(wivrn::vk_bundle & vk, vk::DescriptorSetLayout ds_layout)
|
|
{
|
|
vk::raii::PipelineLayout res(vk.device,
|
|
vk::PipelineLayoutCreateInfo{
|
|
.setLayoutCount = 1,
|
|
.pSetLayouts = &ds_layout,
|
|
});
|
|
vk.name(*res, "layer squasher pipeline layout");
|
|
return res;
|
|
}
|
|
|
|
vk::raii::Pipeline make_pipeline(wivrn::vk_bundle & vk, uint32_t image_array_size, vk::PipelineLayout layout)
|
|
{
|
|
auto specialization = make_specialization_constants(
|
|
int32_t(0), // unused
|
|
VkBool32(true), // do_timewarp
|
|
VkBool32(true), // do_color_correction
|
|
int32_t(image_array_size));
|
|
vk::raii::Pipeline res(
|
|
vk.device,
|
|
nullptr,
|
|
vk::ComputePipelineCreateInfo{
|
|
.stage = {
|
|
.stage = vk::ShaderStageFlagBits::eCompute,
|
|
.module = *vk.load_shader("layer"),
|
|
.pName = "main",
|
|
.pSpecializationInfo = specialization,
|
|
},
|
|
.layout = layout,
|
|
});
|
|
vk.name(*res, "layer squasher pipeline");
|
|
return res;
|
|
}
|
|
|
|
vk::raii::DescriptorPool make_ds_pool(wivrn::vk_bundle & vk, uint32_t image_array_size)
|
|
{
|
|
const uint32_t view_size = 2;
|
|
std::array pool_sizes{
|
|
vk::DescriptorPoolSize{
|
|
.type = vk::DescriptorType::eCombinedImageSampler,
|
|
.descriptorCount = view_size * image_array_size,
|
|
},
|
|
vk::DescriptorPoolSize{
|
|
.type = vk::DescriptorType::eStorageImage,
|
|
.descriptorCount = view_size,
|
|
},
|
|
vk::DescriptorPoolSize{
|
|
.type = vk::DescriptorType::eUniformBuffer,
|
|
.descriptorCount = view_size,
|
|
},
|
|
};
|
|
vk::raii::DescriptorPool res{
|
|
vk.device,
|
|
vk::DescriptorPoolCreateInfo{
|
|
.maxSets = view_size,
|
|
.poolSizeCount = pool_sizes.size(),
|
|
.pPoolSizes = pool_sizes.data(),
|
|
},
|
|
};
|
|
vk.name(*res, "layer squasher descriptor pool");
|
|
return res;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
namespace wivrn
|
|
{
|
|
|
|
layer_squasher::layer_squasher(vk_bundle & vk, vk::Extent3D target_size) :
|
|
partially_bound_desc{bool(std::get<vk::PhysicalDeviceVulkan12Features>(vk.feat).descriptorBindingPartiallyBound)},
|
|
image_array_size{std::min<uint32_t>(vk.physical_device.getProperties().limits.maxPerStageDescriptorSampledImages, RENDER_MAX_IMAGES_SIZE)},
|
|
clamp_to_border_black{vk.device,
|
|
vk::SamplerCreateInfo{
|
|
.magFilter = vk::Filter::eLinear,
|
|
.minFilter = vk::Filter::eLinear,
|
|
.mipmapMode = vk::SamplerMipmapMode::eLinear,
|
|
.addressModeU = vk::SamplerAddressMode::eClampToBorder,
|
|
.addressModeV = vk::SamplerAddressMode::eClampToBorder,
|
|
.addressModeW = vk::SamplerAddressMode::eClampToBorder,
|
|
.borderColor = vk::BorderColor::eFloatOpaqueBlack,
|
|
}},
|
|
clamp_to_edge{vk.device,
|
|
vk::SamplerCreateInfo{
|
|
.magFilter = vk::Filter::eLinear,
|
|
.minFilter = vk::Filter::eLinear,
|
|
.mipmapMode = vk::SamplerMipmapMode::eLinear,
|
|
.addressModeU = vk::SamplerAddressMode::eClampToEdge,
|
|
.addressModeV = vk::SamplerAddressMode::eClampToEdge,
|
|
.addressModeW = vk::SamplerAddressMode::eClampToEdge,
|
|
.borderColor = vk::BorderColor::eFloatOpaqueBlack,
|
|
}},
|
|
ubo{make_ubo(vk.device, 0), make_ubo(vk.device, 1)},
|
|
render_target{vk.device,
|
|
vk::StructureChain{
|
|
vk::ImageCreateInfo{
|
|
.flags = vk::ImageCreateFlagBits::eExtendedUsage | vk::ImageCreateFlagBits::eMutableFormat,
|
|
.imageType = vk::ImageType::e2D,
|
|
.format = formats[0],
|
|
.extent = target_size,
|
|
.mipLevels = 1,
|
|
.arrayLayers = view_count,
|
|
.usage = vk::ImageUsageFlagBits::eStorage | vk::ImageUsageFlagBits::eSampled,
|
|
},
|
|
vk::ImageFormatListCreateInfo{
|
|
.viewFormatCount = formats.size(),
|
|
.pViewFormats = formats.data(),
|
|
},
|
|
|
|
}
|
|
.get(),
|
|
VmaAllocationCreateInfo{
|
|
.usage = VMA_MEMORY_USAGE_AUTO,
|
|
},
|
|
"layer squasher target"},
|
|
render_view_srgb{make_view(vk, render_target, vk::ImageUsageFlagBits::eSampled, 0), make_view(vk, render_target, vk::ImageUsageFlagBits::eSampled, 1)},
|
|
render_view_unorm{make_view(vk, render_target, vk::ImageUsageFlagBits::eStorage, 0), make_view(vk, render_target, vk::ImageUsageFlagBits::eStorage, 1)},
|
|
ds_layout{make_ds_layout(vk, image_array_size, partially_bound_desc)},
|
|
layout{make_layout(vk, *ds_layout)},
|
|
pipeline{make_pipeline(vk, image_array_size, *layout)},
|
|
descriptor_pool{make_ds_pool(vk, image_array_size)}
|
|
{
|
|
{
|
|
std::array layouts = {*ds_layout, *ds_layout};
|
|
auto sets = vk.device.allocateDescriptorSets(vk::DescriptorSetAllocateInfo{
|
|
.descriptorPool = descriptor_pool,
|
|
.descriptorSetCount = uint32_t(descriptor_sets.size()),
|
|
.pSetLayouts = layouts.data(),
|
|
});
|
|
for (int i = 0; i < view_count; ++i)
|
|
descriptor_sets[i] = sets[i].release();
|
|
}
|
|
}
|
|
|
|
std::tuple<std::array<xrt_pose, 2>,
|
|
std::array<xrt_fov, 2>,
|
|
std::array<xrt_rect, 2>>
|
|
layer_squasher::do_layers(
|
|
vk::raii::Device & device,
|
|
vk::raii::CommandBuffer & cmd,
|
|
wivrn_hmd & hmd,
|
|
uint64_t frame_interval_ns,
|
|
const comp_frame & frame,
|
|
const comp_layer_accum & layers,
|
|
const xrt_rect & min_size)
|
|
{
|
|
// get the head/pose to reproject to
|
|
const pose_data poses{hmd, frame_interval_ns, frame, layers};
|
|
const auto extent3D = render_target.info().extent;
|
|
auto fovs = poses.fovs;
|
|
std::array viewports{
|
|
render_viewport_data{
|
|
.w = extent3D.width,
|
|
.h = extent3D.height,
|
|
},
|
|
render_viewport_data{
|
|
.w = extent3D.width,
|
|
.h = extent3D.height,
|
|
},
|
|
};
|
|
|
|
vk::ImageMemoryBarrier im_barrier{
|
|
.dstAccessMask = vk::AccessFlagBits::eShaderWrite,
|
|
.oldLayout = vk::ImageLayout::eUndefined,
|
|
.newLayout = vk::ImageLayout::eGeneral,
|
|
.image = render_target,
|
|
.subresourceRange = {
|
|
.aspectMask = vk::ImageAspectFlagBits::eColor,
|
|
.levelCount = 1,
|
|
.layerCount = view_count,
|
|
},
|
|
};
|
|
|
|
cmd.pipelineBarrier(
|
|
vk::PipelineStageFlagBits::eAllCommands,
|
|
vk::PipelineStageFlagBits::eComputeShader,
|
|
{},
|
|
{},
|
|
{},
|
|
im_barrier);
|
|
cmd.bindPipeline(vk::PipelineBindPoint::eCompute, *pipeline);
|
|
|
|
for (int view = 0; view < view_count; ++view)
|
|
{
|
|
// TODO: staging buffer?
|
|
auto & ubo = *(render_compute_layer_ubo_data *)this->ubo[view].map();
|
|
|
|
// Not the transform of the views, but the inverse: actual view matrices.
|
|
xrt_matrix_4x4 world_view_mat, eye_view_mat;
|
|
math_matrix_4x4_view_from_pose(&poses.world_poses[view], &world_view_mat);
|
|
math_matrix_4x4_view_from_pose(&poses.eye_poses[view], &eye_view_mat);
|
|
|
|
std::array<vk::DescriptorImageInfo, RENDER_MAX_IMAGES_SIZE> src_image_info{};
|
|
for (auto & src: src_image_info)
|
|
src.imageLayout = vk::ImageLayout::eShaderReadOnlyOptimal;
|
|
|
|
xrt_fov all_layers_fov{.angle_left = M_PI,
|
|
.angle_right = -M_PI,
|
|
.angle_up = -M_PI,
|
|
.angle_down = M_PI};
|
|
uint32_t cur_image = 0;
|
|
int cur_layer = 0;
|
|
for (const auto & layer: std::span(layers.layers, layers.layer_count))
|
|
{
|
|
if (not is_layer_view_visible(&layer.data, view))
|
|
continue;
|
|
|
|
const auto & data = layer.data;
|
|
|
|
/*!
|
|
* Stop compositing layers if device's sampled image limit is
|
|
* reached. For most hardware this isn't a problem, most have
|
|
* well over 32 max samplers. But notably the RPi4 only have 16
|
|
* which is a limit we may run into. But if you got 16+ layers
|
|
* on a RPi4 you have more problems then max samplers.
|
|
*/
|
|
uint32_t required_image_samplers;
|
|
switch (data.type)
|
|
{
|
|
case XRT_LAYER_CYLINDER:
|
|
case XRT_LAYER_EQUIRECT2:
|
|
case XRT_LAYER_PROJECTION:
|
|
case XRT_LAYER_QUAD:
|
|
required_image_samplers = 1;
|
|
break;
|
|
case XRT_LAYER_PROJECTION_DEPTH:
|
|
required_image_samplers = 2;
|
|
break;
|
|
default:
|
|
U_LOG_E("Skipping layer %ld, unknown type: %u", std::distance(layers.layers, &layer), data.type);
|
|
continue; // Skip this layer if don't know about it.
|
|
}
|
|
|
|
//! Exit loop if shader cannot receive more image samplers
|
|
if (cur_image + required_image_samplers > image_array_size)
|
|
break;
|
|
|
|
xrt_fov layer_fov;
|
|
switch (data.type)
|
|
{
|
|
case XRT_LAYER_PROJECTION:
|
|
case XRT_LAYER_PROJECTION_DEPTH:
|
|
layer_fov = do_projection_layer(
|
|
layer,
|
|
poses.world_poses[view],
|
|
view,
|
|
cur_layer,
|
|
cur_image,
|
|
src_image_info,
|
|
ubo);
|
|
break;
|
|
case XRT_LAYER_QUAD:
|
|
layer_fov = do_quad_layer(
|
|
layer,
|
|
eye_view_mat,
|
|
world_view_mat,
|
|
view,
|
|
cur_layer,
|
|
cur_image,
|
|
src_image_info,
|
|
ubo);
|
|
break;
|
|
case XRT_LAYER_CYLINDER:
|
|
layer_fov = do_cylinder_layer(
|
|
layer,
|
|
eye_view_mat,
|
|
world_view_mat,
|
|
view,
|
|
cur_layer,
|
|
cur_image,
|
|
src_image_info,
|
|
ubo);
|
|
break;
|
|
case XRT_LAYER_EQUIRECT2:
|
|
layer_fov = do_equirect2_layer(
|
|
layer,
|
|
eye_view_mat,
|
|
world_view_mat,
|
|
view,
|
|
cur_layer,
|
|
cur_image,
|
|
src_image_info,
|
|
ubo);
|
|
break;
|
|
case XRT_LAYER_EQUIRECT1:
|
|
case XRT_LAYER_PASSTHROUGH:
|
|
case XRT_LAYER_CUBE:
|
|
U_LOG_E("unsupported layer type");
|
|
assert(false);
|
|
continue;
|
|
}
|
|
// Add fov to all_layers_fov if there's an intersection
|
|
if (layer_fov.angle_left < fovs[view].angle_right and
|
|
layer_fov.angle_right > fovs[view].angle_left and
|
|
layer_fov.angle_up > fovs[view].angle_down and
|
|
layer_fov.angle_down < fovs[view].angle_up)
|
|
all_layers_fov = {
|
|
.angle_left = std::min(layer_fov.angle_left, all_layers_fov.angle_left),
|
|
.angle_right = std::max(layer_fov.angle_right, all_layers_fov.angle_right),
|
|
.angle_up = std::max(layer_fov.angle_up, all_layers_fov.angle_up),
|
|
.angle_down = std::min(layer_fov.angle_down, all_layers_fov.angle_down),
|
|
};
|
|
ubo.layers[cur_layer].layer_data.unpremultiplied_alpha = is_layer_unpremultiplied(&layer.data);
|
|
apply_bias_and_scale_from_layer(
|
|
&data,
|
|
&ubo.layers[cur_layer].color_scale,
|
|
&ubo.layers[cur_layer].color_bias);
|
|
cur_layer++;
|
|
}
|
|
|
|
all_layers_fov = {
|
|
.angle_left = std::max(fovs[view].angle_left, all_layers_fov.angle_left),
|
|
.angle_right = std::min(fovs[view].angle_right, all_layers_fov.angle_right),
|
|
.angle_up = std::min(fovs[view].angle_up, all_layers_fov.angle_up),
|
|
.angle_down = std::max(fovs[view].angle_down, all_layers_fov.angle_down),
|
|
};
|
|
if (all_layers_fov.angle_up > all_layers_fov.angle_down and
|
|
all_layers_fov.angle_left < all_layers_fov.angle_right)
|
|
{
|
|
fovs[view] = all_layers_fov;
|
|
auto w1 = tan(all_layers_fov.angle_right) - tan(all_layers_fov.angle_left);
|
|
auto h1 = tan(all_layers_fov.angle_up) - tan(all_layers_fov.angle_down);
|
|
auto w2 = tan(fovs[view].angle_right) - tan(fovs[view].angle_left);
|
|
auto h2 = tan(fovs[view].angle_up) - tan(fovs[view].angle_down);
|
|
viewports[view].w = std::max<uint32_t>(min_size.extent.w, viewports[view].w * w1 / w2);
|
|
viewports[view].h = std::max<uint32_t>(min_size.extent.h, viewports[view].h * h1 / h2);
|
|
}
|
|
|
|
ubo.view = viewports[view];
|
|
render_calc_uv_to_tangent_lengths_rect(&fovs[view], &ubo.pre_transform);
|
|
ubo.layer_count.value = cur_layer;
|
|
|
|
for (auto & layer: std::span(ubo.layers).subspan(cur_layer))
|
|
layer.layer_data.layer_type = LAYER_COMP_TYPE_NOOP;
|
|
|
|
vk::DescriptorImageInfo target_image_info{
|
|
.imageView = *render_view_unorm[view],
|
|
.imageLayout = vk::ImageLayout::eGeneral,
|
|
};
|
|
|
|
vk::DescriptorBufferInfo ubo_info{
|
|
.buffer = this->ubo[view],
|
|
.range = vk::WholeSize,
|
|
};
|
|
|
|
std::array writes = {
|
|
vk::WriteDescriptorSet{
|
|
.dstSet = descriptor_sets[view],
|
|
.dstBinding = 0,
|
|
.descriptorCount = uint32_t(cur_image),
|
|
.descriptorType = vk::DescriptorType::eCombinedImageSampler,
|
|
.pImageInfo = src_image_info.data(),
|
|
},
|
|
vk::WriteDescriptorSet{
|
|
.dstSet = descriptor_sets[view],
|
|
.dstBinding = 2,
|
|
.descriptorCount = 1,
|
|
.descriptorType = vk::DescriptorType::eStorageImage,
|
|
.pImageInfo = &target_image_info,
|
|
},
|
|
vk::WriteDescriptorSet{
|
|
.dstSet = descriptor_sets[view],
|
|
.dstBinding = 3,
|
|
.descriptorCount = 1,
|
|
.descriptorType = vk::DescriptorType::eUniformBuffer,
|
|
.pBufferInfo = &ubo_info,
|
|
},
|
|
};
|
|
|
|
if (not partially_bound_desc)
|
|
{
|
|
for (int i = cur_image; i < image_array_size; ++i)
|
|
src_image_info[i] = src_image_info[0];
|
|
writes[0].descriptorCount = image_array_size;
|
|
}
|
|
|
|
if (writes[0].descriptorCount == 0)
|
|
device.updateDescriptorSets(std::span(writes).subspan(1), {});
|
|
else
|
|
device.updateDescriptorSets(writes, {});
|
|
|
|
cmd.bindDescriptorSets(vk::PipelineBindPoint::eCompute, *layout, 0, descriptor_sets[view], {});
|
|
auto [w, h] = calc_dispatch_dims_1_view(viewports[view]);
|
|
cmd.dispatch(w, h, 1);
|
|
}
|
|
|
|
std::array<xrt_rect, 2> rect;
|
|
for (auto [v, r]: std::ranges::zip_view(viewports, rect))
|
|
r = {.extent = {.w = int(v.w), .h = int(v.h)}};
|
|
|
|
return {poses.world_poses, fovs, rect};
|
|
}
|
|
|
|
std::array<vk::ImageView, 2> layer_squasher::get_views()
|
|
{
|
|
return {*render_view_srgb[0], *render_view_srgb[1]};
|
|
}
|
|
|
|
uint32_t layer_squasher::max_layers(const vk::PhysicalDeviceProperties & prop) const
|
|
{
|
|
// We may be limited by samplers, and use up to 2 samplers per layer
|
|
return std::min({XRT_MAX_LAYERS * 2u,
|
|
prop.limits.maxPerStageDescriptorSampledImages,
|
|
prop.limits.maxPerStageDescriptorSamplers}) /
|
|
2u;
|
|
}
|
|
|
|
xrt_fov layer_squasher::do_projection_layer(
|
|
const comp_layer & layer,
|
|
const xrt_pose & world_pose,
|
|
int view,
|
|
int cur_layer,
|
|
uint32_t & cur_image,
|
|
std::span<vk::DescriptorImageInfo> src_image_info,
|
|
render_compute_layer_ubo_data & ubo)
|
|
{
|
|
const xrt_layer_data & layer_data = layer.data;
|
|
const auto & vd = layer_data.type == XRT_LAYER_PROJECTION ? layer_data.proj.v[view] : layer_data.depth.v[view];
|
|
|
|
uint32_t sc_array_index = view % 2;
|
|
|
|
ubo.layers[cur_layer].layer_data.layer_type = LAYER_COMP_TYPE_PROJECTION;
|
|
|
|
// Color
|
|
src_image_info[cur_image].sampler = *clamp_to_border_black;
|
|
src_image_info[cur_image].imageView = get_image_view(
|
|
&get_layer_image(layer, sc_array_index, vd.sub.image_index),
|
|
layer_data.flags,
|
|
vd.sub.array_index);
|
|
ubo.layers[cur_layer].image_info.color_image_index = cur_image++;
|
|
|
|
// Depth
|
|
if (layer_data.type == XRT_LAYER_PROJECTION_DEPTH)
|
|
{
|
|
const auto & dvd = layer_data.depth.d[view];
|
|
|
|
src_image_info[cur_image].sampler = *clamp_to_edge; // Edge to keep depth stable at edges.
|
|
src_image_info[cur_image].imageView = get_image_view(
|
|
&get_layer_depth_image(layer, sc_array_index, dvd.sub.image_index),
|
|
layer_data.flags,
|
|
dvd.sub.array_index);
|
|
ubo.layers[cur_layer].image_info.depth_image_index = cur_image++;
|
|
}
|
|
|
|
// Chroma key - per layer settings
|
|
const xrt_layer_chroma_key_data & ck = (layer_data.type == XRT_LAYER_PROJECTION_DEPTH)
|
|
? layer_data.depth.chroma_key
|
|
: layer_data.proj.chroma_key;
|
|
ubo.layers[cur_layer].chroma_key = {
|
|
.hsv_min_h = ck.hsv_min.h,
|
|
.hsv_min_s = ck.hsv_min.s,
|
|
.hsv_min_v = ck.hsv_min.v,
|
|
.hsv_max_h = ck.hsv_max.h,
|
|
.hsv_max_s = ck.hsv_max.s,
|
|
.hsv_max_v = ck.hsv_max.v,
|
|
.curve = ck.curve,
|
|
.despill = ck.despill,
|
|
};
|
|
|
|
set_post_transform_rect(
|
|
&layer_data,
|
|
&vd.sub.norm_rect,
|
|
false,
|
|
&ubo.layers[cur_layer].post_transforms);
|
|
|
|
// unused if timewarp is off
|
|
if (do_timewarp)
|
|
render_calc_time_warp_matrix(
|
|
&vd.pose,
|
|
&vd.fov,
|
|
&world_pose,
|
|
&ubo.layers[cur_layer].transforms_timewarp);
|
|
|
|
return vd.fov;
|
|
}
|
|
|
|
xrt_fov layer_squasher::do_quad_layer(const comp_layer & layer,
|
|
const xrt_matrix_4x4 & eye_view_mat,
|
|
const xrt_matrix_4x4 & world_view_mat,
|
|
uint32_t view_index,
|
|
int cur_layer,
|
|
uint32_t & cur_image,
|
|
std::span<vk::DescriptorImageInfo> src_image_info,
|
|
render_compute_layer_ubo_data & ubo)
|
|
{
|
|
const xrt_layer_data & layer_data = layer.data;
|
|
const xrt_layer_quad_data & q = layer_data.quad;
|
|
|
|
ubo.layers[cur_layer].layer_data.layer_type = LAYER_COMP_TYPE_QUAD;
|
|
|
|
// Image to use.
|
|
src_image_info[cur_image].sampler = *clamp_to_edge;
|
|
src_image_info[cur_image].imageView = get_image_view(
|
|
&get_layer_image(layer, 0, q.sub.image_index),
|
|
layer_data.flags,
|
|
q.sub.array_index);
|
|
|
|
// Set the normalized post transform values.
|
|
xrt_normalized_rect post_transform{};
|
|
|
|
// Used for Subimage and OpenGL flip.
|
|
set_post_transform_rect(
|
|
&layer_data,
|
|
&q.sub.norm_rect,
|
|
true,
|
|
&post_transform);
|
|
|
|
// Is this layer viewspace or not.
|
|
const xrt_matrix_4x4 & view_mat = is_layer_view_space(&layer_data) ? eye_view_mat : world_view_mat;
|
|
|
|
// Transform quad pose into view space.
|
|
xrt_vec3 quad_position{};
|
|
math_matrix_4x4_transform_vec3(&view_mat, &layer_data.quad.pose.position, &quad_position);
|
|
|
|
// neutral quad layer faces +z, towards the user
|
|
xrt_vec3 normal{.x = 0, .y = 0, .z = 1};
|
|
|
|
// rotation of the quad normal in world space
|
|
math_quat_rotate_vec3(&layer_data.quad.pose.orientation, &normal, &normal);
|
|
|
|
/*
|
|
* normal is a vector that originates on the plane, not on the origin.
|
|
* Instead of using the inverse quad transform to transform it into view space we can
|
|
* simply add up vectors:
|
|
*
|
|
* combined_normal [in world space] = plane_origin [in world space] + normal [in plane
|
|
* space] [with plane in world space]
|
|
*
|
|
* Then combined_normal can be transformed to view space via view matrix and a new
|
|
* normal_view_space retrieved:
|
|
*
|
|
* normal_view_space = combined_normal [in view space] - plane_origin [in view space]
|
|
*/
|
|
xrt_vec3 normal_view_space = normal;
|
|
math_vec3_accum(&layer_data.quad.pose.position, &normal_view_space);
|
|
math_matrix_4x4_transform_vec3(&view_mat, &normal_view_space, &normal_view_space);
|
|
math_vec3_subtract(&quad_position, &normal_view_space);
|
|
|
|
xrt_vec3 scale{1.f, 1.f, 1.f};
|
|
xrt_matrix_4x4 plane_transform_view_space, inverse_quad_transform;
|
|
math_matrix_4x4_model(&layer_data.quad.pose, &scale, &plane_transform_view_space);
|
|
math_matrix_4x4_multiply(&view_mat, &plane_transform_view_space, &plane_transform_view_space);
|
|
math_matrix_4x4_inverse(&plane_transform_view_space, &inverse_quad_transform);
|
|
|
|
// Write all of the UBO data.
|
|
ubo.layers[cur_layer].post_transforms = post_transform;
|
|
ubo.layers[cur_layer].quad_extent.val = layer_data.quad.size;
|
|
ubo.layers[cur_layer].quad_position.val = quad_position;
|
|
ubo.layers[cur_layer].quad_normal.val = normal_view_space;
|
|
ubo.layers[cur_layer].inverse_quad_transform = inverse_quad_transform;
|
|
ubo.layers[cur_layer].image_info.color_image_index = cur_image;
|
|
cur_image++;
|
|
|
|
// Compute effective fov
|
|
struct xrt_matrix_4x4 quad_pose;
|
|
math_matrix_4x4_view_from_pose(&q.pose, &quad_pose);
|
|
math_matrix_4x4_inverse(&quad_pose, &quad_pose);
|
|
math_matrix_4x4_multiply(&view_mat, &quad_pose, &quad_pose);
|
|
|
|
std::array corners{
|
|
xrt_vec3{-q.size.x / 2, -q.size.y / 2, 0},
|
|
xrt_vec3{q.size.x / 2, -q.size.y / 2, 0},
|
|
xrt_vec3{-q.size.x / 2, q.size.y / 2, 0},
|
|
xrt_vec3{q.size.x / 2, q.size.y / 2, 0}};
|
|
|
|
xrt_fov layer_fov = {.angle_left = M_PI,
|
|
.angle_right = -M_PI,
|
|
.angle_up = -M_PI,
|
|
.angle_down = M_PI};
|
|
for (auto & c: corners)
|
|
{
|
|
math_matrix_4x4_transform_vec3(&quad_pose, &c, &c);
|
|
|
|
float x = atan2f(c.x, -c.z);
|
|
float y = atan2f(c.y, -c.z);
|
|
|
|
layer_fov = {
|
|
.angle_left = std::min(layer_fov.angle_left, x),
|
|
.angle_right = std::max(layer_fov.angle_right, x),
|
|
.angle_up = std::max(layer_fov.angle_up, y),
|
|
.angle_down = std::min(layer_fov.angle_down, y),
|
|
};
|
|
}
|
|
return layer_fov;
|
|
}
|
|
|
|
xrt_fov layer_squasher::do_cylinder_layer(const comp_layer & layer,
|
|
const xrt_matrix_4x4 & eye_view_mat,
|
|
const xrt_matrix_4x4 & world_view_mat,
|
|
uint32_t view_index,
|
|
int cur_layer,
|
|
uint32_t & cur_image,
|
|
std::span<vk::DescriptorImageInfo> src_image_info,
|
|
render_compute_layer_ubo_data & ubo)
|
|
{
|
|
const xrt_layer_data & layer_data = layer.data;
|
|
const xrt_layer_cylinder_data & c = layer_data.cylinder;
|
|
|
|
ubo.layers[cur_layer].layer_data.layer_type = LAYER_COMP_TYPE_CYLINDER;
|
|
|
|
// Image to use.
|
|
src_image_info[cur_image].sampler = *clamp_to_edge;
|
|
src_image_info[cur_image].imageView = get_image_view(
|
|
&get_layer_image(layer, 0, c.sub.image_index),
|
|
layer_data.flags,
|
|
c.sub.array_index);
|
|
|
|
// Used for Subimage and OpenGL flip.
|
|
set_post_transform_rect(
|
|
&layer_data,
|
|
&c.sub.norm_rect,
|
|
false,
|
|
&ubo.layers[cur_layer].post_transforms);
|
|
|
|
xrt_vec3 scale{1.f, 1.f, 1.f};
|
|
|
|
xrt_matrix_4x4 model;
|
|
math_matrix_4x4_model(&c.pose, &scale, &model);
|
|
|
|
xrt_matrix_4x4 model_inv;
|
|
math_matrix_4x4_inverse(&model, &model_inv);
|
|
|
|
const xrt_matrix_4x4 & v = is_layer_view_space(&layer_data) ? eye_view_mat : world_view_mat;
|
|
|
|
xrt_matrix_4x4 v_inv;
|
|
math_matrix_4x4_inverse(&v, &v_inv);
|
|
|
|
math_matrix_4x4_multiply(&model_inv, &v_inv, &ubo.layers[cur_layer].mv_inverse);
|
|
|
|
// Simplifies the shader.
|
|
ubo.layers[cur_layer].cylinder_data.radius = c.radius >= INFINITY ? 0.f : c.radius;
|
|
|
|
ubo.layers[cur_layer].cylinder_data.central_angle = c.central_angle;
|
|
ubo.layers[cur_layer].cylinder_data.aspect_ratio = c.aspect_ratio;
|
|
|
|
ubo.layers[cur_layer].image_info.color_image_index = cur_image;
|
|
cur_image++;
|
|
|
|
// Compute effective FOV, simplify by projecting 2 quads instead
|
|
struct xrt_matrix_4x4 quad_pose;
|
|
math_matrix_4x4_view_from_pose(&c.pose, &quad_pose);
|
|
math_matrix_4x4_inverse(&quad_pose, &quad_pose);
|
|
math_matrix_4x4_multiply(&v, &quad_pose, &quad_pose);
|
|
|
|
const float w = sin(c.central_angle / 2) * c.radius;
|
|
const float h = c.radius * c.central_angle / (2 * c.aspect_ratio);
|
|
const float d = c.radius * cos(c.central_angle / 2);
|
|
std::array corners{
|
|
xrt_vec3{-w, -h, -c.radius},
|
|
xrt_vec3{w, -h, -c.radius},
|
|
xrt_vec3{-w, h, -c.radius},
|
|
xrt_vec3{w, h, -c.radius},
|
|
xrt_vec3{-w, -h, -d},
|
|
xrt_vec3{w, -h, -d},
|
|
xrt_vec3{-w, h, -d},
|
|
xrt_vec3{w, h, -d},
|
|
};
|
|
|
|
xrt_fov layer_fov = {.angle_left = M_PI,
|
|
.angle_right = -M_PI,
|
|
.angle_up = -M_PI,
|
|
.angle_down = M_PI};
|
|
for (auto & c: corners)
|
|
{
|
|
math_matrix_4x4_transform_vec3(&quad_pose, &c, &c);
|
|
|
|
float x = atan2f(c.x, -c.z);
|
|
float y = atan2f(c.y, -c.z);
|
|
|
|
layer_fov = {
|
|
.angle_left = std::min(layer_fov.angle_left, x),
|
|
.angle_right = std::max(layer_fov.angle_right, x),
|
|
.angle_up = std::max(layer_fov.angle_up, y),
|
|
.angle_down = std::min(layer_fov.angle_down, y),
|
|
};
|
|
}
|
|
return layer_fov;
|
|
}
|
|
|
|
xrt_fov layer_squasher::do_equirect2_layer(const comp_layer & layer,
|
|
const xrt_matrix_4x4 & eye_view_mat,
|
|
const xrt_matrix_4x4 & world_view_mat,
|
|
uint32_t view_index,
|
|
int cur_layer,
|
|
uint32_t & cur_image,
|
|
std::span<vk::DescriptorImageInfo> src_image_info,
|
|
render_compute_layer_ubo_data & ubo)
|
|
{
|
|
const xrt_layer_data & layer_data = layer.data;
|
|
const xrt_layer_equirect2_data & eq2 = layer_data.equirect2;
|
|
|
|
ubo.layers[cur_layer].layer_data.layer_type = LAYER_COMP_TYPE_EQUIRECT2;
|
|
|
|
// Image to use.
|
|
src_image_info[cur_image].sampler = *clamp_to_edge;
|
|
src_image_info[cur_image].imageView = get_image_view(
|
|
&get_layer_image(layer, 0, eq2.sub.image_index),
|
|
layer_data.flags,
|
|
eq2.sub.array_index);
|
|
|
|
// Used for Subimage and OpenGL flip.
|
|
set_post_transform_rect( //
|
|
&layer_data, // data
|
|
&eq2.sub.norm_rect, // src_norm_rect
|
|
false, // invert_flip
|
|
&ubo.layers[cur_layer].post_transforms); // out_norm_rect
|
|
|
|
xrt_vec3 scale{1.f, 1.f, 1.f};
|
|
|
|
xrt_matrix_4x4 model;
|
|
math_matrix_4x4_model(&eq2.pose, &scale, &model);
|
|
|
|
xrt_matrix_4x4 model_inv;
|
|
math_matrix_4x4_inverse(&model, &model_inv);
|
|
|
|
const xrt_matrix_4x4 & v = is_layer_view_space(&layer_data) ? eye_view_mat : world_view_mat;
|
|
|
|
xrt_matrix_4x4 v_inv;
|
|
math_matrix_4x4_inverse(&v, &v_inv);
|
|
|
|
math_matrix_4x4_multiply(&model_inv, &v_inv, &ubo.layers[cur_layer].mv_inverse);
|
|
|
|
// Simplifies the shader.
|
|
ubo.layers[cur_layer].eq2_data.radius = eq2.radius >= INFINITY ? 0.f : eq2.radius;
|
|
|
|
ubo.layers[cur_layer].eq2_data.central_horizontal_angle = eq2.central_horizontal_angle;
|
|
ubo.layers[cur_layer].eq2_data.upper_vertical_angle = eq2.upper_vertical_angle;
|
|
ubo.layers[cur_layer].eq2_data.lower_vertical_angle = eq2.lower_vertical_angle;
|
|
|
|
ubo.layers[cur_layer].image_info.color_image_index = cur_image;
|
|
cur_image++;
|
|
|
|
// TODO: effective fov
|
|
return {.angle_left = -M_PI,
|
|
.angle_right = M_PI,
|
|
.angle_up = M_PI,
|
|
.angle_down = -M_PI};
|
|
}
|
|
|
|
} // namespace wivrn
|