/* * WiVRn VR streaming * Copyright (C) 2024 galister * Copyright (C) 2025 Patrick Nicolas * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #include "foveation.h" #include "utils/wivrn_vk_bundle.h" #include "vk/specialization_constants.h" #include "wivrn_packets.h" #include "xrt/xrt_defines.h" #include "xrt/xrt_limits.h" #include #include #include #include #include #include #define RENDER_FOVEATION_BUFFER_DIMENSIONS (4096 + 1) namespace { struct ubo_data { uint32_t x[XRT_MAX_VIEWS * RENDER_FOVEATION_BUFFER_DIMENSIONS]; uint32_t y[XRT_MAX_VIEWS * RENDER_FOVEATION_BUFFER_DIMENSIONS]; }; vk::raii::Sampler make_sampler(wivrn::vk_bundle & vk) { vk::raii::Sampler res( vk.device, vk::SamplerCreateInfo{ .magFilter = vk::Filter::eLinear, .minFilter = vk::Filter::eLinear, .mipmapMode = vk::SamplerMipmapMode::eLinear, .addressModeU = vk::SamplerAddressMode::eClampToBorder, .addressModeV = vk::SamplerAddressMode::eClampToBorder, .addressModeW = vk::SamplerAddressMode::eClampToBorder, .borderColor = vk::BorderColor::eFloatOpaqueBlack, }); vk.name(res, "foveation sampler"); return res; } vk::raii::DescriptorSetLayout make_ds_layout(wivrn::vk_bundle & vk) { std::array bindings{ vk::DescriptorSetLayoutBinding{ .binding = 0, .descriptorType = vk::DescriptorType::eCombinedImageSampler, .descriptorCount = 2, .stageFlags = vk::ShaderStageFlagBits::eCompute, }, vk::DescriptorSetLayoutBinding{ .binding = 1, .descriptorType = vk::DescriptorType::eStorageBuffer, .descriptorCount = 1, .stageFlags = vk::ShaderStageFlagBits::eCompute, }, vk::DescriptorSetLayoutBinding{ .binding = 2, .descriptorType = vk::DescriptorType::eStorageImage, .descriptorCount = 1, .stageFlags = vk::ShaderStageFlagBits::eCompute, }, vk::DescriptorSetLayoutBinding{ .binding = 3, .descriptorType = vk::DescriptorType::eStorageImage, .descriptorCount = 1, .stageFlags = vk::ShaderStageFlagBits::eCompute, }, }; vk::raii::DescriptorSetLayout res{ vk.device, vk::DescriptorSetLayoutCreateInfo{ .bindingCount = bindings.size(), .pBindings = bindings.data(), }, }; vk.name(*res, "foveation descriptor set layout"); return res; } vk::raii::PipelineLayout make_layout(wivrn::vk_bundle & vk, vk::DescriptorSetLayout ds_layout) { vk::raii::PipelineLayout res(vk.device, vk::PipelineLayoutCreateInfo{ .setLayoutCount = 1, .pSetLayouts = &ds_layout, }); vk.name(*res, "foveation pipeline layout"); return res; } std::array make_pipelines(wivrn::vk_bundle & vk, vk::PipelineLayout layout, int32_t alpha_width) { auto shader = vk.load_shader("foveation"); auto spc = make_specialization_constants(alpha_width); std::array res{ vk::raii::Pipeline{ vk.device, nullptr, vk::ComputePipelineCreateInfo{ .stage = { .stage = vk::ShaderStageFlagBits::eCompute, .module = *shader, .pName = "main", }, .layout = layout, }, }, vk::raii::Pipeline{ vk.device, nullptr, vk::ComputePipelineCreateInfo{ .stage = { .stage = vk::ShaderStageFlagBits::eCompute, .module = *shader, .pName = "main", .pSpecializationInfo = spc, }, .layout = layout, }, }, }; vk.name(*res[0], "foveation pipeline"); vk.name(*res[1], "foveation+alpha pipeline"); return res; } vk::raii::DescriptorPool make_ds_pool(wivrn::vk_bundle & vk) { std::array pool_sizes{ vk::DescriptorPoolSize{ .type = vk::DescriptorType::eCombinedImageSampler, .descriptorCount = 2, }, vk::DescriptorPoolSize{ .type = vk::DescriptorType::eStorageImage, .descriptorCount = 2, }, vk::DescriptorPoolSize{ .type = vk::DescriptorType::eStorageBuffer, .descriptorCount = 1, }, }; vk::raii::DescriptorPool res{ vk.device, vk::DescriptorPoolCreateInfo{ .maxSets = 5, .poolSizeCount = pool_sizes.size(), .pPoolSizes = pool_sizes.data(), }, }; vk.name(*res, "foveation descriptor pool"); return res; } uint32_t divide_and_round_up(uint32_t a, uint32_t b) { return (a + (b - 1)) / b; } } // namespace // a, b: parameters computed by solve_foveation // λ: pixel ratio between full size and foveated image (in range ]0,1[) // c: coordinates in -1,1 range of the full size image where pixel ratio must be 1:1 // x: coordinates in -1,1 range of the foveated image // result: full size image coordinates in -1,1 range static double defoveate(double a, double b, double λ, double c, double x) { // In order to save encoding, transmit and decoding time, only a portion of the image is encoded in full resolution. // on each axis, foveated coordinates are defined by the following formula. return λ / a * tan(a * x + b) + c; // a and b are defined such as: // edges of the image are not moved // f(-1) = -1 // f( 1) = 1 // the function also enforces pixel ratio 1:1 at fovea // df⁻¹(x)/dx = 1/scale for x = c // We then ensure that source and destination pixel grids match by // rounding to integer pixel ratios: 1:1, 1:2 etc. // Finally, pixel spans are sorted so that we only have increasing // ratios going out from the center. } static std::tuple solve_foveation(float λ, float c) { // Compute a and b for the foveation function such that: // foveate(a, b, scale, c, -1) = -1 (eq. 1) // foveate(a, b, scale, c, 1) = 1 (eq. 2) // // Use eq. 2 to express a as function of b, then replace in eq. 1 // equation that needs to be null is: auto b = [λ, c](double a) { return atan(a * (1 - c) / λ) - a; }; auto eq = [λ, c](double a) { return atan(a * (1 - c) / λ) + atan(a * (1 + c) / λ) - 2 * a; }; // (eq. 3) // function starts positive, reaches a maximum then decreases to -∞ double a0 = 0; // Find a negative value by computing eq(2^n) double a1 = 1; while (eq(a1) > 0) a1 *= 2; // last computed values for f(a0) and f(a1) std::optional f_a0; double f_a1 = eq(a1); int n = 0; double a = 0; while (std::abs(a1 - a0) > 0.0000001 && n++ < 100) { if (not f_a0) { // use binary search a = 0.5 * (a0 + a1); double val = eq(a); if (val > 0) { a0 = a; f_a0 = val; } else { a1 = a; f_a1 = val; } } else { // f(a1) is always defined // when f(a0) is defined, use secant method a = a1 - f_a1 * (a1 - a0) / (f_a1 - *f_a0); a0 = a1; a1 = a; f_a0 = f_a1; f_a1 = eq(a); } } return {a, b(a)}; } static float angles_to_center(float e, float l, float r) { e = tan(e); l = tan(l); r = tan(r); float res = std::clamp((e - l) / (r - l) * 2 - 1, -1.f, 1.f); // If the center isn't in the FoV, fallback to middle of image if (std::isnan(res)) return 0; return res; } static void fill_param_2d( float c, size_t foveated_dim, size_t source_dim, std::vector & out) { float scale = float(foveated_dim) / source_dim; auto [a, b] = solve_foveation(scale, c); uint16_t last = 0; std::vector left; // index 0: 1:1 ratio, then 2:1 etc. std::vector right; for (size_t i = 1; i < foveated_dim; ++i) { double u = (i * 2.) / foveated_dim - 1; auto f = defoveate(a, b, scale, c, u); uint16_t n = std::clamp((f * 0.5 + 0.5) * source_dim + 0.5, 0, source_dim); assert(n > last); size_t count = n - last; auto & vec = u < c ? left : right; if (count > vec.size()) vec.resize(count); vec[count - 1]++; last = n; } assert(last < source_dim); size_t count = source_dim - last; if (count > right.size()) right.resize(count); right[count - 1]++; count = std::max(left.size(), right.size()); out.clear(); out.resize(count - left.size()); out.insert(out.end(), left.rbegin(), left.rend()); if (not right.empty()) out.back() += right.front(); if (right.size() > 1) out.insert(out.end(), right.begin() + 1, right.end()); out.resize(count * 2 - 1); } namespace wivrn { void foveation::compute_params() { for (size_t i = 0; i < 2; ++i) { const auto & fov = last.fovs[i]; size_t extent_w = std::abs(last.src[i].extent.w); if (foveated_size.width < extent_w) { const float center = angles_to_center(last.foveation_angles[i].x, fov.angle_left, fov.angle_right); fill_param_2d(center, foveated_size.width, extent_w, params[i].x); } else params[i].x = {uint16_t(extent_w)}; size_t extent_h = std::abs(last.src[i].extent.h); if (foveated_size.height < extent_h) { const float center = angles_to_center(last.foveation_angles[i].y, fov.angle_up, fov.angle_down); fill_param_2d(center, foveated_size.height, extent_h, params[i].y); } else params[i].y = {uint16_t(extent_h)}; } } foveation::foveation(wivrn::vk_bundle & bundle, vk::Extent3D foveated_size) : foveated_size(foveated_size), gpu_buffer( bundle.device, { .size = sizeof(ubo_data), .usage = vk::BufferUsageFlagBits::eTransferDst | vk::BufferUsageFlagBits::eStorageBuffer, }, VmaAllocationCreateInfo{ .flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT, .usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE, }, "foveation storage buffer"), sampler(make_sampler(bundle)), ds_layout(make_ds_layout(bundle)), layout(make_layout(bundle, ds_layout)), pipeline(make_pipelines(bundle, layout, foveated_size.width / 2)), descriptor_pool(make_ds_pool(bundle)), descriptor_set(bundle.device.allocateDescriptorSets(vk::DescriptorSetAllocateInfo{ .descriptorPool = descriptor_pool, .descriptorSetCount = 1, .pSetLayouts = &*ds_layout, })[0] .release()) { bundle.name(descriptor_set, "foveation descriptor set"); } void foveation::update_foveation(const from_headset::foveation & packet) { std::lock_guard lock(mutex); foveation_angles = packet.angles; } static void fill_ubo( std::span ubo, const std::vector & params, bool flip, size_t offset, size_t size, [[maybe_unused]] int count) { assert(params.size() % 2 == 1); const int n_ratio = (params.size() - 1) / 2; ubo[0] = offset; if (flip) ubo[0] += size; for (auto [i, n]: std::ranges::enumerate_view(params)) { const int n_source = std::abs(n_ratio - int(i)) + 1; for (size_t j = 0; j < n; ++j) { assert(count > 0); if (flip) ubo[1] = ubo[0] - n_source; else ubo[1] = ubo[0] + n_source; ubo = ubo.subspan(1); --count; } } if (not ubo.empty()) std::ranges::fill(ubo, ubo[0]); } template static bool operator==(const T & a, const T & b) { static_assert(std::has_unique_object_representations_v); return std::memcmp(&a, &b, sizeof(T)) == 0; } static bool operator==(const xrt_fov & a, const xrt_fov & b) { return a.angle_left == b.angle_left and a.angle_right == b.angle_right and a.angle_up == b.angle_up and a.angle_down == b.angle_down; } void foveation::update_ubo( vk::raii::CommandBuffer & cmd, bool flip_y, std::array src_rect, std::array src_fov) { // Check if the last value is still valid std::lock_guard lock(mutex); if (last.flip_y == flip_y and last.src[0] == src_rect[0] and last.src[1] == src_rect[1] and last.fovs[0] == src_fov[0] and last.fovs[1] == src_fov[1] and last.foveation_angles[0].x == foveation_angles[0].x and last.foveation_angles[0].y == foveation_angles[0].y and last.foveation_angles[1].x == foveation_angles[1].x and last.foveation_angles[1].y == foveation_angles[1].y) return; last = { .flip_y = flip_y, .src = {src_rect[0], src_rect[1]}, .fovs = {src_fov[0], src_fov[1]}, .foveation_angles = foveation_angles, }; compute_params(); ubo_data ubo; for (size_t view = 0; view < 2; ++view) { bool flip = false; size_t offset, extent; if (src_rect[view].extent.w < 0) { flip = true; offset = src_rect[view].offset.w + src_rect[view].extent.w; extent = -src_rect[view].extent.w; } else { offset = src_rect[view].offset.w; extent = src_rect[view].extent.w; } fill_ubo(std::span(ubo.x + view * RENDER_FOVEATION_BUFFER_DIMENSIONS, RENDER_FOVEATION_BUFFER_DIMENSIONS), params[view].x, flip, offset, extent, foveated_size.width); if (src_rect[view].extent.h < 0) { flip = not flip_y; offset = src_rect[view].offset.h + src_rect[view].extent.h; extent = -src_rect[view].extent.h; } else { flip = flip_y; offset = src_rect[view].offset.h; extent = src_rect[view].extent.h; } fill_ubo(std::span(ubo.y + view * RENDER_FOVEATION_BUFFER_DIMENSIONS, RENDER_FOVEATION_BUFFER_DIMENSIONS), params[view].y, flip, offset, extent, foveated_size.height); } vmaCopyMemoryToAllocation(vk_allocator::instance(), &ubo, gpu_buffer, 0, sizeof(ubo)); std::memcpy(gpu_buffer.data(), &ubo, sizeof(ubo)); } std::array foveation::foveate( vk::raii::Device & device, vk::raii::CommandBuffer & cmd, vk::ImageView y, vk::ImageView cbcr, bool flip_y, std::array src, std::array src_rect, std::array src_fov, bool alpha) { update_ubo(cmd, flip_y, src_rect, src_fov); std::array src_image_info{ vk::DescriptorImageInfo{ .sampler = *sampler, .imageView = src[0], .imageLayout = vk::ImageLayout::eShaderReadOnlyOptimal, }, vk::DescriptorImageInfo{ .sampler = *sampler, .imageView = src[1], .imageLayout = vk::ImageLayout::eShaderReadOnlyOptimal, }, }; vk::DescriptorBufferInfo ubo_info{ .buffer = gpu_buffer, .range = vk::WholeSize, }; vk::DescriptorImageInfo y_info{ .imageView = y, .imageLayout = vk::ImageLayout::eGeneral, }; vk::DescriptorImageInfo cbcr_info{ .imageView = cbcr, .imageLayout = vk::ImageLayout::eGeneral, }; std::array writes = { vk::WriteDescriptorSet{ .dstSet = descriptor_set, .dstBinding = 0, .descriptorCount = src_image_info.size(), .descriptorType = vk::DescriptorType::eCombinedImageSampler, .pImageInfo = src_image_info.data(), }, vk::WriteDescriptorSet{ .dstSet = descriptor_set, .dstBinding = 1, .descriptorCount = 1, .descriptorType = vk::DescriptorType::eStorageBuffer, .pBufferInfo = &ubo_info, }, vk::WriteDescriptorSet{ .dstSet = descriptor_set, .dstBinding = 2, .descriptorCount = 1, .descriptorType = vk::DescriptorType::eStorageImage, .pImageInfo = &y_info, }, vk::WriteDescriptorSet{ .dstSet = descriptor_set, .dstBinding = 3, .descriptorCount = 1, .descriptorType = vk::DescriptorType::eStorageImage, .pImageInfo = &cbcr_info, }, }; device.updateDescriptorSets(writes, {}); cmd.bindPipeline(vk::PipelineBindPoint::eCompute, *pipeline[alpha]); cmd.bindDescriptorSets(vk::PipelineBindPoint::eCompute, *layout, 0, descriptor_set, {}); cmd.dispatch(divide_and_round_up(foveated_size.width, 8), divide_and_round_up(foveated_size.height, 8), 2); return params; } } // namespace wivrn