/* * WiVRn VR streaming * Copyright (C) 2022 Guillaume Meunier * Copyright (C) 2022 Patrick Nicolas * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #define GLM_FORCE_RADIANS #include "stream.h" #include "application.h" #include "decoder/shard_accumulator.h" #include "spdlog/spdlog.h" #include "utils/ranges.h" #include "utils/sync_queue.h" #include "utils/named_thread.h" #include "vk/pipeline.h" #include "vk/shader.h" #include "wivrn_packets.h" #include #include #include #include #include "boost/pfr/core.hpp" #include "audio/audio.h" #include "hardware.h" using namespace xrt::drivers::wivrn; // clang-format off static const std::unordered_map device_ids = { {"/user/hand/left/input/x/click", device_id::X_CLICK}, {"/user/hand/left/input/x/touch", device_id::X_TOUCH}, {"/user/hand/left/input/y/click", device_id::Y_CLICK}, {"/user/hand/left/input/y/touch", device_id::Y_TOUCH}, {"/user/hand/left/input/menu/click", device_id::MENU_CLICK}, {"/user/hand/left/input/squeeze/value", device_id::LEFT_SQUEEZE_VALUE}, {"/user/hand/left/input/trigger/value", device_id::LEFT_TRIGGER_VALUE}, {"/user/hand/left/input/trigger/touch", device_id::LEFT_TRIGGER_TOUCH}, {"/user/hand/left/input/thumbstick", device_id::LEFT_THUMBSTICK_X}, {"/user/hand/left/input/thumbstick/click", device_id::LEFT_THUMBSTICK_CLICK}, {"/user/hand/left/input/thumbstick/touch", device_id::LEFT_THUMBSTICK_TOUCH}, {"/user/hand/left/input/thumbrest/touch", device_id::LEFT_THUMBREST_TOUCH}, {"/user/hand/right/input/a/click", device_id::A_CLICK}, {"/user/hand/right/input/a/touch", device_id::A_TOUCH}, {"/user/hand/right/input/b/click", device_id::B_CLICK}, {"/user/hand/right/input/b/touch", device_id::B_TOUCH}, {"/user/hand/right/input/system/click", device_id::SYSTEM_CLICK}, {"/user/hand/right/input/squeeze/value", device_id::RIGHT_SQUEEZE_VALUE}, {"/user/hand/right/input/trigger/value", device_id::RIGHT_TRIGGER_VALUE}, {"/user/hand/right/input/trigger/touch", device_id::RIGHT_TRIGGER_TOUCH}, {"/user/hand/right/input/thumbstick", device_id::RIGHT_THUMBSTICK_X}, {"/user/hand/right/input/thumbstick/click", device_id::RIGHT_THUMBSTICK_CLICK}, {"/user/hand/right/input/thumbstick/touch", device_id::RIGHT_THUMBSTICK_TOUCH}, {"/user/hand/right/input/thumbrest/touch", device_id::RIGHT_THUMBREST_TOUCH}, }; // clang-format on static const std::array supported_formats = { vk::Format::eR8G8B8A8Srgb, vk::Format::eB8G8R8A8Srgb }; std::shared_ptr scenes::stream::create(std::unique_ptr network_session, bool show_performance_metrics, bool enable_microphone) { std::shared_ptr self{new stream}; spdlog::info("decoder_mutex.native_handle() = {}", (void*)self->decoder_mutex.native_handle()); self->network_session = std::move(network_session); self->show_performance_metrics = show_performance_metrics; from_headset::headset_info_packet info{}; auto view = self->system.view_configuration_views(self->viewconfig)[0]; view = override_view(view, guess_model()); info.recommended_eye_width = view.recommendedImageRectWidth; info.recommended_eye_height = view.recommendedImageRectHeight; auto [flags, views] = self->session.locate_views( XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO, self->instance.now(), application::view()); assert(views.size() == info.fov.size()); for (auto [i, j]: utils::zip(views, info.fov)) { j = i.fov; } if (self->instance.has_extension(XR_FB_DISPLAY_REFRESH_RATE_EXTENSION_NAME)) { info.available_refresh_rates = self->session.get_refresh_rates(); info.preferred_refresh_rate = self->session.get_current_refresh_rate(); } if (info.available_refresh_rates.empty()) spdlog::warn("Unable to detect refresh rates"); info.hand_tracking = application::get_hand_tracking_supported(); audio::get_audio_description(info); if (not enable_microphone) info.microphone = {}; self->network_session->send_control(info); self->network_thread = utils::named_thread("network_thread", &stream::process_packets, self.get()); self->video_thread = utils::named_thread("video_thread", &stream::video, self.get()); self->command_buffer = std::move(self->device.allocateCommandBuffers({ .commandPool = *self->commandpool, .level = vk::CommandBufferLevel::ePrimary, .commandBufferCount = 1})[0]); self->fence = self->device.createFence({.flags = vk::FenceCreateFlagBits::eSignaled}); // Look up the XrActions for haptics self->haptics_actions[0].first = application::get_action("/user/hand/left/output/haptic").first; self->haptics_actions[0].second = application::string_to_path("/user/hand/left"); self->haptics_actions[1].first = application::get_action("/user/hand/right/output/haptic").first; self->haptics_actions[1].second = application::string_to_path("/user/hand/right"); // Look up the XrActions for input for (const auto & [action, action_type, name]: application::inputs()) { auto it = device_ids.find(name); if (it == device_ids.end()) continue; self->input_actions.emplace_back(it->second, action, action_type); } self->swapchain_format = vk::Format::eUndefined; spdlog::info("Supported swapchain formats:"); for (auto format: self->session.get_swapchain_formats()) { spdlog::info(" {}", vk::to_string(format)); } for (auto format: self->session.get_swapchain_formats()) { if (std::find(supported_formats.begin(), supported_formats.end(), format) != supported_formats.end()) { self->swapchain_format = format; break; } } if (self->swapchain_format == vk::Format::eUndefined) throw std::runtime_error("No supported swapchain format"); spdlog::info("Using format {}", vk::to_string(self->swapchain_format)); self->query_pool = vk::raii::QueryPool(self->device, vk::QueryPoolCreateInfo{.queryType = vk::QueryType::eTimestamp, .queryCount = size_gpu_timestamps}); return self; } void scenes::stream::on_focused() { if (show_performance_metrics) { swapchain_imgui = xr::swapchain( session, device, swapchain_format, 1500, 1000); imgui_ctx.emplace(physical_device, device, queue_family_index, queue, view_space, std::span{}, swapchain_imgui, glm::vec2{1.0, 0.6666}); imgui_ctx->set_position({0,0,-1}, {1,0,0,0}); } assert(video_stream_description); setup_reprojection_swapchain(); } void scenes::stream::on_unfocused() { imgui_ctx.reset(); swapchain_imgui = xr::swapchain(); } scenes::stream::~stream() { exit(); if (video_thread.joinable()) video_thread.join(); if (tracking_thread && tracking_thread->joinable()) tracking_thread->join(); if (network_thread.joinable()) network_thread.join(); } void scenes::stream::push_blit_handle(shard_accumulator * decoder, std::shared_ptr handle) { std::shared_ptr removed; if (!application::is_visible()) return; { std::unique_lock lock(decoder_mutex); for (auto & i: decoders) { if (i.decoder.get() == decoder) { std::swap(removed, i.latest_frames[0]); for (size_t j = 1 ; j < i.latest_frames.size() ; ++j) { std::swap(i.latest_frames[j-1], i.latest_frames[j]); } std::swap(i.latest_frames.back(), handle); break; } } if (state_ != state::streaming && std::all_of(decoders.begin(), decoders.end(), [](accumulator_images & i) { return i.latest_frames.back(); })) { state_ = state::streaming; first_frame_time = application::now(); spdlog::info("Stream scene ready at t={}", first_frame_time); } } if (removed and not removed->feedback.blitted) { send_feedback(removed->feedback); } } std::vector scenes::stream::accumulator_images::frames() const { std::vector result; for (const auto & frame: latest_frames) { if (frame) result.push_back(frame->feedback.frame_index); } return result; } std::optional scenes::stream::accumulator_images::common_frame(const std::vector & sets, uint64_t preferred_frame) { if (sets.empty()) return {}; auto common_frames = sets[0].frames(); for (const auto & set: sets) { std::vector tmp; auto x = set.frames(); std::set_intersection( x.begin(), x.end(), common_frames.begin(), common_frames.end(), std::back_inserter(tmp)); common_frames = tmp; if (common_frames.empty()) return {}; } assert(not common_frames.empty()); if (std::ranges::find(common_frames, preferred_frame) != common_frames.end()) return preferred_frame; return common_frames.back(); } std::shared_ptr scenes::stream::accumulator_images::frame(std::optional id) { for (auto it = latest_frames.rbegin(); it != latest_frames.rend(); ++it) { if (not *it) continue; if (id and (*it)->feedback.frame_index != *id) continue; return *it; } return nullptr; } void scenes::stream::render(XrTime predicted_display_time, bool should_render) { if (exiting) application::pop_scene(); if (decoders.empty()) should_render = false; if (!should_render) { // TODO: stop/restart video stream session.begin_frame(); session.end_frame(predicted_display_time, {}); std::unique_lock lock(decoder_mutex); for (auto & i: decoders) { for (auto & frame: i.latest_frames) frame.reset(); } return; } if (state_ == state::stalled) application::pop_scene(); assert(not swapchains.empty()); for (auto & i: decoders) { if (auto sampler = i.decoder->sampler(); sampler and not *i.blit_pipeline) { // Create blit pipeline // Create VkDescriptorSetLayout with an immutable sampler vk::DescriptorSetLayoutBinding sampler_layout_binding{ .binding = 0, .descriptorType = vk::DescriptorType::eCombinedImageSampler, .descriptorCount = 1, .stageFlags = vk::ShaderStageFlagBits::eFragment, .pImmutableSamplers = &sampler, }; vk::DescriptorSetLayoutCreateInfo layout_info{ .bindingCount = 1, .pBindings = &sampler_layout_binding, }; i.descriptor_set_layout = vk::raii::DescriptorSetLayout(device, layout_info); i.descriptor_set = device.allocateDescriptorSets( vk::DescriptorSetAllocateInfo{ .descriptorPool = *blit_descriptor_pool, .descriptorSetCount = 1, .pSetLayouts = &*i.descriptor_set_layout, })[0] .release(); const auto& description = i.decoder->desc(); vk::Extent2D image_size = i.decoder->image_size(); std::array useful_size{ float(description.width) / image_size.width, float(description.height) / image_size.height, }; spdlog::info("useful size: {}x{} with buffer {}x{}", description.width, description.height, image_size.width, image_size.height); std::array specialization_constants_desc{ vk::SpecializationMapEntry{ .constantID = 0, .offset = 0, .size = sizeof(float), }, vk::SpecializationMapEntry{ .constantID = 1, .offset = sizeof(float), .size = sizeof(float), }}; vk::SpecializationInfo specialization_info; specialization_info.setMapEntries(specialization_constants_desc); specialization_info.setData(useful_size); // Create graphics pipeline vk::raii::ShaderModule vertex_shader = load_shader(device, "stream.vert"); vk::raii::ShaderModule fragment_shader = load_shader(device, "stream.frag"); vk::PipelineLayoutCreateInfo pipeline_layout_info{ .setLayoutCount = 1, .pSetLayouts = &*i.descriptor_set_layout, }; i.blit_pipeline_layout = vk::raii::PipelineLayout(device, pipeline_layout_info); vk::pipeline_builder pipeline_info{ .flags = {}, .Stages = {{ .stage = vk::ShaderStageFlagBits::eVertex, .module = *vertex_shader, .pName = "main", .pSpecializationInfo = &specialization_info, }, { .stage = vk::ShaderStageFlagBits::eFragment, .module = *fragment_shader, .pName = "main", }}, .VertexInputState = {.flags = {}}, .VertexBindingDescriptions = {}, .VertexAttributeDescriptions = {}, .InputAssemblyState = {{ .topology = vk::PrimitiveTopology::eTriangleStrip, }}, .ViewportState = {.flags = {}}, .Viewports = {{}}, .Scissors = {{}}, .RasterizationState = {{ .polygonMode = vk::PolygonMode::eFill, .lineWidth = 1, }}, .MultisampleState = {{ .rasterizationSamples = vk::SampleCountFlagBits::e1, }}, .ColorBlendState = {.flags = {}}, .ColorBlendAttachments = {{.colorWriteMask = vk::ColorComponentFlagBits::eR | vk::ColorComponentFlagBits::eG | vk::ColorComponentFlagBits::eB}}, .DynamicState = {.flags = {}}, .DynamicStates = {vk::DynamicState::eViewport, vk::DynamicState::eScissor}, .layout = *i.blit_pipeline_layout, .renderPass = *blit_render_pass, .subpass = 0, }; i.blit_pipeline = vk::raii::Pipeline(device, application::get_pipeline_cache(), pipeline_info); } } if (device.waitForFences(*fence, VK_TRUE, UINT64_MAX) == vk::Result::eTimeout) throw std::runtime_error("Vulkan fence timeout"); device.resetFences(*fence); // We don't need those after vkWaitForFences current_blit_handles.clear(); gpu_timestamps timestamps; if (query_pool_filled) { auto [res, timestamps2] = query_pool.getResults( 0, size_gpu_timestamps, size_gpu_timestamps * sizeof(uint64_t), sizeof(uint64_t), vk::QueryResultFlagBits::e64 | vk::QueryResultFlagBits::eWait); if (res == vk::Result::eSuccess) { boost::pfr::for_each_field(timestamps, [n = 1, ×tamps2](float& t) mutable { t = (timestamps2[n++] - timestamps2[0]) * application::get_physical_device_properties().limits.timestampPeriod / 1e9; }); } } session.begin_frame(); auto [flags, views] = session.locate_views(viewconfig, predicted_display_time, world_space); assert(views.size() == swapchains.size()); std::array image_indices; assert(views.size() == view_count); for (size_t swapchain_index = 0; swapchain_index < views.size(); swapchain_index++) { int image_index = swapchains[swapchain_index].acquire(); swapchains[swapchain_index].wait(); image_indices[swapchain_index] = image_index; } command_buffer.reset(); vk::CommandBufferBeginInfo begin_info; begin_info.flags = vk::CommandBufferUsageFlagBits::eOneTimeSubmit; command_buffer.begin(begin_info); // Keep a reference to the resources needed to blit the images until vkWaitForFences std::vector> current_blit_handles; command_buffer.resetQueryPool(*query_pool, 0, size_gpu_timestamps); command_buffer.writeTimestamp(vk::PipelineStageFlagBits::eTopOfPipe, *query_pool, 0); std::array pose = {views[0].pose, views[1].pose}; std::array fov = {views[0].fov, views[1].fov}; { std::unique_lock lock(decoder_mutex); // Search for the most recent frame available on all decoders. // If no such frame exists, use the latest frame for each decoder auto common_frame = accumulator_images::common_frame(decoders, next_frame); if (common_frame) next_frame = *common_frame + 1; // Blit images from the decoders for (auto & i: decoders) { auto blit_handle = i.frame(common_frame); if (not blit_handle) continue; current_blit_handles.push_back(blit_handle); blit_handle->feedback.blitted = application::now(); if (blit_handle->feedback.blitted - blit_handle->feedback.received_from_decoder > 1'000'000'000) state_ = stream::state::stalled; blit_handle->feedback.displayed = predicted_display_time; blit_handle->feedback.real_pose[0] = views[0].pose; blit_handle->feedback.real_pose[1] = views[1].pose; pose = blit_handle->view_info.pose; fov = blit_handle->view_info.fov; vk::DescriptorImageInfo image_info{ .imageView = *blit_handle->image_view, .imageLayout = vk::ImageLayout::eGeneral, }; vk::WriteDescriptorSet descriptor_write{ .dstSet = i.descriptor_set, .dstBinding = 0, .dstArrayElement = 0, .descriptorCount = 1, .descriptorType = vk::DescriptorType::eCombinedImageSampler, .pImageInfo = &image_info, }; device.updateDescriptorSets(descriptor_write, {}); if (*blit_handle->current_layout != vk::ImageLayout::eGeneral) { vk::ImageMemoryBarrier barrier{ .srcAccessMask = vk::AccessFlagBits::eNone, .dstAccessMask = vk::AccessFlagBits::eMemoryRead, .oldLayout = *blit_handle->current_layout, .newLayout = vk::ImageLayout::eGeneral, .image = blit_handle->image, .subresourceRange = { .aspectMask = vk::ImageAspectFlagBits::eColor, .levelCount = 1, .layerCount = 1, }, }; command_buffer.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands, vk::PipelineStageFlagBits::eAllCommands, {}, {}, {}, barrier); *blit_handle->current_layout = vk::ImageLayout::eGeneral; } } } uint16_t x_offset = 0; for (auto & out: decoder_output) { command_buffer.beginRenderPass( { .renderPass = *blit_render_pass, .framebuffer = *out.frame_buffer, .renderArea = { .offset = {0, 0}, .extent = out.size, }, .clearValueCount = 0, }, vk::SubpassContents::eInline); for (const auto & decoder: decoders) { if (not *decoder.blit_pipeline) continue; command_buffer.bindPipeline(vk::PipelineBindPoint::eGraphics, *decoder.blit_pipeline); const auto & description = decoder.decoder->desc(); int x0 = description.offset_x - x_offset; int y0 = description.offset_y; int x1 = x0 + description.width; int y1 = y0 + description.height; vk::Viewport viewport{ .x = (float)x0, .y = (float)y0, .width = (float)description.width, .height = (float)description.height, .minDepth = 0, .maxDepth = 1}; x0 = std::clamp(x0, 0, out.size.width); x1 = std::clamp(x1, 0, out.size.width); y0 = std::clamp(y0, 0, out.size.height); y1 = std::clamp(y1, 0, out.size.height); vk::Rect2D scissor{ .offset = {.x = x0, .y = y0}, .extent = {.width = (uint32_t)(x1 - x0), .height = (uint32_t)(y1 - y0)}, }; command_buffer.setViewport(0, viewport); command_buffer.setScissor(0, scissor); command_buffer.bindDescriptorSets( vk::PipelineBindPoint::eGraphics, *decoder.blit_pipeline_layout, 0, decoder.descriptor_set, nullptr); command_buffer.draw(3, 1, 0, 0); } command_buffer.endRenderPass(); x_offset += out.size.width; } command_buffer.writeTimestamp(vk::PipelineStageFlagBits::eBottomOfPipe, *query_pool, 1); // Reproject the image to the real pose for (size_t view = 0; view < view_count; view++) { size_t destination_index = view * swapchains[0].images().size() + image_indices[view]; reprojector->reproject(command_buffer, view, destination_index, pose[view].orientation, fov[view], views[view].pose.orientation, views[view].fov); } command_buffer.writeTimestamp(vk::PipelineStageFlagBits::eBottomOfPipe, *query_pool, 2); command_buffer.end(); vk::SubmitInfo submit_info; submit_info.setCommandBuffers(*command_buffer); queue.submit(submit_info, *fence); std::vector layers_base; std::vector layer_view; layer_view.resize(views.size()); for (size_t swapchain_index = 0; swapchain_index < views.size(); swapchain_index++) { swapchains[swapchain_index].release(); layer_view[swapchain_index].type = XR_TYPE_COMPOSITION_LAYER_PROJECTION_VIEW; layer_view[swapchain_index].pose = views[swapchain_index].pose; layer_view[swapchain_index].fov = views[swapchain_index].fov; layer_view[swapchain_index].subImage.swapchain = swapchains[swapchain_index]; layer_view[swapchain_index].subImage.imageRect.offset = {0, 0}; layer_view[swapchain_index].subImage.imageRect.extent.width = swapchains[swapchain_index].width(); layer_view[swapchain_index].subImage.imageRect.extent.height = swapchains[swapchain_index].height(); } float brightness = std::clamp(dbrightness * (predicted_display_time - first_frame_time) / 1.e9, 0, 1); XrCompositionLayerColorScaleBiasKHR color_scale_bias{ .type = XR_TYPE_COMPOSITION_LAYER_COLOR_SCALE_BIAS_KHR, .colorScale = {brightness, brightness, brightness, 1}, .colorBias = {}}; XrCompositionLayerProjection layer{ .type = XR_TYPE_COMPOSITION_LAYER_PROJECTION, .layerFlags = 0, .space = world_space, .viewCount = (uint32_t)layer_view.size(), .views = layer_view.data(), }; if (instance.has_extension(XR_KHR_COMPOSITION_LAYER_COLOR_SCALE_BIAS_EXTENSION_NAME)) layer.next = &color_scale_bias; XrCompositionLayerQuad imgui_layer; if (imgui_ctx) { accumulate_metrics(predicted_display_time, current_blit_handles, timestamps); imgui_layer = plot_performance_metrics(predicted_display_time); } layers_base.push_back(reinterpret_cast(&layer)); if (imgui_ctx) layers_base.push_back(reinterpret_cast(&imgui_layer)); session.end_frame(/*timestamp*/ predicted_display_time, layers_base); // Network operations may be blocking, do them once everything was submitted for (const auto& handle: current_blit_handles) send_feedback(handle->feedback); read_actions(); query_pool_filled = true; } void scenes::stream::exit() { exiting = true; shard_queue.close(); } void scenes::stream::setup(const to_headset::video_stream_description & description) { std::unique_lock lock(decoder_mutex); // FIXME: stop video thread decoders.clear(); if (description.items.empty()) { spdlog::info("Stopping video stream"); return; } video_stream_description = description; const uint32_t video_width = description.width / view_count; const uint32_t video_height = description.height; // Create renderpass { vk::AttachmentDescription color_desc{ .format = vk::Format::eA8B8G8R8SrgbPack32, .samples = vk::SampleCountFlagBits::e1, .loadOp = vk::AttachmentLoadOp::eDontCare, .storeOp = vk::AttachmentStoreOp::eStore, .initialLayout = vk::ImageLayout::eUndefined, .finalLayout = vk::ImageLayout::eShaderReadOnlyOptimal, }; vk::AttachmentReference color_ref{ .attachment = 0, .layout = vk::ImageLayout::eColorAttachmentOptimal, }; vk::SubpassDescription subpass{ .pipelineBindPoint = vk::PipelineBindPoint::eGraphics, }; subpass.setColorAttachments(color_ref); vk::RenderPassCreateInfo renderpass_info{ .flags = {}, }; renderpass_info.setAttachments(color_desc); renderpass_info.setSubpasses(subpass); blit_render_pass = vk::raii::RenderPass(device, renderpass_info); } // Create outputs for the decoders vk::Extent3D decoder_out_size{video_width, video_height, 1}; for (size_t i = 0; i < view_count; i++) { decoder_output[i].format = vk::Format::eA8B8G8R8SrgbPack32; decoder_output[i].size.width = video_width; decoder_output[i].size.height = video_height; vk::ImageCreateInfo image_info{ .flags = vk::ImageCreateFlags{}, .imageType = vk::ImageType::e2D, .format = vk::Format::eA8B8G8R8SrgbPack32, .extent = decoder_out_size, .mipLevels = 1, .arrayLayers = 1, .samples = vk::SampleCountFlagBits::e1, .tiling = vk::ImageTiling::eOptimal, .usage = vk::ImageUsageFlagBits::eSampled | vk::ImageUsageFlagBits::eColorAttachment, .sharingMode = vk::SharingMode::eExclusive, .initialLayout = vk::ImageLayout::eUndefined, }; VmaAllocationCreateInfo alloc_info{ .requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, }; decoder_output[i].image = image_allocation{device, image_info, alloc_info}; vk::ImageViewCreateInfo image_view_info{ .image = vk::Image{decoder_output[i].image}, .viewType = vk::ImageViewType::e2D, .format = vk::Format::eA8B8G8R8SrgbPack32, .components = {}, .subresourceRange = { .aspectMask = vk::ImageAspectFlagBits::eColor, .baseMipLevel = 0, .levelCount = 1, .baseArrayLayer = 0, .layerCount = 1, }, }; decoder_output[i].image_view = vk::raii::ImageView(device, image_view_info); decoder_output[i].frame_buffer = vk::raii::Framebuffer( device, vk::FramebufferCreateInfo{ .renderPass = *blit_render_pass, .attachmentCount = 1, .pAttachments = &*decoder_output[i].image_view, .width = decoder_out_size.width, .height = decoder_out_size.height, .layers = 1, }); } { vk::DescriptorPoolSize pool_size{ .type = vk::DescriptorType::eCombinedImageSampler, .descriptorCount = uint32_t(description.items.size()), }; blit_descriptor_pool = vk::raii::DescriptorPool( device, vk::DescriptorPoolCreateInfo{ .maxSets = uint32_t(description.items.size()), .poolSizeCount = 1, .pPoolSizes = &pool_size, }); } for (const auto & [stream_index, item]: utils::enumerate(description.items)) { spdlog::info("Creating decoder size {}x{} offset {},{}", item.width, item.height, item.offset_x, item.offset_y); try { session.set_refresh_rate(description.fps); } catch (std::exception & e) { spdlog::warn("Failed to set refresh rate to {}: {}", description.fps, e.what()); } accumulator_images dec; dec.decoder = std::make_unique(device, physical_device, item, description.fps, shared_from_this(), stream_index); decoders.push_back(std::move(dec)); } } void scenes::stream::setup_reprojection_swapchain() { std::unique_lock lock(decoder_mutex); swapchains.clear(); const uint32_t video_width = video_stream_description->width / view_count; const uint32_t video_height = video_stream_description->height; const uint32_t swapchain_width = video_width / video_stream_description->foveation[0].x.scale; const uint32_t swapchain_height = video_height / video_stream_description->foveation[0].y.scale; auto views = system.view_configuration_views(viewconfig); swapchains.reserve(views.size()); for (auto view: views) { // Create a swapchain larger than video strem so that reprojection and foveation affect image quality less. const double max_oversample = std::min(double(view.maxImageRectWidth) / swapchain_width, double(view.maxImageRectHeight) / swapchain_height); double oversample = std::min(1.4, max_oversample); XrExtent2Di extent{ .width = int32_t(swapchain_width * oversample), .height = int32_t(swapchain_height * oversample), }; swapchains.emplace_back(session, device, swapchain_format, extent.width, extent.height); spdlog::info("Created stream swapchain {}: {}x{}", swapchains.size(), extent.width, extent.height); } spdlog::info("Initializing reprojector"); vk::Extent2D extent = {(uint32_t)swapchains[0].width(), (uint32_t)swapchains[0].height()}; std::vector swapchain_images; for (auto & swapchain: swapchains) { for (auto & image: swapchain.images()) swapchain_images.push_back(image.image); } std::vector images; for (renderpass_output & i: decoder_output) { images.push_back(i.image); } reprojector.emplace(device, physical_device, images, swapchain_images, extent, swapchains[0].format(), *video_stream_description); } void scenes::stream::video() { #ifdef __ANDROID__ application::instance().setup_jni(); #endif while (not exiting) { try { auto shard = shard_queue.pop(); if (shard.stream_item_idx >= decoders.size()) { // We don't know (yet?) about this stream, ignore packet return; } auto idx = shard.stream_item_idx; decoders[idx].decoder->push_shard(std::move(shard)); } catch (utils::sync_queue_closed &) { break; } catch (std::exception & e) { spdlog::error("Exception in video thread: {}", e.what()); exit(); } } } scene::meta& scenes::stream::get_meta_scene() { static meta m{ "Stream", {} }; return m; }