diff --git a/client/application.cpp b/client/application.cpp index cbbbd154..eda4471b 100644 --- a/client/application.cpp +++ b/client/application.cpp @@ -809,21 +809,6 @@ void application::initialize() vk_cmdpool = vk::raii::CommandPool{vk_device, cmdpool_create_info}; - // Create timestamp queries - query_pool = vk::raii::QueryPool(vk_device, vk::QueryPoolCreateInfo{.queryType = vk::QueryType::eTimestamp, .queryCount = 6}); - query_command_buffers = vk_device.allocateCommandBuffers(vk::CommandBufferAllocateInfo{.commandPool = *vk_cmdpool, .commandBufferCount = 6}); - for(int i = 0; i < 6; i+=2) - { - query_command_buffers[i].begin({}); - query_command_buffers[i].resetQueryPool(*query_pool, i, 2); - query_command_buffers[i].writeTimestamp(vk::PipelineStageFlagBits::eTopOfPipe, *query_pool, i); - query_command_buffers[i].end(); - - query_command_buffers[i+1].begin({}); - query_command_buffers[i+1].writeTimestamp(vk::PipelineStageFlagBits::eBottomOfPipe, *query_pool, i+1); - query_command_buffers[i+1].end(); - } - initialize_actions(); interaction_profile_changed(); @@ -1041,27 +1026,11 @@ void application::loop() auto t1 = std::chrono::steady_clock::now(); - vk_queue.submit(vk::SubmitInfo{ - .commandBufferCount = 1, - .pCommandBuffers = &*query_command_buffers[(current_query%3) * 2], - }, nullptr); - scene->render(framestate.predictedDisplayTime, framestate.shouldRender); last_scene_cpu_time = std::chrono::steady_clock::now() - t1; - vk_queue.submit(vk::SubmitInfo{ - .commandBufferCount = 1, - .pCommandBuffers = &*query_command_buffers[(current_query%3) * 2 + 1], - }, nullptr); - current_query = current_query + 1; - if (current_query > 3) - { - auto [res, timestamps] = query_pool.getResults((current_query%3)*2, 2, 2 * sizeof(uint64_t), sizeof(uint64_t), vk::QueryResultFlagBits::e64 | vk::QueryResultFlagBits::eWait); - if (res == vk::Result::eSuccess) - last_scene_gpu_time = std::chrono::nanoseconds((int)((timestamps[1] - timestamps[0]) * physical_device_properties.limits.timestampPeriod)); - } } else { diff --git a/client/application.h b/client/application.h index 91d41c09..9467a211 100644 --- a/client/application.h +++ b/client/application.h @@ -97,11 +97,6 @@ class application : public singleton // Vulkan memory allocator stuff std::optional allocator; - // Timestamp queries - vk::raii::QueryPool query_pool = nullptr; - std::vector query_command_buffers; - int current_query = 0; - // OpenXR stuff void initialize_actions(); @@ -350,6 +345,11 @@ public: return instance().vk_physical_device; } + static const vk::PhysicalDeviceProperties& get_physical_device_properties() + { + return instance().physical_device_properties; + } + static vk::raii::Device& get_device() { return instance().vk_device; diff --git a/client/decoder/android/android_decoder.cpp b/client/decoder/android/android_decoder.cpp index 8f0ba8b6..46e5188c 100644 --- a/client/decoder/android/android_decoder.cpp +++ b/client/decoder/android/android_decoder.cpp @@ -39,225 +39,12 @@ DEUGLIFY(AMediaFormat) -struct wivrn::android::decoder::pipeline_context -{ - vk::raii::Device& device; - vk::AndroidHardwareBufferFormatPropertiesANDROID ahb_format; - - vk::raii::SamplerYcbcrConversion ycbcr_conversion = nullptr; - vk::raii::Sampler sampler = nullptr; - - vk::raii::DescriptorSetLayout descriptor_set_layout = nullptr; - vk::raii::DescriptorPool descriptor_pool = nullptr; - std::mutex descriptor_pool_mutex; - vk::raii::PipelineLayout layout = nullptr; - vk::raii::Pipeline pipeline = nullptr; - - pipeline_context(vk::raii::Device& device, const AHardwareBuffer_Desc& buffer_desc, vk::AndroidHardwareBufferFormatPropertiesANDROID & ahb_format, vk::raii::RenderPass& renderpass, const to_headset::video_stream_description::item& description) : - device(device), ahb_format(ahb_format) - { - spdlog::info("descriptor_pool_mutex.native_handle() = {}", (void*)descriptor_pool_mutex.native_handle()); - - assert(ahb_format.externalFormat != 0); - spdlog::info("AndroidHardwareBufferProperties"); - spdlog::info(" Vulkan format: {}", vk::to_string(ahb_format.format)); - spdlog::info(" External format: {:#x}", ahb_format.externalFormat); - spdlog::info(" Format features: {}", vk::to_string(ahb_format.formatFeatures)); - spdlog::info(" samplerYcbcrConversionComponents: ({}, {}, {}, {})", - vk::to_string(ahb_format.samplerYcbcrConversionComponents.r), - vk::to_string(ahb_format.samplerYcbcrConversionComponents.g), - vk::to_string(ahb_format.samplerYcbcrConversionComponents.b), - vk::to_string(ahb_format.samplerYcbcrConversionComponents.a)); - spdlog::info(" Suggested YCbCr model: {}", vk::to_string(ahb_format.suggestedYcbcrModel)); - spdlog::info(" Suggested YCbCr range: {}", vk::to_string(ahb_format.suggestedYcbcrRange)); - spdlog::info(" Suggested X chroma offset: {}", vk::to_string(ahb_format.suggestedXChromaOffset)); - spdlog::info(" Suggested Y chroma offset: {}", vk::to_string(ahb_format.suggestedYChromaOffset)); - - vk::Filter yuv_filter; - if (ahb_format.formatFeatures & vk::FormatFeatureFlagBits::eSampledImageYcbcrConversionLinearFilter) - yuv_filter = vk::Filter::eLinear; - else - yuv_filter = vk::Filter::eNearest; - - // Create VkSamplerYcbcrConversion - vk::StructureChain ycbcr_create_info - { - vk::SamplerYcbcrConversionCreateInfo{ - .format = vk::Format::eUndefined, - .ycbcrModel = ahb_format.suggestedYcbcrModel, - .ycbcrRange = ahb_format.suggestedYcbcrRange, - .components = ahb_format.samplerYcbcrConversionComponents, - .xChromaOffset = ahb_format.suggestedXChromaOffset, - .yChromaOffset = ahb_format.suggestedYChromaOffset, - .chromaFilter = yuv_filter, - }, - vk::ExternalFormatANDROID{ - .externalFormat = ahb_format.externalFormat, - }, - }; - - - - // suggested values from decoder don't actually read the metadata, so it's garbage - if (description.range) - ycbcr_create_info.get().ycbcrRange = vk::SamplerYcbcrRange(*description.range); - - if (description.color_model) - ycbcr_create_info.get().ycbcrModel = vk::SamplerYcbcrModelConversion(*description.color_model); - - ycbcr_conversion = vk::raii::SamplerYcbcrConversion(device, ycbcr_create_info.get()); - - // Create VkSampler - vk::StructureChain sampler_info{ - vk::SamplerCreateInfo{ - .magFilter = yuv_filter, - .minFilter = yuv_filter, - .mipmapMode = vk::SamplerMipmapMode::eNearest, - .addressModeU = vk::SamplerAddressMode::eClampToEdge, - .addressModeV = vk::SamplerAddressMode::eClampToEdge, - .addressModeW = vk::SamplerAddressMode::eClampToEdge, - .mipLodBias = 0.0f, - .anisotropyEnable = VK_FALSE, - .maxAnisotropy = 1, - .compareEnable = VK_FALSE, - .compareOp = vk::CompareOp::eNever, - .minLod = 0.0f, - .maxLod = 0.0f, - .borderColor = vk::BorderColor::eFloatOpaqueWhite, // TODO TBC - .unnormalizedCoordinates = VK_FALSE, - }, - vk::SamplerYcbcrConversionInfo{ - .conversion = *ycbcr_conversion, - } - }; - - - sampler = vk::raii::Sampler(device, sampler_info.get()); - - - // Create VkDescriptorSetLayout with an immutable sampler - vk::DescriptorSetLayoutBinding sampler_layout_binding{ - .binding = 0, - .descriptorType = vk::DescriptorType::eCombinedImageSampler, - .descriptorCount = 1, - .stageFlags = vk::ShaderStageFlagBits::eFragment, - }; - sampler_layout_binding.setImmutableSamplers(*sampler); - - vk::DescriptorSetLayoutCreateInfo layout_info; - layout_info.setBindings(sampler_layout_binding); - - descriptor_set_layout = vk::raii::DescriptorSetLayout(device, layout_info); - - - vk::DescriptorPoolSize pool_size{ - .type = vk::DescriptorType::eCombinedImageSampler, - .descriptorCount = 100, - }; - - vk::DescriptorPoolCreateInfo pool_info{ - .flags = vk::DescriptorPoolCreateFlagBits::eFreeDescriptorSet, - .maxSets = pool_size.descriptorCount, - }; - pool_info.setPoolSizes(pool_size); - - descriptor_pool = vk::raii::DescriptorPool(device, pool_info); - - std::array useful_size = - { - float(description.width) / buffer_desc.width, - float(description.height) / buffer_desc.height - }; - spdlog::info("useful size: {}x{} with buffer {}x{}", - description.width, description.height, - buffer_desc.width, buffer_desc.height); - - std::array specialization_constants_desc{ - vk::SpecializationMapEntry{ - .constantID = 0, - .offset = 0, - .size = sizeof(float), - }, - vk::SpecializationMapEntry{ - .constantID = 1, - .offset = sizeof(float), - .size = sizeof(float), - } - }; - - vk::SpecializationInfo specialization_info; - specialization_info.setMapEntries(specialization_constants_desc); - specialization_info.setData(useful_size); - - // Create graphics pipeline - vk::raii::ShaderModule vertex_shader = load_shader(device, "stream.vert"); - vk::raii::ShaderModule fragment_shader = load_shader(device, "stream.frag"); - - - vk::PipelineLayoutCreateInfo pipeline_layout_info; - pipeline_layout_info.setSetLayouts(*descriptor_set_layout); - - layout = vk::raii::PipelineLayout(device, pipeline_layout_info); - - vk::pipeline_builder pipeline_info - { - .flags = {}, - .Stages = {{ - .stage = vk::ShaderStageFlagBits::eVertex, - .module = *vertex_shader, - .pName = "main", - .pSpecializationInfo = &specialization_info, - },{ - .stage = vk::ShaderStageFlagBits::eFragment, - .module = *fragment_shader, - .pName = "main", - }}, - .VertexInputState = {.flags = {}}, - .VertexBindingDescriptions = {}, - .VertexAttributeDescriptions = {}, - .InputAssemblyState = {{ - .topology = vk::PrimitiveTopology::eTriangleStrip, - }}, - .ViewportState = {.flags = {}}, - .RasterizationState = {{ - .polygonMode = vk::PolygonMode::eFill, - .lineWidth = 1, - }}, - .MultisampleState = {{ - .rasterizationSamples = vk::SampleCountFlagBits::e1, - }}, - .ColorBlendState = {.flags = {}}, - .ColorBlendAttachments = {{ - .colorWriteMask = vk::ColorComponentFlagBits::eR | vk::ColorComponentFlagBits::eG | vk::ColorComponentFlagBits::eB - }}, - .DynamicState = {.flags={}}, - .DynamicStates = {vk::DynamicState::eViewport, vk::DynamicState::eScissor}, - .layout = *layout, - .renderPass = *renderpass, - .subpass = 0, - }; - - pipeline = vk::raii::Pipeline(device, application::get_pipeline_cache(), pipeline_info); - } -}; - struct wivrn::android::decoder::mapped_hardware_buffer { - std::shared_ptr pipeline; vk::raii::DeviceMemory memory = nullptr; vk::raii::Image vimage = nullptr; vk::raii::ImageView image_view = nullptr; - vk::raii::DescriptorSet descriptor_set = nullptr; vk::ImageLayout layout = vk::ImageLayout::eUndefined; - - ~mapped_hardware_buffer() - { - assert(pipeline); - { - std::unique_lock lock(pipeline->descriptor_pool_mutex); - descriptor_set = nullptr; - } - } }; namespace @@ -489,54 +276,6 @@ decoder::~decoder() spdlog::info("decoder::~decoder"); } -void decoder::set_blit_targets(std::vector targets, vk::Format format) -{ - // Create renderpass - vk::AttachmentDescription color_desc{ - .format = format, - .samples = vk::SampleCountFlagBits::e1, - .loadOp = vk::AttachmentLoadOp::eLoad, - .storeOp = vk::AttachmentStoreOp::eStore, - .initialLayout = vk::ImageLayout::eColorAttachmentOptimal, - .finalLayout = vk::ImageLayout::eColorAttachmentOptimal, - }; - - vk::AttachmentReference color_ref{ - .attachment = 0, - .layout = vk::ImageLayout::eColorAttachmentOptimal, - }; - - vk::SubpassDescription subpass{ - .pipelineBindPoint = vk::PipelineBindPoint::eGraphics, - }; - subpass.setColorAttachments(color_ref); - - vk::RenderPassCreateInfo renderpass_info{ - .flags = {}, - - }; - renderpass_info.setAttachments(color_desc); - renderpass_info.setSubpasses(subpass); - - renderpass = vk::raii::RenderPass(device, renderpass_info); - - blit_targets = std::move(targets); - - for (auto & i: blit_targets) - { - vk::FramebufferCreateInfo fb_create_info{ - .renderPass = *renderpass, - .attachmentCount = 1, - .pAttachments = &i.image_view, - .width = i.extent.width, - .height = i.extent.height, - .layers = 1 - }; - - i.framebuffer = std::make_shared(device, fb_create_info); - } -} - void decoder::push_data(std::span> data, uint64_t frame_index, bool partial) { if (!media_codec) @@ -652,12 +391,17 @@ void decoder::on_image_available(AImageReader * reader) info->feedback.received_from_decoder = application::now(); assert(info->feedback.frame_index == frame_index); - auto handle = std::make_shared(); - handle->feedback = info->feedback; - handle->timing_info = info->timing_info; - handle->view_info = info->view_info; - handle->vk_data = map_hardware_buffer(image); - handle->aimage = image; + auto vk_data = map_hardware_buffer(image); + + auto handle = std::make_shared( + info->feedback, + info->timing_info, + info->view_info, + vk_data->image_view, + *vk_data->vimage, + &vk_data->layout, + vk_data, + image); if (auto scene = weak_scene.lock()) scene->push_blit_handle(accumulator, std::move(handle)); @@ -669,6 +413,83 @@ void decoder::on_image_available(AImageReader * reader) } } +void decoder::create_sampler(const AHardwareBuffer_Desc& buffer_desc, vk::AndroidHardwareBufferFormatPropertiesANDROID & ahb_format) +{ + assert(ahb_format.externalFormat != 0); + spdlog::info("AndroidHardwareBufferProperties"); + spdlog::info(" Vulkan format: {}", vk::to_string(ahb_format.format)); + spdlog::info(" External format: {:#x}", ahb_format.externalFormat); + spdlog::info(" Format features: {}", vk::to_string(ahb_format.formatFeatures)); + spdlog::info(" samplerYcbcrConversionComponents: ({}, {}, {}, {})", + vk::to_string(ahb_format.samplerYcbcrConversionComponents.r), + vk::to_string(ahb_format.samplerYcbcrConversionComponents.g), + vk::to_string(ahb_format.samplerYcbcrConversionComponents.b), + vk::to_string(ahb_format.samplerYcbcrConversionComponents.a)); + spdlog::info(" Suggested YCbCr model: {}", vk::to_string(ahb_format.suggestedYcbcrModel)); + spdlog::info(" Suggested YCbCr range: {}", vk::to_string(ahb_format.suggestedYcbcrRange)); + spdlog::info(" Suggested X chroma offset: {}", vk::to_string(ahb_format.suggestedXChromaOffset)); + spdlog::info(" Suggested Y chroma offset: {}", vk::to_string(ahb_format.suggestedYChromaOffset)); + + vk::Filter yuv_filter; + if (ahb_format.formatFeatures & vk::FormatFeatureFlagBits::eSampledImageYcbcrConversionLinearFilter) + yuv_filter = vk::Filter::eLinear; + else + yuv_filter = vk::Filter::eNearest; + + // Create VkSamplerYcbcrConversion + vk::StructureChain ycbcr_create_info + { + vk::SamplerYcbcrConversionCreateInfo{ + .format = vk::Format::eUndefined, + .ycbcrModel = ahb_format.suggestedYcbcrModel, + .ycbcrRange = ahb_format.suggestedYcbcrRange, + .components = ahb_format.samplerYcbcrConversionComponents, + .xChromaOffset = ahb_format.suggestedXChromaOffset, + .yChromaOffset = ahb_format.suggestedYChromaOffset, + .chromaFilter = yuv_filter, + }, + vk::ExternalFormatANDROID{ + .externalFormat = ahb_format.externalFormat, + }, + }; + + // suggested values from decoder don't actually read the metadata, so it's garbage + if (description.range) + ycbcr_create_info.get().ycbcrRange = vk::SamplerYcbcrRange(*description.range); + + if (description.color_model) + ycbcr_create_info.get().ycbcrModel = vk::SamplerYcbcrModelConversion(*description.color_model); + + ycbcr_conversion = vk::raii::SamplerYcbcrConversion(device, ycbcr_create_info.get()); + + // Create VkSampler + vk::StructureChain sampler_info{ + vk::SamplerCreateInfo{ + .magFilter = yuv_filter, + .minFilter = yuv_filter, + .mipmapMode = vk::SamplerMipmapMode::eNearest, + .addressModeU = vk::SamplerAddressMode::eClampToEdge, + .addressModeV = vk::SamplerAddressMode::eClampToEdge, + .addressModeW = vk::SamplerAddressMode::eClampToEdge, + .mipLodBias = 0.0f, + .anisotropyEnable = VK_FALSE, + .maxAnisotropy = 1, + .compareEnable = VK_FALSE, + .compareOp = vk::CompareOp::eNever, + .minLod = 0.0f, + .maxLod = 0.0f, + .borderColor = vk::BorderColor::eFloatOpaqueWhite, // TODO TBC + .unnormalizedCoordinates = VK_FALSE, + }, + vk::SamplerYcbcrConversionInfo{ + .conversion = *ycbcr_conversion, + } + }; + + + ycbcr_sampler = vk::raii::Sampler(device, sampler_info.get()); +} + std::shared_ptr decoder::map_hardware_buffer(AImage * image) { AHardwareBuffer * hardware_buffer; @@ -681,11 +502,13 @@ std::shared_ptr decoder::map_hardware_buffer(AI auto [properties, format_properties] = device.getAndroidHardwareBufferPropertiesANDROID(*hardware_buffer); - if (!pipeline || memcmp(&pipeline->ahb_format, &format_properties, sizeof(format_properties))) + if (!*ycbcr_sampler || memcmp(&ahb_format, &format_properties, sizeof(format_properties))) { - pipeline.reset(); - pipeline = std::make_shared(device, buffer_desc, format_properties, renderpass, description); + memcpy(&ahb_format, &format_properties, sizeof(format_properties)); + extent = {buffer_desc.width, buffer_desc.height}; + create_sampler(buffer_desc, ahb_format); hardware_buffer_map.clear(); + // TODO tell the reprojector to recreate the pipeline } auto it = hardware_buffer_map.find(hardware_buffer); @@ -750,49 +573,18 @@ std::shared_ptr decoder::map_hardware_buffer(AI } }, vk::SamplerYcbcrConversionInfo{ - .conversion = *pipeline->ycbcr_conversion + .conversion = *ycbcr_conversion } }; - application::ignore_debug_reports_for(*vimage); vk::raii::ImageView image_view(device, iv_info.get()); application::unignore_debug_reports_for(*vimage); - vk::raii::DescriptorSet descriptor_set = [&]{ - std::unique_lock lock(pipeline->descriptor_pool_mutex); - - vk::raii::DescriptorSets descriptor_sets(device, { - .descriptorPool = *pipeline->descriptor_pool, - .descriptorSetCount = 1, - .pSetLayouts = &*pipeline->descriptor_set_layout, - }); - - return std::move(descriptor_sets[0]); - }(); - - vk::DescriptorImageInfo image_info{ - .imageView = *image_view, - .imageLayout = vk::ImageLayout::eGeneral, - }; - - vk::WriteDescriptorSet descriptor_write{ - .dstSet = *descriptor_set, - .dstBinding = 0, - .dstArrayElement = 0, - .descriptorCount = 1, - .descriptorType = vk::DescriptorType::eCombinedImageSampler, - .pImageInfo = &image_info, - }; - - device.updateDescriptorSets(descriptor_write, {}); - auto handle = std::make_shared(); - handle->pipeline = pipeline; handle->vimage = std::move(vimage); handle->image_view = std::move(image_view); handle->memory = std::move(memory); - handle->descriptor_set = std::move(descriptor_set); hardware_buffer_map[hardware_buffer] = handle; return handle; @@ -818,87 +610,6 @@ void decoder::on_media_output_available(AMediaCodec * media_codec, void * userda // will be consumed by dedicated thread } -void decoder::blit(vk::raii::CommandBuffer& command_buffer, blit_handle & handle, std::span target_indices) -{ - if (handle.vk_data->layout != vk::ImageLayout::eGeneral) - { - vk::ImageMemoryBarrier memory_barrier{ - .srcAccessMask = {}, - .dstAccessMask = vk::AccessFlagBits::eShaderRead, - .oldLayout = vk::ImageLayout::eUndefined, - .newLayout = vk::ImageLayout::eGeneral, - .image = *handle.vk_data->vimage, - .subresourceRange = { - .aspectMask = vk::ImageAspectFlagBits::eColor, - .baseMipLevel = 0, - .levelCount = 1, - .baseArrayLayer = 0, - .layerCount = 1, - } - }; - - command_buffer.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands, vk::PipelineStageFlagBits::eAllCommands, {}, {}, {}, memory_barrier); - - handle.vk_data->layout = vk::ImageLayout::eGeneral; - } - - for (size_t target_index: target_indices) - { - assert(target_index < blit_targets.size()); - - auto & target = blit_targets[target_index]; - if (description.offset_x > target.offset.x + target.extent.width) - continue; - if (description.offset_x + description.width < target.offset.x) - continue; - - int x0 = description.offset_x - target.offset.x; - int y0 = description.offset_y; - int x1 = x0 + description.width; - int y1 = y0 + description.height; - - vk::RenderPassBeginInfo begin_info{ - .renderPass = *renderpass, - .framebuffer = **target.framebuffer, - .renderArea = { - .offset = {0, 0}, - .extent = target.extent, - }, - .clearValueCount = 0, - }; - - command_buffer.beginRenderPass(begin_info, vk::SubpassContents::eInline); - command_buffer.bindPipeline(vk::PipelineBindPoint::eGraphics, *handle.vk_data->pipeline->pipeline); - - vk::Viewport viewport{ - .x = (float)x0, - .y = (float)y0, - .width = (float)description.width, - .height = (float)description.height, - .minDepth = 0, - .maxDepth = 1 - }; - - x0 = std::clamp(x0, 0, target.extent.width); - x1 = std::clamp(x1, 0, target.extent.width); - y0 = std::clamp(y0, 0, target.extent.height); - y1 = std::clamp(y1, 0, target.extent.height); - - vk::Rect2D scissor{ - .offset = {.x = x0, .y = y0}, - .extent = {.width = (uint32_t)(x1 - x0), .height = (uint32_t)(y1 - y0)}, - }; - - command_buffer.setViewport(0, viewport); - command_buffer.setScissor(0, scissor); - - command_buffer.bindDescriptorSets(vk::PipelineBindPoint::eGraphics, *handle.vk_data->pipeline->layout, 0, *handle.vk_data->descriptor_set, {}); - command_buffer.draw(3, 1, 0, 0); - - command_buffer.endRenderPass(); - } -} - decoder::blit_handle::~blit_handle() { AImage_delete(aimage); diff --git a/client/decoder/android/android_decoder.h b/client/decoder/android/android_decoder.h index 4aa0fe42..7e02b57b 100644 --- a/client/decoder/android/android_decoder.h +++ b/client/decoder/android/android_decoder.h @@ -61,19 +61,6 @@ namespace wivrn::android class decoder { public: - struct pipeline_context; - - struct blit_target - { - vk::Image image; - vk::ImageView image_view; - vk::Offset2D offset; - vk::Extent2D extent; - - std::shared_ptr framebuffer; - std::shared_ptr pipeline; - }; - struct mapped_hardware_buffer; struct blit_handle @@ -81,6 +68,9 @@ public: xrt::drivers::wivrn::from_headset::feedback feedback; xrt::drivers::wivrn::to_headset::video_stream_data_shard::timing_info_t timing_info; xrt::drivers::wivrn::to_headset::video_stream_data_shard::view_info_t view_info; + vk::raii::ImageView& image_view; + vk::Image image = nullptr; + vk::ImageLayout * current_layout = nullptr; std::shared_ptr vk_data; @@ -95,6 +85,11 @@ private: vk::raii::Device& device; + vk::AndroidHardwareBufferFormatPropertiesANDROID ahb_format; + vk::raii::SamplerYcbcrConversion ycbcr_conversion = nullptr; + vk::raii::Sampler ycbcr_sampler = nullptr; + vk::Extent2D extent{}; + std::mutex hbm_mutex; AImageReader_ptr image_reader; @@ -126,12 +121,10 @@ private: void push_nals(std::span> data, int64_t timestamp, uint32_t flags); - std::vector blit_targets; - - std::shared_ptr pipeline; std::unordered_map> hardware_buffer_map; vk::raii::RenderPass renderpass = nullptr; + void create_sampler(const AHardwareBuffer_Desc& buffer_desc, vk::AndroidHardwareBufferFormatPropertiesANDROID & ahb_format); std::shared_ptr map_hardware_buffer(AImage *); public: @@ -156,12 +149,15 @@ public: return description; } - static const vk::ImageLayout framebuffer_expected_layout = vk::ImageLayout::eColorAttachmentOptimal; - static const vk::ImageUsageFlagBits framebuffer_usage = vk::ImageUsageFlagBits::eColorAttachment; + vk::Sampler sampler() + { + return *ycbcr_sampler; + } - void set_blit_targets(std::vector targets, vk::Format format); - void blit(vk::raii::CommandBuffer& command_buffer, blit_handle & handle, std::span target_indices); - void blit_finished(blit_handle & handle); + vk::Extent2D image_size() + { + return extent; + } }; } // namespace wivrn::android diff --git a/client/decoder/ffmpeg/ffmpeg_decoder.cpp b/client/decoder/ffmpeg/ffmpeg_decoder.cpp index d6ed6d66..04eb9bbb 100644 --- a/client/decoder/ffmpeg/ffmpeg_decoder.cpp +++ b/client/decoder/ffmpeg/ffmpeg_decoder.cpp @@ -96,7 +96,7 @@ decoder::decoder( .arrayLayers = 1, .samples = vk::SampleCountFlagBits::e1, .tiling = vk::ImageTiling::eLinear, - .usage = vk::ImageUsageFlagBits::eSampled | vk::ImageUsageFlagBits::eTransferSrc, + .usage = vk::ImageUsageFlagBits::eSampled, .initialLayout = vk::ImageLayout::eUndefined, }; @@ -107,11 +107,27 @@ decoder::decoder( decoded_images[i].image = image_allocation(device, image_info, alloc_info); decoded_images[i].image.map(); + extent = vk::Extent2D{description.width, description.height}; vk::ImageSubresource resource; resource.aspectMask = vk::ImageAspectFlagBits::eColor; decoded_images[i].layout = decoded_images[i].image->getSubresourceLayout(resource); + + decoded_images[i].image_view = vk::raii::ImageView(device, { + .image = (vk::Image)decoded_images[i].image, + .viewType = vk::ImageViewType::e2D, + .format = image_info.format, + .subresourceRange = { + .aspectMask = vk::ImageAspectFlagBits::eColor, + .baseMipLevel = 0, + .levelCount = 1, + .baseArrayLayer = 0, + .layerCount = 1, + } + }); + + decoded_images[i].current_layout = vk::ImageLayout::eUndefined; } auto avcodec = avcodec_find_decoder(codec_id(description.codec)); @@ -125,65 +141,17 @@ decoder::decoder( int ret = avcodec_open2(codec.get(), avcodec, nullptr); if (ret < 0) throw std::runtime_error{"avcodec_open2 failed"}; -} -void decoder::set_blit_targets(std::vector targets, vk::Format format) -{ - blit_targets = std::move(targets); -} - -void decoder::blit(vk::raii::CommandBuffer& command_buffer, blit_handle & handle, std::span blit_indices) -{ - std::unique_lock lock(mutex); - - // Transition layout to VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL - // TODO: transition only if needed - vk::ImageMemoryBarrier barrier{ - .srcAccessMask = vk::AccessFlagBits::eTransferWrite, - .dstAccessMask = vk::AccessFlagBits::eTransferWrite, - .oldLayout = vk::ImageLayout::eUndefined, - .newLayout = vk::ImageLayout::eTransferSrcOptimal, - .image = handle.image, - .subresourceRange = { - .aspectMask = vk::ImageAspectFlagBits::eColor, - .levelCount = 1, - .layerCount = 1, - }, - }; - - command_buffer.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands, vk::PipelineStageFlagBits::eAllCommands, vk::DependencyFlags{}, {}, {}, barrier); - - for (int blit_index: blit_indices) - { - uint32_t left = blit_targets[blit_index].offset.x; - uint32_t right = left + blit_targets[blit_index].extent.width; - - uint32_t width = blit_targets[blit_index].extent.width; - uint32_t height = blit_targets[blit_index].extent.height; - - // check for intersection - if (description.offset_x >= right or description.offset_x + description.width <= left) - continue; - - vk::ImageBlit blit; - - blit.srcSubresource.aspectMask = vk::ImageAspectFlagBits::eColor; - blit.srcSubresource.layerCount = 1; - blit.srcOffsets[0].x = std::max(0, left - description.offset_x); - blit.srcOffsets[0].y = 0; - blit.srcOffsets[1].x = std::min(description.width, right - description.offset_x); - blit.srcOffsets[1].y = std::min(description.height, height - description.offset_y); - blit.srcOffsets[1].z = 1; - - blit.dstSubresource.aspectMask = vk::ImageAspectFlagBits::eColor; - blit.dstSubresource.layerCount = 1; - blit.dstOffsets[0].x = std::max(0, description.offset_x - left); - blit.dstOffsets[0].y = description.offset_y; - blit.dstOffsets[1].x = std::min(width, description.offset_x + description.width - left); - blit.dstOffsets[1].y = std::min(height, description.offset_y + description.height); - blit.dstOffsets[1].z = 1; - command_buffer.blitImage(handle.image, vk::ImageLayout::eTransferSrcOptimal, blit_targets[blit_index].image, vk::ImageLayout::eTransferDstOptimal, blit, vk::Filter::eLinear); - } + rgb_sampler = vk::raii::Sampler(device, { + .flags = {}, + .magFilter = vk::Filter::eLinear, + .minFilter = vk::Filter::eLinear, + .mipmapMode = vk::SamplerMipmapMode::eNearest, + .addressModeU = vk::SamplerAddressMode::eClampToEdge, + .addressModeV = vk::SamplerAddressMode::eClampToEdge, + .addressModeW = vk::SamplerAddressMode::eClampToEdge, + .unnormalizedCoordinates = false, + }); } void decoder::push_data(std::span> data, uint64_t frame_index, bool partial) @@ -243,18 +211,21 @@ void decoder::frame_completed(const xrt::drivers::wivrn::from_headset::feedback decoded_images[index].frame_index = frame_index; int dstStride = decoded_images[index].layout.rowPitch; - uint8_t * out = (uint8_t *)decoded_images[index].image.map(); + uint8_t * out = decoded_images[index].image.data(); res = sws_scale(sws.get(), frame->data, frame->linesize, 0, frame->height, &out, &dstStride); if (res == 0) throw std::runtime_error{"sws_scale failed"}; - auto handle = std::make_shared(); - handle->feedback = feedback; - handle->timing_info = timing_info; - handle->view_info = view_info; - handle->image = decoded_images[index].image; - handle->image_index = index; - handle->self = this; + auto handle = std::make_shared( + feedback, + timing_info, + view_info, + decoded_images[index].image_view, + (vk::Image)decoded_images[index].image, + &decoded_images[index].current_layout, + index, + this + ); if (auto scene = weak_scene.lock()) scene->push_blit_handle(accumulator, std::move(handle)); diff --git a/client/decoder/ffmpeg/ffmpeg_decoder.h b/client/decoder/ffmpeg/ffmpeg_decoder.h index 14cc8bc2..d03c92c3 100644 --- a/client/decoder/ffmpeg/ffmpeg_decoder.h +++ b/client/decoder/ffmpeg/ffmpeg_decoder.h @@ -48,12 +48,19 @@ namespace ffmpeg class decoder { public: - struct blit_target + struct blit_handle { - vk::Image image; - vk::ImageView image_view; - vk::Offset2D offset; - vk::Extent2D extent; + xrt::drivers::wivrn::from_headset::feedback feedback; + xrt::drivers::wivrn::to_headset::video_stream_data_shard::timing_info_t timing_info; + xrt::drivers::wivrn::to_headset::video_stream_data_shard::view_info_t view_info; + vk::raii::ImageView& image_view; + vk::Image image = nullptr; + vk::ImageLayout * current_layout = nullptr; + + int image_index; + decoder * self; + + ~blit_handle(); }; private: @@ -63,10 +70,15 @@ private: image_allocation image; vk::SubresourceLayout layout; uint64_t frame_index; + vk::raii::ImageView image_view = nullptr; + vk::ImageLayout current_layout = vk::ImageLayout::eUndefined; }; vk::raii::Device& device; + vk::raii::Sampler rgb_sampler = nullptr; + std::array decoded_images; + vk::Extent2D extent{}; std::vector free_images; xrt::drivers::wivrn::to_headset::video_stream_description::item description; @@ -78,12 +90,10 @@ private: std::weak_ptr weak_scene; shard_accumulator * accumulator; - std::vector blit_targets; std::mutex mutex; public: - decoder( - vk::raii::Device& device, + decoder(vk::raii::Device& device, vk::raii::PhysicalDevice& physical_device, const xrt::drivers::wivrn::to_headset::video_stream_description::item & description, float fps, @@ -91,6 +101,9 @@ public: std::weak_ptr scene, shard_accumulator * accumulator); + decoder(const decoder &) = delete; + decoder(decoder &&) = delete; + void push_data(std::span> data, uint64_t frame_index, bool partial); void frame_completed(const xrt::drivers::wivrn::from_headset::feedback & feedback, const xrt::drivers::wivrn::to_headset::video_stream_data_shard::timing_info_t & timing_info, const xrt::drivers::wivrn::to_headset::video_stream_data_shard::view_info_t & view_info); @@ -100,22 +113,14 @@ public: return description; } - struct blit_handle + vk::Sampler sampler() { - xrt::drivers::wivrn::from_headset::feedback feedback; - xrt::drivers::wivrn::to_headset::video_stream_data_shard::timing_info_t timing_info; - xrt::drivers::wivrn::to_headset::video_stream_data_shard::view_info_t view_info; - int image_index; - VkImage image; - decoder * self; + return *rgb_sampler; + } - ~blit_handle(); - }; - - static const vk::ImageLayout framebuffer_expected_layout = vk::ImageLayout::eTransferDstOptimal; - static const vk::ImageUsageFlagBits framebuffer_usage = vk::ImageUsageFlagBits::eTransferDst; - - void set_blit_targets(std::vector targets, vk::Format format); - void blit(vk::raii::CommandBuffer& command_buffer, blit_handle & handle, std::span target_indices); + vk::Extent2D image_size() + { + return extent; + } }; } // namespace ffmpeg diff --git a/client/decoder/shard_accumulator.h b/client/decoder/shard_accumulator.h index cd3fa5fc..c9ca323f 100644 --- a/client/decoder/shard_accumulator.h +++ b/client/decoder/shard_accumulator.h @@ -93,23 +93,17 @@ public: return decoder->desc(); } + vk::Sampler sampler() + { + return decoder->sampler(); + } + + vk::Extent2D image_size() + { + return decoder->image_size(); + } + using blit_handle = decoder_impl::blit_handle; - using blit_target = decoder_impl::blit_target; - - - static const vk::ImageLayout framebuffer_expected_layout = decoder_impl::framebuffer_expected_layout; - static const vk::ImageUsageFlagBits framebuffer_usage = decoder_impl::framebuffer_usage; - - void set_blit_targets(std::vector targets, vk::Format format) - { - static_assert(std::copy_constructible); - decoder->set_blit_targets(std::move(targets), format); - } - - void blit(vk::raii::CommandBuffer& command_buffer, blit_handle & handle, std::span blit_target_indices) - { - return decoder->blit(command_buffer, handle, blit_target_indices); - } private: void try_submit_frame(std::optional shard_idx); diff --git a/client/scenes/stream.cpp b/client/scenes/stream.cpp index 2e60aa52..5326a16c 100644 --- a/client/scenes/stream.cpp +++ b/client/scenes/stream.cpp @@ -16,8 +16,6 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ - -#include #define GLM_FORCE_RADIANS #include "stream.h" @@ -28,14 +26,16 @@ #include "utils/ranges.h" #include "utils/sync_queue.h" #include "utils/named_thread.h" +#include "vk/pipeline.h" +#include "vk/shader.h" #include "wivrn_packets.h" #include #include #include -#include +#include +#include "boost/pfr/core.hpp" #include "audio/audio.h" #include "hardware.h" -#include "utils/contains.h" using namespace xrt::drivers::wivrn; @@ -123,7 +123,7 @@ std::shared_ptr scenes::stream::create(std::unique_ptrcommandpool, .level = vk::CommandBufferLevel::ePrimary, .commandBufferCount = 1})[0]); - self->fence = self->device.createFence({}); + self->fence = self->device.createFence({.flags = vk::FenceCreateFlagBits::eSignaled}); // Look up the XrActions for haptics self->haptics_actions[0].first = application::get_action("/user/hand/left/output/haptic").first; @@ -164,6 +164,8 @@ std::shared_ptr scenes::stream::create(std::unique_ptrswapchain_format)); + self->query_pool = vk::raii::QueryPool(self->device, vk::QueryPoolCreateInfo{.queryType = vk::QueryType::eTimestamp, .queryCount = size_gpu_timestamps}); + return self; } @@ -319,6 +321,140 @@ void scenes::stream::render(XrTime predicted_display_time, bool should_render) if (state_ == state::stalled) application::pop_scene(); + for (auto & i: decoders) + { + if (auto sampler = i.decoder->sampler(); sampler and not *i.blit_pipeline) + { + // Create blit pipeline + // Create VkDescriptorSetLayout with an immutable sampler + vk::DescriptorSetLayoutBinding sampler_layout_binding{ + .binding = 0, + .descriptorType = vk::DescriptorType::eCombinedImageSampler, + .descriptorCount = 1, + .stageFlags = vk::ShaderStageFlagBits::eFragment, + .pImmutableSamplers = &sampler, + }; + + vk::DescriptorSetLayoutCreateInfo layout_info{ + .bindingCount = 1, + .pBindings = &sampler_layout_binding, + }; + + i.descriptor_set_layout = vk::raii::DescriptorSetLayout(device, layout_info); + i.descriptor_set = device.allocateDescriptorSets( + vk::DescriptorSetAllocateInfo{ + .descriptorPool = *blit_descriptor_pool, + .descriptorSetCount = 1, + .pSetLayouts = &*i.descriptor_set_layout, + })[0] + .release(); + + const auto& description = i.decoder->desc(); + vk::Extent2D image_size = i.decoder->image_size(); + std::array useful_size{ + float(description.width) / image_size.width, + float(description.height) / image_size.height, + }; + spdlog::info("useful size: {}x{} with buffer {}x{}", + description.width, + description.height, + image_size.width, + image_size.height); + + std::array specialization_constants_desc{ + vk::SpecializationMapEntry{ + .constantID = 0, + .offset = 0, + .size = sizeof(float), + }, + vk::SpecializationMapEntry{ + .constantID = 1, + .offset = sizeof(float), + .size = sizeof(float), + }}; + + vk::SpecializationInfo specialization_info; + specialization_info.setMapEntries(specialization_constants_desc); + specialization_info.setData(useful_size); + + // Create graphics pipeline + vk::raii::ShaderModule vertex_shader = load_shader(device, "stream.vert"); + vk::raii::ShaderModule fragment_shader = load_shader(device, "stream.frag"); + + vk::PipelineLayoutCreateInfo pipeline_layout_info{ + .setLayoutCount = 1, + .pSetLayouts = &*i.descriptor_set_layout, + }; + + i.blit_pipeline_layout = vk::raii::PipelineLayout(device, pipeline_layout_info); + + vk::pipeline_builder pipeline_info{ + .flags = {}, + .Stages = {{ + .stage = vk::ShaderStageFlagBits::eVertex, + .module = *vertex_shader, + .pName = "main", + .pSpecializationInfo = &specialization_info, + }, + { + .stage = vk::ShaderStageFlagBits::eFragment, + .module = *fragment_shader, + .pName = "main", + }}, + .VertexInputState = {.flags = {}}, + .VertexBindingDescriptions = {}, + .VertexAttributeDescriptions = {}, + .InputAssemblyState = {{ + .topology = vk::PrimitiveTopology::eTriangleStrip, + }}, + .ViewportState = {.flags = {}}, + .Viewports = {{}}, + .Scissors = {{}}, + .RasterizationState = {{ + .polygonMode = vk::PolygonMode::eFill, + .lineWidth = 1, + }}, + .MultisampleState = {{ + .rasterizationSamples = vk::SampleCountFlagBits::e1, + }}, + .ColorBlendState = {.flags = {}}, + .ColorBlendAttachments = {{.colorWriteMask = vk::ColorComponentFlagBits::eR | vk::ColorComponentFlagBits::eG | vk::ColorComponentFlagBits::eB}}, + .DynamicState = {.flags = {}}, + .DynamicStates = {vk::DynamicState::eViewport, vk::DynamicState::eScissor}, + .layout = *i.blit_pipeline_layout, + .renderPass = *blit_render_pass, + .subpass = 0, + }; + + i.blit_pipeline = vk::raii::Pipeline(device, application::get_pipeline_cache(), pipeline_info); + } + } + + if (device.waitForFences(*fence, VK_TRUE, UINT64_MAX) == vk::Result::eTimeout) + throw std::runtime_error("Vulkan fence timeout"); + device.resetFences(*fence); + + // We don't need those after vkWaitForFences + current_blit_handles.clear(); + + gpu_timestamps timestamps; + if (query_pool_filled) + { + auto [res, timestamps2] = query_pool.getResults( + 0, + size_gpu_timestamps, + size_gpu_timestamps * sizeof(uint64_t), + sizeof(uint64_t), + vk::QueryResultFlagBits::e64 | vk::QueryResultFlagBits::eWait); + + if (res == vk::Result::eSuccess) + { + boost::pfr::for_each_field(timestamps, [n = 1, ×tamps2](float& t) mutable { + t = (timestamps2[n++] - timestamps2[0]) * application::get_physical_device_properties().limits.timestampPeriod / 1e9; + }); + } + } + session.begin_frame(); auto [flags, views] = session.locate_views(viewconfig, predicted_display_time, world_space); @@ -340,33 +476,12 @@ void scenes::stream::render(XrTime predicted_display_time, bool should_render) begin_info.flags = vk::CommandBufferUsageFlagBits::eOneTimeSubmit; command_buffer.begin(begin_info); - // Transition the layout of the decoder framebuffer to the one the decoders expect - std::vector image_barriers; - for (size_t i = 0; i < decoder_output.size(); i++) - { - vk::ImageMemoryBarrier barrier - { - .srcAccessMask = vk::AccessFlagBits::eNone, - .dstAccessMask = vk::AccessFlagBits::eShaderRead, - .oldLayout = vk::ImageLayout::eUndefined, - .newLayout = shard_accumulator::framebuffer_expected_layout, - .image = vk::Image{decoder_output[i].image}, - .subresourceRange = { - .aspectMask = vk::ImageAspectFlagBits::eColor, - .baseMipLevel = 0, - .levelCount = 1, - .baseArrayLayer = 0, - .layerCount = 1, - }, - }; - - image_barriers.push_back(barrier); - } - command_buffer.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands, vk::PipelineStageFlagBits::eAllCommands, vk::DependencyFlags{}, {}, {}, image_barriers); - // Keep a reference to the resources needed to blit the images until vkWaitForFences std::vector> current_blit_handles; + command_buffer.resetQueryPool(*query_pool, 0, size_gpu_timestamps); + command_buffer.writeTimestamp(vk::PipelineStageFlagBits::eTopOfPipe, *query_pool, 0); + std::array pose = {views[0].pose, views[1].pose}; std::array fov = {views[0].fov, views[1].fov}; { @@ -377,7 +492,6 @@ void scenes::stream::render(XrTime predicted_display_time, bool should_render) if (common_frame) next_frame = *common_frame + 1; // Blit images from the decoders - // TODO be smarter: group blits per eye, so that the framebuffer can use OP_DONT_CARE instead of OP_LOAD and use the same renderpass if possible for (auto & i: decoders) { auto blit_handle = i.frame(common_frame); @@ -394,33 +508,104 @@ void scenes::stream::render(XrTime predicted_display_time, bool should_render) pose = blit_handle->view_info.pose; fov = blit_handle->view_info.fov; - int indices[] = {0, 1}; - i.decoder->blit(command_buffer, *blit_handle, indices); // TODO blit indices no longer needed here + vk::DescriptorImageInfo image_info{ + .imageView = *blit_handle->image_view, + .imageLayout = vk::ImageLayout::eGeneral, + }; + + vk::WriteDescriptorSet descriptor_write{ + .dstSet = i.descriptor_set, + .dstBinding = 0, + .dstArrayElement = 0, + .descriptorCount = 1, + .descriptorType = vk::DescriptorType::eCombinedImageSampler, + .pImageInfo = &image_info, + }; + + device.updateDescriptorSets(descriptor_write, {}); + if (*blit_handle->current_layout != vk::ImageLayout::eGeneral) + { + vk::ImageMemoryBarrier barrier{ + .srcAccessMask = vk::AccessFlagBits::eNone, + .dstAccessMask = vk::AccessFlagBits::eMemoryRead, + .oldLayout = *blit_handle->current_layout, + .newLayout = vk::ImageLayout::eGeneral, + .image = blit_handle->image, + .subresourceRange = { + .aspectMask = vk::ImageAspectFlagBits::eColor, + .levelCount = 1, + .layerCount = 1, + }, + }; + command_buffer.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands, vk::PipelineStageFlagBits::eAllCommands, {}, {}, {}, barrier); + *blit_handle->current_layout = vk::ImageLayout::eGeneral; + } } } - // Transition the output of the decoder to VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL - image_barriers.clear(); - for (size_t i = 0; i < decoder_output.size(); i++) - { - vk::ImageMemoryBarrier barrier{ - .srcAccessMask = vk::AccessFlagBits::eNone, - .dstAccessMask = vk::AccessFlagBits::eShaderRead, - .oldLayout = shard_accumulator::framebuffer_expected_layout, - .newLayout = vk::ImageLayout::eShaderReadOnlyOptimal, - .image = vk::Image{decoder_output[i].image}, - .subresourceRange = { - .aspectMask = vk::ImageAspectFlagBits::eColor, - .baseMipLevel = 0, - .levelCount = 1, - .baseArrayLayer = 0, - .layerCount = 1, - }, - }; - image_barriers.push_back(barrier); + uint16_t x_offset = 0; + for (auto & out: decoder_output) + { + command_buffer.beginRenderPass( + { + .renderPass = *blit_render_pass, + .framebuffer = *out.frame_buffer, + .renderArea = { + .offset = {0, 0}, + .extent = out.size, + }, + .clearValueCount = 0, + }, + vk::SubpassContents::eInline); + + for (const auto & decoder: decoders) + { + if (not *decoder.blit_pipeline) + continue; + + command_buffer.bindPipeline(vk::PipelineBindPoint::eGraphics, *decoder.blit_pipeline); + + const auto & description = decoder.decoder->desc(); + int x0 = description.offset_x - x_offset; + int y0 = description.offset_y; + int x1 = x0 + description.width; + int y1 = y0 + description.height; + + vk::Viewport viewport{ + .x = (float)x0, + .y = (float)y0, + .width = (float)description.width, + .height = (float)description.height, + .minDepth = 0, + .maxDepth = 1}; + + x0 = std::clamp(x0, 0, out.size.width); + x1 = std::clamp(x1, 0, out.size.width); + y0 = std::clamp(y0, 0, out.size.height); + y1 = std::clamp(y1, 0, out.size.height); + + vk::Rect2D scissor{ + .offset = {.x = x0, .y = y0}, + .extent = {.width = (uint32_t)(x1 - x0), .height = (uint32_t)(y1 - y0)}, + }; + + command_buffer.setViewport(0, viewport); + command_buffer.setScissor(0, scissor); + + command_buffer.bindDescriptorSets( + vk::PipelineBindPoint::eGraphics, + *decoder.blit_pipeline_layout, + 0, + decoder.descriptor_set, + nullptr); + command_buffer.draw(3, 1, 0, 0); + } + command_buffer.endRenderPass(); + x_offset += out.size.width; } - command_buffer.pipelineBarrier(vk::PipelineStageFlagBits::eColorAttachmentOutput, vk::PipelineStageFlagBits::eFragmentShader, vk::DependencyFlags{}, {}, {}, image_barriers); + + command_buffer.writeTimestamp(vk::PipelineStageFlagBits::eBottomOfPipe, *query_pool, 1); // Reproject the image to the real pose for (size_t view = 0; view < view_count; view++) @@ -429,6 +614,8 @@ void scenes::stream::render(XrTime predicted_display_time, bool should_render) reprojector->reproject(command_buffer, view, destination_index, pose[view].orientation, fov[view], views[view].pose.orientation, views[view].fov); } + command_buffer.writeTimestamp(vk::PipelineStageFlagBits::eBottomOfPipe, *query_pool, 2); + command_buffer.end(); vk::SubmitInfo submit_info; submit_info.setCommandBuffers(*command_buffer); @@ -474,7 +661,7 @@ void scenes::stream::render(XrTime predicted_display_time, bool should_render) XrCompositionLayerQuad imgui_layer; if (imgui_ctx) { - accumulate_metrics(predicted_display_time, current_blit_handles); + accumulate_metrics(predicted_display_time, current_blit_handles, timestamps); imgui_layer = plot_performance_metrics(predicted_display_time); } @@ -489,14 +676,8 @@ void scenes::stream::render(XrTime predicted_display_time, bool should_render) for (const auto& handle: current_blit_handles) send_feedback(handle->feedback); - if (device.waitForFences(*fence, VK_TRUE, UINT64_MAX) == vk::Result::eTimeout) - throw std::runtime_error("Vulkan fence timeout"); - device.resetFences(*fence); - - // We don't need those after vkWaitForFences - current_blit_handles.clear(); - read_actions(); + query_pool_filled = true; } void scenes::stream::exit() @@ -543,6 +724,36 @@ void scenes::stream::setup(const to_headset::video_stream_description & descript spdlog::info("Created stream swapchain {}: {}x{}", swapchains.size(), extent.width, extent.height); } + // Create renderpass + { + vk::AttachmentDescription color_desc{ + .format = vk::Format::eA8B8G8R8SrgbPack32, + .samples = vk::SampleCountFlagBits::e1, + .loadOp = vk::AttachmentLoadOp::eDontCare, + .storeOp = vk::AttachmentStoreOp::eStore, + .initialLayout = vk::ImageLayout::eUndefined, + .finalLayout = vk::ImageLayout::eShaderReadOnlyOptimal, + }; + + vk::AttachmentReference color_ref{ + .attachment = 0, + .layout = vk::ImageLayout::eColorAttachmentOptimal, + }; + + vk::SubpassDescription subpass{ + .pipelineBindPoint = vk::PipelineBindPoint::eGraphics, + }; + subpass.setColorAttachments(color_ref); + + vk::RenderPassCreateInfo renderpass_info{ + .flags = {}, + }; + renderpass_info.setAttachments(color_desc); + renderpass_info.setSubpasses(subpass); + + blit_render_pass = vk::raii::RenderPass(device, renderpass_info); + } + // Create outputs for the decoders vk::Extent3D decoder_out_size{video_width, video_height, 1}; for (size_t i = 0; i < view_count; i++) @@ -560,7 +771,7 @@ void scenes::stream::setup(const to_headset::video_stream_description & descript .arrayLayers = 1, .samples = vk::SampleCountFlagBits::e1, .tiling = vk::ImageTiling::eOptimal, - .usage = vk::ImageUsageFlagBits::eSampled | shard_accumulator::framebuffer_usage, + .usage = vk::ImageUsageFlagBits::eSampled | vk::ImageUsageFlagBits::eColorAttachment, .sharingMode = vk::SharingMode::eExclusive, .initialLayout = vk::ImageLayout::eUndefined, }; @@ -586,19 +797,31 @@ void scenes::stream::setup(const to_headset::video_stream_description & descript }; decoder_output[i].image_view = vk::raii::ImageView(device, image_view_info); + + decoder_output[i].frame_buffer = vk::raii::Framebuffer( + device, + vk::FramebufferCreateInfo{ + .renderPass = *blit_render_pass, + .attachmentCount = 1, + .pAttachments = &*decoder_output[i].image_view, + .width = decoder_out_size.width, + .height = decoder_out_size.height, + .layers = 1, + }); } - std::vector blit_targets; - blit_targets.resize(view_count); - - for (size_t i = 0; i < view_count; i++) { - blit_targets[i].image = vk::Image{decoder_output[i].image}; - blit_targets[i].image_view = *decoder_output[i].image_view; - blit_targets[i].extent.width = video_width; - blit_targets[i].extent.height = video_height; - blit_targets[i].offset.x = video_width * i; - blit_targets[i].offset.y = 0; + vk::DescriptorPoolSize pool_size{ + .type = vk::DescriptorType::eCombinedImageSampler, + .descriptorCount = uint32_t(description.items.size()), + }; + blit_descriptor_pool = vk::raii::DescriptorPool( + device, + vk::DescriptorPoolCreateInfo{ + .maxSets = uint32_t(description.items.size()), + .poolSizeCount = 1, + .pPoolSizes = &pool_size, + }); } for (const auto & [stream_index, item]: utils::enumerate(description.items)) @@ -616,7 +839,6 @@ void scenes::stream::setup(const to_headset::video_stream_description & descript accumulator_images dec; dec.decoder = std::make_unique(device, physical_device, item, description.fps, shared_from_this(), stream_index); - dec.decoder->set_blit_targets(blit_targets, vk::Format::eA8B8G8R8SrgbPack32); decoders.push_back(std::move(dec)); } diff --git a/client/scenes/stream.h b/client/scenes/stream.h index 5d82398b..2db9c224 100644 --- a/client/scenes/stream.h +++ b/client/scenes/stream.h @@ -51,6 +51,10 @@ private: struct accumulator_images { std::unique_ptr decoder; + vk::raii::DescriptorSetLayout descriptor_set_layout = nullptr; + vk::DescriptorSet descriptor_set = nullptr; + vk::raii::PipelineLayout blit_pipeline_layout = nullptr; + vk::raii::Pipeline blit_pipeline = nullptr; // latest frames from oldest to most recent std::array, 2> latest_frames; @@ -65,6 +69,7 @@ private: vk::Format format; image_allocation image; vk::raii::ImageView image_view = nullptr; + vk::raii::Framebuffer frame_buffer = nullptr; }; std::unique_ptr network_session; @@ -78,6 +83,8 @@ private: std::mutex decoder_mutex; uint64_t next_frame; // Preferred index for next frame std::vector decoders; // Locked by decoder_mutex + vk::raii::DescriptorPool blit_descriptor_pool = nullptr; + vk::raii::RenderPass blit_render_pass = nullptr; std::array decoder_output{}; @@ -102,6 +109,9 @@ private: std::optional imgui_ctx; + // Keep a reference to the resources needed to blit the images until vkWaitForFences + std::vector> current_blit_handles; + stream() = default; public: @@ -146,19 +156,40 @@ private: void setup(const to_headset::video_stream_description &); void exit(); + vk::raii::QueryPool query_pool = nullptr; + bool query_pool_filled = false; + uint64_t bytes_sent = 0; uint64_t bytes_received = 0; float bandwidth_rx = 0; float bandwidth_tx = 0; - struct global_metric + struct gpu_timestamps { - float cpu_time = 0; + float gpu_barrier = 0; float gpu_time = 0; + }; + + struct global_metric //: gpu_timestamps + { + float gpu_barrier; + float gpu_time; + float cpu_time = 0; float bandwidth_rx = 0; float bandwidth_tx = 0; + }; + struct plot + { + int subplot; + const char * title; + const char * unit; + float scenes::stream::global_metric::* data; + }; + + static const inline int size_gpu_timestamps = 1 + sizeof(gpu_timestamps) / sizeof(float); + struct decoder_metric { // All times are in seconds relative to encode_begin @@ -178,7 +209,7 @@ private: XrTime last_metric_time = 0; int metrics_offset = 0; - void accumulate_metrics(XrTime predicted_display_time, const std::vector>& blit_handles); + void accumulate_metrics(XrTime predicted_display_time, const std::vector>& blit_handles, const gpu_timestamps& timestamps); XrCompositionLayerQuad plot_performance_metrics(XrTime predicted_display_time); }; } // namespace scenes diff --git a/client/scenes/stream_stats.cpp b/client/scenes/stream_stats.cpp index e10d2e1d..d84edd5f 100644 --- a/client/scenes/stream_stats.cpp +++ b/client/scenes/stream_stats.cpp @@ -24,6 +24,7 @@ #include "application.h" #include "implot.h" #include "utils/ranges.h" +#include #include #include @@ -79,7 +80,7 @@ ImPlotPoint getter(int index, void * data_) } } -void scenes::stream::accumulate_metrics(XrTime predicted_display_time, const std::vector>& blit_handles) +void scenes::stream::accumulate_metrics(XrTime predicted_display_time, const std::vector>& blit_handles, const gpu_timestamps& timestamps) { uint64_t rx = network_session->bytes_received(); uint64_t tx = network_session->bytes_sent(); @@ -93,12 +94,10 @@ void scenes::stream::accumulate_metrics(XrTime predicted_display_time, const std bytes_received = rx; bytes_sent = tx; - global_metrics[metrics_offset] = global_metric{ - .cpu_time = application::get_cpu_time().count() * 1e-9f, - .gpu_time = application::get_gpu_time().count() * 1e-9f, - .bandwidth_rx = bandwidth_rx * 8, - .bandwidth_tx = bandwidth_tx * 8, - }; + *(gpu_timestamps*)&global_metrics[metrics_offset] = timestamps; + global_metrics[metrics_offset].cpu_time = application::get_cpu_time().count() * 1e-9f; + global_metrics[metrics_offset].bandwidth_rx = bandwidth_rx * 8; + global_metrics[metrics_offset].bandwidth_tx = bandwidth_tx * 8; if (decoder_metrics.size() != blit_handles.size()) decoder_metrics.resize(blit_handles.size()); @@ -142,10 +141,11 @@ XrCompositionLayerQuad scenes::stream::plot_performance_metrics(XrTime predicted ImVec2 window_size = ImGui::GetWindowSize() - ImVec2(2,2) * style.WindowPadding; static const std::array plots = { - std::make_tuple("CPU time", "s", &global_metric::cpu_time), - std::make_tuple("GPU time", "s", &global_metric::gpu_time), - std::make_tuple("Download", "bit/s", &global_metric::bandwidth_rx), - std::make_tuple("Upload", "bit/s", &global_metric::bandwidth_tx), + plot(0, "CPU time", "s", &global_metric::cpu_time), + plot(1, "GPU blit", "s", &global_metric::gpu_barrier), + plot(1, "GPU time", "s", &global_metric::gpu_time), + plot(2, "Download", "bit/s", &global_metric::bandwidth_rx), + plot(3, "Upload", "bit/s", &global_metric::bandwidth_tx), }; int n_plots = plots.size() + decoders.size(); @@ -165,7 +165,7 @@ XrCompositionLayerQuad scenes::stream::plot_performance_metrics(XrTime predicted ImPlot::PushStyleColor(ImPlotCol_AxisBgHovered, IM_COL32(0, 0, 0, 0)); int n = 0; - for(auto [title, unit, data]: plots) + for(auto [subplot, title, unit, data]: plots) { if (ImPlot::BeginPlot(title, plot_size, ImPlotFlags_CanvasOnly | ImPlotFlags_NoChild)) { @@ -173,7 +173,7 @@ XrCompositionLayerQuad scenes::stream::plot_performance_metrics(XrTime predicted float max_v = compute_plot_max_value(&(global_metrics.data()->*data), global_metrics.size(), sizeof(global_metric)); auto [ multiplier, prefix ] = compute_plot_unit(max_v); - if (axis_scale[n] == 0) + if (axis_scale[n] == 0 || std::isnan(axis_scale[n])) axis_scale[n] = max_v; else axis_scale[n] = 0.99 * axis_scale[n] + 0.01 * max_v; diff --git a/common/vk/allocation.h b/common/vk/allocation.h index 48cf1d17..d66158b6 100644 --- a/common/vk/allocation.h +++ b/common/vk/allocation.h @@ -22,6 +22,7 @@ #include "vk_allocator.h" #include "vk_mem_alloc.h" #include +#include template struct basic_allocation_traits @@ -177,7 +178,7 @@ public: mapped = nullptr; } - template + template U * data() { return reinterpret_cast(map());