From 410b112fcf52b153ca944af57d57eee1e458d9cd Mon Sep 17 00:00:00 2001 From: Patrick Nicolas Date: Mon, 10 Jun 2024 18:39:16 +0200 Subject: [PATCH] Support passthrough for stream --- client/decoder/android/android_decoder.cpp | 4 +- client/decoder/android/android_decoder.h | 1 + client/scenes/lobby.cpp | 14 +- client/scenes/stream.cpp | 251 +++++++++++------- client/scenes/stream.h | 6 +- client/scenes/stream_network.cpp | 4 +- client/scenes/stream_reprojection.cpp | 3 - client/scenes/stream_stats.cpp | 4 +- client/shaders/reprojection.glsl | 2 +- client/shaders/stream.glsl | 25 +- client/utils/overloaded.h | 31 +++ client/xr/session.cpp | 7 +- common/wivrn_packets.h | 13 +- .../0012-store-alpha-channel-in-layer-1.patch | 94 +++++++ server/driver/configuration.cpp | 48 ++-- server/driver/configuration.h | 1 + server/driver/wivrn_comp_target.cpp | 39 +-- server/driver/wivrn_comp_target.h | 6 +- server/driver/wivrn_hmd.cpp | 6 +- server/encoder/encoder_settings.cpp | 59 +++- .../encoder/ffmpeg/video_encoder_ffmpeg.cpp | 8 +- server/encoder/ffmpeg/video_encoder_ffmpeg.h | 6 +- server/encoder/ffmpeg/video_encoder_va.cpp | 10 +- server/encoder/ffmpeg/video_encoder_va.h | 4 +- server/encoder/video_encoder.cpp | 49 ++-- server/encoder/video_encoder.h | 17 +- server/encoder/video_encoder_nvenc.cpp | 26 +- server/encoder/video_encoder_nvenc.h | 6 +- server/encoder/video_encoder_vulkan.cpp | 23 +- server/encoder/video_encoder_vulkan.h | 9 +- server/encoder/video_encoder_vulkan_h264.cpp | 17 +- server/encoder/video_encoder_vulkan_h264.h | 10 +- server/encoder/video_encoder_x264.cpp | 22 +- server/encoder/video_encoder_x264.h | 6 +- 34 files changed, 571 insertions(+), 260 deletions(-) create mode 100644 client/utils/overloaded.h create mode 100644 patches/monado/0012-store-alpha-channel-in-layer-1.patch diff --git a/client/decoder/android/android_decoder.cpp b/client/decoder/android/android_decoder.cpp index 066e8cde..cb29ad93 100644 --- a/client/decoder/android/android_decoder.cpp +++ b/client/decoder/android/android_decoder.cpp @@ -85,7 +85,7 @@ decoder::decoder( uint8_t stream_index, std::weak_ptr weak_scene, shard_accumulator * accumulator) : - description(description), fps(fps), device(device), weak_scene(weak_scene), accumulator(accumulator) + description(description), stream_index(stream_index), fps(fps), device(device), weak_scene(weak_scene), accumulator(accumulator) { spdlog::info("hbm_mutex.native_handle() = {}", (void *)hbm_mutex.native_handle()); @@ -372,7 +372,7 @@ void decoder::create_sampler(const AHardwareBuffer_Desc & buffer_desc, vk::Andro }, }; - ycbcr_sampler = vk::raii::Sampler(device, sampler_info.get()); + ycbcr_sampler = vk::raii::Sampler(device, sampler_info.get()); } std::shared_ptr decoder::map_hardware_buffer(AImage * image) diff --git a/client/decoder/android/android_decoder.h b/client/decoder/android/android_decoder.h index b538f0f4..ecfeb308 100644 --- a/client/decoder/android/android_decoder.h +++ b/client/decoder/android/android_decoder.h @@ -85,6 +85,7 @@ public: private: wivrn::to_headset::video_stream_description::item description; + uint8_t stream_index; float fps; vk::raii::Device & device; diff --git a/client/scenes/lobby.cpp b/client/scenes/lobby.cpp index bf04d7db..4db1456d 100644 --- a/client/scenes/lobby.cpp +++ b/client/scenes/lobby.cpp @@ -28,6 +28,7 @@ #include "render/scene_data.h" #include "render/scene_renderer.h" #include "stream.h" +#include "utils/overloaded.h" #include "version.h" #include "wivrn_client.h" #include "wivrn_packets.h" @@ -573,17 +574,6 @@ static std::vector render_layer(std::vector(render_layer(views, color_swapchains, depth_swapchains, renderer, data, clear_color)); } -namespace -{ -template -struct overloaded : Ts... -{ - using Ts::operator()...; -}; -template -overloaded(Ts...) -> overloaded; -} // namespace - // Return the vector v such that dot(v, x) > 0 iff x is on the side where the composition layer is visible static glm::vec4 compute_ray_limits(const XrPosef & pose, float margin = 0) { @@ -873,7 +863,7 @@ void scenes::lobby::render(const XrFrameState & frame_state) if (application::get_config().passthrough_enabled) { std::visit( - overloaded{ + utils::overloaded{ [&](std::monostate &) { assert(false); }, diff --git a/client/scenes/stream.cpp b/client/scenes/stream.cpp index 153ecff1..1325789a 100644 --- a/client/scenes/stream.cpp +++ b/client/scenes/stream.cpp @@ -16,6 +16,7 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ +#include "utils/overloaded.h" #define GLM_FORCE_RADIANS #include "stream.h" @@ -127,6 +128,7 @@ std::shared_ptr scenes::stream::create(std::unique_ptrsystem.passthrough_supported() != xr::system::passthrough_type::no_passthrough; audio::get_audio_description(info); if (not(config.check_feature(feature::microphone))) @@ -259,7 +261,7 @@ void scenes::stream::push_blit_handle(shard_accumulator * decoder, std::shared_p { std::shared_lock lock(decoder_mutex); std::unique_lock frame_lock(frames_mutex); - const auto stream = handle->feedback.stream_index; + auto stream = handle->feedback.stream_index; if (stream < decoders.size()) { assert(decoder == decoders[stream].decoder.get()); @@ -267,8 +269,8 @@ void scenes::stream::push_blit_handle(shard_accumulator * decoder, std::shared_p std::swap(handle, decoders[stream].latest_frames[handle->feedback.frame_index % decoders[stream].latest_frames.size()]); } - if (state_ != state::streaming && std::all_of(decoders.begin(), decoders.end(), [](accumulator_images & i) { - return i.latest_frames.back(); + if (state_ != state::streaming and std::ranges::all_of(decoders, [](accumulator_images & i) { + return i.alpha() or not i.frames().empty(); })) { state_ = state::streaming; @@ -282,6 +284,11 @@ void scenes::stream::push_blit_handle(shard_accumulator * decoder, std::shared_p } } +bool scenes::stream::accumulator_images::alpha() const +{ + return decoder->desc().channels == wivrn::to_headset::video_stream_description::channels_t::alpha; +} + std::vector scenes::stream::accumulator_images::frames() const { std::vector result; @@ -300,8 +307,11 @@ std::vector> scenes::stream::com std::unique_lock lock(frames_mutex); thread_local std::vector common_frames; common_frames.clear(); + const bool alpha = decoders[0].latest_frames[0] and decoders[0].latest_frames[0]->view_info.alpha; for (size_t i = 0; i < decoders.size(); ++i) { + if (decoders[i].alpha() and not alpha) + continue; if (i == 0) { for (const auto & h: decoders[i].latest_frames) @@ -324,7 +334,8 @@ std::vector> scenes::stream::com // clang-format on } } - std::optional frame_index; + std::vector> result; + result.reserve(decoders.size()); if (not common_frames.empty()) { auto min = std::ranges::min_element(common_frames, @@ -336,7 +347,14 @@ std::vector> scenes::stream::com }); assert(*min); - frame_index = (*min)->feedback.frame_index; + auto frame_index = (*min)->feedback.frame_index; + for (const auto & decoder: decoders) + { + if (alpha or not decoder.alpha()) + result.emplace_back(decoder.frame(frame_index)); + else + result.emplace_back(nullptr); + } } else { @@ -353,22 +371,35 @@ std::vector> scenes::stream::com } spdlog::warn(frames); } + + for (const auto & decoder: decoders) + { + if (alpha or not decoder.alpha()) + { + auto min = std::ranges::min_element(decoder.latest_frames, + std::ranges::less{}, + [display_time](auto frame) { + if (not frame) + return std::numeric_limits::max(); + return std::abs(frame->view_info.display_time - display_time); + }); + result.emplace_back(*min); + } + else + result.emplace_back(nullptr); + } } - std::vector> result; - result.reserve(decoders.size()); - for (const auto & decoder: decoders) - result.push_back(decoder.frame(frame_index)); return result; } -std::shared_ptr scenes::stream::accumulator_images::frame(std::optional id) const +std::shared_ptr scenes::stream::accumulator_images::frame(uint64_t id) const { for (auto it = latest_frames.rbegin(); it != latest_frames.rend(); ++it) { if (not *it) continue; - if (id and (*it)->feedback.frame_index != *id) + if ((*it)->feedback.frame_index != id) continue; return *it; @@ -461,15 +492,25 @@ void scenes::stream::render(const XrFrameState & frame_state) vert_specialization_info.setMapEntries(specialization_constants_desc); vert_specialization_info.setData(useful_size); - VkBool32 do_srgb = need_srgb_conversion(guess_model()); - vk::SpecializationMapEntry frag_specialization_constant_desc{ - .constantID = 0, - .offset = 0, - .size = sizeof(do_srgb), + std::array constants = { + need_srgb_conversion(guess_model()), + i.alpha(), + }; + std::array frag_specialization_constant_desc = { + vk::SpecializationMapEntry{ + .constantID = 0, + .offset = 0, + .size = sizeof(constants[0]), + }, + vk::SpecializationMapEntry{ + .constantID = 1, + .offset = sizeof(constants[0]), + .size = sizeof(constants[1]), + }, }; vk::SpecializationInfo frag_specialization_info; frag_specialization_info.setMapEntries(frag_specialization_constant_desc); - frag_specialization_info.setData(do_srgb); + frag_specialization_info.setData(constants); // Create graphics pipeline vk::raii::ShaderModule vertex_shader = load_shader(device, "stream.vert"); @@ -511,7 +552,9 @@ void scenes::stream::render(const XrFrameState & frame_state) .MultisampleState = {{ .rasterizationSamples = vk::SampleCountFlagBits::e1, }}, - .ColorBlendAttachments = {{.colorWriteMask = vk::ColorComponentFlagBits::eR | vk::ColorComponentFlagBits::eG | vk::ColorComponentFlagBits::eB}}, + .ColorBlendAttachments = { + {.colorWriteMask = i.alpha() ? vk::ColorComponentFlagBits::eA + : vk::ColorComponentFlagBits::eA | vk::ColorComponentFlagBits::eR | vk::ColorComponentFlagBits::eG | vk::ColorComponentFlagBits::eB}}, .DynamicStates = {vk::DynamicState::eViewport, vk::DynamicState::eScissor}, .layout = *i.blit_pipeline_layout, .renderPass = *blit_render_pass, @@ -566,68 +609,67 @@ void scenes::stream::render(const XrFrameState & frame_state) command_buffer.begin(begin_info); // Keep a reference to the resources needed to blit the images until vkWaitForFences - std::vector> blit_handles; command_buffer.resetQueryPool(*query_pool, 0, size_gpu_timestamps); command_buffer.writeTimestamp(vk::PipelineStageFlagBits::eTopOfPipe, *query_pool, 0); + // Search for frame with desired display time on all decoders + // If no such frame exists, use the latest frame for each decoder + current_blit_handles = common_frame(frame_state.predictedDisplayTime); std::array pose{}; std::array fov{}; std::array foveation{}; + bool use_alpha = false; + + // Blit images from the decoders + for (auto [i, blit_handle]: std::views::zip(decoders, current_blit_handles)) { - // Search for frame with desired display time on all decoders - // If no such frame exists, use the latest frame for each decoder - blit_handles = common_frame(frame_state.predictedDisplayTime); + if (not blit_handle) + continue; - // Blit images from the decoders - for (auto [i, blit_handle]: std::views::zip(decoders, blit_handles)) + blit_handle->feedback.blitted = application::now(); + if (blit_handle->feedback.blitted - blit_handle->feedback.received_from_decoder > 1'000'000'000) + state_ = stream::state::stalled; + ++blit_handle->feedback.times_displayed; + blit_handle->feedback.displayed = frame_state.predictedDisplayTime; + + pose = blit_handle->view_info.pose; + fov = blit_handle->view_info.fov; + foveation = blit_handle->view_info.foveation; + use_alpha = blit_handle->view_info.alpha; + + vk::DescriptorImageInfo image_info{ + .imageView = *blit_handle->image_view, + .imageLayout = vk::ImageLayout::eGeneral, + }; + + vk::WriteDescriptorSet descriptor_write{ + .dstSet = i.descriptor_set, + .dstBinding = 0, + .dstArrayElement = 0, + .descriptorCount = 1, + .descriptorType = vk::DescriptorType::eCombinedImageSampler, + .pImageInfo = &image_info, + }; + + device.updateDescriptorSets(descriptor_write, {}); + if (*blit_handle->current_layout != vk::ImageLayout::eGeneral) { - if (not blit_handle) - continue; - - blit_handle->feedback.blitted = application::now(); - if (blit_handle->feedback.blitted - blit_handle->feedback.received_from_decoder > 1'000'000'000) - state_ = stream::state::stalled; - ++blit_handle->feedback.times_displayed; - blit_handle->feedback.displayed = frame_state.predictedDisplayTime; - - pose = blit_handle->view_info.pose; - fov = blit_handle->view_info.fov; - foveation = blit_handle->view_info.foveation; - - vk::DescriptorImageInfo image_info{ - .imageView = *blit_handle->image_view, - .imageLayout = vk::ImageLayout::eGeneral, + vk::ImageMemoryBarrier barrier{ + .srcAccessMask = vk::AccessFlagBits::eNone, + .dstAccessMask = vk::AccessFlagBits::eMemoryRead, + .oldLayout = *blit_handle->current_layout, + .newLayout = vk::ImageLayout::eGeneral, + .image = blit_handle->image, + .subresourceRange = { + .aspectMask = vk::ImageAspectFlagBits::eColor, + .levelCount = 1, + .layerCount = 1, + }, }; - vk::WriteDescriptorSet descriptor_write{ - .dstSet = i.descriptor_set, - .dstBinding = 0, - .dstArrayElement = 0, - .descriptorCount = 1, - .descriptorType = vk::DescriptorType::eCombinedImageSampler, - .pImageInfo = &image_info, - }; - - device.updateDescriptorSets(descriptor_write, {}); - if (*blit_handle->current_layout != vk::ImageLayout::eGeneral) - { - vk::ImageMemoryBarrier barrier{ - .srcAccessMask = vk::AccessFlagBits::eNone, - .dstAccessMask = vk::AccessFlagBits::eMemoryRead, - .oldLayout = *blit_handle->current_layout, - .newLayout = vk::ImageLayout::eGeneral, - .image = blit_handle->image, - .subresourceRange = { - .aspectMask = vk::ImageAspectFlagBits::eColor, - .levelCount = 1, - .layerCount = 1, - }, - }; - - command_buffer.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands, vk::PipelineStageFlagBits::eAllCommands, {}, {}, {}, barrier); - *blit_handle->current_layout = vk::ImageLayout::eGeneral; - } + command_buffer.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands, vk::PipelineStageFlagBits::eAllCommands, {}, {}, {}, barrier); + *blit_handle->current_layout = vk::ImageLayout::eGeneral; } } @@ -650,20 +692,22 @@ void scenes::stream::render(const XrFrameState & frame_state) { if (not *decoder.blit_pipeline) continue; + if (decoder.alpha() and not use_alpha) + continue; command_buffer.bindPipeline(vk::PipelineBindPoint::eGraphics, *decoder.blit_pipeline); const auto & description = decoder.decoder->desc(); int x0 = description.offset_x - x_offset; int y0 = description.offset_y; - int x1 = x0 + description.width; - int y1 = y0 + description.height; + int x1 = x0 + description.width * description.subsampling; + int y1 = y0 + description.height * description.subsampling; vk::Viewport viewport{ .x = (float)x0, .y = (float)y0, - .width = (float)description.width, - .height = (float)description.height, + .width = float(description.width * description.subsampling), + .height = float(description.height * description.subsampling), .minDepth = 0, .maxDepth = 1, }; @@ -710,9 +754,27 @@ void scenes::stream::render(const XrFrameState & frame_state) submit_info.setCommandBuffers(*command_buffer); queue.submit(submit_info, *fence); + XrEnvironmentBlendMode blend_mode = XR_ENVIRONMENT_BLEND_MODE_OPAQUE; std::vector layers_base; std::vector layer_view(view_count); + if (use_alpha) + { + session.enable_passthrough(system); + std::visit( + utils::overloaded{ + [&](std::monostate &) { + assert(false); + }, + [&](xr::passthrough_alpha_blend & p) { + blend_mode = XR_ENVIRONMENT_BLEND_MODE_ALPHA_BLEND; + }, + [&](auto & p) { + layers_base.push_back(p.layer()); + }}, + session.get_passthrough()); + } + for (size_t swapchain_index = 0; swapchain_index < view_count; swapchain_index++) { swapchains[swapchain_index].release(); @@ -738,7 +800,7 @@ void scenes::stream::render(const XrFrameState & frame_state) XrCompositionLayerProjection layer{ .type = XR_TYPE_COMPOSITION_LAYER_PROJECTION, - .layerFlags = 0, + .layerFlags = XR_COMPOSITION_LAYER_BLEND_TEXTURE_SOURCE_ALPHA_BIT, .space = application::space(xr::spaces::world), .viewCount = (uint32_t)layer_view.size(), .views = layer_view.data(), @@ -747,7 +809,7 @@ void scenes::stream::render(const XrFrameState & frame_state) std::vector imgui_layers; if (imgui_ctx and plots_visible) { - accumulate_metrics(frame_state.predictedDisplayTime, blit_handles, timestamps); + accumulate_metrics(frame_state.predictedDisplayTime, current_blit_handles, timestamps); imgui_layers = plot_performance_metrics(frame_state.predictedDisplayTime); } @@ -761,7 +823,7 @@ void scenes::stream::render(const XrFrameState & frame_state) try { - session.end_frame(frame_state.predictedDisplayTime, layers_base); + session.end_frame(frame_state.predictedDisplayTime, layers_base, blend_mode); } catch (std::system_error & e) { @@ -774,8 +836,8 @@ void scenes::stream::render(const XrFrameState & frame_state) // Network operations may be blocking, do them once everything was submitted { std::vector packets; - packets.reserve(blit_handles.size()); - for (const auto & handle: blit_handles) + packets.reserve(current_blit_handles.size()); + for (const auto & handle: current_blit_handles) { if (handle) { @@ -929,37 +991,36 @@ void scenes::stream::setup(const to_headset::video_stream_description & descript { vk::DescriptorPoolSize pool_size{ .type = vk::DescriptorType::eCombinedImageSampler, - .descriptorCount = uint32_t(description.items.size()), + .descriptorCount = uint32_t(2 * description.items.size()), }; blit_descriptor_pool = vk::raii::DescriptorPool( device, vk::DescriptorPoolCreateInfo{ - .maxSets = uint32_t(description.items.size()), + .maxSets = uint32_t(2 * description.items.size()), .poolSizeCount = 1, .pPoolSizes = &pool_size, }); } + if (instance.has_extension(XR_FB_DISPLAY_REFRESH_RATE_EXTENSION_NAME)) + { + try + { + session.set_refresh_rate(description.fps); + } + catch (std::exception & e) + { + spdlog::warn("Failed to set refresh rate to {}: {}", description.fps, e.what()); + } + } + for (const auto & [stream_index, item]: utils::enumerate(description.items)) { spdlog::info("Creating decoder size {}x{} offset {},{}", item.width, item.height, item.offset_x, item.offset_y); - if (instance.has_extension(XR_FB_DISPLAY_REFRESH_RATE_EXTENSION_NAME)) - { - try - { - session.set_refresh_rate(description.fps); - } - catch (std::exception & e) - { - spdlog::warn("Failed to set refresh rate to {}: {}", description.fps, e.what()); - } - } - - accumulator_images dec; - dec.decoder = std::make_unique(device, physical_device, item, description.fps, shared_from_this(), stream_index); - - decoders.push_back(std::move(dec)); + decoders.push_back(accumulator_images{ + .decoder = std::make_unique(device, physical_device, item, description.fps, shared_from_this(), stream_index), + }); } } diff --git a/client/scenes/stream.h b/client/scenes/stream.h index 2906d874..2a282d4a 100644 --- a/client/scenes/stream.h +++ b/client/scenes/stream.h @@ -27,6 +27,7 @@ #include "wifi_lock.h" #include "wivrn_client.h" #include "wivrn_packets.h" +#include "xr/passthrough.h" #include #include #include @@ -60,7 +61,8 @@ private: // latest frames from oldest to most recent std::array, image_buffer_size> latest_frames; - std::shared_ptr frame(std::optional id) const; + std::shared_ptr frame(uint64_t id) const; + bool alpha() const; std::vector frames() const; }; @@ -99,6 +101,8 @@ private: std::optional reprojector; + std::optional passthrough; + vk::raii::Fence fence = nullptr; vk::raii::CommandBuffer command_buffer = nullptr; diff --git a/client/scenes/stream_network.cpp b/client/scenes/stream_network.cpp index be8b69b9..971ae639 100644 --- a/client/scenes/stream_network.cpp +++ b/client/scenes/stream_network.cpp @@ -45,12 +45,12 @@ void scenes::stream::process_packets() void scenes::stream::operator()(to_headset::video_stream_data_shard && shard) { std::shared_lock lock(decoder_mutex); - if (shard.stream_item_idx >= decoders.size()) + uint8_t idx = shard.stream_item_idx; + if (idx >= decoders.size()) { // We don't know (yet?) about this stream, ignore packet return; } - auto idx = shard.stream_item_idx; decoders[idx].decoder->push_shard(std::move(shard)); } diff --git a/client/scenes/stream_reprojection.cpp b/client/scenes/stream_reprojection.cpp index d65b58c2..0d5b7ee1 100644 --- a/client/scenes/stream_reprojection.cpp +++ b/client/scenes/stream_reprojection.cpp @@ -390,7 +390,6 @@ void stream_reprojection::reproject(vk::raii::CommandBuffer & command_buffer, in ubo[source]->xc.y = foveation_parameters[source].y.center; } - vk::ClearValue clear(vk::ClearColorValue(0, 0, 0, 0)); vk::RenderPassBeginInfo begin_info{ .renderPass = *renderpass, .framebuffer = *framebuffers[destination], @@ -398,8 +397,6 @@ void stream_reprojection::reproject(vk::raii::CommandBuffer & command_buffer, in .offset = {0, 0}, .extent = extent, }, - .clearValueCount = 1, - .pClearValues = &clear, }; command_buffer.pipelineBarrier( diff --git a/client/scenes/stream_stats.cpp b/client/scenes/stream_stats.cpp index c39ef94d..660281b0 100644 --- a/client/scenes/stream_stats.cpp +++ b/client/scenes/stream_stats.cpp @@ -111,6 +111,8 @@ void scenes::stream::accumulate_metrics(XrTime predicted_display_time, const std { if (metrics.size() != global_metrics.size()) metrics.resize(global_metrics.size()); + if (not bh) + continue; // clang-format off metrics[metrics_offset] = bh ? decoder_metric{ @@ -155,7 +157,7 @@ std::vector scenes::stream::plot_performance_metrics(XrT // clang-format on }; - int n_plots = plots.size() + decoders.size(); + int n_plots = plots.size() + decoder_metrics.size(); axis_scale.resize(n_plots); int n_cols = 2; diff --git a/client/shaders/reprojection.glsl b/client/shaders/reprojection.glsl index a61cecd5..f143af10 100644 --- a/client/shaders/reprojection.glsl +++ b/client/shaders/reprojection.glsl @@ -83,7 +83,7 @@ layout(location = 0) out vec4 outColor; void main() { - outColor = vec4(texture(texSampler, inUV).rgb, 1); + outColor = texture(texSampler, inUV).rgba; } #endif diff --git a/client/shaders/stream.glsl b/client/shaders/stream.glsl index 7bbbf6e8..84b4ca78 100644 --- a/client/shaders/stream.glsl +++ b/client/shaders/stream.glsl @@ -21,8 +21,8 @@ #ifdef VERT_SHADER -layout (constant_id = 0) const float useful_size_x = 1.0; -layout (constant_id = 1) const float useful_size_y = 1.0; +layout(constant_id = 0) const float useful_size_x = 1.0; +layout(constant_id = 1) const float useful_size_y = 1.0; vec2 positions[3] = vec2[]( vec2(-1.0, -1.0), @@ -46,6 +46,7 @@ void main() #ifdef FRAG_SHADER layout(binding = 0) uniform sampler2D texSampler; layout(constant_id = 0) const bool do_srgb = true; +layout(constant_id = 1) const bool is_alpha = false; layout(location = 0) in vec2 inUV; @@ -53,19 +54,18 @@ layout(location = 0) out vec4 outColor; float sRGB_to_linear(float x) { - if (x <= 0.04045) - return x / 12.92; - return pow((x + 0.055) / 1.055, 2.4); + if (x <= 0.04045) + return x / 12.92; + return pow((x + 0.055) / 1.055, 2.4); } vec4 sRGB_to_linear_rgba(vec4 x) { return vec4( - sRGB_to_linear(x.r), - sRGB_to_linear(x.g), - sRGB_to_linear(x.b), - x.a - ); + sRGB_to_linear(x.r), + sRGB_to_linear(x.g), + sRGB_to_linear(x.b), + x.r); } void main() @@ -77,6 +77,11 @@ void main() else { outColor = texture(texSampler, inUV); + outColor.a = outColor.r; + } + if (!is_alpha) + { + outColor.a = 1; } } #endif diff --git a/client/utils/overloaded.h b/client/utils/overloaded.h new file mode 100644 index 00000000..0ea8b4bf --- /dev/null +++ b/client/utils/overloaded.h @@ -0,0 +1,31 @@ +/* + * WiVRn VR streaming + * Copyright (C) 2024 Guillaume Meunier + * Copyright (C) 2024 Patrick Nicolas + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once + +namespace utils +{ +template +struct overloaded : Ts... +{ + using Ts::operator()...; +}; +template +overloaded(Ts...) -> overloaded; +} // namespace utils diff --git a/client/xr/session.cpp b/client/xr/session.cpp index 5ef6d391..d2d670be 100644 --- a/client/xr/session.cpp +++ b/client/xr/session.cpp @@ -21,8 +21,8 @@ #include "application.h" #include "details/enumerate.h" -#include "xr/instance.h" #include "utils/contains.h" +#include "xr/instance.h" #include "xr/system.h" #include #include @@ -347,6 +347,9 @@ void xr::session::sync_actions(XrActionSet action_set, const std::string & subac void xr::session::enable_passthrough(xr::system & system) { + if (not std::holds_alternative(passthrough)) + return; + if (system.passthrough_supported() == xr::system::passthrough_type::no_passthrough) return; @@ -366,5 +369,7 @@ void xr::session::enable_passthrough(xr::system & system) void xr::session::disable_passthrough() { + if (std::holds_alternative(passthrough)) + return; passthrough.emplace(); } diff --git a/common/wivrn_packets.h b/common/wivrn_packets.h index 1d4f44d1..ef7ea644 100644 --- a/common/wivrn_packets.h +++ b/common/wivrn_packets.h @@ -114,6 +114,7 @@ struct headset_info_packet bool eye_gaze; bool face_tracking2_fb; bool palm_pose; + bool passthrough; std::vector supported_codecs; // from preferred to least preferred }; @@ -232,7 +233,7 @@ struct timesync_response struct feedback { uint64_t frame_index; - uint8_t stream_index; + uint8_t stream_index; // 0-127 for yuv, 128-255 for alpha // Timestamps XrTime received_first_packet; @@ -290,6 +291,11 @@ struct audio_stream_description struct video_stream_description { + enum class channels_t + { + colour, + alpha, + }; struct item { // useful dimensions of the video stream @@ -301,6 +307,8 @@ struct video_stream_description uint16_t offset_x; uint16_t offset_y; video_codec codec; + channels_t channels; + uint8_t subsampling; // applies to width/height only, offsets are in full size pixels std::optional range; std::optional color_model; }; @@ -322,6 +330,7 @@ public: end_of_frame = 1 << 2, }; // Identifier of stream in video_stream_description + // elements > 127 are for alpha uint8_t stream_item_idx; // Counter increased for each frame uint64_t frame_idx; @@ -338,6 +347,8 @@ public: std::array pose; std::array fov; std::array foveation; + // True when the frame contains an alpha channel + bool alpha; }; std::optional view_info; diff --git a/patches/monado/0012-store-alpha-channel-in-layer-1.patch b/patches/monado/0012-store-alpha-channel-in-layer-1.patch new file mode 100644 index 00000000..fcef6b1f --- /dev/null +++ b/patches/monado/0012-store-alpha-channel-in-layer-1.patch @@ -0,0 +1,94 @@ +From 3a5642d8993d18f0a886dc6b7748ea718380e771 Mon Sep 17 00:00:00 2001 +From: Patrick Nicolas +Date: Sun, 3 Nov 2024 13:40:00 +0100 +Subject: [PATCH] store alpha channel in layer 1 + +--- + src/xrt/compositor/shaders/clear.comp | 4 ++-- + src/xrt/compositor/shaders/distortion.comp | 18 ++++++++++++------ + 2 files changed, 14 insertions(+), 8 deletions(-) + +diff --git a/src/xrt/compositor/shaders/clear.comp b/src/xrt/compositor/shaders/clear.comp +index 036da401a..db8e5178e 100644 +--- a/src/xrt/compositor/shaders/clear.comp ++++ b/src/xrt/compositor/shaders/clear.comp +@@ -10,7 +10,7 @@ + + layout(local_size_x = 8, local_size_y = 8, local_size_z = 1) in; + +-layout(set = 0, binding = 2) uniform writeonly restrict image2D target; ++layout(set = 0, binding = 2) uniform writeonly restrict image2DArray target; + layout(set = 0, binding = 3) uniform restrict Config + { + ivec4 views[2]; +@@ -35,5 +35,5 @@ void main() + + vec4 colour = vec4(from_linear_to_srgb(vec3(0.1)), 1.0); + +- imageStore(target, ivec2(offset.x + ix, offset.y + iy), colour); ++ imageStore(target, ivec3(offset.x + ix, offset.y + iy, 0), colour); + } +diff --git a/src/xrt/compositor/shaders/distortion.comp b/src/xrt/compositor/shaders/distortion.comp +index 461aeb4ec..237221a54 100644 +--- a/src/xrt/compositor/shaders/distortion.comp ++++ b/src/xrt/compositor/shaders/distortion.comp +@@ -18,7 +18,7 @@ layout(local_size_x = 8, local_size_y = 8, local_size_z = 1) in; + + layout(set = 0, binding = 0) uniform sampler2D source[2]; + layout(set = 0, binding = 1) uniform sampler2D distortion[6]; +-layout(set = 0, binding = 2) uniform writeonly restrict image2D luma; ++layout(set = 0, binding = 2) uniform writeonly restrict image2DArray luma; + layout(set = 0, binding = 3, std140) uniform restrict Config + { + ivec4 views[2]; +@@ -26,7 +26,7 @@ layout(set = 0, binding = 3, std140) uniform restrict Config + vec4 post_transform[2]; + mat4 transform[2]; + } ubo; +-layout(set = 0, binding = 4) uniform writeonly restrict image2D chroma; ++layout(set = 0, binding = 4) uniform writeonly restrict image2DArray chroma; + + const mat3 color_space = mat3( + // clang-format off +@@ -142,6 +142,7 @@ void main() + ivec2 offset = ivec2(ubo.views[iz].xy); + ivec2 extent = ivec2(ubo.views[iz].zw); + vec2 cbcrs[4]; ++ vec4 alpha; + for (int k = 0; k < 2; k += 1) { + uint iy = gl_GlobalInvocationID.y * 2 + k; + for (int j = 0; j < 2; j += 1) { +@@ -156,15 +157,16 @@ void main() + uv = transform_uv(uv, iz); + + // Sample the source +- vec3 colour = textureLod(source[iz], uv, mix(lod.x, lod.y, 0.5)).rgb; ++ vec4 colour = textureLod(source[iz], uv, mix(lod.x, lod.y, 0.5)); + +- colour = rgb_to_ycbcr(from_linear_to_srgb(colour)); ++ colour.rgb = rgb_to_ycbcr(from_linear_to_srgb(colour.rgb)); + + int i = k * 2 + j; + cbcrs[i] = colour.yz; ++ alpha[i] = colour.a; + + if (ix < extent.x && iy < extent.y) { +- imageStore(luma, ivec2(offset.x + ix, offset.y + iy), vec4(colour.x)); ++ imageStore(luma, ivec3(offset.x + ix, offset.y + iy, 0), vec4(colour.x)); + } + } + } +@@ -172,5 +174,9 @@ void main() + return; + } + vec2 cbcr = mix(mix(cbcrs[0], cbcrs[1], 0.5), mix(cbcrs[2], cbcrs[3], 0.5), 0.5); +- imageStore(chroma, ivec2(offset.xy/2 + gl_GlobalInvocationID.xy), vec4(cbcr.x, cbcr.y, 0, 0)); ++ imageStore(chroma, ivec3(offset.xy/2 + gl_GlobalInvocationID.xy, 0), vec4(cbcr.x, cbcr.y, 0, 0)); ++ ++ imageStore(chroma, ivec3(offset.xy/2 + gl_GlobalInvocationID.xy, 1), vec4(0.5, 0.5, 0, 0)); ++ alpha = vec4(mix(mix(alpha[0], alpha[1], 0.5), mix(alpha[2], alpha[3], 0.5), 0.5)); ++ imageStore(luma, ivec3(offset.xy/2 + gl_GlobalInvocationID.xy, 1), alpha); + } +-- +2.47.0 + diff --git a/server/driver/configuration.cpp b/server/driver/configuration.cpp index 4abfbb52..49a141e4 100644 --- a/server/driver/configuration.cpp +++ b/server/driver/configuration.cpp @@ -60,6 +60,29 @@ const std::filesystem::path & configuration::get_config_file() return config_file; } +configuration::encoder parse_encoder(const nlohmann::json & item) +{ + configuration::encoder e; + if (item.contains("encoder")) + e.name = item["encoder"]; + +#define SET_IF(property) \ + if (item.contains(#property)) \ + e.property = item[#property]; + + SET_IF(width); + SET_IF(height); + SET_IF(offset_x); + SET_IF(offset_y); + SET_IF(group); + SET_IF(codec); + if (e.codec == wivrn::video_codec(-1)) + throw std::runtime_error("invalid codec value " + item["codec"].get()); + SET_IF(options); + SET_IF(device); + return e; +} + configuration configuration::read_user_configuration() { configuration result; @@ -86,29 +109,12 @@ configuration configuration::read_user_configuration() if (json.contains("encoders")) { for (const auto & encoder: json["encoders"]) - { - configuration::encoder e; - if (encoder.contains("encoder")) - e.name = encoder["encoder"]; - -#define SET_IF(property) \ - if (encoder.contains(#property)) \ - e.property = encoder[#property]; - - SET_IF(width); - SET_IF(height); - SET_IF(offset_x); - SET_IF(offset_y); - SET_IF(group); - SET_IF(codec); - if (e.codec == wivrn::video_codec(-1)) - throw std::runtime_error("invalid codec value " + encoder["codec"].get()); - SET_IF(options); - SET_IF(device); - result.encoders.push_back(e); - } + result.encoders.push_back(parse_encoder(encoder)); } + if (json.contains("encoder-passthrough")) + result.encoder_passthrough = parse_encoder(json["encoder-passthrough"]); + if (json.contains("application")) { if (json["application"].is_string()) diff --git a/server/driver/configuration.h b/server/driver/configuration.h index 057ad4c0..c91804c2 100644 --- a/server/driver/configuration.h +++ b/server/driver/configuration.h @@ -46,6 +46,7 @@ struct configuration }; std::vector encoders; + std::optional encoder_passthrough; std::optional bitrate; std::optional> scale; std::vector application; diff --git a/server/driver/wivrn_comp_target.cpp b/server/driver/wivrn_comp_target.cpp index a29749fc..3a74b12e 100644 --- a/server/driver/wivrn_comp_target.cpp +++ b/server/driver/wivrn_comp_target.cpp @@ -114,7 +114,7 @@ static void destroy_images(struct wivrn_comp_target * cn) target_fini_semaphores(cn); } -static void comp_wivrn_present_thread(std::stop_token stop_token, wivrn_comp_target * cn, int index, std::vector> encoders); +static void comp_wivrn_present_thread(std::stop_token stop_token, wivrn_comp_target * cn, int index, std::vector> encoders); static void create_encoders(wivrn_comp_target * cn) { @@ -129,7 +129,7 @@ static void create_encoders(wivrn_comp_target * cn) desc.height = cn->height; desc.foveation = cn->cnx.set_foveated_size(desc.width, desc.height); - std::map>> thread_params; + std::map>> thread_params; #if WIVRN_USE_VULKAN_ENCODE if (std::ranges::any_of(cn->settings, [](const auto & item) { return item.encoder_name == encoder_vulkan; })) @@ -162,7 +162,7 @@ static void create_encoders(wivrn_comp_target * cn) { uint8_t stream_index = cn->encoders.size(); auto & encoder = cn->encoders.emplace_back( - VideoEncoder::Create(*cn->wivrn_bundle, settings, stream_index, desc.width, desc.height, desc.fps)); + video_encoder::create(*cn->wivrn_bundle, settings, stream_index, desc.width, desc.height, desc.fps)); desc.items.push_back(settings); thread_params[settings.group].emplace_back(encoder); @@ -193,7 +193,7 @@ static VkResult create_images(struct wivrn_comp_target * cn, vk::ImageUsageFlags cn->images = U_TYPED_ARRAY_CALLOC(struct comp_target_image, cn->image_count); #if WIVRN_USE_VULKAN_ENCODE - auto [video_profiles, encoder_flags] = VideoEncoder::get_create_image_info(cn->settings); + auto [video_profiles, encoder_flags] = video_encoder::get_create_image_info(cn->settings); vk::VideoProfileListInfoKHR video_profile_list{ .profileCount = uint32_t(video_profiles.size()), @@ -202,7 +202,6 @@ static VkResult create_images(struct wivrn_comp_target * cn, vk::ImageUsageFlags #endif cn->psc.images.resize(cn->image_count); - std::vector rgb; for (uint32_t i = 0; i < cn->image_count; i++) { std::array formats = { @@ -233,7 +232,7 @@ static VkResult create_images(struct wivrn_comp_target * cn, vk::ImageUsageFlags .depth = 1, }, .mipLevels = 1, - .arrayLayers = 1, + .arrayLayers = 2, // colour then alpha .samples = vk::SampleCountFlagBits::e1, .tiling = vk::ImageTiling::eOptimal, .usage = flags @@ -247,7 +246,6 @@ static VkResult create_images(struct wivrn_comp_target * cn, vk::ImageUsageFlags .usage = VMA_MEMORY_USAGE_AUTO, }); cn->images[i].handle = image; - rgb.push_back(image); } for (uint32_t i = 0; i < cn->image_count; i++) @@ -260,24 +258,24 @@ static VkResult create_images(struct wivrn_comp_target * cn, vk::ImageUsageFlags { .pNext = &usage, .image = item.image, - .viewType = vk::ImageViewType::e2D, + .viewType = vk::ImageViewType::e2DArray, .format = vk::Format::eR8Unorm, .subresourceRange = { .aspectMask = vk::ImageAspectFlagBits::ePlane0, .levelCount = 1, - .layerCount = 1, + .layerCount = 2, }, }); item.image_view_cbcr = vk::raii::ImageView(device, { .pNext = &usage, .image = item.image, - .viewType = vk::ImageViewType::e2D, + .viewType = vk::ImageViewType::e2DArray, .format = vk::Format::eR8G8Unorm, .subresourceRange = { .aspectMask = vk::ImageAspectFlagBits::ePlane1, .levelCount = 1, - .layerCount = 1, + .layerCount = 2, }, }); cn->images[i].view = *item.image_view_y; @@ -460,7 +458,7 @@ static VkResult comp_wivrn_acquire(struct comp_target * ct, uint32_t * out_index }; } -static void comp_wivrn_present_thread(std::stop_token stop_token, wivrn_comp_target * cn, int index, std::vector> encoders) +static void comp_wivrn_present_thread(std::stop_token stop_token, wivrn_comp_target * cn, int index, std::vector> encoders) { auto & vk = *cn->wivrn_bundle; U_LOG_I("Starting encoder thread %d", index); @@ -506,7 +504,8 @@ static void comp_wivrn_present_thread(std::stop_token stop_token, wivrn_comp_tar { for (auto & encoder: encoders) { - encoder->Encode(cn->cnx, view_info, frame_index); + if (encoder->channels == to_headset::video_stream_description::channels_t::colour or view_info.alpha) + encoder->Encode(cn->cnx, view_info, frame_index); } } catch (std::exception & e) @@ -604,9 +603,12 @@ static VkResult comp_wivrn_present(struct comp_target * ct, cn->wivrn_bundle->device.resetFences(*cn->psc.fence); psc_image.status = pseudo_swapchain::status_t::encoding; auto info = cn->pacer.present_to_info(desired_present_time_ns); + const bool do_alpha = cn->c->base.layer_accum.data.env_blend_mode == XRT_BLEND_MODE_ALPHA_BLEND; for (auto & encoder: cn->encoders) { + if (encoder->channels == to_headset::video_stream_description::channels_t::alpha and not do_alpha) + continue; #if WIVRN_USE_VULKAN_ENCODE encoder->present_image(psc_image.image, video_command_buffer, *cn->psc.images[index].video_fence, info.frame_id); #endif @@ -658,6 +660,9 @@ static VkResult comp_wivrn_present(struct comp_target * ct, auto & view_info = cn->psc.view_info; view_info.foveation = cn->cnx.get_foveation_parameters(); view_info.display_time = cn->cnx.get_offset().to_headset(info.predicted_display_time); + if (view_info.alpha != do_alpha) + cn->pacer.reset(); + view_info.alpha = do_alpha; for (int eye = 0; eye < 2; ++eye) { const auto & frame_params = cn->c->base.frame_params; @@ -788,11 +793,13 @@ void wivrn_comp_target::on_feedback(const from_headset::feedback & feedback, con { if (not o) return; - pacer.on_feedback(feedback, o); + uint8_t stream = feedback.stream_index; if (psc.status & 1) return; - if (feedback.stream_index < encoders.size()) - encoders[feedback.stream_index]->on_feedback(feedback); + if (encoders.size() <= stream) + return; + encoders[stream]->on_feedback(feedback); + pacer.on_feedback(feedback, o); } void wivrn_comp_target::reset_encoders() diff --git a/server/driver/wivrn_comp_target.h b/server/driver/wivrn_comp_target.h index 2c4408fd..57962e23 100644 --- a/server/driver/wivrn_comp_target.h +++ b/server/driver/wivrn_comp_target.h @@ -24,6 +24,7 @@ #include "encoder/encoder_settings.h" #include "utils/wivrn_vk_bundle.h" #include "vk/allocation.h" +#include "wivrn_foveation.h" #include "wivrn_pacer.h" #include "wivrn_packets.h" @@ -37,9 +38,8 @@ namespace wivrn { -class wivrn_foveation_renderer; class wivrn_session; -class VideoEncoder; +class video_encoder; struct pseudo_swapchain { @@ -101,7 +101,7 @@ struct wivrn_comp_target : public comp_target std::vector settings; to_headset::video_stream_description desc{}; std::list encoder_threads; - std::vector> encoders; + std::vector> encoders; wivrn::wivrn_session & cnx; std::unique_ptr foveation_renderer = nullptr; diff --git a/server/driver/wivrn_hmd.cpp b/server/driver/wivrn_hmd.cpp index b4b0db7c..a97a64a0 100644 --- a/server/driver/wivrn_hmd.cpp +++ b/server/driver/wivrn_hmd.cpp @@ -226,8 +226,10 @@ wivrn_hmd::wivrn_hmd(wivrn::wivrn_session * cnx, // Setup info. hmd->view_count = 2; - hmd->blend_modes[0] = XRT_BLEND_MODE_OPAQUE; - hmd->blend_mode_count = 1; + hmd->blend_modes[hmd->blend_mode_count++] = XRT_BLEND_MODE_OPAQUE; + if (info.passthrough) + hmd->blend_modes[hmd->blend_mode_count++] = XRT_BLEND_MODE_ALPHA_BLEND; + hmd->distortion.models = XRT_DISTORTION_MODEL_NONE; hmd->distortion.preferred = XRT_DISTORTION_MODEL_NONE; hmd->screens[0].w_pixels = eye_width * 2; diff --git a/server/encoder/encoder_settings.cpp b/server/encoder/encoder_settings.cpp index ef9b600a..f15ba6a6 100644 --- a/server/encoder/encoder_settings.cpp +++ b/server/encoder/encoder_settings.cpp @@ -43,6 +43,10 @@ namespace wivrn // TODO: size independent bitrate static const uint64_t default_bitrate = 50'000'000; +// subsampling is also in monado's distortion shader +static const uint8_t passthrough_subsampling = 2; +static const double passthrough_bitrate_factor = 0.05; + #define WIVRN_SPLIT_ENCODERS 1 static bool is_nvidia(vk::PhysicalDevice physical_device) @@ -88,6 +92,7 @@ void print_encoders(const std::vector & encoders) } std::string codec(magic_enum::enum_name(encoder.codec)); U_LOG_I("\t%s (%s)", encoder.encoder_name.c_str(), codec.c_str()); + U_LOG_I("\tchannels: %s", std::string(magic_enum::enum_name(encoder.channels)).c_str()); U_LOG_I("\tsize:%dx%d offset:%dx%d", encoder.width, encoder.height, @@ -102,7 +107,7 @@ static void check_scale(std::string_view encoder_name, video_codec codec, uint16 #if WIVRN_USE_NVENC if (encoder_name == encoder_nvenc) { - auto max = VideoEncoderNvenc::get_max_size(codec); + auto max = video_encoder_nvenc::get_max_size(codec); if (width * scale[0] > max[0]) { scale[0] = double(max[0] - 1) / width; @@ -135,7 +140,7 @@ static constexpr auto ffmpeg_version() static std::optional filter_codecs_vaapi(wivrn_vk_bundle & bundle, const std::vector & codecs) { - VideoEncoderFFMPEG::mute_logs mute; + video_encoder_ffmpeg::mute_logs mute; encoder_settings s{ { .width = 800, @@ -159,7 +164,7 @@ static std::optional filter_codecs_vaapi(wivrn_vk_bundle & b try { s.codec = codec; - video_encoder_va test(bundle, s, 60); + video_encoder_va test(bundle, s, 60, 0); return codec; } catch (...) @@ -295,10 +300,27 @@ std::vector get_encoder_settings(wivrn_vk_bundle & bundle, uin } if (config.encoders.empty()) config.encoders = get_encoder_default_settings(bundle, info.supported_codecs); + if (not config.encoder_passthrough) + config.encoder_passthrough.emplace(); + + config.encoder_passthrough->width = 1; + config.encoder_passthrough->height = 1; + config.encoder_passthrough->offset_x = 0; + config.encoder_passthrough->offset_y = 0; + fill_defaults(bundle, info.supported_codecs, *config.encoder_passthrough); + uint64_t bitrate = config.bitrate.value_or(default_bitrate); std::array default_scale; default_scale.fill(info.eye_gaze ? 0.35 : 0.5); auto scale = config.scale.value_or(default_scale); + + check_scale(config.encoder_passthrough->name, + *config.encoder_passthrough->codec, + // passthrough stream is subsampled + width / passthrough_subsampling, + height / passthrough_subsampling, + scale); + for (auto & encoder: config.encoders) { fill_defaults(bundle, info.supported_codecs, encoder); @@ -330,6 +352,8 @@ std::vector get_encoder_settings(wivrn_vk_bundle & bundle, uin for (const auto & encoder: config.encoders) { wivrn::encoder_settings settings{}; + settings.channels = to_headset::video_stream_description::channels_t::colour; + settings.subsampling = 1; settings.encoder_name = encoder.name; settings.width = std::ceil(encoder.width.value_or(1) * width); settings.height = std::ceil(encoder.height.value_or(1) * height); @@ -355,6 +379,35 @@ std::vector get_encoder_settings(wivrn_vk_bundle & bundle, uin res.push_back(settings); } split_bitrate(res, bitrate); + + // passthrough encoder + { + const auto & encoder = *config.encoder_passthrough; + wivrn::encoder_settings settings{}; + settings.channels = to_headset::video_stream_description::channels_t::alpha; + settings.subsampling = passthrough_subsampling; + settings.encoder_name = encoder.name; + settings.width = width / settings.subsampling; + settings.height = height / settings.subsampling; + settings.width += settings.width % 2; + settings.height += settings.height % 2; + settings.video_width = settings.width; + settings.video_height = settings.height; + settings.codec = *encoder.codec; + if (encoder.group) + settings.group = *encoder.group; + else + { + auto [it, inserted] = groups.emplace(encoder.name, next_group); + settings.group = it->second; + if (inserted) + ++next_group; + } + settings.options = encoder.options; + settings.device = encoder.device; + settings.bitrate = bitrate * passthrough_bitrate_factor; + res.push_back(settings); + } return res; } } // namespace wivrn diff --git a/server/encoder/ffmpeg/video_encoder_ffmpeg.cpp b/server/encoder/ffmpeg/video_encoder_ffmpeg.cpp index 24bdac77..b447e860 100644 --- a/server/encoder/ffmpeg/video_encoder_ffmpeg.cpp +++ b/server/encoder/ffmpeg/video_encoder_ffmpeg.cpp @@ -64,9 +64,9 @@ bool set_log_level() namespace wivrn { -bool VideoEncoderFFMPEG::once = set_log_level(); +bool video_encoder_ffmpeg::once = set_log_level(); -std::optional VideoEncoderFFMPEG::encode(bool idr, std::chrono::steady_clock::time_point target_timestamp, uint8_t slot) +std::optional video_encoder_ffmpeg::encode(bool idr, std::chrono::steady_clock::time_point target_timestamp, uint8_t slot) { push_frame(idr, target_timestamp, slot); std::shared_ptr enc_pkt(av_packet_alloc(), [](AVPacket * d) { av_packet_free(&d); }); @@ -87,13 +87,13 @@ std::optional VideoEncoderFFMPEG::encode(bool idr, st throw std::runtime_error("frame encoding failed, code " + std::to_string(err)); } -VideoEncoderFFMPEG::mute_logs::mute_logs() +video_encoder_ffmpeg::mute_logs::mute_logs() { if (not getenv("FFMPEG_LOG_LEVEL")) av_log_set_level(AV_LOG_QUIET); } -VideoEncoderFFMPEG::mute_logs::~mute_logs() +video_encoder_ffmpeg::mute_logs::~mute_logs() { set_log_level(); } diff --git a/server/encoder/ffmpeg/video_encoder_ffmpeg.h b/server/encoder/ffmpeg/video_encoder_ffmpeg.h index f0394b2d..bacb2a92 100644 --- a/server/encoder/ffmpeg/video_encoder_ffmpeg.h +++ b/server/encoder/ffmpeg/video_encoder_ffmpeg.h @@ -26,7 +26,7 @@ namespace wivrn { -class VideoEncoderFFMPEG : public wivrn::VideoEncoder +class video_encoder_ffmpeg : public wivrn::video_encoder { public: std::optional encode(bool idr, std::chrono::steady_clock::time_point target_timestamp, uint8_t slot) override; @@ -38,8 +38,8 @@ public: }; protected: - VideoEncoderFFMPEG() : - wivrn::VideoEncoder(true) {} + video_encoder_ffmpeg(uint8_t stream_idx, to_headset::video_stream_description::channels_t channels) : + wivrn::video_encoder(stream_idx, channels, true) {} virtual void push_frame(bool idr, std::chrono::steady_clock::time_point pts, uint8_t slot) = 0; diff --git a/server/encoder/ffmpeg/video_encoder_va.cpp b/server/encoder/ffmpeg/video_encoder_va.cpp index 80ce9c6d..203decf8 100644 --- a/server/encoder/ffmpeg/video_encoder_va.cpp +++ b/server/encoder/ffmpeg/video_encoder_va.cpp @@ -131,7 +131,11 @@ vk::Format drm_to_vulkan_fmt(uint32_t drm_fourcc) } // namespace -video_encoder_va::video_encoder_va(wivrn_vk_bundle & vk, wivrn::encoder_settings & settings, float fps) : +video_encoder_va::video_encoder_va(wivrn_vk_bundle & vk, + wivrn::encoder_settings & settings, + float fps, + uint8_t stream_idx) : + video_encoder_ffmpeg(stream_idx, settings.channels), synchronization2(vk.vk.features.synchronization_2) { auto drm_hw_ctx = make_drm_hw_ctx(vk.physical_device, settings.device); @@ -377,7 +381,6 @@ void video_encoder_va::present_image(vk::Image y_cbcr, vk::raii::CommandBuffer & .subresourceRange = {.aspectMask = vk::ImageAspectFlagBits::eColor, .baseMipLevel = 0, .levelCount = 1, - .baseArrayLayer = 0, .layerCount = 1}, }, vk::ImageMemoryBarrier{ @@ -389,7 +392,6 @@ void video_encoder_va::present_image(vk::Image y_cbcr, vk::raii::CommandBuffer & .subresourceRange = {.aspectMask = vk::ImageAspectFlagBits::eColor, .baseMipLevel = 0, .levelCount = 1, - .baseArrayLayer = 0, .layerCount = 1}, }, }; @@ -409,6 +411,7 @@ void video_encoder_va::present_image(vk::Image y_cbcr, vk::raii::CommandBuffer & vk::ImageCopy{ .srcSubresource = { .aspectMask = vk::ImageAspectFlagBits::ePlane0, + .baseArrayLayer = uint32_t(channels), .layerCount = 1, }, .srcOffset = { @@ -433,6 +436,7 @@ void video_encoder_va::present_image(vk::Image y_cbcr, vk::raii::CommandBuffer & vk::ImageCopy{ .srcSubresource = { .aspectMask = vk::ImageAspectFlagBits::ePlane1, + .baseArrayLayer = uint32_t(channels), .layerCount = 1, }, .srcOffset = { diff --git a/server/encoder/ffmpeg/video_encoder_va.h b/server/encoder/ffmpeg/video_encoder_va.h index 2c3bdbe8..2a4bd427 100644 --- a/server/encoder/ffmpeg/video_encoder_va.h +++ b/server/encoder/ffmpeg/video_encoder_va.h @@ -32,7 +32,7 @@ namespace wivrn struct encoder_settings; struct wivrn_vk_bundle; -class video_encoder_va : public VideoEncoderFFMPEG +class video_encoder_va : public video_encoder_ffmpeg { struct in_t { @@ -48,7 +48,7 @@ class video_encoder_va : public VideoEncoderFFMPEG bool synchronization2 = false; public: - video_encoder_va(wivrn_vk_bundle &, wivrn::encoder_settings & settings, float fps); + video_encoder_va(wivrn_vk_bundle &, wivrn::encoder_settings & settings, float fps, uint8_t stream_index); void present_image(vk::Image y_cbcr, vk::raii::CommandBuffer & cmd_buf, uint8_t slot) override; diff --git a/server/encoder/video_encoder.cpp b/server/encoder/video_encoder.cpp index 84b2377e..522bfe3b 100644 --- a/server/encoder/video_encoder.cpp +++ b/server/encoder/video_encoder.cpp @@ -46,7 +46,7 @@ namespace wivrn { -VideoEncoder::sender::sender() : +video_encoder::sender::sender() : thread([this](std::stop_token t) { while (not t.stop_requested()) { @@ -73,34 +73,34 @@ VideoEncoder::sender::sender() : { } -void VideoEncoder::sender::push(data && d) +void video_encoder::sender::push(data && d) { std::unique_lock lock(mutex); pending.push_back(std::move(d)); cv.notify_all(); } -void VideoEncoder::sender::wait_idle(VideoEncoder * encoder) +void video_encoder::sender::wait_idle(video_encoder * encoder) { std::unique_lock lock(mutex); while (std::ranges::any_of(pending, [=](auto & data) { return data.encoder == encoder; })) cv.wait_for(lock, std::chrono::milliseconds(100)); } -std::shared_ptr VideoEncoder::sender::get() +std::shared_ptr video_encoder::sender::get() { - static std::weak_ptr instance; + static std::weak_ptr instance; static std::mutex m; std::unique_lock lock(m); auto s = instance.lock(); if (s) return s; - s.reset(new VideoEncoder::sender()); + s.reset(new video_encoder::sender()); instance = s; return s; } -std::unique_ptr VideoEncoder::Create( +std::unique_ptr video_encoder::create( wivrn_vk_bundle & wivrn_vk, encoder_settings & settings, uint8_t stream_idx, @@ -109,7 +109,7 @@ std::unique_ptr VideoEncoder::Create( float fps) { using namespace std::string_literals; - std::unique_ptr res; + std::unique_ptr res; settings.range = VK_SAMPLER_YCBCR_RANGE_ITU_FULL; settings.color_model = VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_709; if (settings.encoder_name == encoder_vulkan) @@ -118,7 +118,7 @@ std::unique_ptr VideoEncoder::Create( switch (settings.codec) { case video_codec::h264: - res = video_encoder_vulkan_h264::create(wivrn_vk, settings, fps); + res = video_encoder_vulkan_h264::create(wivrn_vk, settings, fps, stream_idx); break; case video_codec::h265: throw std::runtime_error("h265 not supported for vulkan video encode"); @@ -134,7 +134,7 @@ std::unique_ptr VideoEncoder::Create( if (settings.encoder_name == encoder_x264) { #if WIVRN_USE_X264 - res = std::make_unique(wivrn_vk, settings, fps); + res = std::make_unique(wivrn_vk, settings, fps, stream_idx); #else throw std::runtime_error("x264 encoder not enabled"); #endif @@ -142,7 +142,7 @@ std::unique_ptr VideoEncoder::Create( if (settings.encoder_name == encoder_nvenc) { #if WIVRN_USE_NVENC - res = std::make_unique(wivrn_vk, settings, fps); + res = std::make_unique(wivrn_vk, settings, fps, stream_idx); #else throw std::runtime_error("nvenc support not enabled"); #endif @@ -150,14 +150,13 @@ std::unique_ptr VideoEncoder::Create( if (settings.encoder_name == encoder_vaapi) { #if WIVRN_USE_VAAPI - res = std::make_unique(wivrn_vk, settings, fps); + res = std::make_unique(wivrn_vk, settings, fps, stream_idx); #else throw std::runtime_error("vaapi support not enabled"); #endif } if (not res) throw std::runtime_error("Failed to create encoder " + settings.encoder_name); - res->stream_idx = stream_idx; auto wivrn_dump_video = std::getenv("WIVRN_DUMP_VIDEO"); if (wivrn_dump_video) @@ -182,7 +181,7 @@ std::unique_ptr VideoEncoder::Create( } #if WIVRN_USE_VULKAN_ENCODE -std::pair, vk::ImageUsageFlags> VideoEncoder::get_create_image_info(const std::vector & settings) +std::pair, vk::ImageUsageFlags> video_encoder::get_create_image_info(const std::vector & settings) { std::pair, vk::ImageUsageFlags> result; for (const auto & item: settings) @@ -209,29 +208,31 @@ std::pair, vk::ImageUsageFlags> VideoEncode static const uint64_t idr_throttle = 100; -VideoEncoder::VideoEncoder(bool async_send) : +video_encoder::video_encoder(uint8_t stream_idx, to_headset::video_stream_description::channels_t channels, bool async_send) : + stream_idx(stream_idx), + channels(channels), last_idr_frame(-idr_throttle), shared_sender(async_send ? sender::get() : nullptr) {} -VideoEncoder::~VideoEncoder() +video_encoder::~video_encoder() { if (shared_sender) shared_sender->wait_idle(this); } -void VideoEncoder::on_feedback(const from_headset::feedback & feedback) +void video_encoder::on_feedback(const from_headset::feedback & feedback) { if (not feedback.sent_to_decoder) sync_needed = true; } -void VideoEncoder::reset() +void video_encoder::reset() { sync_needed = true; } -void VideoEncoder::present_image(vk::Image y_cbcr, vk::raii::CommandBuffer & cmd_buf) +void video_encoder::present_image(vk::Image y_cbcr, vk::raii::CommandBuffer & cmd_buf) { // Wait for encoder to be done busy[next_present].wait(true); @@ -241,14 +242,14 @@ void VideoEncoder::present_image(vk::Image y_cbcr, vk::raii::CommandBuffer & cmd next_present = (next_present + 1) % num_slots; } -void VideoEncoder::present_image(vk::Image y_cbcr, vk::raii::CommandBuffer & video_cmd_buf, vk::Fence fence, uint64_t frame_index) +void video_encoder::present_image(vk::Image y_cbcr, vk::raii::CommandBuffer & video_cmd_buf, vk::Fence fence, uint64_t frame_index) { present_image(y_cbcr, video_cmd_buf, fence, next_present, frame_index); } -void VideoEncoder::Encode(wivrn_session & cnx, - const to_headset::video_stream_data_shard::view_info_t & view_info, - uint64_t frame_index) +void video_encoder::Encode(wivrn_session & cnx, + const to_headset::video_stream_data_shard::view_info_t & view_info, + uint64_t frame_index) { assert(busy[next_encode].load()); if (shared_sender) @@ -303,7 +304,7 @@ void VideoEncoder::Encode(wivrn_session & cnx, std::rethrow_exception(ex); } -void VideoEncoder::SendData(std::span data, bool end_of_frame) +void video_encoder::SendData(std::span data, bool end_of_frame) { std::lock_guard lock(mutex); if (end_of_frame) diff --git a/server/encoder/video_encoder.h b/server/encoder/video_encoder.h index 5096fd70..ca85930c 100644 --- a/server/encoder/video_encoder.h +++ b/server/encoder/video_encoder.h @@ -45,12 +45,12 @@ inline const char * encoder_vaapi = "vaapi"; inline const char * encoder_x264 = "x264"; inline const char * encoder_vulkan = "vulkan"; -class VideoEncoder +class video_encoder { protected: struct data { - VideoEncoder * encoder; + video_encoder * encoder; std::span span; std::shared_ptr mem; }; @@ -68,11 +68,12 @@ private: public: void push(data &&); static std::shared_ptr get(); - void wait_idle(VideoEncoder *); + void wait_idle(video_encoder *); }; -protected: - uint8_t stream_idx; +public: + const uint8_t stream_idx; + const to_headset::video_stream_description::channels_t channels; static const uint8_t num_slots = 2; private: @@ -98,7 +99,7 @@ private: std::shared_ptr shared_sender; public: - static std::unique_ptr Create( + static std::unique_ptr create( wivrn_vk_bundle &, encoder_settings & settings, uint8_t stream_idx, @@ -110,8 +111,8 @@ public: static std::pair, vk::ImageUsageFlags> get_create_image_info(const std::vector &); #endif - VideoEncoder(bool async_send = false); - virtual ~VideoEncoder(); + video_encoder(uint8_t stream_idx, to_headset::video_stream_description::channels_t channels, bool async_send); + virtual ~video_encoder(); void present_image(vk::Image y_cbcr, vk::raii::CommandBuffer & cmd_buf); // for vulkan video (command buffer is on a video queue) diff --git a/server/encoder/video_encoder_nvenc.cpp b/server/encoder/video_encoder_nvenc.cpp index dfc20a12..390d9370 100644 --- a/server/encoder/video_encoder_nvenc.cpp +++ b/server/encoder/video_encoder_nvenc.cpp @@ -64,19 +64,19 @@ namespace wivrn { -void VideoEncoderNvenc::deleter::operator()(CudaFunctions * fn) +void video_encoder_nvenc::deleter::operator()(CudaFunctions * fn) { cuda_free_functions(&fn); } -void VideoEncoderNvenc::deleter::operator()(NvencFunctions * fn) +void video_encoder_nvenc::deleter::operator()(NvencFunctions * fn) { nvenc_free_functions(&fn); } static auto init() { - std::unique_ptr cuda_fn; - std::unique_ptr nvenc_fn; + std::unique_ptr cuda_fn; + std::unique_ptr nvenc_fn; void * session_handle; { @@ -134,8 +134,12 @@ static auto encode_guid(video_codec codec) throw std::out_of_range("Invalid codec " + std::to_string(codec)); } -VideoEncoderNvenc::VideoEncoderNvenc(wivrn_vk_bundle & vk, encoder_settings & settings, float fps) : - VideoEncoder(true), +video_encoder_nvenc::video_encoder_nvenc( + wivrn_vk_bundle & vk, + encoder_settings & settings, + float fps, + uint8_t stream_idx) : + video_encoder(stream_idx, settings.channels, true), vk(vk), fps(fps), bitrate(settings.bitrate) @@ -320,13 +324,13 @@ VideoEncoderNvenc::VideoEncoderNvenc(wivrn_vk_bundle & vk, encoder_settings & se CU_CHECK(cuda_fn->cuCtxPopCurrent(NULL)); } -VideoEncoderNvenc::~VideoEncoderNvenc() +video_encoder_nvenc::~video_encoder_nvenc() { if (session_handle) fn.nvEncDestroyEncoder(session_handle); } -void VideoEncoderNvenc::present_image(vk::Image y_cbcr, vk::raii::CommandBuffer & cmd_buf, uint8_t slot) +void video_encoder_nvenc::present_image(vk::Image y_cbcr, vk::raii::CommandBuffer & cmd_buf, uint8_t slot) { cmd_buf.copyImageToBuffer( y_cbcr, @@ -336,6 +340,7 @@ void VideoEncoderNvenc::present_image(vk::Image y_cbcr, vk::raii::CommandBuffer .bufferRowLength = width, .imageSubresource = { .aspectMask = vk::ImageAspectFlagBits::ePlane0, + .baseArrayLayer = uint32_t(channels), .layerCount = 1, }, .imageOffset = { @@ -356,6 +361,7 @@ void VideoEncoderNvenc::present_image(vk::Image y_cbcr, vk::raii::CommandBuffer .bufferRowLength = uint32_t(width / 2), .imageSubresource = { .aspectMask = vk::ImageAspectFlagBits::ePlane1, + .baseArrayLayer = uint32_t(channels), .layerCount = 1, }, .imageOffset = { @@ -369,7 +375,7 @@ void VideoEncoderNvenc::present_image(vk::Image y_cbcr, vk::raii::CommandBuffer }}); } -std::optional VideoEncoderNvenc::encode(bool idr, std::chrono::steady_clock::time_point pts, uint8_t slot) +std::optional video_encoder_nvenc::encode(bool idr, std::chrono::steady_clock::time_point pts, uint8_t slot) { CU_CHECK(cuda_fn->cuCtxPushCurrent(cuda)); @@ -412,7 +418,7 @@ std::optional VideoEncoderNvenc::encode(bool idr, std::chron }; } -std::array VideoEncoderNvenc::get_max_size(video_codec codec) +std::array video_encoder_nvenc::get_max_size(video_codec codec) { auto [cuda_fn, nvenc_fn, fn, cuda, session_handle] = init(); std::array result; diff --git a/server/encoder/video_encoder_nvenc.h b/server/encoder/video_encoder_nvenc.h index ce5dbd32..75fb9ab0 100644 --- a/server/encoder/video_encoder_nvenc.h +++ b/server/encoder/video_encoder_nvenc.h @@ -29,7 +29,7 @@ namespace wivrn { -class VideoEncoderNvenc : public VideoEncoder +class video_encoder_nvenc : public video_encoder { public: struct deleter @@ -64,8 +64,8 @@ private: int bitrate; public: - VideoEncoderNvenc(wivrn_vk_bundle & vk, encoder_settings & settings, float fps); - ~VideoEncoderNvenc(); + video_encoder_nvenc(wivrn_vk_bundle & vk, encoder_settings & settings, float fps, uint8_t stream_idx); + ~video_encoder_nvenc(); void present_image(vk::Image y_cbcr, vk::raii::CommandBuffer & cmd_buf, uint8_t slot) override; std::optional encode(bool idr, std::chrono::steady_clock::time_point pts, uint8_t slot) override; diff --git a/server/encoder/video_encoder_vulkan.cpp b/server/encoder/video_encoder_vulkan.cpp index 3217de23..8d1a9c75 100644 --- a/server/encoder/video_encoder_vulkan.cpp +++ b/server/encoder/video_encoder_vulkan.cpp @@ -17,6 +17,7 @@ */ #include "video_encoder_vulkan.h" +#include "encoder/encoder_settings.h" #include "util/u_logging.h" #include "utils/wivrn_vk_bundle.h" #include @@ -51,8 +52,14 @@ vk::VideoFormatPropertiesKHR wivrn::video_encoder_vulkan::select_video_format( throw std::runtime_error("No suitable image format found"); } -wivrn::video_encoder_vulkan::video_encoder_vulkan(wivrn_vk_bundle & vk, vk::Rect2D rect, vk::VideoEncodeCapabilitiesKHR in_encode_caps, float fps, uint64_t bitrate) : - VideoEncoder(true), vk(vk), encode_caps(patch_capabilities(in_encode_caps)), rect(rect), fps(fps) +wivrn::video_encoder_vulkan::video_encoder_vulkan( + wivrn_vk_bundle & vk, + vk::Rect2D rect, + vk::VideoEncodeCapabilitiesKHR in_encode_caps, + float fps, + uint8_t stream_idx, + const encoder_settings & settings) : + video_encoder(stream_idx, settings.channels, true), vk(vk), encode_caps(patch_capabilities(in_encode_caps)), rect(rect), fps(fps) { // Initialize Rate control U_LOG_D("Supported rate control modes: %s", vk::to_string(encode_caps.rateControlModes).c_str()); @@ -60,17 +67,17 @@ wivrn::video_encoder_vulkan::video_encoder_vulkan(wivrn_vk_bundle & vk, vk::Rect if (encode_caps.rateControlModes & (vk::VideoEncodeRateControlModeFlagBitsKHR::eCbr | vk::VideoEncodeRateControlModeFlagBitsKHR::eVbr)) { U_LOG_D("Maximum bitrate: %ld", encode_caps.maxBitrate / 1'000'000); - if (encode_caps.maxBitrate < bitrate) + if (encode_caps.maxBitrate < settings.bitrate) { U_LOG_W("Configured bitrate %ldMB/s is higher than max supported %ld", - bitrate / 1'000'000, + settings.bitrate / 1'000'000, encode_caps.maxBitrate / 1'000'000); } } rate_control_layer = vk::VideoEncodeRateControlLayerInfoKHR{ - .averageBitrate = std::min(bitrate, encode_caps.maxBitrate), - .maxBitrate = std::min(2 * bitrate, encode_caps.maxBitrate), + .averageBitrate = std::min(settings.bitrate, encode_caps.maxBitrate), + .maxBitrate = std::min(2 * settings.bitrate, encode_caps.maxBitrate), .frameRateNumerator = uint32_t(fps * 1'000'000), .frameRateDenominator = 1'000'000, }; @@ -227,7 +234,7 @@ void wivrn::video_encoder_vulkan::init(const vk::VideoCapabilitiesKHR & video_ca .subresourceRange = {.aspectMask = vk::ImageAspectFlagBits::eColor, .baseMipLevel = 0, .levelCount = 1, - .baseArrayLayer = 0, + .baseArrayLayer = uint32_t(channels), .layerCount = 1}, }; @@ -320,7 +327,7 @@ std::vector wivrn::video_encoder_vulkan::get_encoded_parameters(void * return encoded; } -std::optional wivrn::video_encoder_vulkan::encode(bool idr, std::chrono::steady_clock::time_point target_timestamp, uint8_t encode_slot) +std::optional wivrn::video_encoder_vulkan::encode(bool idr, std::chrono::steady_clock::time_point target_timestamp, uint8_t encode_slot) { if (idr) send_idr_data(); diff --git a/server/encoder/video_encoder_vulkan.h b/server/encoder/video_encoder_vulkan.h index e56bb5de..db17d9a0 100644 --- a/server/encoder/video_encoder_vulkan.h +++ b/server/encoder/video_encoder_vulkan.h @@ -26,7 +26,7 @@ namespace wivrn { -class video_encoder_vulkan : public VideoEncoder +class video_encoder_vulkan : public video_encoder { wivrn_vk_bundle & vk; const vk::VideoEncodeCapabilitiesKHR encode_caps; @@ -75,7 +75,12 @@ protected: const uint8_t num_dpb_slots = 5; std::optional rate_control; - video_encoder_vulkan(wivrn_vk_bundle & vk, vk::Rect2D rect, vk::VideoEncodeCapabilitiesKHR encode_caps, float fps, uint64_t bitrate); + video_encoder_vulkan(wivrn_vk_bundle & vk, + vk::Rect2D rect, + vk::VideoEncodeCapabilitiesKHR encode_caps, + float fps, + uint8_t stream_idx, + const encoder_settings & settings); void init(const vk::VideoCapabilitiesKHR & video_caps, const vk::VideoProfileInfoKHR & video_profile, diff --git a/server/encoder/video_encoder_vulkan_h264.cpp b/server/encoder/video_encoder_vulkan_h264.cpp index 5b50d7eb..71275f42 100644 --- a/server/encoder/video_encoder_vulkan_h264.cpp +++ b/server/encoder/video_encoder_vulkan_h264.cpp @@ -68,8 +68,14 @@ static StdVideoH264LevelIdc compute_level(const StdVideoH264SequenceParameterSet return STD_VIDEO_H264_LEVEL_IDC_6_2; } -wivrn::video_encoder_vulkan_h264::video_encoder_vulkan_h264(wivrn_vk_bundle & vk, vk::Rect2D rect, vk::VideoEncodeCapabilitiesKHR encode_caps, float fps, uint64_t bitrate) : - video_encoder_vulkan(vk, rect, encode_caps, fps, bitrate), +wivrn::video_encoder_vulkan_h264::video_encoder_vulkan_h264( + wivrn_vk_bundle & vk, + vk::Rect2D rect, + vk::VideoEncodeCapabilitiesKHR encode_caps, + float fps, + uint8_t stream_idx, + const encoder_settings & settings) : + video_encoder_vulkan(vk, rect, encode_caps, fps, stream_idx, settings), sps{ .flags = { @@ -139,7 +145,7 @@ wivrn::video_encoder_vulkan_h264::video_encoder_vulkan_h264(wivrn_vk_bundle & vk .pScalingLists = nullptr, } { - sps.level_idc = compute_level(sps, fps, num_dpb_slots, bitrate); + sps.level_idc = compute_level(sps, fps, num_dpb_slots, settings.bitrate); if (not std::ranges::any_of(vk.device_extensions, [](std::string_view ext) { return ext == VK_KHR_VIDEO_ENCODE_H264_EXTENSION_NAME; })) { throw std::runtime_error("Vulkan video encode H264 extension not available"); @@ -165,7 +171,8 @@ std::vector wivrn::video_encoder_vulkan_h264::setup_slot_info(size_t dpb std::unique_ptr wivrn::video_encoder_vulkan_h264::create( wivrn_vk_bundle & vk, encoder_settings & settings, - float fps) + float fps, + uint8_t stream_idx) { vk::Rect2D rect{ .offset = { @@ -184,7 +191,7 @@ std::unique_ptr wivrn::video_encoder_vulkan_h2 vk::VideoEncodeCapabilitiesKHR, vk::VideoEncodeH264CapabilitiesKHR>(video_profile_info.get()); - std::unique_ptr self(new video_encoder_vulkan_h264(vk, rect, encode_caps, fps, settings.bitrate)); + std::unique_ptr self(new video_encoder_vulkan_h264(vk, rect, encode_caps, fps, stream_idx, settings)); vk::VideoEncodeH264SessionParametersAddInfoKHR h264_add_info{}; h264_add_info.setStdSPSs(self->sps); diff --git a/server/encoder/video_encoder_vulkan_h264.h b/server/encoder/video_encoder_vulkan_h264.h index 797b69a1..2ff5653b 100644 --- a/server/encoder/video_encoder_vulkan_h264.h +++ b/server/encoder/video_encoder_vulkan_h264.h @@ -47,7 +47,12 @@ class video_encoder_vulkan_h264 : public video_encoder_vulkan vk::VideoEncodeH264GopRemainingFrameInfoKHR gop_info; vk::VideoEncodeH264RateControlInfoKHR rate_control_h264; - video_encoder_vulkan_h264(wivrn_vk_bundle & vk, vk::Rect2D rect, vk::VideoEncodeCapabilitiesKHR encode_caps, float fps, uint64_t bitrate); + video_encoder_vulkan_h264(wivrn_vk_bundle & vk, + vk::Rect2D rect, + vk::VideoEncodeCapabilitiesKHR encode_caps, + float fps, + uint8_t stream_idx, + const encoder_settings & settings); protected: std::vector setup_slot_info(size_t dpb_size) override; @@ -60,7 +65,8 @@ protected: public: static std::unique_ptr create(wivrn_vk_bundle & vk, encoder_settings & settings, - float fps); + float fps, + uint8_t stream_idx); std::vector get_sps_pps(); diff --git a/server/encoder/video_encoder_x264.cpp b/server/encoder/video_encoder_x264.cpp index 7f40f9eb..ebcddd8b 100644 --- a/server/encoder/video_encoder_x264.cpp +++ b/server/encoder/video_encoder_x264.cpp @@ -28,9 +28,9 @@ namespace wivrn { -void VideoEncoderX264::ProcessCb(x264_t * h, x264_nal_t * nal, void * opaque) +void video_encoder_x264::ProcessCb(x264_t * h, x264_nal_t * nal, void * opaque) { - VideoEncoderX264 * self = (VideoEncoderX264 *)opaque; + video_encoder_x264 * self = (video_encoder_x264 *)opaque; std::vector data(nal->i_payload * 3 / 2 + 5 + 64, 0); x264_nal_encode(h, data.data(), nal); data.resize(nal->i_payload); @@ -50,7 +50,7 @@ void VideoEncoderX264::ProcessCb(x264_t * h, x264_nal_t * nal, void * opaque) } } -void VideoEncoderX264::ProcessNal(pending_nal && nal) +void video_encoder_x264::ProcessNal(pending_nal && nal) { std::lock_guard lock(mutex); if (nal.first_mb == next_mb) @@ -70,7 +70,7 @@ void VideoEncoderX264::ProcessNal(pending_nal && nal) } } -void VideoEncoderX264::InsertInPendingNal(pending_nal && nal) +void video_encoder_x264::InsertInPendingNal(pending_nal && nal) { auto it = pending_nals.begin(); auto end = pending_nals.end(); @@ -85,10 +85,12 @@ void VideoEncoderX264::InsertInPendingNal(pending_nal && nal) pending_nals.push_back(std::move(nal)); } -VideoEncoderX264::VideoEncoderX264( +video_encoder_x264::video_encoder_x264( wivrn_vk_bundle & vk, encoder_settings & settings, - float fps) + float fps, + uint8_t stream_idx) : + video_encoder(stream_idx, settings.channels, false) { if (settings.codec != h264) { @@ -181,7 +183,7 @@ VideoEncoderX264::VideoEncoderX264( } } -void VideoEncoderX264::present_image(vk::Image y_cbcr, vk::raii::CommandBuffer & cmd_buf, uint8_t slot) +void video_encoder_x264::present_image(vk::Image y_cbcr, vk::raii::CommandBuffer & cmd_buf, uint8_t slot) { cmd_buf.copyImageToBuffer( y_cbcr, @@ -191,6 +193,7 @@ void VideoEncoderX264::present_image(vk::Image y_cbcr, vk::raii::CommandBuffer & .bufferRowLength = chroma_width * 2, .imageSubresource = { .aspectMask = vk::ImageAspectFlagBits::ePlane0, + .baseArrayLayer = uint32_t(channels), .layerCount = 1, }, .imageOffset = { @@ -210,6 +213,7 @@ void VideoEncoderX264::present_image(vk::Image y_cbcr, vk::raii::CommandBuffer & .bufferRowLength = chroma_width, .imageSubresource = { .aspectMask = vk::ImageAspectFlagBits::ePlane1, + .baseArrayLayer = uint32_t(channels), .layerCount = 1, }, .imageOffset = { @@ -223,7 +227,7 @@ void VideoEncoderX264::present_image(vk::Image y_cbcr, vk::raii::CommandBuffer & }}); } -std::optional VideoEncoderX264::encode(bool idr, std::chrono::steady_clock::time_point pts, uint8_t slot) +std::optional video_encoder_x264::encode(bool idr, std::chrono::steady_clock::time_point pts, uint8_t slot) { int num_nal; x264_nal_t * nal; @@ -244,7 +248,7 @@ std::optional VideoEncoderX264::encode(bool idr, std::chrono return {}; } -VideoEncoderX264::~VideoEncoderX264() +video_encoder_x264::~video_encoder_x264() { x264_encoder_close(enc); } diff --git a/server/encoder/video_encoder_x264.h b/server/encoder/video_encoder_x264.h index c5f9f5ec..09e8e5f9 100644 --- a/server/encoder/video_encoder_x264.h +++ b/server/encoder/video_encoder_x264.h @@ -30,7 +30,7 @@ namespace wivrn { -class VideoEncoderX264 : public VideoEncoder +class video_encoder_x264 : public video_encoder { x264_param_t param = {}; x264_t * enc; @@ -61,13 +61,13 @@ class VideoEncoderX264 : public VideoEncoder std::list pending_nals; public: - VideoEncoderX264(wivrn_vk_bundle & vk, encoder_settings & settings, float fps); + video_encoder_x264(wivrn_vk_bundle & vk, encoder_settings & settings, float fps, uint8_t stream_idx); void present_image(vk::Image y_cbcr, vk::raii::CommandBuffer & cmd_buf, uint8_t slot) override; std::optional encode(bool idr, std::chrono::steady_clock::time_point pts, uint8_t slot) override; - ~VideoEncoderX264(); + ~video_encoder_x264(); private: static void ProcessCb(x264_t * h, x264_nal_t * nal, void * opaque);