diff --git a/docs/configuration.md b/docs/configuration.md index 5f1f06f0..af642e79 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -39,7 +39,7 @@ One of `h264`, `h265`, `av1`, `raw`. Not all encoders support every codec: - `x264` encoder only supports `h264` codec -- `vulkan` encoder supports `h264` and `h265` codecs +- `vulkan` encoder supports `h264`, `h265` and `av1` codecs, but `av1` is never selected automatically: it must be requested by setting `encoder` to `vulkan` - `raw` encoder only supports `raw` codec - `nvenc` and `vaapi` support all codecs, except `raw` diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index dcfc2117..b09a78a0 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -184,6 +184,7 @@ if (WIVRN_BUILD_SERVER) if(WIVRN_USE_VULKAN_ENCODE) target_sources(wivrn-server PRIVATE encoder/video_encoder_vulkan.cpp + encoder/video_encoder_vulkan_av1.cpp encoder/video_encoder_vulkan_h264.cpp encoder/video_encoder_vulkan_h265.cpp ) diff --git a/server/encoder/encoder_settings.cpp b/server/encoder/encoder_settings.cpp index ca3f6b80..16010fef 100644 --- a/server/encoder/encoder_settings.cpp +++ b/server/encoder/encoder_settings.cpp @@ -204,8 +204,12 @@ class prober U_LOG_I("GPU does not support H.265 Vulkan video encode"); return res; } - case av1: - U_LOG_D("Vulkan video encode for AV1 is not implemented in WiVRn"); + case av1: { + auto res = vk.has_device_ext(VK_KHR_VIDEO_ENCODE_AV1_EXTENSION_NAME) and flags & vk::VideoCodecOperationFlagBitsKHR::eEncodeAv1; + if (not res) + U_LOG_I("GPU does not support AV1 Vulkan video encode"); + return res; + } case raw: return false; } @@ -240,6 +244,9 @@ public: { for (auto codec: codecs) { + if (codec == video_codec::av1 and config.name.empty()) + continue; + if (has_vk(codec)) return {encoder_vulkan, codec}; } diff --git a/server/encoder/video_encoder.cpp b/server/encoder/video_encoder.cpp index 253babc5..0c700f47 100644 --- a/server/encoder/video_encoder.cpp +++ b/server/encoder/video_encoder.cpp @@ -39,6 +39,7 @@ #include "video_encoder_x264.h" #endif #if WIVRN_USE_VULKAN_ENCODE +#include "video_encoder_vulkan_av1.h" #include "video_encoder_vulkan_h264.h" #include "video_encoder_vulkan_h265.h" #endif @@ -120,7 +121,8 @@ std::unique_ptr video_encoder::create( res = video_encoder_vulkan_h265::create(wivrn_vk, settings, stream_idx); break; case video_codec::av1: - throw std::runtime_error("av1 not supported for vulkan video encode"); + res = video_encoder_vulkan_av1::create(wivrn_vk, settings, stream_idx); + break; case video_codec::raw: throw std::runtime_error("raw codec only supported on raw encoder"); } diff --git a/server/encoder/video_encoder_vulkan.cpp b/server/encoder/video_encoder_vulkan.cpp index 076a4c8c..e71780af 100644 --- a/server/encoder/video_encoder_vulkan.cpp +++ b/server/encoder/video_encoder_vulkan.cpp @@ -821,7 +821,7 @@ void wivrn::video_encoder_vulkan::present_image(vk::Image y_cbcr, vk::SemaphoreS } video_cmd_buf.beginVideoCodingKHR({ - .pNext = (session_initialized and rate_control) ? &rate_control.value() : nullptr, + .pNext = begin_coding_next((session_initialized and rate_control) ? &rate_control.value() : nullptr), .videoSession = *video_session, .videoSessionParameters = *video_session_parameters, .referenceSlotCount = ref_slot ? 2u : 1u, diff --git a/server/encoder/video_encoder_vulkan.h b/server/encoder/video_encoder_vulkan.h index d888aab9..3a05afb1 100644 --- a/server/encoder/video_encoder_vulkan.h +++ b/server/encoder/video_encoder_vulkan.h @@ -111,6 +111,14 @@ protected: virtual void * encode_info_next(uint32_t frame_num, size_t slot, std::optional reference_slot) = 0; virtual vk::ExtensionProperties std_header_version() = 0; + // Codec-specific structures to prepend to the vkCmdBeginVideoCodingKHR pNext chain. + // Note that this chain accepts structures the vkCmdControlVideoCodingKHR one does not, + // so it must not be reused for rate control updates. + virtual const void * begin_coding_next(const void * next) + { + return next; + } + public: void present_image(vk::Image y_cbcr, vk::SemaphoreSubmitInfo, uint8_t slot, uint64_t frame_index) override; std::optional encode(uint8_t slot, uint64_t frame_index) override; diff --git a/server/encoder/video_encoder_vulkan_av1.cpp b/server/encoder/video_encoder_vulkan_av1.cpp new file mode 100644 index 00000000..f1605b41 --- /dev/null +++ b/server/encoder/video_encoder_vulkan_av1.cpp @@ -0,0 +1,633 @@ +/* + * WiVRn VR streaming + * Copyright (C) 2024 Patrick Nicolas + * Copyright (C) 2026 Philipp Schlegel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#include "video_encoder_vulkan_av1.h" + +#include "encoder/encoder_settings.h" +#include "util/u_logging.h" +#include "utils/wivrn_vk_bundle.h" + +#include +#include +#include +#include +#include + +namespace +{ +uint32_t align_div(uint32_t value, uint32_t divisor) +{ + return (value + divisor - 1) / divisor; +} + +uint8_t bits_for(uint32_t value) +{ + return std::max(std::bit_width(value), 1); +} + +// AV1 spec, annex A.3 "Levels", limited to the levels that are actually defined. +// MaxDisplayRate is omitted, it is always smaller than MaxDecodeRate and we never +// display more samples than we decode. +struct av1_level_limits +{ + StdVideoAV1Level level; + uint32_t max_pic_size; // samples + uint32_t max_h_size; // samples + uint32_t max_v_size; // samples + uint64_t max_decode_rate; +}; + +constexpr av1_level_limits av1_levels[] = { + {STD_VIDEO_AV1_LEVEL_2_0, 147456, 2048, 1152, 5529600}, + {STD_VIDEO_AV1_LEVEL_2_1, 278784, 2816, 1584, 10454400}, + {STD_VIDEO_AV1_LEVEL_3_0, 665856, 4352, 2448, 24969600}, + {STD_VIDEO_AV1_LEVEL_3_1, 1065024, 5504, 3096, 39938400}, + {STD_VIDEO_AV1_LEVEL_4_0, 2359296, 6144, 3456, 77856768}, + {STD_VIDEO_AV1_LEVEL_4_1, 2359296, 6144, 3456, 155713536}, + {STD_VIDEO_AV1_LEVEL_5_0, 8912896, 8192, 4352, 273715200}, + {STD_VIDEO_AV1_LEVEL_5_1, 8912896, 8192, 4352, 547430400}, + {STD_VIDEO_AV1_LEVEL_5_2, 8912896, 8192, 4352, 1094860800}, + {STD_VIDEO_AV1_LEVEL_5_3, 8912896, 8192, 4352, 1176502272}, + {STD_VIDEO_AV1_LEVEL_6_0, 35651584, 16384, 8704, 1176502272}, + {STD_VIDEO_AV1_LEVEL_6_1, 35651584, 16384, 8704, 2189721600}, + {STD_VIDEO_AV1_LEVEL_6_2, 35651584, 16384, 8704, 4379443200}, + {STD_VIDEO_AV1_LEVEL_6_3, 35651584, 16384, 8704, 4706009088}, +}; + +// Smallest level able to carry the stream, clamped to what the encoder supports. +// Reporting the driver's maximum instead would make decoders reject or downgrade +// the stream for no reason. +StdVideoAV1Level select_level(uint32_t width, uint32_t height, float fps, StdVideoAV1Level max_level) +{ + const uint64_t pic_size = uint64_t(width) * height; + const uint64_t decode_rate = uint64_t(double(pic_size) * std::max(fps, 1.f)); + + for (const auto & l: av1_levels) + { + if (l.level > max_level) + break; + if (pic_size <= l.max_pic_size and width <= l.max_h_size and height <= l.max_v_size and decode_rate <= l.max_decode_rate) + return l.level; + } + + U_LOG_W("AV1: no level supports %ux%u@%.1f, falling back to the encoder maximum", width, height, fps); + return max_level; +} +} // namespace + +wivrn::video_encoder_vulkan_av1::video_encoder_vulkan_av1( + wivrn::vk_bundle & vk, + const vk::VideoCapabilitiesKHR & video_caps, + const vk::VideoEncodeCapabilitiesKHR & encode_caps, + const vk::VideoEncodeAV1CapabilitiesKHR & encode_av1_caps, + uint8_t stream_idx, + const encoder_settings & settings) : + video_encoder_vulkan(vk, video_caps, encode_caps, stream_idx, settings) +{ + if (not vk.has_device_ext(VK_KHR_VIDEO_ENCODE_AV1_EXTENSION_NAME)) + throw std::runtime_error("Vulkan video encode AV1 extension not available"); + + if (settings.bit_depth != 8 && settings.bit_depth != 10) + throw std::runtime_error("av1 encoder supports 8-bit or 10-bit only"); + + configure_from_caps(encode_av1_caps); + + level = select_level(aligned_extent.width, aligned_extent.height, settings.fps, encode_av1_caps.maxLevel); + + color_config = { + .flags = { + .mono_chrome = 0, + .color_range = 1, + .separate_uv_delta_q = 0, + .color_description_present_flag = 1, + }, + .BitDepth = static_cast(settings.bit_depth), + .subsampling_x = 1, + .subsampling_y = 1, + .reserved1 = 0, + .color_primaries = STD_VIDEO_AV1_COLOR_PRIMARIES_BT_709, + .transfer_characteristics = STD_VIDEO_AV1_TRANSFER_CHARACTERISTICS_BT_709, + .matrix_coefficients = STD_VIDEO_AV1_MATRIX_COEFFICIENTS_BT_709, + .chroma_sample_position = STD_VIDEO_AV1_CHROMA_SAMPLE_POSITION_COLOCATED, + }; + + seq_header = { + .flags = { + .still_picture = 0, + .reduced_still_picture_header = 0, + .use_128x128_superblock = superblock_size == 128 ? 1u : 0u, + .enable_filter_intra = 0, + .enable_intra_edge_filter = 0, + .enable_interintra_compound = 0, + .enable_masked_compound = 0, + .enable_warped_motion = 0, + .enable_dual_filter = 0, + .enable_order_hint = 1, + .enable_jnt_comp = 0, + .enable_ref_frame_mvs = 0, + .frame_id_numbers_present_flag = 0, + .enable_superres = 0, + .enable_cdef = 0, + .enable_restoration = 0, + .film_grain_params_present = 0, + .timing_info_present_flag = 0, + .initial_display_delay_present_flag = 0, + }, + .seq_profile = STD_VIDEO_AV1_PROFILE_MAIN, + .frame_width_bits_minus_1 = static_cast(bits_for(aligned_extent.width - 1) - 1), + .frame_height_bits_minus_1 = static_cast(bits_for(aligned_extent.height - 1) - 1), + .max_frame_width_minus_1 = static_cast(aligned_extent.width - 1), + .max_frame_height_minus_1 = static_cast(aligned_extent.height - 1), + .delta_frame_id_length_minus_2 = 0, + .additional_frame_id_length_minus_1 = 0, + .order_hint_bits_minus_1 = static_cast(order_hint_bits - 1), + .seq_force_integer_mv = STD_VIDEO_AV1_SELECT_INTEGER_MV, + .seq_force_screen_content_tools = STD_VIDEO_AV1_SELECT_SCREEN_CONTENT_TOOLS, + .reserved1 = {}, + .pColorConfig = &color_config, + .pTimingInfo = nullptr, + }; + + operating_point = { + .flags = { + .decoder_model_present_for_this_op = 0, + .low_delay_mode_flag = 1, + .initial_display_delay_present_for_this_op = 0, + }, + .operating_point_idc = 0, + .seq_level_idx = static_cast(level), + .seq_tier = 0, + .decoder_buffer_delay = 0, + .encoder_buffer_delay = 0, + .initial_display_delay_minus_1 = 0, + }; + + const uint32_t sb_cols = align_div(aligned_extent.width, superblock_size); + const uint32_t sb_rows = align_div(aligned_extent.height, superblock_size); + const uint32_t mi_cols = align_div(aligned_extent.width, 4); + const uint32_t mi_rows = align_div(aligned_extent.height, 4); + const bool uniform_tile_spacing = bool(std_flags & vk::VideoEncodeAV1StdFlagBitsKHR::eUniformTileSpacingFlagSet); + + // A single tile is enough for the resolutions WiVRn streams, but AV1 caps a tile at 4096 + // samples wide (annex A.3), a wider stream would not be decodable. + if (sb_cols * superblock_size > 4096) + throw std::runtime_error("av1 encoder supports at most 4096 samples per tile row"); + + mi_col_starts = {0u, static_cast(mi_cols)}; + mi_row_starts = {0u, static_cast(mi_rows)}; + if (!tile_widths_sb.empty()) + tile_widths_sb[0] = static_cast(sb_cols - 1); + if (!tile_heights_sb.empty()) + tile_heights_sb[0] = static_cast(sb_rows - 1); + + tile_info = { + .flags = { + .uniform_tile_spacing_flag = uniform_tile_spacing ? 1u : 0u, + }, + .TileCols = 1, + .TileRows = 1, + .context_update_tile_id = 0, + .tile_size_bytes_minus_1 = 0, + .reserved1 = {}, + .pMiColStarts = uniform_tile_spacing ? nullptr : mi_col_starts.data(), + .pMiRowStarts = uniform_tile_spacing ? nullptr : mi_row_starts.data(), + .pWidthInSbsMinus1 = uniform_tile_spacing ? nullptr : tile_widths_sb.data(), + .pHeightInSbsMinus1 = uniform_tile_spacing ? nullptr : tile_heights_sb.data(), + }; + + quantization = { + .flags = { + .using_qmatrix = 0, + .diff_uv_delta = 0, + }, + .base_q_idx = static_cast(encode_av1_caps.maxQIndex), + .DeltaQYDc = 0, + .DeltaQUDc = 0, + .DeltaQUAc = 0, + .DeltaQVDc = 0, + .DeltaQVAc = 0, + .qm_y = 0, + .qm_u = 0, + .qm_v = 0, + }; + + loop_filter = { + .flags = { + .loop_filter_delta_enabled = 0, + .loop_filter_delta_update = 0, + }, + .loop_filter_level = {}, + .loop_filter_sharpness = 0, + .update_ref_delta = 0, + .loop_filter_ref_deltas = {}, + .update_mode_delta = 0, + .loop_filter_mode_deltas = {}, + }; + + cdef = { + .cdef_damping_minus_3 = 0, + .cdef_bits = 0, + .cdef_y_pri_strength = {}, + .cdef_y_sec_strength = {}, + .cdef_uv_pri_strength = {}, + .cdef_uv_sec_strength = {}, + }; + + loop_restoration = { + .FrameRestorationType = {STD_VIDEO_AV1_FRAME_RESTORATION_TYPE_NONE, + STD_VIDEO_AV1_FRAME_RESTORATION_TYPE_NONE, + STD_VIDEO_AV1_FRAME_RESTORATION_TYPE_NONE}, + .LoopRestorationSize = {0, 0, 0}, + }; + + global_motion = { + .GmType = {}, + .gm_params = {}, + }; + + segmentation = {}; + + rate_control_layer.pNext = &rate_control_layer_av1; +} + +void wivrn::video_encoder_vulkan_av1::configure_from_caps(const vk::VideoEncodeAV1CapabilitiesKHR & encode_av1_caps) +{ + U_LOG_D("AV1: capabilities: stdSyntaxFlags=0x%x, superblockSizes=0x%x, singleReferenceNameMask=0x%x, maxSingleReferenceCount=%u, maxUnidirectionalCompoundReferenceCount=%u, maxBidirectionalCompoundReferenceCount=%u, maxQIndex=%u, minQIndex=%u, requiresGopRemainingFrames=%u, maxOperatingPoints=%u", + static_cast(encode_av1_caps.stdSyntaxFlags), + static_cast(encode_av1_caps.superblockSizes), + encode_av1_caps.singleReferenceNameMask, + encode_av1_caps.maxSingleReferenceCount, + encode_av1_caps.maxUnidirectionalCompoundReferenceCount, + encode_av1_caps.maxBidirectionalCompoundReferenceCount, + encode_av1_caps.maxQIndex, + encode_av1_caps.minQIndex, + encode_av1_caps.requiresGopRemainingFrames, + encode_av1_caps.maxOperatingPoints); + std_flags = encode_av1_caps.stdSyntaxFlags; + if (encode_av1_caps.superblockSizes & vk::VideoEncodeAV1SuperblockSizeFlagBitsKHR::e64) + superblock_size = 64; + else + superblock_size = 128; + + single_reference_name_mask = encode_av1_caps.singleReferenceNameMask; + max_single_reference_count = encode_av1_caps.maxSingleReferenceCount; + max_q_index = encode_av1_caps.maxQIndex; + min_q_index = encode_av1_caps.minQIndex; + + // The single reference of an inter frame must use a reference name the encoder + // advertises, bit i standing for LAST_FRAME + i. + // VUID-vkCmdEncodeVideoKHR-predictionMode-10329 + ref_name_index = -1; + for (size_t i = 0; i < reference_name_slot_indices.size(); ++i) + { + if (single_reference_name_mask & (1u << i)) + { + ref_name_index = int32_t(i); + break; + } + } + // WiVRn streams key frame + inter frames referencing a single acknowledged frame, an + // encoder that cannot do single reference prediction is of no use to us. + if (ref_name_index < 0 or max_single_reference_count == 0) + throw std::runtime_error("av1 encoder does not support single reference prediction"); + if (ref_name_index != 0) + U_LOG_W("AV1: LAST_FRAME not supported (mask=0x%x), using reference name index %d", single_reference_name_mask, ref_name_index); + + rate_control_av1 = vk::VideoEncodeAV1RateControlInfoKHR{ + .flags = vk::VideoEncodeAV1RateControlFlagBitsKHR::eRegularGop, + .gopFrameCount = std::numeric_limits::max(), + .keyFramePeriod = std::numeric_limits::max(), + .consecutiveBipredictiveFrameCount = 0, + // VUID-VkVideoEncodeAV1RateControlInfoKHR-temporalLayerCount-10299 + .temporalLayerCount = std::min(1u, encode_av1_caps.maxTemporalLayerCount), + }; + + rate_control_layer_av1 = vk::VideoEncodeAV1RateControlLayerInfoKHR{ + .useMinQIndex = VK_FALSE, + .minQIndex = { + .intraQIndex = encode_av1_caps.minQIndex, + .predictiveQIndex = encode_av1_caps.minQIndex, + .bipredictiveQIndex = encode_av1_caps.minQIndex, + }, + .useMaxQIndex = VK_FALSE, + .maxQIndex = { + .intraQIndex = encode_av1_caps.maxQIndex, + .predictiveQIndex = encode_av1_caps.maxQIndex, + .bipredictiveQIndex = encode_av1_caps.maxQIndex, + }, + .useMaxFrameSize = VK_FALSE, + .maxFrameSize = {}, + }; +} + +std::vector wivrn::video_encoder_vulkan_av1::setup_slot_info(size_t dpb_size) +{ + dpb_std_info.resize(dpb_size, {}); + dpb_std_slots.reserve(dpb_size); + std::vector res; + for (size_t i = 0; i < dpb_size; ++i) + { + dpb_std_slots.push_back({ + .pStdReferenceInfo = &dpb_std_info[i], + }); + res.push_back(&dpb_std_slots[i]); + } + + return res; +} + +static auto get_video_caps(vk::raii::PhysicalDevice & phys_dev, int bit_depth) +{ + if (!(bit_depth == 8 || bit_depth == 10)) + throw std::runtime_error("av1 encoder supports 8-bit or 10-bit only"); + + vk::StructureChain video_profile_info{ + vk::VideoProfileInfoKHR{ + .videoCodecOperation = vk::VideoCodecOperationFlagBitsKHR::eEncodeAv1, + .chromaSubsampling = vk::VideoChromaSubsamplingFlagBitsKHR::e420, + .lumaBitDepth = bit_depth == 10 ? vk::VideoComponentBitDepthFlagBitsKHR::e10 : vk::VideoComponentBitDepthFlagBitsKHR::e8, + .chromaBitDepth = bit_depth == 10 ? vk::VideoComponentBitDepthFlagBitsKHR::e10 : vk::VideoComponentBitDepthFlagBitsKHR::e8, + }, + vk::VideoEncodeAV1ProfileInfoKHR{ + .stdProfile = STD_VIDEO_AV1_PROFILE_MAIN, + }, + vk::VideoEncodeUsageInfoKHR{ + .videoUsageHints = vk::VideoEncodeUsageFlagBitsKHR::eStreaming, + .videoContentHints = vk::VideoEncodeContentFlagBitsKHR::eRendered, + .tuningMode = vk::VideoEncodeTuningModeKHR::eUltraLowLatency, + }}; + + try + { + auto [video_caps, encode_caps, encode_av1_caps] = + phys_dev.getVideoCapabilitiesKHR< + vk::VideoCapabilitiesKHR, + vk::VideoEncodeCapabilitiesKHR, + vk::VideoEncodeAV1CapabilitiesKHR>(video_profile_info.get()); + + video_caps.maxDpbSlots = std::min(video_caps.maxDpbSlots, uint32_t(STD_VIDEO_AV1_NUM_REF_FRAMES)); + return std::make_tuple(video_caps, encode_caps, encode_av1_caps, video_profile_info); + } + catch (...) + {} + + video_profile_info.unlink(); + auto [video_caps, encode_caps, encode_av1_caps] = + phys_dev.getVideoCapabilitiesKHR< + vk::VideoCapabilitiesKHR, + vk::VideoEncodeCapabilitiesKHR, + vk::VideoEncodeAV1CapabilitiesKHR>(video_profile_info.get()); + video_caps.maxDpbSlots = std::min(video_caps.maxDpbSlots, uint32_t(STD_VIDEO_AV1_NUM_REF_FRAMES)); + return std::make_tuple(video_caps, encode_caps, encode_av1_caps, video_profile_info); +} + +std::unique_ptr wivrn::video_encoder_vulkan_av1::create( + wivrn::vk_bundle & vk, + const encoder_settings & settings, + uint8_t stream_idx) +{ + auto [video_caps, encode_caps, encode_av1_caps, video_profile_info] = get_video_caps(vk.physical_device, settings.bit_depth); + + std::unique_ptr self( + new video_encoder_vulkan_av1(vk, video_caps, encode_caps, encode_av1_caps, stream_idx, settings)); + + vk::VideoEncodeAV1SessionParametersCreateInfoKHR session_params_info{ + .pStdSequenceHeader = &self->seq_header, + .pStdDecoderModelInfo = nullptr, + .stdOperatingPointCount = 0, + .pStdOperatingPoints = nullptr, + }; + + self->quality_level_info = vk::VideoEncodeQualityLevelInfoKHR{ + .qualityLevel = 0, // Use default quality level + }; + session_params_info.pNext = &self->quality_level_info; + + // Let the session be sized for whatever the encoder supports, the level that matters to the + // decoder is the one written to the operating point. + vk::VideoEncodeAV1SessionCreateInfoKHR session_create_info{ + .useMaxLevel = false, + .maxLevel = encode_av1_caps.maxLevel, + }; + + // VkVideoEncodeAV1RateControlInfoKHR belongs to the same pNext chain as the generic + // rate control info, in both vkCmdBeginVideoCodingKHR and vkCmdControlVideoCodingKHR. + if (self->rate_control) + { + self->rate_control_enabled = true; + self->rate_control->pNext = &self->rate_control_av1; + } + + // VkVideoEncodeAV1GopRemainingFrameInfoKHR only extends VkVideoBeginCodingInfoKHR, so it + // is added by begin_coding_next() rather than chained onto the rate control info, which is + // also used for vkCmdControlVideoCodingKHR. + // VUID-vkCmdBeginVideoCodingKHR-pBeginInfo-10282 + if (self->rate_control_enabled and (encode_av1_caps.requiresGopRemainingFrames or encode_av1_caps.prefersGopRemainingFrames)) + { + self->gop_info = vk::VideoEncodeAV1GopRemainingFrameInfoKHR{ + .useGopRemainingFrames = true, + .gopRemainingIntra = 0, + .gopRemainingPredictive = std::numeric_limits::max(), + .gopRemainingBipredictive = 0, + }; + } + + if (encode_av1_caps.maxOperatingPoints > 0) + { + session_params_info.stdOperatingPointCount = 1; + session_params_info.pStdOperatingPoints = &self->operating_point; + } + +#ifdef VK_KHR_video_encode_intra_refresh + vk::VideoEncodeIntraRefreshCapabilitiesKHR intra_caps{}; + if (std::get(vk.feat).videoEncodeIntraRefresh) + intra_caps = std::get( + vk.physical_device.getVideoCapabilitiesKHR(video_profile_info.get())); +#endif + self->init(video_caps, +#ifdef VK_KHR_video_encode_intra_refresh + intra_caps, +#endif + video_profile_info.get(), + &session_create_info, + &session_params_info); + return self; +} + +void wivrn::video_encoder_vulkan_av1::send_idr_data() +{ + U_LOG_D("AV1: send_idr_data() called"); + // Fetch encoded session parameters (sequence header OBU). + try + { + auto data = get_encoded_parameters(nullptr); + U_LOG_D("AV1: get_encoded_parameters returned %zu bytes", data.size()); + if (!data.empty()) + SendData(data, false, true); + else + U_LOG_W("AV1: get_encoded_parameters returned empty data"); + } + catch (const std::exception & e) + { + U_LOG_E("AV1: Exception in get_encoded_parameters: %s", e.what()); + } +} + +void * wivrn::video_encoder_vulkan_av1::encode_info_next(uint32_t frame_num, size_t slot, std::optional ref_slot) +{ + const bool has_ref = ref_slot.has_value(); + const bool is_keyframe = not has_ref; + + // Initialize reference name slot indices to -1 (no reference) + std::ranges::fill(reference_name_slot_indices, -1); + + if (has_ref) + reference_name_slot_indices[ref_name_index] = *ref_slot; + + std_picture_info = {}; + std_picture_info.flags.error_resilient_mode = is_keyframe ? 1u : 0u; + std_picture_info.flags.disable_cdf_update = 0; + std_picture_info.flags.use_superres = 0; + // The coded frame is always aligned_extent, which is what the sequence header declares as + // max_frame_{width,height}, so the frame size is never overridden. Only the render size + // differs, when the requested extent needed padding to satisfy the encoder granularity. + // VUID-vkCmdEncodeVideoKHR-flags-10322 + std_picture_info.flags.frame_size_override_flag = 0; + std_picture_info.flags.render_and_frame_size_different = + (aligned_extent.width != extent.width or aligned_extent.height != extent.height) ? 1u : 0u; + std_picture_info.flags.allow_screen_content_tools = 0; + std_picture_info.flags.is_filter_switchable = 1; + // AV1 spec 5.9.2: both are inferred for intra frames, match what the decoder will derive + std_picture_info.flags.force_integer_mv = is_keyframe ? 1u : 0u; + std_picture_info.flags.buffer_removal_time_present_flag = 0; + std_picture_info.flags.allow_intrabc = 0; + std_picture_info.flags.frame_refs_short_signaling = 0; + std_picture_info.flags.allow_high_precision_mv = is_keyframe ? 0u : 1u; + std_picture_info.flags.is_motion_mode_switchable = 0; + std_picture_info.flags.use_ref_frame_mvs = 0; + std_picture_info.flags.disable_frame_end_update_cdf = 0; + std_picture_info.flags.allow_warped_motion = 0; + std_picture_info.flags.reduced_tx_set = 0; + std_picture_info.flags.skip_mode_present = 0; + std_picture_info.flags.delta_q_present = 0; + std_picture_info.flags.delta_lf_present = 0; + std_picture_info.flags.delta_lf_multi = 0; + std_picture_info.flags.segmentation_enabled = 0; + std_picture_info.flags.segmentation_update_map = 0; + std_picture_info.flags.segmentation_temporal_update = 0; + std_picture_info.flags.segmentation_update_data = 0; + std_picture_info.flags.UsesLr = 0; + std_picture_info.flags.usesChromaLr = 0; + std_picture_info.flags.show_frame = 1; + std_picture_info.flags.showable_frame = std_picture_info.flags.show_frame ? (is_keyframe ? 0u : 1u) : 1u; + std_picture_info.frame_type = is_keyframe ? STD_VIDEO_AV1_FRAME_TYPE_KEY : STD_VIDEO_AV1_FRAME_TYPE_INTER; + // Both are only coded when the decoder model / frame ids are signalled, which the sequence + // header disables, and the decoder then infers 0 for them (AV1 spec 5.9.2). + std_picture_info.frame_presentation_time = 0; + std_picture_info.current_frame_id = 0; + std_picture_info.order_hint = static_cast(frame_num & ((1u << order_hint_bits) - 1)); + // primary_ref_frame indexes ref_frame_idx[], it selects the reference to load the CDFs and + // the loop filter / segmentation state from. Only usable if the encoder supports values + // other than PRIMARY_REF_NONE. + if (has_ref and (std_flags & vk::VideoEncodeAV1StdFlagBitsKHR::ePrimaryRefFrame)) + std_picture_info.primary_ref_frame = uint8_t(ref_name_index); + else + std_picture_info.primary_ref_frame = STD_VIDEO_AV1_PRIMARY_REF_NONE; + std_picture_info.refresh_frame_flags = static_cast(is_keyframe ? 0xFF : (1u << slot)); + std_picture_info.coded_denom = 0; + std_picture_info.render_width_minus_1 = static_cast(extent.width - 1); + std_picture_info.render_height_minus_1 = static_cast(extent.height - 1); + std_picture_info.interpolation_filter = STD_VIDEO_AV1_INTERPOLATION_FILTER_SWITCHABLE; + std_picture_info.TxMode = STD_VIDEO_AV1_TX_MODE_SELECT; + std_picture_info.delta_q_res = 0; + std_picture_info.delta_lf_res = 0; + std::fill(std_picture_info.ref_order_hint, std_picture_info.ref_order_hint + STD_VIDEO_AV1_NUM_REF_FRAMES, 0); + std::fill(std_picture_info.ref_frame_idx, std_picture_info.ref_frame_idx + STD_VIDEO_AV1_REFS_PER_FRAME, static_cast(-1)); + std::fill(std_picture_info.delta_frame_id_minus_1, std_picture_info.delta_frame_id_minus_1 + STD_VIDEO_AV1_REFS_PER_FRAME, 0); + std_picture_info.pTileInfo = &tile_info; + std_picture_info.pQuantization = &quantization; + std_picture_info.pSegmentation = (std_picture_info.flags.segmentation_enabled) ? &segmentation : nullptr; + std_picture_info.pLoopFilter = &loop_filter; + std_picture_info.pCDEF = &cdef; + std_picture_info.pLoopRestoration = &loop_restoration; + std_picture_info.pGlobalMotion = &global_motion; + std_picture_info.pExtensionHeader = nullptr; + std_picture_info.pBufferRemovalTimes = nullptr; + + // ref_frame_idx[] maps a reference name to a DPB slot, and must be filled in for the same + // reference name that referenceNameSlotIndices uses. + if (has_ref) + std_picture_info.ref_frame_idx[ref_name_index] = static_cast(*ref_slot); + + const size_t dpb_ref_count = std::min(dpb_std_info.size(), STD_VIDEO_AV1_NUM_REF_FRAMES); + for (size_t i = 0; i < dpb_ref_count; ++i) + std_picture_info.ref_order_hint[i] = dpb_std_info[i].OrderHint; + + picture_info = vk::VideoEncodeAV1PictureInfoKHR{}; + // WiVRn always predicts from the primary reference, never uses it for CDFs only. + // VUID-VkVideoEncodeAV1PictureInfoKHR-flags-10289 + picture_info.primaryReferenceCdfOnly = VK_FALSE; + // Key frames are the only intra frames we produce, everything else predicts from the single + // acknowledged reference frame. Encoders without single reference support are rejected in + // configure_from_caps(). + // VUID-vkCmdEncodeVideoKHR-pStdPictureInfo-10327 + picture_info.predictionMode = is_keyframe ? vk::VideoEncodeAV1PredictionModeKHR::eIntraOnly + : vk::VideoEncodeAV1PredictionModeKHR::eSingleReference; + + // Set rate control group based on frame type + // Note: WiVRn currently doesn't support B-frames, so only INTRA and PREDICTIVE groups + picture_info.rateControlGroup = is_keyframe ? vk::VideoEncodeAV1RateControlGroupKHR::eIntra + : vk::VideoEncodeAV1RateControlGroupKHR::ePredictive; + // Must be zero while rate control is active, and within the reported range otherwise. + // VUID-vkCmdEncodeVideoKHR-constantQIndex-10320, VUID-vkCmdEncodeVideoKHR-constantQIndex-10321 + picture_info.constantQIndex = rate_control_enabled ? 0 : std::midpoint(min_q_index, max_q_index); + picture_info.pStdPictureInfo = &std_picture_info; + picture_info.referenceNameSlotIndices = reference_name_slot_indices; + picture_info.generateObuExtensionHeader = VK_FALSE; + + auto & i = dpb_std_info[slot]; + i = {}; + i.flags.disable_frame_end_update_cdf = 0; + i.flags.segmentation_enabled = 0; + // Frame ids are not signalled, see current_frame_id above + i.RefFrameId = 0; + i.frame_type = std_picture_info.frame_type; + i.OrderHint = std_picture_info.order_hint; + i.pExtensionHeader = nullptr; + + return &picture_info; +} + +const void * wivrn::video_encoder_vulkan_av1::begin_coding_next(const void * next) +{ + if (not gop_info) + return next; + + gop_info->pNext = next; + return &*gop_info; +} + +vk::ExtensionProperties wivrn::video_encoder_vulkan_av1::std_header_version() +{ + vk::ExtensionProperties std_header_version{ + .specVersion = VK_STD_VULKAN_VIDEO_CODEC_AV1_ENCODE_SPEC_VERSION, + }; + strcpy(std_header_version.extensionName, + VK_STD_VULKAN_VIDEO_CODEC_AV1_ENCODE_EXTENSION_NAME); + return std_header_version; +} diff --git a/server/encoder/video_encoder_vulkan_av1.h b/server/encoder/video_encoder_vulkan_av1.h new file mode 100644 index 00000000..aae6412f --- /dev/null +++ b/server/encoder/video_encoder_vulkan_av1.h @@ -0,0 +1,101 @@ +/* + * WiVRn VR streaming + * Copyright (C) 2024 Patrick Nicolas + * Copyright (C) 2026 Philipp Schlegel + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once + +#include "video_encoder_vulkan.h" + +#include +#include +#include +#include + +#include + +#include +#include + +namespace wivrn +{ +class video_encoder_vulkan_av1 : public video_encoder_vulkan +{ + StdVideoAV1ColorConfig color_config{}; + StdVideoAV1SequenceHeader seq_header{}; + + StdVideoAV1TileInfo tile_info{}; + StdVideoAV1Quantization quantization{}; + StdVideoAV1LoopFilter loop_filter{}; + StdVideoAV1CDEF cdef{}; + StdVideoAV1LoopRestoration loop_restoration{}; + StdVideoAV1GlobalMotion global_motion{}; + StdVideoAV1Segmentation segmentation{}; + + StdVideoEncodeAV1PictureInfo std_picture_info{}; + StdVideoEncodeAV1OperatingPointInfo operating_point{}; + std::array reference_name_slot_indices{}; + std::array mi_col_starts{}; + std::array mi_row_starts{}; + std::array tile_widths_sb{}; + std::array tile_heights_sb{}; + + std::vector dpb_std_info; + std::vector dpb_std_slots; + + vk::VideoEncodeAV1PictureInfoKHR picture_info{}; + + std::optional gop_info; + vk::VideoEncodeQualityLevelInfoKHR quality_level_info{}; + vk::VideoEncodeAV1RateControlInfoKHR rate_control_av1{}; + vk::VideoEncodeAV1RateControlLayerInfoKHR rate_control_layer_av1{}; + + vk::VideoEncodeAV1StdFlagsKHR std_flags{}; + uint32_t single_reference_name_mask = 0; + uint32_t max_single_reference_count = 0; + uint32_t max_q_index = 0; + uint32_t min_q_index = 0; + uint32_t superblock_size = 64; + uint8_t order_hint_bits = 7; + // Reference name (LAST_FRAME + index) used for the single reference of inter frames + int32_t ref_name_index = 0; + bool rate_control_enabled = false; + StdVideoAV1Level level = STD_VIDEO_AV1_LEVEL_2_0; + + video_encoder_vulkan_av1(wivrn::vk_bundle & vk, + const vk::VideoCapabilitiesKHR & video_caps, + const vk::VideoEncodeCapabilitiesKHR & encode_caps, + const vk::VideoEncodeAV1CapabilitiesKHR & encode_av1_caps, + uint8_t stream_idx, + const encoder_settings & settings); + + void configure_from_caps(const vk::VideoEncodeAV1CapabilitiesKHR & encode_av1_caps); + +protected: + std::vector setup_slot_info(size_t dpb_size) override; + + void * encode_info_next(uint32_t frame_num, size_t slot, std::optional ref_slot) override; + vk::ExtensionProperties std_header_version() override; + const void * begin_coding_next(const void * next) override; + + void send_idr_data() override; + +public: + static std::unique_ptr create(wivrn::vk_bundle & vk, + const encoder_settings & settings, + uint8_t stream_idx); +}; +} // namespace wivrn diff --git a/server/utils/wivrn_vk_bundle.cpp b/server/utils/wivrn_vk_bundle.cpp index 74656def..7a0b179b 100644 --- a/server/utils/wivrn_vk_bundle.cpp +++ b/server/utils/wivrn_vk_bundle.cpp @@ -269,6 +269,9 @@ wivrn::vk_bundle::vk_bundle() : #ifdef VK_KHR_video_encode_h265 VK_KHR_VIDEO_ENCODE_H265_EXTENSION_NAME, #endif +#ifdef VK_KHR_video_encode_av1 + VK_KHR_VIDEO_ENCODE_AV1_EXTENSION_NAME, +#endif #ifdef VK_KHR_video_encode_intra_refresh VK_KHR_VIDEO_ENCODE_INTRA_REFRESH_EXTENSION_NAME, #endif @@ -334,6 +337,10 @@ wivrn::vk_bundle::vk_bundle() : if (has_device_ext(VK_KHR_VIDEO_ENCODE_INTRA_REFRESH_EXTENSION_NAME)) std::get(feat).videoEncodeIntraRefresh = std::get(physical_device.getFeatures2()).videoEncodeIntraRefresh; #endif +#ifdef VK_KHR_video_encode_av1 + if (has_device_ext(VK_KHR_VIDEO_ENCODE_AV1_EXTENSION_NAME)) + std::get(feat).videoEncodeAV1 = std::get(physical_device.getFeatures2()).videoEncodeAV1; +#endif #ifdef VK_KHR_unified_image_layouts if (has_device_ext(VK_KHR_UNIFIED_IMAGE_LAYOUTS_EXTENSION_NAME)) { diff --git a/server/utils/wivrn_vk_bundle.h b/server/utils/wivrn_vk_bundle.h index 280909a9..54701b6d 100644 --- a/server/utils/wivrn_vk_bundle.h +++ b/server/utils/wivrn_vk_bundle.h @@ -131,6 +131,9 @@ struct vk_bundle #ifdef VK_KHR_video_encode_intra_refresh vk::PhysicalDeviceVideoEncodeIntraRefreshFeaturesKHR, #endif +#ifdef VK_KHR_video_encode_av1 + vk::PhysicalDeviceVideoEncodeAV1FeaturesKHR, +#endif #ifdef VK_KHR_unified_image_layouts vk::PhysicalDeviceUnifiedImageLayoutsFeaturesKHR, #endif