/* * WiVRn VR streaming * Copyright (C) 2024 Patrick Nicolas * Copyright (C) 2026 Philipp Schlegel * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #include "video_encoder_vulkan_av1.h" #include "encoder/encoder_settings.h" #include "util/u_logging.h" #include "utils/wivrn_vk_bundle.h" #include #include #include #include #include namespace { uint32_t align_div(uint32_t value, uint32_t divisor) { return (value + divisor - 1) / divisor; } uint8_t bits_for(uint32_t value) { return std::max(std::bit_width(value), 1); } // AV1 spec, annex A.3 "Levels", limited to the levels that are actually defined. // MaxDisplayRate is omitted, it is always smaller than MaxDecodeRate and we never // display more samples than we decode. struct av1_level_limits { StdVideoAV1Level level; uint32_t max_pic_size; // samples uint32_t max_h_size; // samples uint32_t max_v_size; // samples uint64_t max_decode_rate; }; constexpr av1_level_limits av1_levels[] = { {STD_VIDEO_AV1_LEVEL_2_0, 147456, 2048, 1152, 5529600}, {STD_VIDEO_AV1_LEVEL_2_1, 278784, 2816, 1584, 10454400}, {STD_VIDEO_AV1_LEVEL_3_0, 665856, 4352, 2448, 24969600}, {STD_VIDEO_AV1_LEVEL_3_1, 1065024, 5504, 3096, 39938400}, {STD_VIDEO_AV1_LEVEL_4_0, 2359296, 6144, 3456, 77856768}, {STD_VIDEO_AV1_LEVEL_4_1, 2359296, 6144, 3456, 155713536}, {STD_VIDEO_AV1_LEVEL_5_0, 8912896, 8192, 4352, 273715200}, {STD_VIDEO_AV1_LEVEL_5_1, 8912896, 8192, 4352, 547430400}, {STD_VIDEO_AV1_LEVEL_5_2, 8912896, 8192, 4352, 1094860800}, {STD_VIDEO_AV1_LEVEL_5_3, 8912896, 8192, 4352, 1176502272}, {STD_VIDEO_AV1_LEVEL_6_0, 35651584, 16384, 8704, 1176502272}, {STD_VIDEO_AV1_LEVEL_6_1, 35651584, 16384, 8704, 2189721600}, {STD_VIDEO_AV1_LEVEL_6_2, 35651584, 16384, 8704, 4379443200}, {STD_VIDEO_AV1_LEVEL_6_3, 35651584, 16384, 8704, 4706009088}, }; // Smallest level able to carry the stream, clamped to what the encoder supports. // Reporting the driver's maximum instead would make decoders reject or downgrade // the stream for no reason. StdVideoAV1Level select_level(uint32_t width, uint32_t height, float fps, StdVideoAV1Level max_level) { const uint64_t pic_size = uint64_t(width) * height; const uint64_t decode_rate = uint64_t(double(pic_size) * std::max(fps, 1.f)); for (const auto & l: av1_levels) { if (l.level > max_level) break; if (pic_size <= l.max_pic_size and width <= l.max_h_size and height <= l.max_v_size and decode_rate <= l.max_decode_rate) return l.level; } U_LOG_W("AV1: no level supports %ux%u@%.1f, falling back to the encoder maximum", width, height, fps); return max_level; } } // namespace wivrn::video_encoder_vulkan_av1::video_encoder_vulkan_av1( wivrn::vk_bundle & vk, const vk::VideoCapabilitiesKHR & video_caps, const vk::VideoEncodeCapabilitiesKHR & encode_caps, const vk::VideoEncodeAV1CapabilitiesKHR & encode_av1_caps, uint8_t stream_idx, const encoder_settings & settings) : video_encoder_vulkan(vk, video_caps, encode_caps, stream_idx, settings) { if (not vk.has_device_ext(VK_KHR_VIDEO_ENCODE_AV1_EXTENSION_NAME)) throw std::runtime_error("Vulkan video encode AV1 extension not available"); if (settings.bit_depth != 8 && settings.bit_depth != 10) throw std::runtime_error("av1 encoder supports 8-bit or 10-bit only"); configure_from_caps(encode_av1_caps); level = select_level(aligned_extent.width, aligned_extent.height, settings.fps, encode_av1_caps.maxLevel); color_config = { .flags = { .mono_chrome = 0, .color_range = 1, .separate_uv_delta_q = 0, .color_description_present_flag = 1, }, .BitDepth = static_cast(settings.bit_depth), .subsampling_x = 1, .subsampling_y = 1, .reserved1 = 0, .color_primaries = STD_VIDEO_AV1_COLOR_PRIMARIES_BT_709, .transfer_characteristics = STD_VIDEO_AV1_TRANSFER_CHARACTERISTICS_BT_709, .matrix_coefficients = STD_VIDEO_AV1_MATRIX_COEFFICIENTS_BT_709, .chroma_sample_position = STD_VIDEO_AV1_CHROMA_SAMPLE_POSITION_COLOCATED, }; seq_header = { .flags = { .still_picture = 0, .reduced_still_picture_header = 0, .use_128x128_superblock = superblock_size == 128 ? 1u : 0u, .enable_filter_intra = 0, .enable_intra_edge_filter = 0, .enable_interintra_compound = 0, .enable_masked_compound = 0, .enable_warped_motion = 0, .enable_dual_filter = 0, .enable_order_hint = 1, .enable_jnt_comp = 0, .enable_ref_frame_mvs = 0, .frame_id_numbers_present_flag = 0, .enable_superres = 0, .enable_cdef = 0, .enable_restoration = 0, .film_grain_params_present = 0, .timing_info_present_flag = 0, .initial_display_delay_present_flag = 0, }, .seq_profile = STD_VIDEO_AV1_PROFILE_MAIN, .frame_width_bits_minus_1 = static_cast(bits_for(aligned_extent.width - 1) - 1), .frame_height_bits_minus_1 = static_cast(bits_for(aligned_extent.height - 1) - 1), .max_frame_width_minus_1 = static_cast(aligned_extent.width - 1), .max_frame_height_minus_1 = static_cast(aligned_extent.height - 1), .delta_frame_id_length_minus_2 = 0, .additional_frame_id_length_minus_1 = 0, .order_hint_bits_minus_1 = static_cast(order_hint_bits - 1), .seq_force_integer_mv = STD_VIDEO_AV1_SELECT_INTEGER_MV, .seq_force_screen_content_tools = STD_VIDEO_AV1_SELECT_SCREEN_CONTENT_TOOLS, .reserved1 = {}, .pColorConfig = &color_config, .pTimingInfo = nullptr, }; operating_point = { .flags = { .decoder_model_present_for_this_op = 0, .low_delay_mode_flag = 1, .initial_display_delay_present_for_this_op = 0, }, .operating_point_idc = 0, .seq_level_idx = static_cast(level), .seq_tier = 0, .decoder_buffer_delay = 0, .encoder_buffer_delay = 0, .initial_display_delay_minus_1 = 0, }; const uint32_t sb_cols = align_div(aligned_extent.width, superblock_size); const uint32_t sb_rows = align_div(aligned_extent.height, superblock_size); const uint32_t mi_cols = align_div(aligned_extent.width, 4); const uint32_t mi_rows = align_div(aligned_extent.height, 4); const bool uniform_tile_spacing = bool(std_flags & vk::VideoEncodeAV1StdFlagBitsKHR::eUniformTileSpacingFlagSet); // A single tile is enough for the resolutions WiVRn streams, but AV1 caps a tile at 4096 // samples wide (annex A.3), a wider stream would not be decodable. if (sb_cols * superblock_size > 4096) throw std::runtime_error("av1 encoder supports at most 4096 samples per tile row"); mi_col_starts = {0u, static_cast(mi_cols)}; mi_row_starts = {0u, static_cast(mi_rows)}; if (!tile_widths_sb.empty()) tile_widths_sb[0] = static_cast(sb_cols - 1); if (!tile_heights_sb.empty()) tile_heights_sb[0] = static_cast(sb_rows - 1); tile_info = { .flags = { .uniform_tile_spacing_flag = uniform_tile_spacing ? 1u : 0u, }, .TileCols = 1, .TileRows = 1, .context_update_tile_id = 0, .tile_size_bytes_minus_1 = 0, .reserved1 = {}, .pMiColStarts = uniform_tile_spacing ? nullptr : mi_col_starts.data(), .pMiRowStarts = uniform_tile_spacing ? nullptr : mi_row_starts.data(), .pWidthInSbsMinus1 = uniform_tile_spacing ? nullptr : tile_widths_sb.data(), .pHeightInSbsMinus1 = uniform_tile_spacing ? nullptr : tile_heights_sb.data(), }; quantization = { .flags = { .using_qmatrix = 0, .diff_uv_delta = 0, }, .base_q_idx = static_cast(encode_av1_caps.maxQIndex), .DeltaQYDc = 0, .DeltaQUDc = 0, .DeltaQUAc = 0, .DeltaQVDc = 0, .DeltaQVAc = 0, .qm_y = 0, .qm_u = 0, .qm_v = 0, }; loop_filter = { .flags = { .loop_filter_delta_enabled = 0, .loop_filter_delta_update = 0, }, .loop_filter_level = {}, .loop_filter_sharpness = 0, .update_ref_delta = 0, .loop_filter_ref_deltas = {}, .update_mode_delta = 0, .loop_filter_mode_deltas = {}, }; cdef = { .cdef_damping_minus_3 = 0, .cdef_bits = 0, .cdef_y_pri_strength = {}, .cdef_y_sec_strength = {}, .cdef_uv_pri_strength = {}, .cdef_uv_sec_strength = {}, }; loop_restoration = { .FrameRestorationType = {STD_VIDEO_AV1_FRAME_RESTORATION_TYPE_NONE, STD_VIDEO_AV1_FRAME_RESTORATION_TYPE_NONE, STD_VIDEO_AV1_FRAME_RESTORATION_TYPE_NONE}, .LoopRestorationSize = {0, 0, 0}, }; global_motion = { .GmType = {}, .gm_params = {}, }; segmentation = {}; rate_control_layer.pNext = &rate_control_layer_av1; } void wivrn::video_encoder_vulkan_av1::configure_from_caps(const vk::VideoEncodeAV1CapabilitiesKHR & encode_av1_caps) { U_LOG_D("AV1: capabilities: stdSyntaxFlags=0x%x, superblockSizes=0x%x, singleReferenceNameMask=0x%x, maxSingleReferenceCount=%u, maxUnidirectionalCompoundReferenceCount=%u, maxBidirectionalCompoundReferenceCount=%u, maxQIndex=%u, minQIndex=%u, requiresGopRemainingFrames=%u, maxOperatingPoints=%u", static_cast(encode_av1_caps.stdSyntaxFlags), static_cast(encode_av1_caps.superblockSizes), encode_av1_caps.singleReferenceNameMask, encode_av1_caps.maxSingleReferenceCount, encode_av1_caps.maxUnidirectionalCompoundReferenceCount, encode_av1_caps.maxBidirectionalCompoundReferenceCount, encode_av1_caps.maxQIndex, encode_av1_caps.minQIndex, encode_av1_caps.requiresGopRemainingFrames, encode_av1_caps.maxOperatingPoints); std_flags = encode_av1_caps.stdSyntaxFlags; if (encode_av1_caps.superblockSizes & vk::VideoEncodeAV1SuperblockSizeFlagBitsKHR::e64) superblock_size = 64; else superblock_size = 128; single_reference_name_mask = encode_av1_caps.singleReferenceNameMask; max_single_reference_count = encode_av1_caps.maxSingleReferenceCount; max_q_index = encode_av1_caps.maxQIndex; min_q_index = encode_av1_caps.minQIndex; // The single reference of an inter frame must use a reference name the encoder // advertises, bit i standing for LAST_FRAME + i. // VUID-vkCmdEncodeVideoKHR-predictionMode-10329 ref_name_index = -1; for (size_t i = 0; i < reference_name_slot_indices.size(); ++i) { if (single_reference_name_mask & (1u << i)) { ref_name_index = int32_t(i); break; } } // WiVRn streams key frame + inter frames referencing a single acknowledged frame, an // encoder that cannot do single reference prediction is of no use to us. if (ref_name_index < 0 or max_single_reference_count == 0) throw std::runtime_error("av1 encoder does not support single reference prediction"); if (ref_name_index != 0) U_LOG_W("AV1: LAST_FRAME not supported (mask=0x%x), using reference name index %d", single_reference_name_mask, ref_name_index); rate_control_av1 = vk::VideoEncodeAV1RateControlInfoKHR{ .flags = vk::VideoEncodeAV1RateControlFlagBitsKHR::eRegularGop, .gopFrameCount = std::numeric_limits::max(), .keyFramePeriod = std::numeric_limits::max(), .consecutiveBipredictiveFrameCount = 0, // VUID-VkVideoEncodeAV1RateControlInfoKHR-temporalLayerCount-10299 .temporalLayerCount = std::min(1u, encode_av1_caps.maxTemporalLayerCount), }; rate_control_layer_av1 = vk::VideoEncodeAV1RateControlLayerInfoKHR{ .useMinQIndex = VK_FALSE, .minQIndex = { .intraQIndex = encode_av1_caps.minQIndex, .predictiveQIndex = encode_av1_caps.minQIndex, .bipredictiveQIndex = encode_av1_caps.minQIndex, }, .useMaxQIndex = VK_FALSE, .maxQIndex = { .intraQIndex = encode_av1_caps.maxQIndex, .predictiveQIndex = encode_av1_caps.maxQIndex, .bipredictiveQIndex = encode_av1_caps.maxQIndex, }, .useMaxFrameSize = VK_FALSE, .maxFrameSize = {}, }; } std::vector wivrn::video_encoder_vulkan_av1::setup_slot_info(size_t dpb_size) { dpb_std_info.resize(dpb_size, {}); dpb_std_slots.reserve(dpb_size); std::vector res; for (size_t i = 0; i < dpb_size; ++i) { dpb_std_slots.push_back({ .pStdReferenceInfo = &dpb_std_info[i], }); res.push_back(&dpb_std_slots[i]); } return res; } static auto get_video_caps(vk::raii::PhysicalDevice & phys_dev, int bit_depth) { if (!(bit_depth == 8 || bit_depth == 10)) throw std::runtime_error("av1 encoder supports 8-bit or 10-bit only"); vk::StructureChain video_profile_info{ vk::VideoProfileInfoKHR{ .videoCodecOperation = vk::VideoCodecOperationFlagBitsKHR::eEncodeAv1, .chromaSubsampling = vk::VideoChromaSubsamplingFlagBitsKHR::e420, .lumaBitDepth = bit_depth == 10 ? vk::VideoComponentBitDepthFlagBitsKHR::e10 : vk::VideoComponentBitDepthFlagBitsKHR::e8, .chromaBitDepth = bit_depth == 10 ? vk::VideoComponentBitDepthFlagBitsKHR::e10 : vk::VideoComponentBitDepthFlagBitsKHR::e8, }, vk::VideoEncodeAV1ProfileInfoKHR{ .stdProfile = STD_VIDEO_AV1_PROFILE_MAIN, }, vk::VideoEncodeUsageInfoKHR{ .videoUsageHints = vk::VideoEncodeUsageFlagBitsKHR::eStreaming, .videoContentHints = vk::VideoEncodeContentFlagBitsKHR::eRendered, .tuningMode = vk::VideoEncodeTuningModeKHR::eUltraLowLatency, }}; try { auto [video_caps, encode_caps, encode_av1_caps] = phys_dev.getVideoCapabilitiesKHR< vk::VideoCapabilitiesKHR, vk::VideoEncodeCapabilitiesKHR, vk::VideoEncodeAV1CapabilitiesKHR>(video_profile_info.get()); video_caps.maxDpbSlots = std::min(video_caps.maxDpbSlots, uint32_t(STD_VIDEO_AV1_NUM_REF_FRAMES)); return std::make_tuple(video_caps, encode_caps, encode_av1_caps, video_profile_info); } catch (...) {} video_profile_info.unlink(); auto [video_caps, encode_caps, encode_av1_caps] = phys_dev.getVideoCapabilitiesKHR< vk::VideoCapabilitiesKHR, vk::VideoEncodeCapabilitiesKHR, vk::VideoEncodeAV1CapabilitiesKHR>(video_profile_info.get()); video_caps.maxDpbSlots = std::min(video_caps.maxDpbSlots, uint32_t(STD_VIDEO_AV1_NUM_REF_FRAMES)); return std::make_tuple(video_caps, encode_caps, encode_av1_caps, video_profile_info); } std::unique_ptr wivrn::video_encoder_vulkan_av1::create( wivrn::vk_bundle & vk, const encoder_settings & settings, uint8_t stream_idx) { auto [video_caps, encode_caps, encode_av1_caps, video_profile_info] = get_video_caps(vk.physical_device, settings.bit_depth); std::unique_ptr self( new video_encoder_vulkan_av1(vk, video_caps, encode_caps, encode_av1_caps, stream_idx, settings)); vk::VideoEncodeAV1SessionParametersCreateInfoKHR session_params_info{ .pStdSequenceHeader = &self->seq_header, .pStdDecoderModelInfo = nullptr, .stdOperatingPointCount = 0, .pStdOperatingPoints = nullptr, }; self->quality_level_info = vk::VideoEncodeQualityLevelInfoKHR{ .qualityLevel = 0, // Use default quality level }; session_params_info.pNext = &self->quality_level_info; // Let the session be sized for whatever the encoder supports, the level that matters to the // decoder is the one written to the operating point. vk::VideoEncodeAV1SessionCreateInfoKHR session_create_info{ .useMaxLevel = false, .maxLevel = encode_av1_caps.maxLevel, }; // VkVideoEncodeAV1RateControlInfoKHR belongs to the same pNext chain as the generic // rate control info, in both vkCmdBeginVideoCodingKHR and vkCmdControlVideoCodingKHR. if (self->rate_control) { self->rate_control_enabled = true; self->rate_control->pNext = &self->rate_control_av1; } // VkVideoEncodeAV1GopRemainingFrameInfoKHR only extends VkVideoBeginCodingInfoKHR, so it // is added by begin_coding_next() rather than chained onto the rate control info, which is // also used for vkCmdControlVideoCodingKHR. // VUID-vkCmdBeginVideoCodingKHR-pBeginInfo-10282 if (self->rate_control_enabled and (encode_av1_caps.requiresGopRemainingFrames or encode_av1_caps.prefersGopRemainingFrames)) { self->gop_info = vk::VideoEncodeAV1GopRemainingFrameInfoKHR{ .useGopRemainingFrames = true, .gopRemainingIntra = 0, .gopRemainingPredictive = std::numeric_limits::max(), .gopRemainingBipredictive = 0, }; } if (encode_av1_caps.maxOperatingPoints > 0) { session_params_info.stdOperatingPointCount = 1; session_params_info.pStdOperatingPoints = &self->operating_point; } #ifdef VK_KHR_video_encode_intra_refresh vk::VideoEncodeIntraRefreshCapabilitiesKHR intra_caps{}; if (std::get(vk.feat).videoEncodeIntraRefresh) intra_caps = std::get( vk.physical_device.getVideoCapabilitiesKHR(video_profile_info.get())); #endif self->init(video_caps, #ifdef VK_KHR_video_encode_intra_refresh intra_caps, #endif video_profile_info.get(), &session_create_info, &session_params_info); return self; } void wivrn::video_encoder_vulkan_av1::send_idr_data() { U_LOG_D("AV1: send_idr_data() called"); // Fetch encoded session parameters (sequence header OBU). try { auto data = get_encoded_parameters(nullptr); U_LOG_D("AV1: get_encoded_parameters returned %zu bytes", data.size()); if (!data.empty()) SendData(data, false, true); else U_LOG_W("AV1: get_encoded_parameters returned empty data"); } catch (const std::exception & e) { U_LOG_E("AV1: Exception in get_encoded_parameters: %s", e.what()); } } void * wivrn::video_encoder_vulkan_av1::encode_info_next(uint32_t frame_num, size_t slot, std::optional ref_slot) { const bool has_ref = ref_slot.has_value(); const bool is_keyframe = not has_ref; // Initialize reference name slot indices to -1 (no reference) std::ranges::fill(reference_name_slot_indices, -1); if (has_ref) reference_name_slot_indices[ref_name_index] = *ref_slot; std_picture_info = {}; std_picture_info.flags.error_resilient_mode = is_keyframe ? 1u : 0u; std_picture_info.flags.disable_cdf_update = 0; std_picture_info.flags.use_superres = 0; // The coded frame is always aligned_extent, which is what the sequence header declares as // max_frame_{width,height}, so the frame size is never overridden. Only the render size // differs, when the requested extent needed padding to satisfy the encoder granularity. // VUID-vkCmdEncodeVideoKHR-flags-10322 std_picture_info.flags.frame_size_override_flag = 0; std_picture_info.flags.render_and_frame_size_different = (aligned_extent.width != extent.width or aligned_extent.height != extent.height) ? 1u : 0u; std_picture_info.flags.allow_screen_content_tools = 0; std_picture_info.flags.is_filter_switchable = 1; // AV1 spec 5.9.2: both are inferred for intra frames, match what the decoder will derive std_picture_info.flags.force_integer_mv = is_keyframe ? 1u : 0u; std_picture_info.flags.buffer_removal_time_present_flag = 0; std_picture_info.flags.allow_intrabc = 0; std_picture_info.flags.frame_refs_short_signaling = 0; std_picture_info.flags.allow_high_precision_mv = is_keyframe ? 0u : 1u; std_picture_info.flags.is_motion_mode_switchable = 0; std_picture_info.flags.use_ref_frame_mvs = 0; std_picture_info.flags.disable_frame_end_update_cdf = 0; std_picture_info.flags.allow_warped_motion = 0; std_picture_info.flags.reduced_tx_set = 0; std_picture_info.flags.skip_mode_present = 0; std_picture_info.flags.delta_q_present = 0; std_picture_info.flags.delta_lf_present = 0; std_picture_info.flags.delta_lf_multi = 0; std_picture_info.flags.segmentation_enabled = 0; std_picture_info.flags.segmentation_update_map = 0; std_picture_info.flags.segmentation_temporal_update = 0; std_picture_info.flags.segmentation_update_data = 0; std_picture_info.flags.UsesLr = 0; std_picture_info.flags.usesChromaLr = 0; std_picture_info.flags.show_frame = 1; std_picture_info.flags.showable_frame = std_picture_info.flags.show_frame ? (is_keyframe ? 0u : 1u) : 1u; std_picture_info.frame_type = is_keyframe ? STD_VIDEO_AV1_FRAME_TYPE_KEY : STD_VIDEO_AV1_FRAME_TYPE_INTER; // Both are only coded when the decoder model / frame ids are signalled, which the sequence // header disables, and the decoder then infers 0 for them (AV1 spec 5.9.2). std_picture_info.frame_presentation_time = 0; std_picture_info.current_frame_id = 0; std_picture_info.order_hint = static_cast(frame_num & ((1u << order_hint_bits) - 1)); // primary_ref_frame indexes ref_frame_idx[], it selects the reference to load the CDFs and // the loop filter / segmentation state from. Only usable if the encoder supports values // other than PRIMARY_REF_NONE. if (has_ref and (std_flags & vk::VideoEncodeAV1StdFlagBitsKHR::ePrimaryRefFrame)) std_picture_info.primary_ref_frame = uint8_t(ref_name_index); else std_picture_info.primary_ref_frame = STD_VIDEO_AV1_PRIMARY_REF_NONE; std_picture_info.refresh_frame_flags = static_cast(is_keyframe ? 0xFF : (1u << slot)); std_picture_info.coded_denom = 0; std_picture_info.render_width_minus_1 = static_cast(extent.width - 1); std_picture_info.render_height_minus_1 = static_cast(extent.height - 1); std_picture_info.interpolation_filter = STD_VIDEO_AV1_INTERPOLATION_FILTER_SWITCHABLE; std_picture_info.TxMode = STD_VIDEO_AV1_TX_MODE_SELECT; std_picture_info.delta_q_res = 0; std_picture_info.delta_lf_res = 0; std::fill(std_picture_info.ref_order_hint, std_picture_info.ref_order_hint + STD_VIDEO_AV1_NUM_REF_FRAMES, 0); std::fill(std_picture_info.ref_frame_idx, std_picture_info.ref_frame_idx + STD_VIDEO_AV1_REFS_PER_FRAME, static_cast(-1)); std::fill(std_picture_info.delta_frame_id_minus_1, std_picture_info.delta_frame_id_minus_1 + STD_VIDEO_AV1_REFS_PER_FRAME, 0); std_picture_info.pTileInfo = &tile_info; std_picture_info.pQuantization = &quantization; std_picture_info.pSegmentation = (std_picture_info.flags.segmentation_enabled) ? &segmentation : nullptr; std_picture_info.pLoopFilter = &loop_filter; std_picture_info.pCDEF = &cdef; std_picture_info.pLoopRestoration = &loop_restoration; std_picture_info.pGlobalMotion = &global_motion; std_picture_info.pExtensionHeader = nullptr; std_picture_info.pBufferRemovalTimes = nullptr; // ref_frame_idx[] maps a reference name to a DPB slot, and must be filled in for the same // reference name that referenceNameSlotIndices uses. if (has_ref) std_picture_info.ref_frame_idx[ref_name_index] = static_cast(*ref_slot); const size_t dpb_ref_count = std::min(dpb_std_info.size(), STD_VIDEO_AV1_NUM_REF_FRAMES); for (size_t i = 0; i < dpb_ref_count; ++i) std_picture_info.ref_order_hint[i] = dpb_std_info[i].OrderHint; picture_info = vk::VideoEncodeAV1PictureInfoKHR{}; // WiVRn always predicts from the primary reference, never uses it for CDFs only. // VUID-VkVideoEncodeAV1PictureInfoKHR-flags-10289 picture_info.primaryReferenceCdfOnly = VK_FALSE; // Key frames are the only intra frames we produce, everything else predicts from the single // acknowledged reference frame. Encoders without single reference support are rejected in // configure_from_caps(). // VUID-vkCmdEncodeVideoKHR-pStdPictureInfo-10327 picture_info.predictionMode = is_keyframe ? vk::VideoEncodeAV1PredictionModeKHR::eIntraOnly : vk::VideoEncodeAV1PredictionModeKHR::eSingleReference; // Set rate control group based on frame type // Note: WiVRn currently doesn't support B-frames, so only INTRA and PREDICTIVE groups picture_info.rateControlGroup = is_keyframe ? vk::VideoEncodeAV1RateControlGroupKHR::eIntra : vk::VideoEncodeAV1RateControlGroupKHR::ePredictive; // Must be zero while rate control is active, and within the reported range otherwise. // VUID-vkCmdEncodeVideoKHR-constantQIndex-10320, VUID-vkCmdEncodeVideoKHR-constantQIndex-10321 picture_info.constantQIndex = rate_control_enabled ? 0 : std::midpoint(min_q_index, max_q_index); picture_info.pStdPictureInfo = &std_picture_info; picture_info.referenceNameSlotIndices = reference_name_slot_indices; picture_info.generateObuExtensionHeader = VK_FALSE; auto & i = dpb_std_info[slot]; i = {}; i.flags.disable_frame_end_update_cdf = 0; i.flags.segmentation_enabled = 0; // Frame ids are not signalled, see current_frame_id above i.RefFrameId = 0; i.frame_type = std_picture_info.frame_type; i.OrderHint = std_picture_info.order_hint; i.pExtensionHeader = nullptr; return &picture_info; } const void * wivrn::video_encoder_vulkan_av1::begin_coding_next(const void * next) { if (not gop_info) return next; gop_info->pNext = next; return &*gop_info; } vk::ExtensionProperties wivrn::video_encoder_vulkan_av1::std_header_version() { vk::ExtensionProperties std_header_version{ .specVersion = VK_STD_VULKAN_VIDEO_CODEC_AV1_ENCODE_SPEC_VERSION, }; strcpy(std_header_version.extensionName, VK_STD_VULKAN_VIDEO_CODEC_AV1_ENCODE_EXTENSION_NAME); return std_header_version; }