diff --git a/client/decoder/android/android_decoder.cpp b/client/decoder/android/android_decoder.cpp index 2747b1d5..9d9ee08d 100644 --- a/client/decoder/android/android_decoder.cpp +++ b/client/decoder/android/android_decoder.cpp @@ -21,7 +21,6 @@ #include "application.h" #include "scenes/stream.h" #include "utils/named_thread.h" -#include #include #include #include @@ -57,6 +56,8 @@ const char * mime(xrt::drivers::wivrn::video_codec codec) return "video/avc"; case c::h265: return "video/hevc"; + case c::av1: + return "video/AV1"; } assert(false); } @@ -70,109 +71,6 @@ void check(media_status_t status, const char * msg) } } -namespace nal_h264 -{ -static const int sps = 7; -static const int pps = 8; -} // namespace nal_h264 - -namespace nal_h265 -{ -static const int vps = 32; -static const int sps = 33; -static const int pps = 34; -static const int aud = 35; -static const int filler = 38; -}; // namespace nal_h265 - -enum class nal_class -{ - csd, - data, - garbage -}; - -nal_class get_nal_class_h264(uint8_t * nal) -{ - uint8_t nal_type = (nal[2] == 0 ? nal[4] : nal[3]) & 0x1F; - // spdlog::info("H264 NAL type {}", (int)nal_type); - switch (nal_type) - { - case nal_h264::sps: - case nal_h264::pps: - return nal_class::csd; - default: - return nal_class::data; - } -} - -nal_class get_nal_class_h265(uint8_t * nal) -{ - uint8_t nal_type = ((nal[2] == 0 ? nal[4] : nal[3]) >> 1) & 0x3F; - // spdlog::info("H265 NAL type {}", (int)nal_type); - switch (nal_type) - { - case nal_h265::vps: - case nal_h265::sps: - case nal_h265::pps: - return nal_class::csd; - case nal_h265::aud: - case nal_h265::filler: - return nal_class::garbage; - default: - return nal_class::data; - } -} - -nal_class get_nal_class(uint8_t * nal, xrt::drivers::wivrn::video_codec codec) -{ - switch (codec) - { - case xrt::drivers::wivrn::video_codec::h264: - return get_nal_class_h264(nal); - - case xrt::drivers::wivrn::video_codec::h265: - return get_nal_class_h265(nal); - } -} - -std::pair, std::vector> filter_csd(std::span packet, - xrt::drivers::wivrn::video_codec codec) -{ - if (packet.size() < 4) - return {}; - - std::array header = {{0, 0, 1}}; - auto end = packet.end(); - auto header_start = packet.begin(); - - std::pair, std::vector> out; - - while (header_start != end) - { - auto next_header = std::search(header_start + 3, end, header.begin(), header.end()); - if (next_header != end and next_header[-1] == 0) - { - next_header--; - } - - switch (get_nal_class(&*header_start, codec)) - { - case nal_class::csd: - out.first.insert(out.first.end(), header_start, next_header); - break; - case nal_class::data: - out.second.insert(out.second.end(), header_start, next_header); - break; - case nal_class::garbage: - break; - } - - header_start = next_header; - } - - return out; -} } // namespace namespace wivrn::android @@ -292,16 +190,6 @@ void decoder::push_data(std::span> data, uint64_t frame { if (!media_codec) { - std::vector contiguous_data; - for (const auto & d: data) - contiguous_data.insert(contiguous_data.end(), d.begin(), d.end()); - auto [csd, not_csd] = filter_csd(contiguous_data, description.codec); - if (csd.empty()) - { - // We can't initizale the encoder, drop the data - return; - } - AMediaFormat_ptr format(AMediaFormat_new()); AMediaFormat_setString(format.get(), AMEDIAFORMAT_KEY_MIME, mime(description.codec)); // AMediaFormat_setInt32(format.get(), "vendor.qti-ext-dec-low-latency.enable", 1); // Qualcomm low @@ -310,7 +198,6 @@ void decoder::push_data(std::span> data, uint64_t frame AMediaFormat_setInt32(format.get(), AMEDIAFORMAT_KEY_HEIGHT, description.height); AMediaFormat_setInt32(format.get(), AMEDIAFORMAT_KEY_OPERATING_RATE, std::ceil(fps)); AMediaFormat_setInt32(format.get(), AMEDIAFORMAT_KEY_PRIORITY, 0); - // AMediaFormat_setBuffer(format.get(), AMEDIAFORMAT_KEY_CSD_0, csd.data(), csd.size()); media_codec.reset(AMediaCodec_createDecoderByType(mime(description.codec))); diff --git a/client/decoder/ffmpeg/ffmpeg_decoder.cpp b/client/decoder/ffmpeg/ffmpeg_decoder.cpp index dbd4634f..d4672afd 100644 --- a/client/decoder/ffmpeg/ffmpeg_decoder.cpp +++ b/client/decoder/ffmpeg/ffmpeg_decoder.cpp @@ -54,6 +54,8 @@ static AVCodecID codec_id(xrt::drivers::wivrn::video_codec codec) return AV_CODEC_ID_H264; case c::h265: return AV_CODEC_ID_HEVC; + case c::av1: + return AV_CODEC_ID_AV1; } assert(false); __builtin_unreachable(); diff --git a/common/wivrn_packets.h b/common/wivrn_packets.h index fab3f373..6d09a4df 100644 --- a/common/wivrn_packets.h +++ b/common/wivrn_packets.h @@ -81,6 +81,7 @@ enum video_codec h264, h265, hevc = h265, + av1, }; struct audio_data diff --git a/docs/configuration.md b/docs/configuration.md index 70e9e216..97f8eb84 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -41,7 +41,7 @@ Identifier of the encoder, one of `x264` (software encoding), `nvenc` (Nvidia ha ### `codec` Default value: `h265` -One of `h264` or `h265`. If using `x264` encoder, value is ignored and `h264` is used. +One of `h264`, `h265` or `av1`. If using `x264` encoder, value is ignored and `h264` is used. `av1` is only supported on `vaapi` encoder. ### `width`, `height`, `offset_x`, `offset_y` (advanced) Default values: full image (`width` = 1, `height` = 1, `offset_x` = 0, `offset_y` = 0) diff --git a/server/driver/configuration.cpp b/server/driver/configuration.cpp index 33f69744..97eff477 100644 --- a/server/driver/configuration.cpp +++ b/server/driver/configuration.cpp @@ -46,6 +46,8 @@ NLOHMANN_JSON_SERIALIZE_ENUM( {h264, "avc"}, {h265, "h265"}, {h265, "hevc"}, + {av1, "av1"}, + {av1, "AV1"}, }) } diff --git a/server/encoder/encoder_settings.cpp b/server/encoder/encoder_settings.cpp index 4e795928..367b09c2 100644 --- a/server/encoder/encoder_settings.cpp +++ b/server/encoder/encoder_settings.cpp @@ -61,6 +61,7 @@ static void split_bitrate(std::vector & e w *= 2; break; case xrt::drivers::wivrn::h265: + case xrt::drivers::wivrn::av1: break; } encoder.bitrate = w; diff --git a/server/encoder/ffmpeg/ffmpeg_helper.cpp b/server/encoder/ffmpeg/ffmpeg_helper.cpp index 97d92ab4..93291d7d 100644 --- a/server/encoder/ffmpeg/ffmpeg_helper.cpp +++ b/server/encoder/ffmpeg/ffmpeg_helper.cpp @@ -99,11 +99,6 @@ void AvDeleter::operator()(AVCodecContext * x) avcodec_free_context(&x); } -void AvDeleter::operator()(AVFilterGraph * x) -{ - avfilter_graph_free(&x); -} - void AvDeleter::operator()(AVPacket * x) { av_packet_free(&x); diff --git a/server/encoder/ffmpeg/ffmpeg_helper.h b/server/encoder/ffmpeg/ffmpeg_helper.h index d22a2216..72d9bff1 100644 --- a/server/encoder/ffmpeg/ffmpeg_helper.h +++ b/server/encoder/ffmpeg/ffmpeg_helper.h @@ -24,7 +24,6 @@ #include struct AVBufferRef; -struct AVFilterGraph; struct AVFrame; struct AVCodecContext; struct AVPacket; @@ -52,15 +51,12 @@ struct AvDeleter void operator()(AVCodecContext *); void - operator()(AVFilterGraph *); - void operator()(AVPacket *); }; using av_buffer_ptr = std::unique_ptr; using av_frame_ptr = std::unique_ptr; using av_codec_context_ptr = std::unique_ptr; -using av_filter_graph_ptr = std::unique_ptr; using av_packet_ptr = std::unique_ptr; av_frame_ptr diff --git a/server/encoder/ffmpeg/video_encoder_va.cpp b/server/encoder/ffmpeg/video_encoder_va.cpp index 14b17987..46ebb77f 100644 --- a/server/encoder/ffmpeg/video_encoder_va.cpp +++ b/server/encoder/ffmpeg/video_encoder_va.cpp @@ -52,6 +52,8 @@ encoder(VideoEncoderFFMPEG::Codec codec) return "h264_vaapi"; case VideoEncoderFFMPEG::Codec::h265: return "hevc_vaapi"; + case VideoEncoderFFMPEG::Codec::av1: + return "av1_vaapi"; } throw std::runtime_error("invalid codec " + std::to_string(int(codec))); } @@ -192,6 +194,9 @@ video_encoder_va::video_encoder_va(wivrn_vk_bundle & vk, xrt::drivers::wivrn::en case Codec::h265: encoder_ctx->profile = FF_PROFILE_HEVC_MAIN; break; + case Codec::av1: + encoder_ctx->profile = FF_PROFILE_AV1_MAIN; + break; } for (auto option: settings.options) { diff --git a/server/encoder/video_encoder.cpp b/server/encoder/video_encoder.cpp index 1daf6551..a6806041 100644 --- a/server/encoder/video_encoder.cpp +++ b/server/encoder/video_encoder.cpp @@ -142,6 +142,9 @@ std::unique_ptr VideoEncoder::Create( case h265: file += ".h265"; break; + case av1: + file += ".av1"; + break; } res->video_dump.open(file); } diff --git a/server/encoder/video_encoder_nvenc.cpp b/server/encoder/video_encoder_nvenc.cpp index ca4dafb9..4df6babc 100644 --- a/server/encoder/video_encoder_nvenc.cpp +++ b/server/encoder/video_encoder_nvenc.cpp @@ -128,6 +128,8 @@ static auto encode_guid(video_codec codec) return NV_ENC_CODEC_H264_GUID; case h265: return NV_ENC_CODEC_HEVC_GUID; + case av1: + throw std::runtime_error("AV1 not implemented on nvenc"); } throw std::out_of_range("Invalid codec " + std::to_string(codec)); } @@ -170,6 +172,9 @@ VideoEncoderNvenc::VideoEncoderNvenc(wivrn_vk_bundle & vk, encoder_settings & se case video_codec::h265: printf("%d HEVC presets\n", count); break; + + case video_codec::av1: + break; } for (GUID & i: presets) @@ -216,6 +221,8 @@ VideoEncoderNvenc::VideoEncoderNvenc(wivrn_vk_bundle & vk, encoder_settings & se params.encodeCodecConfig.hevcConfig.idrPeriod = NVENC_INFINITE_GOPLENGTH; params.encodeCodecConfig.hevcConfig.hevcVUIParameters.videoFullRangeFlag = 1; break; + case video_codec::av1: + break; } settings.range = VK_SAMPLER_YCBCR_RANGE_ITU_FULL; settings.color_model = VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_709;