diff --git a/client/decoder/android/android_decoder.cpp b/client/decoder/android/android_decoder.cpp index 2a091ad4..0d5f4217 100644 --- a/client/decoder/android/android_decoder.cpp +++ b/client/decoder/android/android_decoder.cpp @@ -61,6 +61,7 @@ const char * mime(wivrn::video_codec codec) return "video/av01"; } assert(false); + __builtin_unreachable(); } void check(media_status_t status, const char * msg) @@ -72,6 +73,26 @@ void check(media_status_t status, const char * msg) } } +struct android_blit_handle : public decoder::blit_handle +{ + std::shared_ptr vk_data; + std::shared_ptr image_reader; + AImage_ptr aimage; + android_blit_handle( + const wivrn::from_headset::feedback & feedback, + const wivrn::to_headset::video_stream_data_shard::view_info_t & view_info, + vk::ImageView image_view, + vk::Image image, + vk::ImageLayout & current_layout, + decltype(vk_data) vk_data, + decltype(image_reader) image_reader, + decltype(aimage) aimage) : + decoder::blit_handle(feedback, view_info, image_view, image, current_layout), + vk_data(vk_data), + image_reader(image_reader), + aimage(std::move(aimage)) {} +}; + } // namespace namespace wivrn::android @@ -85,7 +106,7 @@ decoder::decoder( uint8_t stream_index, std::weak_ptr weak_scene, shard_accumulator * accumulator) : - description(description), stream_index(stream_index), fps(fps), device(device), weak_scene(weak_scene), accumulator(accumulator) + wivrn::decoder(description), stream_index(stream_index), fps(fps), device(device), weak_scene(weak_scene), accumulator(accumulator) { spdlog::info("hbm_mutex.native_handle() = {}", (void *)hbm_mutex.native_handle()); @@ -222,7 +243,7 @@ void decoder::push_data(std::span> data, uint64_t frame current_input_buffer = input_buffer{}; } -void decoder::frame_completed(wivrn::from_headset::feedback & feedback, const wivrn::to_headset::video_stream_data_shard::view_info_t & view_info) +void decoder::frame_completed(const wivrn::from_headset::feedback & feedback, const wivrn::to_headset::video_stream_data_shard::view_info_t & view_info) { if (not media_codec) { @@ -255,7 +276,7 @@ void decoder::on_image_available(AImageReader * reader) assert(reader == image_reader.get()); // Executed on image reader thread - decltype(blit_handle::aimage) image; + decltype(android_blit_handle::aimage) image; try { AImage * tmp; @@ -280,13 +301,13 @@ void decoder::on_image_available(AImageReader * reader) auto vk_data = map_hardware_buffer(image.get()); - auto handle = std::make_shared( + auto handle = std::make_shared( info->feedback, info->view_info, - vk_data->image_view, + *vk_data->image_view, *vk_data->vimage, - &vk_data->layout, - vk_data, + vk_data->layout, + std::move(vk_data), image_reader, std::move(image)); @@ -514,11 +535,6 @@ void decoder::on_media_output_available(AMediaCodec * media_codec, void * userda }); } -void decoder::blit_handle::deleter::operator()(AImage * aimage) -{ - AImage_delete(aimage); -} - static bool hardware_accelerated(AMediaCodec * media_codec) { // MediaCodecInfo has isHardwareAccelerated, but this does not exist in NDK. @@ -542,9 +558,8 @@ static bool hardware_accelerated(AMediaCodec * media_codec) return true; } -std::vector decoder::supported_codecs() +void decoder::supported_codecs(std::vector & result) { - std::vector result; // Make sure we update this code when codecs are changed static_assert(magic_enum::enum_count() == 3); @@ -563,8 +578,6 @@ std::vector decoder::supported_codecs() spdlog::info("video codec {}: {}supported", magic_enum::enum_name(codec), supported ? "" : "NOT "); } - - return result; } } // namespace wivrn::android diff --git a/client/decoder/android/android_decoder.h b/client/decoder/android/android_decoder.h index 63926935..27d26c1f 100644 --- a/client/decoder/android/android_decoder.h +++ b/client/decoder/android/android_decoder.h @@ -19,6 +19,7 @@ #pragma once +#include "decoder/decoder.h" #include "utils/sync_queue.h" #include "wivrn_packets.h" #include @@ -59,31 +60,12 @@ class stream; namespace wivrn::android { -class decoder +class decoder : public wivrn::decoder { public: struct mapped_hardware_buffer; - struct blit_handle - { - struct deleter - { - void operator()(AImage *); - }; - wivrn::from_headset::feedback feedback; - wivrn::to_headset::video_stream_data_shard::view_info_t view_info; - vk::raii::ImageView & image_view; - vk::Image image = nullptr; - vk::ImageLayout * current_layout = nullptr; - - std::shared_ptr vk_data; - - std::shared_ptr image_reader; - AImage_ptr aimage; - }; - private: - wivrn::to_headset::video_stream_description::item description; uint8_t stream_index; float fps; @@ -146,33 +128,20 @@ public: uint8_t stream_index, std::weak_ptr scene, shard_accumulator * accumulator); - - decoder(const decoder &) = delete; - decoder(decoder &&) = delete; ~decoder(); - void push_data(std::span> data, uint64_t frame_index, bool partial); + void push_data(std::span> data, uint64_t frame_index, bool partial) override; void frame_completed( - wivrn::from_headset::feedback & feedback, - const wivrn::to_headset::video_stream_data_shard::view_info_t & view_info); + const wivrn::from_headset::feedback & feedback, + const wivrn::to_headset::video_stream_data_shard::view_info_t & view_info) override; - const auto & desc() const - { - return description; - } - - vk::Sampler sampler() + vk::Sampler sampler() override { return *ycbcr_sampler; } - vk::Extent2D image_size() - { - return extent; - } - - static std::vector supported_codecs(); + static void supported_codecs(std::vector &); }; } // namespace wivrn::android diff --git a/client/decoder/decoder.cpp b/client/decoder/decoder.cpp new file mode 100644 index 00000000..3fcd7e54 --- /dev/null +++ b/client/decoder/decoder.cpp @@ -0,0 +1,67 @@ +/* + * WiVRn VR streaming + * Copyright (C) 2025 Patrick Nicolas + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "decoder.h" + +#ifdef __ANDROID__ +#include "decoder/android/android_decoder.h" +#else +#include "decoder/ffmpeg/ffmpeg_decoder.h" +#endif + +wivrn::decoder::~decoder() = default; + +std::shared_ptr wivrn::decoder::make( + vk::raii::Device & device, + vk::raii::PhysicalDevice & phys_dev, + const wivrn::to_headset::video_stream_description::item & description, + float fps, + uint8_t stream_index, + std::weak_ptr scene, + shard_accumulator * acc) +{ +#ifdef __ANDROID__ + return std::make_shared( + device, + phys_dev, + description, + fps, + stream_index, + scene, + acc); +#else + return std::make_shared( + device, + phys_dev, + description, + stream_index, + scene, + acc); +#endif +} + +std::vector wivrn::decoder::supported_codecs() +{ + std::vector res; +#ifdef __ANDROID__ + android::decoder::supported_codecs(res); +#else + ffmpeg::decoder::supported_codecs(res); +#endif + return res; +} diff --git a/client/decoder/decoder.h b/client/decoder/decoder.h new file mode 100644 index 00000000..1886759a --- /dev/null +++ b/client/decoder/decoder.h @@ -0,0 +1,74 @@ +/* + * WiVRn VR streaming + * Copyright (C) 2025 Patrick Nicolas + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once + +#include "wivrn_packets.h" + +#include +#include +#include +#include + +namespace scenes +{ +class stream; +} + +namespace wivrn +{ +class shard_accumulator; +class decoder +{ +public: + struct blit_handle + { + wivrn::from_headset::feedback feedback; + wivrn::to_headset::video_stream_data_shard::view_info_t view_info; + vk::ImageView image_view = nullptr; + vk::Image image = nullptr; + vk::ImageLayout & current_layout; + }; + + const wivrn::to_headset::video_stream_description::item description; + +protected: + decoder(const wivrn::to_headset::video_stream_description::item & description) : + description(description) {} + +public: + static std::shared_ptr make( + vk::raii::Device &, + vk::raii::PhysicalDevice &, + const wivrn::to_headset::video_stream_description::item & description, + float fps, + uint8_t stream_index, + std::weak_ptr, + shard_accumulator *); + virtual ~decoder(); + virtual void push_data(std::span> data, uint64_t frame_index, bool partial) = 0; + + virtual void frame_completed( + const from_headset::feedback & feedback, + const to_headset::video_stream_data_shard::view_info_t & view_info) = 0; + + virtual vk::Sampler sampler() = 0; + + static std::vector supported_codecs(); +}; +} // namespace wivrn diff --git a/client/decoder/ffmpeg/ffmpeg_decoder.cpp b/client/decoder/ffmpeg/ffmpeg_decoder.cpp index 09416a28..afa61e4c 100644 --- a/client/decoder/ffmpeg/ffmpeg_decoder.cpp +++ b/client/decoder/ffmpeg/ffmpeg_decoder.cpp @@ -61,24 +61,53 @@ static AVCodecID codec_id(wivrn::video_codec codec) __builtin_unreachable(); } -decoder::blit_handle::~blit_handle() +struct decoder::ffmpeg_blit_handle : public wivrn::decoder::blit_handle { - std::unique_lock lock(self->mutex); - if (image_index < 0) - return; + int image_index; + wivrn::ffmpeg::decoder * self; - self->free_images.push_back(image_index); -} + ffmpeg_blit_handle( + const wivrn::from_headset::feedback & feedback, + const wivrn::to_headset::video_stream_data_shard::view_info_t & view_info, + vk::ImageView image_view, + vk::Image image, + vk::ImageLayout & current_layout, + int image_index, + wivrn::ffmpeg::decoder * self) : + wivrn::decoder::blit_handle{ + feedback, + view_info, + image_view, + image, + current_layout, + }, + image_index(image_index), + self(self) + {} + + ~ffmpeg_blit_handle() + { + std::unique_lock lock(self->mutex); + if (image_index < 0) + return; + + self->free_images.push_back(image_index); + } +}; decoder::decoder( vk::raii::Device & device, vk::raii::PhysicalDevice & physical_device, const wivrn::to_headset::video_stream_description::item & description, - float fps, uint8_t stream_index, std::weak_ptr scene, shard_accumulator * accumulator) : - device(device), description(description), codec(nullptr, free_codec_context), sws(nullptr, sws_freeContext), weak_scene(scene), accumulator(accumulator) + wivrn::decoder(description), + device(device), + codec(nullptr, free_codec_context), + sws(nullptr, sws_freeContext), + weak_scene(scene), + accumulator(accumulator) { free_images.resize(image_count); @@ -221,12 +250,12 @@ void decoder::frame_completed(const wivrn::from_headset::feedback & feedback, co if (res == 0) throw std::runtime_error{"sws_scale failed"}; - auto handle = std::make_shared( + auto handle = std::make_shared( feedback, view_info, - decoded_images[index].image_view, - (vk::Image)decoded_images[index].image, - &decoded_images[index].current_layout, + *decoded_images[index].image_view, + decoded_images[index].image, + decoded_images[index].current_layout, index, this); @@ -234,13 +263,11 @@ void decoder::frame_completed(const wivrn::from_headset::feedback & feedback, co scene->push_blit_handle(accumulator, std::move(handle)); } -std::vector decoder::supported_codecs() +void decoder::supported_codecs(std::vector & res) { - return { - wivrn::video_codec::h264, - wivrn::video_codec::h265, - wivrn::video_codec::av1, - }; + res.push_back(wivrn::video_codec::h264); + res.push_back(wivrn::video_codec::h265); + res.push_back(wivrn::video_codec::av1); } } // namespace wivrn::ffmpeg diff --git a/client/decoder/ffmpeg/ffmpeg_decoder.h b/client/decoder/ffmpeg/ffmpeg_decoder.h index 33190af8..360de157 100644 --- a/client/decoder/ffmpeg/ffmpeg_decoder.h +++ b/client/decoder/ffmpeg/ffmpeg_decoder.h @@ -19,6 +19,7 @@ #pragma once +#include "decoder/decoder.h" #include "vk/allocation.h" #include "wivrn_packets.h" #include @@ -46,24 +47,10 @@ class stream; namespace wivrn::ffmpeg { -class decoder +class decoder : public wivrn::decoder { -public: - struct blit_handle - { - wivrn::from_headset::feedback feedback; - wivrn::to_headset::video_stream_data_shard::view_info_t view_info; - vk::raii::ImageView & image_view; - vk::Image image = nullptr; - vk::ImageLayout * current_layout = nullptr; - - int image_index; - decoder * self; - - ~blit_handle(); - }; - private: + struct ffmpeg_blit_handle; static const int image_count = 12; struct image { @@ -81,8 +68,6 @@ private: vk::Extent2D extent{}; std::vector free_images; - wivrn::to_headset::video_stream_description::item description; - std::unique_ptr codec; std::unique_ptr sws; std::vector packet; @@ -96,26 +81,17 @@ public: decoder(vk::raii::Device & device, vk::raii::PhysicalDevice & physical_device, const wivrn::to_headset::video_stream_description::item & description, - float fps, uint8_t stream_index, std::weak_ptr scene, shard_accumulator * accumulator); - decoder(const decoder &) = delete; - decoder(decoder &&) = delete; - - void push_data(std::span> data, uint64_t frame_index, bool partial); + void push_data(std::span> data, uint64_t frame_index, bool partial) override; void frame_completed( const wivrn::from_headset::feedback & feedback, - const wivrn::to_headset::video_stream_data_shard::view_info_t & view_info); + const wivrn::to_headset::video_stream_data_shard::view_info_t & view_info) override; - const auto & desc() const - { - return description; - } - - vk::Sampler sampler() + vk::Sampler sampler() override { return *rgb_sampler; } @@ -125,6 +101,6 @@ public: return extent; } - static std::vector supported_codecs(); + static void supported_codecs(std::vector &); }; } // namespace wivrn::ffmpeg diff --git a/client/decoder/shard_accumulator.cpp b/client/decoder/shard_accumulator.cpp index 900c05ee..a7ec0636 100644 --- a/client/decoder/shard_accumulator.cpp +++ b/client/decoder/shard_accumulator.cpp @@ -186,7 +186,7 @@ void shard_accumulator::try_submit_frame(uint16_t shard_idx) payload.emplace_back(data_shards[idx]->payload); bool frame_complete = last_idx == data_shards.size() and data_shards.back()->flags & video_stream_data_shard::end_of_frame; - decoder->push_data(payload, data_shards[shard_idx]->frame_idx, not frame_complete); + decoder_->push_data(payload, data_shards[shard_idx]->frame_idx, not frame_complete); if (not frame_complete) return; @@ -206,7 +206,7 @@ void shard_accumulator::try_submit_frame(uint16_t shard_idx) } // Try to extract a frame - decoder->frame_completed(current.feedback, *data_shards.front()->view_info); + decoder_->frame_completed(current.feedback, *data_shards.front()->view_info); send_feedback(current.feedback); diff --git a/client/decoder/shard_accumulator.h b/client/decoder/shard_accumulator.h index 406c0c49..bcc7730b 100644 --- a/client/decoder/shard_accumulator.h +++ b/client/decoder/shard_accumulator.h @@ -19,32 +19,30 @@ #pragma once -#include -#include - -#ifdef __ANDROID__ -#include "decoder/android/android_decoder.h" -using decoder_impl = ::wivrn::android::decoder; -#else -#include "decoder/ffmpeg/ffmpeg_decoder.h" -using decoder_impl = ::wivrn::ffmpeg::decoder; -#endif - +#include "decoder.h" #include "wivrn_packets.h" + +#include #include #include +#include namespace xr { class instance; } +namespace scenes +{ +class stream; +} + namespace wivrn { class shard_accumulator { - std::shared_ptr decoder; + std::shared_ptr decoder_; public: using data_shard = wivrn::to_headset::video_stream_data_shard; @@ -86,7 +84,7 @@ public: float fps, std::weak_ptr scene, uint8_t stream_index) : - decoder(std::make_shared(device, physical_device, description, fps, stream_index, scene, this)), + decoder_(decoder::make(device, physical_device, description, fps, stream_index, scene, this)), current(stream_index), next(stream_index), weak_scene(scene), @@ -99,20 +97,15 @@ public: auto & desc() const { - return decoder->desc(); + return decoder_->description; } vk::Sampler sampler() { - return decoder->sampler(); + return decoder_->sampler(); } - vk::Extent2D image_size() - { - return decoder->image_size(); - } - - using blit_handle = decoder_impl::blit_handle; + using blit_handle = decoder::blit_handle; private: void try_submit_frame(std::optional shard_idx); diff --git a/client/scenes/stream.cpp b/client/scenes/stream.cpp index aba0b167..ecf72112 100644 --- a/client/scenes/stream.cpp +++ b/client/scenes/stream.cpp @@ -299,7 +299,7 @@ std::shared_ptr scenes::stream::create(std::unique_ptrview_info.foveation; use_alpha = blit_handle->view_info.alpha; - if (*blit_handle->current_layout == vk::ImageLayout::eUndefined) + if (blit_handle->current_layout == vk::ImageLayout::eUndefined) { vk::ImageMemoryBarrier barrier{ .srcAccessMask = vk::AccessFlagBits::eNone, @@ -864,7 +864,7 @@ void scenes::stream::render(const XrFrameState & frame_state) }; command_buffer.pipelineBarrier(vk::PipelineStageFlagBits::eAllCommands, vk::PipelineStageFlagBits::eAllCommands, {}, {}, {}, barrier); - *blit_handle->current_layout = vk::ImageLayout::eGeneral; + blit_handle->current_layout = vk::ImageLayout::eGeneral; } } @@ -877,7 +877,7 @@ void scenes::stream::render(const XrFrameState & frame_state) { if (not blit_handle) continue; - b.push_image(command_buffer, j, decoders[j].decoder->sampler(), *blit_handle->image_view, *blit_handle->current_layout); + b.push_image(command_buffer, j, decoders[j].decoder->sampler(), blit_handle->image_view, blit_handle->current_layout); } blitted[i] = b.end(command_buffer); }