diff --git a/client/application.cpp b/client/application.cpp index 24082765..e04d4cf2 100644 --- a/client/application.cpp +++ b/client/application.cpp @@ -759,13 +759,14 @@ void application::initialize_vulkan() spdlog::info(" {} (version {})", extension_name, spec_version); vk_device_extensions.push_back(VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME); + vk_device_extensions.push_back(VK_KHR_SAMPLER_YCBCR_CONVERSION_EXTENSION_NAME); optional_device_extensions.emplace(VK_IMG_FILTER_CUBIC_EXTENSION_NAME); optional_device_extensions.emplace(VK_KHR_CREATE_RENDERPASS_2_EXTENSION_NAME); optional_device_extensions.emplace(VK_KHR_FRAGMENT_SHADING_RATE_EXTENSION_NAME); + optional_device_extensions.emplace(VK_KHR_TIMELINE_SEMAPHORE_EXTENSION_NAME); #ifdef __ANDROID__ vk_device_extensions.push_back(VK_ANDROID_EXTERNAL_MEMORY_ANDROID_HARDWARE_BUFFER_EXTENSION_NAME); - vk_device_extensions.push_back(VK_KHR_SAMPLER_YCBCR_CONVERSION_EXTENSION_NAME); vk_device_extensions.push_back(VK_KHR_EXTERNAL_MEMORY_EXTENSION_NAME); vk_device_extensions.push_back(VK_EXT_QUEUE_FAMILY_FOREIGN_EXTENSION_NAME); vk_device_extensions.push_back(VK_KHR_DEDICATED_ALLOCATION_EXTENSION_NAME); @@ -881,11 +882,12 @@ void application::initialize_vulkan() .pEnabledFeatures = &device_features, }, vk::PhysicalDeviceFragmentShadingRateFeaturesKHR{}, -#ifdef __ANDROID__ vk::PhysicalDeviceSamplerYcbcrConversionFeaturesKHR{ - .samplerYcbcrConversion = VK_TRUE, + .samplerYcbcrConversion = true, + }, + vk::PhysicalDeviceTimelineSemaphoreFeaturesKHR{ + .timelineSemaphore = true, }, -#endif }; if (utils::contains(vk_device_extensions, VK_KHR_CREATE_RENDERPASS_2_EXTENSION_NAME) and @@ -904,6 +906,17 @@ void application::initialize_vulkan() device_create_info.unlink(); } + if (utils::contains(vk_device_extensions, VK_KHR_TIMELINE_SEMAPHORE_EXTENSION_NAME)) + { + auto [_, feat] = vk_physical_device.getFeatures2(); + auto & create_feat = device_create_info.get(); + create_feat.timelineSemaphore = feat.timelineSemaphore; + } + else + { + device_create_info.unlink(); + } + vk_device = xr_system_id.create_device(vk_physical_device, device_create_info.get()); *vk_queue.lock() = vk_device.getQueue(vk_queue_family_index, 0); diff --git a/client/decoder/android/android_decoder.cpp b/client/decoder/android/android_decoder.cpp index bde67eff..f276c348 100644 --- a/client/decoder/android/android_decoder.cpp +++ b/client/decoder/android/android_decoder.cpp @@ -59,6 +59,8 @@ const char * mime(wivrn::video_codec codec) return "video/hevc"; case c::av1: return "video/av01"; + case c::raw: + break; } assert(false); __builtin_unreachable(); @@ -562,7 +564,7 @@ static bool hardware_accelerated(AMediaCodec * media_codec) void decoder::supported_codecs(std::vector & result) { // Make sure we update this code when codecs are changed - static_assert(magic_enum::enum_count() == 3); + static_assert(magic_enum::enum_count() == 4); // In order or preference, from preferred to least preferred for (auto codec: { diff --git a/client/decoder/decoder.cpp b/client/decoder/decoder.cpp index 3fcd7e54..d92f657f 100644 --- a/client/decoder/decoder.cpp +++ b/client/decoder/decoder.cpp @@ -23,36 +23,54 @@ #else #include "decoder/ffmpeg/ffmpeg_decoder.h" #endif +#include "decoder/raw_decoder.h" wivrn::decoder::~decoder() = default; std::shared_ptr wivrn::decoder::make( vk::raii::Device & device, vk::raii::PhysicalDevice & phys_dev, + uint32_t vk_queue_family_index, const wivrn::to_headset::video_stream_description::item & description, float fps, uint8_t stream_index, std::weak_ptr scene, shard_accumulator * acc) { + switch (description.codec) + { + case h264: + case h265: + case av1: #ifdef __ANDROID__ - return std::make_shared( - device, - phys_dev, - description, - fps, - stream_index, - scene, - acc); + return std::make_shared( + device, + phys_dev, + description, + fps, + stream_index, + scene, + acc); #else - return std::make_shared( - device, - phys_dev, - description, - stream_index, - scene, - acc); + return std::make_shared( + device, + phys_dev, + description, + stream_index, + scene, + acc); #endif + case raw: + return std::make_shared( + device, + phys_dev, + vk_queue_family_index, + description, + stream_index, + scene, + acc); + } + __builtin_unreachable(); } std::vector wivrn::decoder::supported_codecs() @@ -63,5 +81,6 @@ std::vector wivrn::decoder::supported_codecs() #else ffmpeg::decoder::supported_codecs(res); #endif + res.push_back(video_codec::raw); return res; } diff --git a/client/decoder/decoder.h b/client/decoder/decoder.h index 44497327..da927082 100644 --- a/client/decoder/decoder.h +++ b/client/decoder/decoder.h @@ -43,6 +43,8 @@ public: vk::ImageView image_view = nullptr; vk::Image image = nullptr; vk::ImageLayout & current_layout; + vk::Semaphore semaphore = nullptr; + uint64_t * semaphore_val = nullptr; }; const wivrn::to_headset::video_stream_description::item description; @@ -56,6 +58,7 @@ public: static std::shared_ptr make( vk::raii::Device &, vk::raii::PhysicalDevice &, + uint32_t vk_queue_family_index, const wivrn::to_headset::video_stream_description::item & description, float fps, uint8_t stream_index, diff --git a/client/decoder/ffmpeg/ffmpeg_decoder.cpp b/client/decoder/ffmpeg/ffmpeg_decoder.cpp index 7f960b37..6b62d532 100644 --- a/client/decoder/ffmpeg/ffmpeg_decoder.cpp +++ b/client/decoder/ffmpeg/ffmpeg_decoder.cpp @@ -56,6 +56,8 @@ static AVCodecID codec_id(wivrn::video_codec codec) return AV_CODEC_ID_HEVC; case c::av1: return AV_CODEC_ID_AV1; + case c::raw: + break; } assert(false); __builtin_unreachable(); diff --git a/client/decoder/raw_decoder.cpp b/client/decoder/raw_decoder.cpp new file mode 100644 index 00000000..74c12596 --- /dev/null +++ b/client/decoder/raw_decoder.cpp @@ -0,0 +1,353 @@ +/* + * WiVRn VR streaming + * Copyright (C) 2025 Patrick Nicolas + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "raw_decoder.h" + +#include "application.h" +#include "scenes/stream.h" + +namespace +{ +struct raw_blit_handle : public wivrn::decoder::blit_handle +{ + std::atomic_bool & free; + + raw_blit_handle( + const wivrn::from_headset::feedback & feedback, + const wivrn::to_headset::video_stream_data_shard::view_info_t & view_info, + vk::ImageView image_view, + vk::Image image, + vk::ImageLayout & current_layout, + vk::Semaphore semaphore, + uint64_t & semaphore_val, + std::atomic_bool & free) : + wivrn::decoder::blit_handle{feedback, view_info, image_view, image, current_layout, semaphore, &semaphore_val}, + free(free) {} + ~raw_blit_handle() + { + free = true; + }; +}; + +vk::raii::Sampler make_sampler( + vk::raii::Device & device, + vk::raii::SamplerYcbcrConversion & ycbcr_conversion, + to_headset::video_stream_description::channels_t channels) +{ + vk::StructureChain info{ + vk::SamplerCreateInfo{ + .magFilter = vk::Filter::eNearest, + .minFilter = vk::Filter::eNearest, + .mipmapMode = vk::SamplerMipmapMode::eNearest, + .addressModeU = vk::SamplerAddressMode::eClampToEdge, + .addressModeV = vk::SamplerAddressMode::eClampToEdge, + .addressModeW = vk::SamplerAddressMode::eClampToEdge, + .maxAnisotropy = 1, + }, + vk::SamplerYcbcrConversionInfo{ + .conversion = *ycbcr_conversion, + }, + }; + switch (channels) + { + case wivrn::to_headset::video_stream_description::channels_t::colour: + break; + case wivrn::to_headset::video_stream_description::channels_t::alpha: + info.unlink(); + break; + } + return vk::raii::Sampler(device, info.get()); +} +} // namespace + +namespace wivrn +{ +raw_decoder::raw_decoder( + vk::raii::Device & device, + vk::raii::PhysicalDevice & physical_device, + uint32_t vk_queue_family_index, + const wivrn::to_headset::video_stream_description::item & description, + uint8_t stream_index, + std::weak_ptr scene, + shard_accumulator * accumulator) : + decoder(description), + device(device), + ycbcr_conversion(device, + { + .format = vk::Format::eG8B8R82Plane420Unorm, + .ycbcrModel = vk::SamplerYcbcrModelConversion(description.color_model.value_or(VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_709)), + .ycbcrRange = vk::SamplerYcbcrRange(description.range.value_or(VK_SAMPLER_YCBCR_RANGE_ITU_FULL)), + .chromaFilter = vk::Filter::eNearest, + }), + sampler_(make_sampler(device, ycbcr_conversion, description.channels)), + command_pool(device, vk::CommandPoolCreateInfo{ + .flags = vk::CommandPoolCreateFlagBits::eResetCommandBuffer, + .queueFamilyIndex = vk_queue_family_index, + }), + cmd(device.allocateCommandBuffers(vk::CommandBufferAllocateInfo{ + .commandPool = *command_pool, + .commandBufferCount = 1, + })[0] + .release()), + fence(device, vk::FenceCreateInfo{.flags = vk::FenceCreateFlagBits::eSignaled}), + weak_scene(scene), + accumulator(accumulator) +{ + extent_ = vk::Extent2D{ + .width = description.width, + .height = description.height, + }; + vk::DeviceSize buffer_size = description.width * description.height; + vk::Format format{}; + switch (description.channels) + { + case to_headset::video_stream_description::channels_t::colour: + buffer_size += (description.width * description.height) / 2; + format = vk::Format::eG8B8R82Plane420Unorm; + break; + case to_headset::video_stream_description::channels_t::alpha: + format = vk::Format::eR8Unorm; + break; + } + for (auto & i: input) + { + i = buffer_allocation( + device, + { + .size = buffer_size, + .usage = vk::BufferUsageFlagBits::eTransferSrc, + }, + { + .flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT, + .usage = VMA_MEMORY_USAGE_AUTO, + }, + "raw stream buffer"); + } + input_pos = (uint8_t *)input[0].map(); + + for (auto & item: image_pool) + { + item.image = image_allocation( + device, + vk::ImageCreateInfo{ + .imageType = vk::ImageType::e2D, + .format = format, + .extent = { + .width = description.width, + .height = description.height, + .depth = 1, + }, + .mipLevels = 1, + .arrayLayers = 1, + .tiling = vk::ImageTiling::eOptimal, + .usage = vk::ImageUsageFlagBits::eSampled | vk::ImageUsageFlagBits::eTransferDst, + }, + { + .usage = VMA_MEMORY_USAGE_AUTO, + }, + "raw image"); + + vk::SamplerYcbcrConversionInfo conv{ + .conversion = *ycbcr_conversion, + }; + vk::ImageViewCreateInfo view_info{ + .pNext = format == vk::Format::eG8B8R82Plane420Unorm ? &conv : nullptr, + .image = item.image, + .viewType = vk::ImageViewType::e2D, + .format = format, + .subresourceRange = { + .aspectMask = vk::ImageAspectFlagBits::eColor, + .levelCount = 1, + .layerCount = 1, + }, + }; + item.view_full = vk::raii::ImageView(device, view_info); + + item.semaphore = vk::raii::Semaphore( + device, + vk::StructureChain{ + vk::SemaphoreCreateInfo{}, + vk::SemaphoreTypeCreateInfo{ + .semaphoreType = vk::SemaphoreType::eTimeline, + }, + } + .get()); + } +} + +void raw_decoder::push_data(std::span> data, uint64_t frame_index, bool partial) +{ + if (frame_index != current_frame) + { + input_pos = (uint8_t *)input[0].map(); + current_frame = frame_index; + } + for (const auto & item: data) + { + memcpy(input_pos, item.data(), item.size_bytes()); + input_pos += item.size(); + } +} + +void raw_decoder::frame_completed( + const from_headset::feedback & feedback, + const to_headset::video_stream_data_shard::view_info_t & view_info) +{ + auto item = get_free(); + if (not item) + { + spdlog::warn("No image available in pool, discard frame"); + return; + } + + auto handle = std::make_shared( + feedback, + view_info, + *item->view_full, + item->image, + item->current_layout, + *item->semaphore, + item->semaphore_val, + item->free); + + auto res = device.waitForFences(*fence, true, UINT64_MAX); + if (res != vk::Result::eSuccess) + spdlog::warn("waitForFences failed"); + + cmd.reset(); + cmd.begin({.flags = vk::CommandBufferUsageFlagBits::eOneTimeSubmit}); + + if (item->current_layout != vk::ImageLayout::eTransferDstOptimal) + { + item->current_layout = vk::ImageLayout::eTransferDstOptimal; + vk::ImageMemoryBarrier barrier{ + .srcAccessMask = vk::AccessFlagBits::eNone, + .dstAccessMask = vk::AccessFlagBits::eMemoryWrite, + .oldLayout = vk::ImageLayout::eUndefined, + .newLayout = item->current_layout, + .image = item->image, + .subresourceRange = { + .aspectMask = vk::ImageAspectFlagBits::eColor, + .levelCount = 1, + .layerCount = 1, + }, + }; + cmd.pipelineBarrier( + vk::PipelineStageFlagBits::eAllCommands, + vk::PipelineStageFlagBits::eTransfer, + {}, + {}, + {}, + barrier); + } + + std::array regions{ + vk::BufferImageCopy{ + .imageSubresource = { + .aspectMask = vk::ImageAspectFlagBits::ePlane0, + .layerCount = 1, + }, + .imageExtent = { + .width = description.width, + .height = description.height, + .depth = 1, + }, + }, + vk::BufferImageCopy{ + .bufferOffset = vk::DeviceSize(description.width * description.height), + .imageSubresource = { + .aspectMask = vk::ImageAspectFlagBits::ePlane1, + .layerCount = 1, + }, + .imageExtent = { + .width = description.width / 2u, + .height = description.height / 2u, + .depth = 1, + }, + }, + }; + + cmd.copyBufferToImage( + input[0], + item->image, + item->current_layout, + description.channels == to_headset::video_stream_description::channels_t::colour ? 2 : 1, + regions.data()); + + vk::ImageMemoryBarrier barrier{ + .srcAccessMask = vk::AccessFlagBits::eMemoryWrite, + .dstAccessMask = vk::AccessFlagBits::eMemoryRead, + .oldLayout = item->current_layout, + .newLayout = vk::ImageLayout::eShaderReadOnlyOptimal, + .image = item->image, + .subresourceRange = { + .aspectMask = vk::ImageAspectFlagBits::eColor, + .levelCount = 1, + .layerCount = 1, + }, + }; + item->current_layout = barrier.newLayout; + cmd.pipelineBarrier( + vk::PipelineStageFlagBits::eTransfer, + vk::PipelineStageFlagBits::eFragmentShader, + {}, + {}, + {}, + barrier); + + cmd.end(); + + device.resetFences(*fence); + application::get_queue().lock()->submit( + vk::StructureChain{ + vk::SubmitInfo{ + .commandBufferCount = 1, + .pCommandBuffers = &cmd, + .signalSemaphoreCount = 1, + .pSignalSemaphores = &*item->semaphore, + }, + vk::TimelineSemaphoreSubmitInfo{ + .signalSemaphoreValueCount = 1, + .pSignalSemaphoreValues = &++item->semaphore_val, + }, + } + .get(), + *fence); + + if (auto scene = weak_scene.lock()) + scene->push_blit_handle(accumulator, std::move(handle)); + + std::swap(input[0], input[1]); + input_pos = (uint8_t *)input[0].map(); +} + +raw_decoder::image * raw_decoder::get_free() +{ + for (auto & item: image_pool) + { + if (item.free.exchange(false)) + return &item; + } + return nullptr; +} + +std::vector raw_decoder::supported_codecs() +{ + return {video_codec::raw}; +} +} // namespace wivrn diff --git a/client/decoder/raw_decoder.h b/client/decoder/raw_decoder.h new file mode 100644 index 00000000..01eb9269 --- /dev/null +++ b/client/decoder/raw_decoder.h @@ -0,0 +1,96 @@ +/* + * WiVRn VR streaming + * Copyright (C) 2025 Patrick Nicolas + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once + +#include "decoder.h" + +#include "vk/allocation.h" + +#include + +namespace wivrn +{ +class raw_decoder : public decoder +{ +private: + static const int image_count = 5; + struct image + { + image_allocation image; + vk::raii::ImageView view_full = nullptr; + vk::ImageLayout current_layout = vk::ImageLayout::eUndefined; + std::atomic_bool free = true; + vk::raii::Semaphore semaphore = nullptr; + uint64_t semaphore_val = 0; + }; + + vk::raii::Device & device; + vk::raii::SamplerYcbcrConversion ycbcr_conversion; + vk::raii::Sampler sampler_; + + vk::raii::CommandPool command_pool; + vk::CommandBuffer cmd; + vk::raii::Fence fence; + + std::array image_pool; + + std::weak_ptr weak_scene; + shard_accumulator * accumulator; + + uint64_t current_frame = 0; + std::array input; + uint8_t * input_pos; + from_headset::feedback feedback; + to_headset::video_stream_data_shard::view_info_t view_info; + +public: + raw_decoder(vk::raii::Device & device, + vk::raii::PhysicalDevice & physical_device, + uint32_t vk_queue_family_index, + const wivrn::to_headset::video_stream_description::item & description, + uint8_t stream_index, + std::weak_ptr scene, + shard_accumulator * accumulator); + + void push_data(std::span> data, uint64_t frame_index, bool partial) override; + + void frame_completed( + const wivrn::from_headset::feedback & feedback, + const wivrn::to_headset::video_stream_data_shard::view_info_t & view_info) override; + + vk::Sampler sampler() override + { + return *sampler_; + } + + vk::Extent2D image_size() + { + return { + description.width, + description.height, + }; + } + + static std::vector supported_codecs(); + +private: + image * get_free(); +}; + +} // namespace wivrn diff --git a/client/decoder/shard_accumulator.h b/client/decoder/shard_accumulator.h index a0145e22..6f099534 100644 --- a/client/decoder/shard_accumulator.h +++ b/client/decoder/shard_accumulator.h @@ -80,11 +80,12 @@ public: vk::raii::Device & device, vk::raii::PhysicalDevice & physical_device, xr::instance & instance, + uint32_t vk_queue_family_index, const wivrn::to_headset::video_stream_description::item & description, float fps, std::weak_ptr scene, uint8_t stream_index) : - decoder_(decoder::make(device, physical_device, description, fps, stream_index, scene, this)), + decoder_(decoder::make(device, physical_device, vk_queue_family_index, description, fps, stream_index, scene, this)), current(stream_index), next(stream_index), weak_scene(scene), diff --git a/client/scenes/stream.cpp b/client/scenes/stream.cpp index e79c1305..a4ac22fd 100644 --- a/client/scenes/stream.cpp +++ b/client/scenes/stream.cpp @@ -927,6 +927,28 @@ void scenes::stream::render(const XrFrameState & frame_state) command_buffer.end(); vk::SubmitInfo submit_info; submit_info.setCommandBuffers(*command_buffer); + + std::vector semaphores; + std::vector semaphore_vals; + std::vector wait_stages; + for (auto b: current_blit_handles) + { + if (b and b->semaphore) + { + assert(b->semaphore_val); + semaphores.push_back(b->semaphore); + semaphore_vals.push_back(*b->semaphore_val); + wait_stages.push_back(vk::PipelineStageFlagBits::eFragmentShader); + } + } + submit_info.setWaitDstStageMask(wait_stages); + submit_info.setWaitSemaphores(semaphores); + vk::TimelineSemaphoreSubmitInfo sem_info{ + .waitSemaphoreValueCount = uint32_t(semaphore_vals.size()), + .pWaitSemaphoreValues = semaphore_vals.data(), + }; + submit_info.pNext = &sem_info; + queue.lock()->submit(submit_info, *fence); #if WIVRN_FEATURE_RENDERDOC renderdoc_end(*vk_instance); @@ -1105,7 +1127,7 @@ void scenes::stream::setup(const to_headset::video_stream_description & descript spdlog::info("Creating decoder size {}x{} offset {},{}", item.width, item.height, item.offset_x, item.offset_y); decoders.push_back(accumulator_images{ - .decoder = std::make_unique(device, physical_device, instance, item, description.fps, shared_from_this(), stream_index), + .decoder = std::make_unique(device, physical_device, instance, queue_family_index, item, description.fps, shared_from_this(), stream_index), }); } } diff --git a/common/wivrn_packets.h b/common/wivrn_packets.h index ce852ddd..ed7f75e6 100644 --- a/common/wivrn_packets.h +++ b/common/wivrn_packets.h @@ -41,7 +41,7 @@ namespace wivrn // Default port for server to listen, both TCP and UDP static const int default_port = 9757; -static constexpr int protocol_revision = 1; +static constexpr int protocol_revision = 2; enum class device_id : uint8_t { @@ -168,6 +168,7 @@ enum video_codec h265, hevc = h265, av1, + raw, }; struct audio_data diff --git a/docs/configuration.md b/docs/configuration.md index 5c2785b3..e74a45be 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -61,8 +61,8 @@ Identifier of the encoder, one of ### `codec` Default value: first supported by both headset and encoder of `av1`, `h264`, `h265`. -One of `h264`, `h265` or `av1`. -Not all encoders support every codec, `x264` and `vulkan` only support `h264`. +One of `h264`, `h265`, `av1`, `raw`. +Not all encoders support every codec, `x264` and `vulkan` only support `h264`. For `raw` codec, only `raw` encoder can be used. ### `width`, `height`, `offset_x`, `offset_y` (advanced) Default values: full image (`width` = 1, `height` = 1, `offset_x` = 0, `offset_y` = 0) diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index 3794c3c6..bb7a6906 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -86,6 +86,7 @@ if (WIVRN_BUILD_SERVER) encoder/encoder_settings.cpp encoder/video_encoder.cpp + encoder/video_encoder_raw.cpp driver/app_pacer.cpp driver/clock_offset.cpp diff --git a/server/driver/configuration.cpp b/server/driver/configuration.cpp index 678ff349..19c0494d 100644 --- a/server/driver/configuration.cpp +++ b/server/driver/configuration.cpp @@ -64,6 +64,7 @@ NLOHMANN_JSON_SERIALIZE_ENUM( {h265, "hevc"}, {av1, "av1"}, {av1, "AV1"}, + {raw, "raw"}, }) NLOHMANN_JSON_SERIALIZE_ENUM( diff --git a/server/encoder/encoder_settings.cpp b/server/encoder/encoder_settings.cpp index 16464648..da6c942a 100644 --- a/server/encoder/encoder_settings.cpp +++ b/server/encoder/encoder_settings.cpp @@ -70,6 +70,7 @@ static void split_bitrate(std::vector & encoders, uint6 break; case wivrn::h265: case wivrn::av1: + case wivrn::raw: break; } encoder.bitrate = w; @@ -296,6 +297,9 @@ static void fill_defaults(wivrn_vk_bundle & bundle, const std::vectorprofile = FF_PROFILE_AV1_MAIN; break; + case video_codec::raw: + throw std::runtime_error("raw codec not supported"); } for (auto option: settings.options) { diff --git a/server/encoder/video_encoder.cpp b/server/encoder/video_encoder.cpp index a45d084e..864cb197 100644 --- a/server/encoder/video_encoder.cpp +++ b/server/encoder/video_encoder.cpp @@ -42,6 +42,7 @@ #include "video_encoder_vulkan_h264.h" // #include "video_encoder_vulkan_h265.h" #endif +#include "video_encoder_raw.h" namespace wivrn { @@ -126,6 +127,8 @@ std::unique_ptr video_encoder::create( // break; case video_codec::av1: throw std::runtime_error("av1 not supported for vulkan video encode"); + case video_codec::raw: + throw std::runtime_error("raw codec only supported on raw encoder"); } #else throw std::runtime_error("Vulkan video encode not enabled"); @@ -155,6 +158,12 @@ std::unique_ptr video_encoder::create( throw std::runtime_error("vaapi support not enabled"); #endif } + + if (settings.encoder_name == encoder_raw) + { + res = std::make_unique(wivrn_vk, settings, fps, stream_idx); + } + if (not res) throw std::runtime_error("Failed to create encoder " + settings.encoder_name); @@ -174,6 +183,9 @@ std::unique_ptr video_encoder::create( case av1: file += ".av1"; break; + case raw: + file += ".yuv"; + break; } res->video_dump.open(file); } diff --git a/server/encoder/video_encoder.h b/server/encoder/video_encoder.h index 639f56b6..dadbe075 100644 --- a/server/encoder/video_encoder.h +++ b/server/encoder/video_encoder.h @@ -43,6 +43,7 @@ inline const char * encoder_nvenc = "nvenc"; inline const char * encoder_vaapi = "vaapi"; inline const char * encoder_x264 = "x264"; inline const char * encoder_vulkan = "vulkan"; +inline const char * encoder_raw = "raw"; class video_encoder { diff --git a/server/encoder/video_encoder_nvenc.cpp b/server/encoder/video_encoder_nvenc.cpp index 3b1e6e4d..c376a613 100644 --- a/server/encoder/video_encoder_nvenc.cpp +++ b/server/encoder/video_encoder_nvenc.cpp @@ -74,6 +74,8 @@ static auto encode_guid(video_codec codec) return NV_ENC_CODEC_HEVC_GUID; case av1: return NV_ENC_CODEC_AV1_GUID; + case raw: + break; } throw std::out_of_range("Invalid codec " + std::to_string(codec)); } @@ -164,6 +166,8 @@ video_encoder_nvenc::video_encoder_nvenc( params.encodeCodecConfig.av1Config.maxNumRefFramesInDPB = 0; params.encodeCodecConfig.av1Config.idrPeriod = NVENC_INFINITE_GOPLENGTH; break; + case video_codec::raw: + throw std::runtime_error("raw codec not supported for nvenc"); } NV_ENC_INITIALIZE_PARAMS params2{ diff --git a/server/encoder/video_encoder_raw.cpp b/server/encoder/video_encoder_raw.cpp new file mode 100644 index 00000000..583eae5c --- /dev/null +++ b/server/encoder/video_encoder_raw.cpp @@ -0,0 +1,121 @@ +/* + * WiVRn VR streaming + * Copyright (C) 2025 Patrick Nicolas + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "video_encoder_raw.h" + +#include "encoder/encoder_settings.h" +#include "utils/wivrn_vk_bundle.h" + +wivrn::video_encoder_raw::video_encoder_raw( + wivrn_vk_bundle & vk, + encoder_settings & settings, + float fps, + uint8_t stream_idx) : + video_encoder(stream_idx, settings.channels, settings.bitrate_multiplier, true) +{ + if (settings.bit_depth != 8) + throw std::runtime_error("Raw encoding is only supported for 8 bit"); + rect = vk::Rect2D{ + .offset = { + .x = settings.offset_x, + .y = settings.offset_y, + }, + .extent = { + .width = settings.width, + .height = settings.height, + }, + }; + + vk::DeviceSize buffer_size = settings.width * settings.height; + switch (settings.channels) + { + case to_headset::video_stream_description::channels_t::colour: + buffer_size += buffer_size / 2; + break; + case to_headset::video_stream_description::channels_t::alpha: + break; + } + for (auto & slot: buffers) + { + slot = buffer_allocation( + vk.device, + { + .size = buffer_size, + .usage = vk::BufferUsageFlagBits::eTransferDst, + }, + { + .flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT, + .usage = VMA_MEMORY_USAGE_AUTO, + }, + "raw stream buffer"); + } +} + +std::pair wivrn::video_encoder_raw::present_image(vk::Image y_cbcr, vk::raii::CommandBuffer & cmd_buf, uint8_t slot, uint64_t) +{ + std::array regions{ + vk::BufferImageCopy{ + .imageSubresource = { + .aspectMask = vk::ImageAspectFlagBits::ePlane0, + .baseArrayLayer = uint32_t(channels), + .layerCount = 1, + }, + .imageOffset = { + .x = rect.offset.x, + .y = rect.offset.y, + }, + .imageExtent = { + .width = rect.extent.width, + .height = rect.extent.height, + .depth = 1, + }, + }, + vk::BufferImageCopy{ + .bufferOffset = rect.extent.width * rect.extent.height, + .imageSubresource = { + .aspectMask = vk::ImageAspectFlagBits::ePlane1, + .baseArrayLayer = uint32_t(channels), + .layerCount = 1, + }, + .imageOffset = { + .x = rect.offset.x / 2, + .y = rect.offset.y / 2, + }, + .imageExtent = { + .width = rect.extent.width / 2, + .height = rect.extent.height / 2, + .depth = 1, + }, + }, + }; + cmd_buf.copyImageToBuffer( + y_cbcr, + vk::ImageLayout::eTransferSrcOptimal, + buffers[slot], + std::span(regions.data(), + channels == to_headset::video_stream_description::channels_t::colour ? 2 : 1)); + return {false, nullptr}; +} + +std::optional wivrn::video_encoder_raw::encode(bool idr, std::chrono::steady_clock::time_point pts, uint8_t slot) +{ + return wivrn::video_encoder::data{ + .encoder = this, + .span = std::span((uint8_t *)buffers[slot].map(), buffers[slot].info().size), + }; +} diff --git a/server/encoder/video_encoder_raw.h b/server/encoder/video_encoder_raw.h new file mode 100644 index 00000000..8779214a --- /dev/null +++ b/server/encoder/video_encoder_raw.h @@ -0,0 +1,41 @@ +/* + * WiVRn VR streaming + * Copyright (C) 2025 Patrick Nicolas + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once + +#include "video_encoder.h" +#include "vk/allocation.h" + +#include + +namespace wivrn +{ + +class video_encoder_raw : public video_encoder +{ + std::array buffers; + vk::Rect2D rect; + +public: + video_encoder_raw(wivrn_vk_bundle & vk, encoder_settings & settings, float fps, uint8_t stream_idx); + + std::pair present_image(vk::Image y_cbcr, vk::raii::CommandBuffer & cmd_buf, uint8_t slot, uint64_t frame_index) override; + + std::optional encode(bool idr, std::chrono::steady_clock::time_point pts, uint8_t slot) override; +}; +} // namespace wivrn