/* * WiVRn VR streaming * Copyright (C) 2022 Guillaume Meunier * Copyright (C) 2022 Patrick Nicolas * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #include "ffmpeg_decoder.h" #include "scenes/stream.h" #include "spdlog/spdlog.h" #include "utils/check.h" #include extern "C" { #include #include #include #include #include } namespace ffmpeg { static void free_codec_context(AVCodecContext * ctx) { avcodec_free_context(&ctx); } static void free_frame(AVFrame * frame) { av_frame_free(&frame); } static AVCodecID codec_id(xrt::drivers::wivrn::video_codec codec) { using c = xrt::drivers::wivrn::video_codec; switch (codec) { case c::h264: return AV_CODEC_ID_H264; case c::h265: return AV_CODEC_ID_HEVC; } assert(false); __builtin_unreachable(); } decoder::blit_handle::~blit_handle() { std::lock_guard lock(self->mutex); if (image_index < 0) return; self->free_images.push_back(image_index); } decoder::decoder( VkDevice device, VkPhysicalDevice physical_device, const xrt::drivers::wivrn::to_headset::video_stream_description::item & description, float fps, std::weak_ptr scene, shard_accumulator * accumulator) : device(device), description(description), codec(nullptr, free_codec_context), sws(nullptr, sws_freeContext), weak_scene(scene), accumulator(accumulator) { free_images.resize(image_count); for (int i = 0; i < image_count; i++) { free_images[i] = i; VkImageCreateInfo image_info{}; image_info.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; image_info.imageType = VK_IMAGE_TYPE_2D; image_info.format = VK_FORMAT_A8B8G8R8_SRGB_PACK32; image_info.extent.width = description.width; image_info.extent.height = description.height; image_info.extent.depth = 1; image_info.mipLevels = 1; image_info.arrayLayers = 1; image_info.samples = VK_SAMPLE_COUNT_1_BIT; image_info.tiling = VK_IMAGE_TILING_LINEAR; image_info.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT; image_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; image_info.queueFamilyIndexCount = 0; image_info.pQueueFamilyIndices = NULL; image_info.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; decoded_images[i].image = vk::image(device, image_info); decoded_images[i].memory = vk::device_memory(device, physical_device, decoded_images[i].image, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); decoded_images[i].memory.map_memory(); VkImageSubresource resource{}; resource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; vkGetImageSubresourceLayout(device, decoded_images[i].image, &resource, &decoded_images[i].layout); } auto avcodec = avcodec_find_decoder(codec_id(description.codec)); if (avcodec == nullptr) { throw std::runtime_error{"avcodec_find_decoder failed"}; } codec.reset(avcodec_alloc_context3(avcodec)); int ret = avcodec_open2(codec.get(), avcodec, nullptr); if (ret < 0) throw std::runtime_error{"avcodec_open2 failed"}; } void decoder::set_blit_targets(std::vector targets, VkFormat format) { blit_targets = std::move(targets); } void decoder::blit(VkCommandBuffer command_buffer, blit_handle & handle, std::span blit_indices) { std::unique_lock lock(mutex); // Transition layout to VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL // TODO: transition only if needed VkImageMemoryBarrier barrier{}; barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; barrier.image = handle.image; barrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; barrier.subresourceRange.levelCount = 1; barrier.subresourceRange.layerCount = 1; barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; vkCmdPipelineBarrier(command_buffer, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); for (int blit_index: blit_indices) { uint32_t left = blit_targets[blit_index].offset.x; uint32_t right = left + blit_targets[blit_index].extent.width; uint32_t width = blit_targets[blit_index].extent.width; uint32_t height = blit_targets[blit_index].extent.height; // check for intersection if (description.offset_x >= right or description.offset_x + description.width <= left) continue; VkImageBlit blit{}; blit.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; blit.srcSubresource.layerCount = 1; blit.srcOffsets[0].x = std::max(0, left - description.offset_x); blit.srcOffsets[0].y = 0; blit.srcOffsets[1].x = std::min(description.width, right - description.offset_x); blit.srcOffsets[1].y = std::min(description.height, height - description.offset_y); blit.srcOffsets[1].z = 1; blit.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; blit.dstSubresource.layerCount = 1; blit.dstOffsets[0].x = std::max(0, description.offset_x - left); blit.dstOffsets[0].y = description.offset_y; blit.dstOffsets[1].x = std::min(width, description.offset_x + description.width - left); blit.dstOffsets[1].y = std::min(height, description.offset_y + description.height); blit.dstOffsets[1].z = 1; vkCmdBlitImage(command_buffer, handle.image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, blit_targets[blit_index].image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &blit, VK_FILTER_LINEAR); } } void decoder::push_data(std::span> data, uint64_t frame_index, bool partial) { for (const auto& d: data) packet.insert(packet.end(), d.begin(), d.end()); this->frame_index = frame_index; } void decoder::frame_completed(const xrt::drivers::wivrn::from_headset::feedback & feedback, const xrt::drivers::wivrn::to_headset::video_stream_data_shard::view_info_t & view_info) { spdlog::trace("ffmpeg decoder:frame_completed {}", frame_index); AVPacket packet{}; packet.pts = AV_NOPTS_VALUE; packet.dts = AV_NOPTS_VALUE; packet.data = this->packet.data(); packet.size = this->packet.size(); packet.pos = -1; int res = AVERROR(EAGAIN); while (res == AVERROR(EAGAIN)) { res = avcodec_send_packet(codec.get(), &packet); if (res == AVERROR(EAGAIN)) { spdlog::warn("EAGAIN in avcodec_send_packet"); frame_completed(feedback, view_info); } } if (res < 0) throw std::runtime_error{"avcodec_send_packet failed"}; std::unique_ptr frame(av_frame_alloc(), free_frame); res = avcodec_receive_frame(codec.get(), frame.get()); if (res == AVERROR(EAGAIN)) return; if (res != 0) throw std::runtime_error{"avcodec_receive_frame failed"}; this->packet.clear(); if (!sws) { sws.reset(sws_getContext(frame->width, frame->height, (AVPixelFormat)frame->format, description.width, description.height, AV_PIX_FMT_RGB0, SWS_BILINEAR, nullptr, nullptr, nullptr)); } std::unique_lock lock(mutex); if (free_images.empty()) { spdlog::warn("No free image"); return; } int index = free_images.back(); free_images.pop_back(); lock.unlock(); decoded_images[index].frame_index = frame_index; int dstStride = decoded_images[index].layout.rowPitch; uint8_t * out = (uint8_t *)decoded_images[index].memory.data(); res = sws_scale(sws.get(), frame->data, frame->linesize, 0, frame->height, &out, &dstStride); if (res == 0) throw std::runtime_error{"sws_scale failed"}; auto handle = std::make_shared(); handle->feedback = feedback; handle->view_info = view_info; handle->image = decoded_images[index].image; handle->image_index = index; handle->self = this; if (auto scene = weak_scene.lock()) scene->push_blit_handle(accumulator, std::move(handle)); } } // namespace ffmpeg