Files

284 lines
8.2 KiB
C++

/*
* WiVRn VR streaming
* Copyright (C) 2022 Guillaume Meunier <[email protected]>
* Copyright (C) 2022 Patrick Nicolas <[email protected]>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
#include "ffmpeg_decoder.h"
#include "scenes/stream.h"
#include "spdlog/spdlog.h"
#include <cassert>
#include <vulkan/vulkan.hpp>
extern "C"
{
#include <libavcodec/avcodec.h>
#include <libavutil/buffer.h>
#include <libavutil/hwcontext.h>
#include <libavutil/hwcontext_vulkan.h>
#include <libswscale/swscale.h>
}
namespace wivrn::ffmpeg
{
static void free_codec_context(AVCodecContext * ctx)
{
avcodec_free_context(&ctx);
}
static void free_frame(AVFrame * frame)
{
av_frame_free(&frame);
}
static AVCodecID codec_id(wivrn::video_codec codec)
{
using c = wivrn::video_codec;
switch (codec)
{
case c::h264:
return AV_CODEC_ID_H264;
case c::h265:
return AV_CODEC_ID_HEVC;
case c::av1:
return AV_CODEC_ID_AV1;
case c::raw:
break;
}
assert(false);
__builtin_unreachable();
}
struct decoder::ffmpeg_blit_handle : public wivrn::decoder::blit_handle
{
int image_index;
wivrn::ffmpeg::decoder * self;
ffmpeg_blit_handle(
const wivrn::from_headset::feedback & feedback,
const wivrn::to_headset::video_stream_data_shard::view_info_t & view_info,
vk::ImageView image_view,
vk::Image image,
vk::Extent2D extent,
vk::ImageLayout & current_layout,
int image_index,
wivrn::ffmpeg::decoder * self) :
wivrn::decoder::blit_handle{
feedback,
view_info,
image_view,
image,
extent,
current_layout,
},
image_index(image_index),
self(self)
{}
~ffmpeg_blit_handle()
{
std::unique_lock lock(self->mutex);
if (image_index < 0)
return;
self->free_images.push_back(image_index);
}
};
decoder::decoder(
vk::raii::Device & device,
vk::raii::PhysicalDevice & physical_device,
const wivrn::to_headset::video_stream_description & description,
uint8_t stream_index,
std::weak_ptr<scenes::stream> scene,
shard_accumulator * accumulator) :
device(device),
codec(nullptr, free_codec_context),
sws(nullptr, sws_freeContext),
weak_scene(scene),
accumulator(accumulator),
extent{
.width = description.width,
.height = description.height / (stream_index == 2 ? 2u : 1u),
}
{
free_images.resize(image_count);
for (int i = 0; i < image_count; i++)
{
free_images[i] = i;
vk::ImageCreateInfo image_info{
.imageType = vk::ImageType::e2D,
.format = vk::Format::eA8B8G8R8SrgbPack32,
.extent = {
.width = extent.width,
.height = extent.height,
.depth = 1,
},
.mipLevels = 1,
.arrayLayers = 1,
.samples = vk::SampleCountFlagBits::e1,
.tiling = vk::ImageTiling::eLinear,
.usage = vk::ImageUsageFlagBits::eSampled,
.initialLayout = vk::ImageLayout::eUndefined,
};
VmaAllocationCreateInfo alloc_info{
.requiredFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT};
decoded_images[i].image = image_allocation(device, image_info, alloc_info);
decoded_images[i].image.map();
vk::ImageSubresource resource;
resource.aspectMask = vk::ImageAspectFlagBits::eColor;
decoded_images[i].layout = decoded_images[i].image->getSubresourceLayout(resource);
decoded_images[i].image_view = vk::raii::ImageView(
device,
{
.image = (vk::Image)decoded_images[i].image,
.viewType = vk::ImageViewType::e2D,
.format = image_info.format,
.subresourceRange = {
.aspectMask = vk::ImageAspectFlagBits::eColor,
.baseMipLevel = 0,
.levelCount = 1,
.baseArrayLayer = 0,
.layerCount = 1,
},
});
decoded_images[i].current_layout = vk::ImageLayout::eUndefined;
}
auto avcodec = avcodec_find_decoder(codec_id(description.codec[stream_index]));
if (avcodec == nullptr)
{
throw std::runtime_error{"avcodec_find_decoder failed"};
}
codec.reset(avcodec_alloc_context3(avcodec));
int ret = avcodec_open2(codec.get(), avcodec, nullptr);
if (ret < 0)
throw std::runtime_error{"avcodec_open2 failed"};
rgb_sampler = vk::raii::Sampler(
device,
{
.flags = {},
.magFilter = vk::Filter::eLinear,
.minFilter = vk::Filter::eLinear,
.mipmapMode = vk::SamplerMipmapMode::eNearest,
.addressModeU = vk::SamplerAddressMode::eClampToEdge,
.addressModeV = vk::SamplerAddressMode::eClampToEdge,
.addressModeW = vk::SamplerAddressMode::eClampToEdge,
.unnormalizedCoordinates = false,
});
}
void decoder::push_data(std::span<std::span<const uint8_t>> data, uint64_t frame_index, bool partial)
{
if (frame_index != this->frame_index)
packet.clear();
for (const auto & d: data)
packet.insert(packet.end(), d.begin(), d.end());
this->frame_index = frame_index;
}
void decoder::frame_completed(const wivrn::from_headset::feedback & feedback, const wivrn::to_headset::video_stream_data_shard::view_info_t & view_info)
{
spdlog::trace("ffmpeg decoder:frame_completed {}", frame_index);
AVPacket packet{};
packet.pts = AV_NOPTS_VALUE;
packet.dts = AV_NOPTS_VALUE;
packet.data = this->packet.data();
packet.size = this->packet.size();
packet.pos = -1;
int res = AVERROR(EAGAIN);
while (res == AVERROR(EAGAIN))
{
res = avcodec_send_packet(codec.get(), &packet);
if (res == AVERROR(EAGAIN))
{
spdlog::warn("EAGAIN in avcodec_send_packet");
frame_completed(feedback, view_info);
}
}
if (res < 0)
throw std::runtime_error{"avcodec_send_packet failed"};
std::unique_ptr<AVFrame, void (*)(AVFrame *)> frame(av_frame_alloc(), free_frame);
res = avcodec_receive_frame(codec.get(), frame.get());
if (res == AVERROR(EAGAIN))
return;
if (res != 0)
throw std::runtime_error{"avcodec_receive_frame failed"};
this->packet.clear();
if (!sws)
{
sws.reset(sws_getContext(frame->width, frame->height, (AVPixelFormat)frame->format, extent.width, extent.height, AV_PIX_FMT_RGB0, SWS_BILINEAR, nullptr, nullptr, nullptr));
}
std::unique_lock lock(mutex);
if (free_images.empty())
{
spdlog::warn("No free image");
return;
}
int index = free_images.back();
free_images.pop_back();
lock.unlock();
decoded_images[index].frame_index = frame_index;
int dstStride = decoded_images[index].layout.rowPitch;
uint8_t * out = decoded_images[index].image.data<uint8_t>();
res = sws_scale(sws.get(), frame->data, frame->linesize, 0, frame->height, &out, &dstStride);
if (res == 0)
throw std::runtime_error{"sws_scale failed"};
auto handle = std::make_shared<ffmpeg_blit_handle>(
feedback,
view_info,
*decoded_images[index].image_view,
decoded_images[index].image,
extent,
decoded_images[index].current_layout,
index,
this);
if (auto scene = weak_scene.lock())
scene->push_blit_handle(accumulator, std::move(handle));
}
void decoder::supported_codecs(std::vector<wivrn::video_codec> & res)
{
// As long as we do software decoding, prefer faster ones
res.push_back(wivrn::video_codec::h264);
res.push_back(wivrn::video_codec::h265);
res.push_back(wivrn::video_codec::av1);
}
} // namespace wivrn::ffmpeg