mirror of
https://github.com/WiVRn/WiVRn.git
synced 2026-11-20 09:01:39 -08:00
Initial timesync algorithm assumed that latency was symmetrical, but it appears that it is not. Add a mechanism to estimate where latency happens, which leads to much more stable values: about 200µs variation between estimations.
358 lines
12 KiB
C++
358 lines
12 KiB
C++
/*
|
|
* WiVRn VR streaming
|
|
* Copyright (C) 2022 Guillaume Meunier <[email protected]>
|
|
* Copyright (C) 2022 Patrick Nicolas <[email protected]>
|
|
*
|
|
* This program is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation, either version 3 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
#include "VideoEncoderVA.h"
|
|
#include "ffmpeg_helper.h"
|
|
#include "vk/vk_helpers.h"
|
|
#include <cassert>
|
|
#include <chrono>
|
|
#include <filesystem>
|
|
#include <optional>
|
|
#include <stdexcept>
|
|
|
|
extern "C"
|
|
{
|
|
#include <libavcodec/avcodec.h>
|
|
#include <libavfilter/avfilter.h>
|
|
#include <libavfilter/buffersink.h>
|
|
#include <libavfilter/buffersrc.h>
|
|
#include <libavutil/hwcontext.h>
|
|
#include <libavutil/hwcontext_drm.h>
|
|
#include <libavutil/hwcontext_vaapi.h>
|
|
#include <libavutil/opt.h>
|
|
}
|
|
|
|
namespace
|
|
{
|
|
|
|
const char *
|
|
encoder(VideoEncoderFFMPEG::Codec codec)
|
|
{
|
|
switch (codec)
|
|
{
|
|
case VideoEncoderFFMPEG::Codec::h264:
|
|
return "h264_vaapi";
|
|
case VideoEncoderFFMPEG::Codec::h265:
|
|
return "hevc_vaapi";
|
|
}
|
|
throw std::runtime_error("invalid codec " + std::to_string(int(codec)));
|
|
}
|
|
|
|
void set_hwframe_ctx(AVCodecContext * ctx, AVBufferRef * hw_device_ctx)
|
|
{
|
|
av_buffer_ptr hw_frames_ref(av_hwframe_ctx_alloc(hw_device_ctx));
|
|
int err = 0;
|
|
|
|
if (not hw_frames_ref)
|
|
{
|
|
throw std::runtime_error("Failed to create VAAPI frame context.");
|
|
}
|
|
auto frames_ctx = (AVHWFramesContext *)(hw_frames_ref->data);
|
|
frames_ctx->format = AV_PIX_FMT_VAAPI;
|
|
frames_ctx->sw_format = AV_PIX_FMT_NV12;
|
|
frames_ctx->width = ctx->width;
|
|
frames_ctx->height = ctx->height;
|
|
frames_ctx->initial_pool_size = 3;
|
|
if ((err = av_hwframe_ctx_init(hw_frames_ref.get())) < 0)
|
|
{
|
|
throw std::system_error(err, av_error_category(), "Failed to initialize VAAPI frame context");
|
|
}
|
|
ctx->hw_frames_ctx = av_buffer_ref(hw_frames_ref.get());
|
|
if (!ctx->hw_frames_ctx)
|
|
err = AVERROR(ENOMEM);
|
|
}
|
|
|
|
std::optional<std::filesystem::path>
|
|
get_render_device(vk_bundle * vk)
|
|
{
|
|
VkPhysicalDeviceDrmPropertiesEXT drmProps{};
|
|
drmProps.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRM_PROPERTIES_EXT;
|
|
|
|
VkPhysicalDeviceProperties2 props{};
|
|
props.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2;
|
|
props.pNext = &drmProps;
|
|
|
|
vk->vkGetPhysicalDeviceProperties2(vk->physical_device, &props);
|
|
|
|
if (!drmProps.hasRender)
|
|
{
|
|
U_LOG_E("Failed to find render DRM device");
|
|
throw std::runtime_error("Failed to find render DRM device");
|
|
}
|
|
std::filesystem::path path = "/dev/dri/renderD" + std::to_string(drmProps.renderMinor);
|
|
if (not std::filesystem::exists(path))
|
|
{
|
|
U_LOG_W("DRI device %s does not exist, reverting to default", path.c_str());
|
|
return std::nullopt;
|
|
}
|
|
return path;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
VideoEncoderVA::VideoEncoderVA(vk_bundle * vk, const xrt::drivers::wivrn::encoder_settings & settings, float fps) :
|
|
vk(vk), width(settings.width), height(settings.height), offset_x(settings.offset_x), offset_y(settings.offset_y)
|
|
{
|
|
codec = settings.codec;
|
|
/* VAAPI Encoding pipeline
|
|
* The encoding pipeline has 3 frame types:
|
|
* - input vulkan frames, only used to initialize the mapped frames
|
|
* - mapped frames, one per input frame, same format, and point to the same memory on the device
|
|
* - encoder frame, with a format compatible with the encoder, created by the filter
|
|
* Each frame type has a corresponding hardware frame context, the vulkan one is provided
|
|
*
|
|
* The pipeline is simply made of a scale_vaapi object, that does the conversion between formats
|
|
* and the encoder that takes the converted frame and produces packets.
|
|
*/
|
|
|
|
AVBufferRef * tmp;
|
|
const auto device = get_render_device(vk);
|
|
int err = av_hwdevice_ctx_create(&tmp, AV_HWDEVICE_TYPE_VAAPI, device ? device->c_str() : NULL, NULL, 0);
|
|
if (err < 0)
|
|
{
|
|
throw std::system_error(err, av_error_category(), "Failed create VAAPI device" + (device ? (" " + device->string()) : std::string()));
|
|
}
|
|
hw_ctx_vaapi.reset(tmp);
|
|
|
|
const char * encoder_name = encoder(codec);
|
|
const AVCodec * codec = avcodec_find_encoder_by_name(encoder_name);
|
|
if (codec == nullptr)
|
|
{
|
|
throw std::runtime_error(std::string("Failed to find encoder ") + encoder_name);
|
|
}
|
|
|
|
encoder_ctx.reset(avcodec_alloc_context3(codec));
|
|
if (not encoder_ctx)
|
|
{
|
|
throw std::runtime_error("failed to allocate VAAPI encoder");
|
|
}
|
|
|
|
AVDictionary * opts = nullptr;
|
|
av_dict_set(&opts, "async_depth", "1", 0);
|
|
for (auto option: settings.options)
|
|
{
|
|
av_dict_set(&opts, option.first.c_str(), option.second.c_str(), 0);
|
|
}
|
|
switch (this->codec)
|
|
{
|
|
case Codec::h264:
|
|
encoder_ctx->profile = FF_PROFILE_H264_MAIN;
|
|
break;
|
|
case Codec::h265:
|
|
encoder_ctx->profile = FF_PROFILE_HEVC_MAIN;
|
|
break;
|
|
}
|
|
|
|
encoder_ctx->width = width;
|
|
encoder_ctx->height = height;
|
|
encoder_ctx->time_base = {std::chrono::steady_clock::duration::period::num,
|
|
std::chrono::steady_clock::duration::period::den};
|
|
encoder_ctx->framerate = AVRational{(int)fps, 1};
|
|
encoder_ctx->sample_aspect_ratio = AVRational{1, 1};
|
|
encoder_ctx->pix_fmt = AV_PIX_FMT_VAAPI;
|
|
encoder_ctx->max_b_frames = 0;
|
|
encoder_ctx->bit_rate = settings.bitrate;
|
|
|
|
set_hwframe_ctx(encoder_ctx.get(), hw_ctx_vaapi.get());
|
|
|
|
err = avcodec_open2(encoder_ctx.get(), codec, &opts);
|
|
av_dict_free(&opts);
|
|
if (err < 0)
|
|
{
|
|
throw std::system_error(err, av_error_category(), "Cannot open video encoder codec");
|
|
}
|
|
|
|
if (encoder_ctx->delay != 0)
|
|
{
|
|
U_LOG_W("Encoder %d reports a %d frame delay, reprojection will fail", stream_idx, encoder_ctx->delay);
|
|
}
|
|
}
|
|
|
|
void VideoEncoderVA::PushFrame(uint32_t frame_index, bool idr, std::chrono::steady_clock::time_point pts)
|
|
{
|
|
assert(frame_index < mapped_frames.size());
|
|
av_frame_ptr encoder_frame = make_av_frame();
|
|
int err = av_buffersrc_add_frame_flags(filter_in, mapped_frames[frame_index].get(), AV_BUFFERSRC_FLAG_PUSH | AV_BUFFERSRC_FLAG_KEEP_REF);
|
|
if (err != 0)
|
|
{
|
|
throw std::system_error(err, av_error_category(), "av_buffersrc_add_frame failed");
|
|
}
|
|
err = av_buffersink_get_frame(filter_out, encoder_frame.get());
|
|
if (err != 0)
|
|
{
|
|
throw std::system_error(err, av_error_category(), "av_buffersink_get_frame failed");
|
|
}
|
|
|
|
encoder_frame->pict_type = idr ? AV_PICTURE_TYPE_I : AV_PICTURE_TYPE_NONE;
|
|
encoder_frame->pts = pts.time_since_epoch().count();
|
|
err = avcodec_send_frame(encoder_ctx.get(), encoder_frame.get());
|
|
if (err)
|
|
{
|
|
throw std::system_error(err, av_error_category(), "avcodec_send_frame failed");
|
|
}
|
|
}
|
|
|
|
void VideoEncoderVA::SetImages(
|
|
int width,
|
|
int height,
|
|
VkFormat format,
|
|
int num_images,
|
|
VkImage * images,
|
|
VkImageView * views,
|
|
VkDeviceMemory * memory)
|
|
{
|
|
mapped_frames.clear();
|
|
av_buffer_ptr va_ctx(av_hwframe_ctx_alloc(hw_ctx_vaapi.get()));
|
|
if (!va_ctx)
|
|
{
|
|
throw std::runtime_error("Failed to create VAAPI frame context.");
|
|
}
|
|
AVHWFramesContext * va_frames_ctx = (AVHWFramesContext *)(va_ctx->data);
|
|
va_frames_ctx->format = AV_PIX_FMT_VAAPI;
|
|
va_frames_ctx->sw_format = vk_format_to_av_format(format);
|
|
va_frames_ctx->width = width;
|
|
va_frames_ctx->height = height;
|
|
va_frames_ctx->initial_pool_size = num_images;
|
|
int err = av_hwframe_ctx_init(va_ctx.get());
|
|
if (err)
|
|
{
|
|
throw std::system_error(err, av_error_category(), "Failed to create VAAPI frame context");
|
|
}
|
|
|
|
mapped_frames.reserve(num_images);
|
|
for (int i = 0; i < num_images; ++i)
|
|
{
|
|
VkMemoryRequirements req;
|
|
vk->vkGetImageMemoryRequirements(vk->device, images[i], &req);
|
|
|
|
int fd;
|
|
VkMemoryGetFdInfoKHR export_info{};
|
|
export_info.sType = VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR;
|
|
export_info.memory = memory[i];
|
|
export_info.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT;
|
|
vk->vkGetMemoryFdKHR(vk->device, &export_info, &fd);
|
|
|
|
auto drm_frame = make_av_frame();
|
|
drm_frame->format = AV_PIX_FMT_DRM_PRIME;
|
|
drm_frame->width = width;
|
|
drm_frame->height = height;
|
|
drm_frame->buf[0] = av_buffer_alloc(sizeof(AVDRMFrameDescriptor));
|
|
drm_frame->data[0] = drm_frame->buf[0]->data;
|
|
auto & drm_frame_desc = *(AVDRMFrameDescriptor *)drm_frame->data[0];
|
|
drm_frame_desc.nb_objects = 1;
|
|
drm_frame_desc.objects[0].fd = fd;
|
|
drm_frame_desc.objects[0].size = req.size;
|
|
drm_frame_desc.nb_layers = 1;
|
|
drm_frame_desc.layers[0].format = vk_format_to_fourcc(format);
|
|
drm_frame_desc.layers[0].nb_planes = 1;
|
|
drm_frame_desc.layers[0].planes[0].object_index = 0;
|
|
|
|
// DRM format modifiers seem to be broken on ffmpeg or radeon, or both
|
|
bool drm_tiling = use_drm_format_modifiers and vk->has_EXT_image_drm_format_modifier;
|
|
if (drm_tiling)
|
|
{
|
|
VkImageDrmFormatModifierPropertiesEXT modifiers{};
|
|
modifiers.sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_PROPERTIES_EXT;
|
|
vk->vkGetImageDrmFormatModifierPropertiesEXT(vk->device, images[i], &modifiers);
|
|
drm_frame_desc.objects[0].format_modifier = modifiers.drmFormatModifier;
|
|
}
|
|
// if we don't have modifiers, we did linear tiling
|
|
VkImageSubresource subresource{(use_drm_format_modifiers and vk->has_EXT_image_drm_format_modifier)
|
|
? VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT
|
|
: VK_IMAGE_ASPECT_COLOR_BIT,
|
|
0,
|
|
0};
|
|
VkSubresourceLayout layout;
|
|
vk->vkGetImageSubresourceLayout(vk->device, images[i], &subresource, &layout);
|
|
drm_frame_desc.layers[0].planes[0].offset = layout.offset;
|
|
drm_frame_desc.layers[0].planes[0].pitch = layout.rowPitch;
|
|
|
|
auto va_frame = make_av_frame();
|
|
err = av_hwframe_get_buffer(va_ctx.get(), va_frame.get(), 0);
|
|
if (err)
|
|
throw std::system_error(err, av_error_category(), "Failed to create VAAPI frame");
|
|
err = av_hwframe_map(va_frame.get(), drm_frame.get(), AV_HWFRAME_MAP_DIRECT);
|
|
if (err)
|
|
throw std::system_error(err, av_error_category(), "Failed to map DRM frame to VAAPI frame");
|
|
va_frame->crop_left = offset_x;
|
|
va_frame->crop_right = width - this->width - offset_x;
|
|
va_frame->crop_top = offset_y;
|
|
va_frame->crop_bottom = height - this->height - offset_y;
|
|
mapped_frames.push_back(std::move(va_frame));
|
|
}
|
|
|
|
InitFilterGraph();
|
|
}
|
|
|
|
void VideoEncoderVA::InitFilterGraph()
|
|
{
|
|
assert(not mapped_frames.empty());
|
|
|
|
filter_graph.reset(avfilter_graph_alloc());
|
|
|
|
AVFilterInOut * outputs = avfilter_inout_alloc();
|
|
AVFilterInOut * inputs = avfilter_inout_alloc();
|
|
|
|
std::stringstream buffer_filter_args;
|
|
buffer_filter_args << "video_size=" << mapped_frames[0]->width << "x" << mapped_frames[0]->height;
|
|
buffer_filter_args << ":pix_fmt=" << mapped_frames[0]->format;
|
|
buffer_filter_args << ":time_base=" << encoder_ctx->time_base.num << "/" << encoder_ctx->time_base.den;
|
|
int err;
|
|
if ((err = avfilter_graph_create_filter(&filter_in, avfilter_get_by_name("buffer"), "in", buffer_filter_args.str().c_str(), NULL, filter_graph.get())))
|
|
{
|
|
throw std::system_error(err, av_error_category(), "filter_in creation failed");
|
|
}
|
|
AVBufferSrcParameters * par = av_buffersrc_parameters_alloc();
|
|
memset(par, 0, sizeof(*par));
|
|
par->format = AV_PIX_FMT_NONE;
|
|
par->hw_frames_ctx = av_buffer_ref(mapped_frames[0]->hw_frames_ctx);
|
|
av_buffersrc_parameters_set(filter_in, par);
|
|
av_free(par);
|
|
|
|
if ((err = avfilter_graph_create_filter(&filter_out, avfilter_get_by_name("buffersink"), "out", NULL, NULL, filter_graph.get())))
|
|
{
|
|
throw std::system_error(err, av_error_category(), "filter_out creation failed");
|
|
}
|
|
|
|
outputs->name = av_strdup("in");
|
|
outputs->filter_ctx = filter_in;
|
|
outputs->pad_idx = 0;
|
|
outputs->next = NULL;
|
|
|
|
inputs->name = av_strdup("out");
|
|
inputs->filter_ctx = filter_out;
|
|
inputs->pad_idx = 0;
|
|
inputs->next = NULL;
|
|
|
|
std::string scale_param = "scale_vaapi=format=nv12:w=" + std::to_string(encoder_ctx->width) +
|
|
":h=" + std::to_string(encoder_ctx->height);
|
|
err = avfilter_graph_parse_ptr(filter_graph.get(), scale_param.c_str(), &inputs, &outputs, NULL);
|
|
avfilter_inout_free(&outputs);
|
|
avfilter_inout_free(&inputs);
|
|
if (err < 0)
|
|
{
|
|
throw std::system_error(err, av_error_category(), "avfilter_graph_parse_ptr failed");
|
|
}
|
|
|
|
if ((err = avfilter_graph_config(filter_graph.get(), NULL)))
|
|
{
|
|
throw std::system_error(err, av_error_category(), "avfilter_graph_config failed");
|
|
}
|
|
}
|