From b97c8b29b53c19a08526e69cd1ceaac9ac96c617 Mon Sep 17 00:00:00 2001 From: Patrick Nicolas Date: Wed, 29 Jan 2025 21:27:02 +0100 Subject: [PATCH] Add encode and network timings in feedback --- client/decoder/android/android_decoder.cpp | 4 +--- client/decoder/android/android_decoder.h | 3 --- client/decoder/ffmpeg/ffmpeg_decoder.cpp | 5 ++--- client/decoder/ffmpeg/ffmpeg_decoder.h | 2 -- client/decoder/shard_accumulator.cpp | 6 +++++- client/scenes/stream_stats.cpp | 12 ++++++------ common/wivrn_packets.h | 4 ++++ 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/client/decoder/android/android_decoder.cpp b/client/decoder/android/android_decoder.cpp index 7d53a48e..2a091ad4 100644 --- a/client/decoder/android/android_decoder.cpp +++ b/client/decoder/android/android_decoder.cpp @@ -222,7 +222,7 @@ void decoder::push_data(std::span> data, uint64_t frame current_input_buffer = input_buffer{}; } -void decoder::frame_completed(wivrn::from_headset::feedback & feedback, const wivrn::to_headset::video_stream_data_shard::timing_info_t & timing_info, const wivrn::to_headset::video_stream_data_shard::view_info_t & view_info) +void decoder::frame_completed(wivrn::from_headset::feedback & feedback, const wivrn::to_headset::video_stream_data_shard::view_info_t & view_info) { if (not media_codec) { @@ -234,7 +234,6 @@ void decoder::frame_completed(wivrn::from_headset::feedback & feedback, const wi // nothing required for decoder, mediacodec will callback when done frame_infos.push(frame_info{ .feedback = feedback, - .timing_info = timing_info, .view_info = view_info, }); } @@ -283,7 +282,6 @@ void decoder::on_image_available(AImageReader * reader) auto handle = std::make_shared( info->feedback, - info->timing_info, info->view_info, vk_data->image_view, *vk_data->vimage, diff --git a/client/decoder/android/android_decoder.h b/client/decoder/android/android_decoder.h index ecfeb308..63926935 100644 --- a/client/decoder/android/android_decoder.h +++ b/client/decoder/android/android_decoder.h @@ -71,7 +71,6 @@ public: void operator()(AImage *); }; wivrn::from_headset::feedback feedback; - wivrn::to_headset::video_stream_data_shard::timing_info_t timing_info; wivrn::to_headset::video_stream_data_shard::view_info_t view_info; vk::raii::ImageView & image_view; vk::Image image = nullptr; @@ -123,7 +122,6 @@ private: struct frame_info { wivrn::from_headset::feedback feedback; - wivrn::to_headset::video_stream_data_shard::timing_info_t timing_info; wivrn::to_headset::video_stream_data_shard::view_info_t view_info; }; utils::sync_queue frame_infos; @@ -157,7 +155,6 @@ public: void frame_completed( wivrn::from_headset::feedback & feedback, - const wivrn::to_headset::video_stream_data_shard::timing_info_t & timing_info, const wivrn::to_headset::video_stream_data_shard::view_info_t & view_info); const auto & desc() const diff --git a/client/decoder/ffmpeg/ffmpeg_decoder.cpp b/client/decoder/ffmpeg/ffmpeg_decoder.cpp index 014ebed3..09416a28 100644 --- a/client/decoder/ffmpeg/ffmpeg_decoder.cpp +++ b/client/decoder/ffmpeg/ffmpeg_decoder.cpp @@ -166,7 +166,7 @@ void decoder::push_data(std::span> data, uint64_t frame this->frame_index = frame_index; } -void decoder::frame_completed(const wivrn::from_headset::feedback & feedback, const wivrn::to_headset::video_stream_data_shard::timing_info_t & timing_info, const wivrn::to_headset::video_stream_data_shard::view_info_t & view_info) +void decoder::frame_completed(const wivrn::from_headset::feedback & feedback, const wivrn::to_headset::video_stream_data_shard::view_info_t & view_info) { spdlog::trace("ffmpeg decoder:frame_completed {}", frame_index); AVPacket packet{}; @@ -183,7 +183,7 @@ void decoder::frame_completed(const wivrn::from_headset::feedback & feedback, co if (res == AVERROR(EAGAIN)) { spdlog::warn("EAGAIN in avcodec_send_packet"); - frame_completed(feedback, timing_info, view_info); + frame_completed(feedback, view_info); } } if (res < 0) @@ -223,7 +223,6 @@ void decoder::frame_completed(const wivrn::from_headset::feedback & feedback, co auto handle = std::make_shared( feedback, - timing_info, view_info, decoded_images[index].image_view, (vk::Image)decoded_images[index].image, diff --git a/client/decoder/ffmpeg/ffmpeg_decoder.h b/client/decoder/ffmpeg/ffmpeg_decoder.h index 34f00bbc..33190af8 100644 --- a/client/decoder/ffmpeg/ffmpeg_decoder.h +++ b/client/decoder/ffmpeg/ffmpeg_decoder.h @@ -52,7 +52,6 @@ public: struct blit_handle { wivrn::from_headset::feedback feedback; - wivrn::to_headset::video_stream_data_shard::timing_info_t timing_info; wivrn::to_headset::video_stream_data_shard::view_info_t view_info; vk::raii::ImageView & image_view; vk::Image image = nullptr; @@ -109,7 +108,6 @@ public: void frame_completed( const wivrn::from_headset::feedback & feedback, - const wivrn::to_headset::video_stream_data_shard::timing_info_t & timing_info, const wivrn::to_headset::video_stream_data_shard::view_info_t & view_info); const auto & desc() const diff --git a/client/decoder/shard_accumulator.cpp b/client/decoder/shard_accumulator.cpp index d167e398..6b6106ce 100644 --- a/client/decoder/shard_accumulator.cpp +++ b/client/decoder/shard_accumulator.cpp @@ -194,6 +194,10 @@ void shard_accumulator::try_submit_frame(uint16_t shard_idx) current.feedback.received_last_packet = application::now(); current.feedback.sent_to_decoder = current.feedback.received_last_packet; data_shard::timing_info_t timing_info = data_shards.back()->timing_info.value_or(data_shard::timing_info_t{}); + current.feedback.encode_begin = timing_info.encode_begin; + current.feedback.encode_end = timing_info.encode_end; + current.feedback.send_begin = timing_info.send_begin; + current.feedback.send_end = timing_info.send_end; if (not data_shards.front()->view_info) { @@ -202,7 +206,7 @@ void shard_accumulator::try_submit_frame(uint16_t shard_idx) } // Try to extract a frame - decoder->frame_completed(current.feedback, timing_info, *data_shards.front()->view_info); + decoder->frame_completed(current.feedback, *data_shards.front()->view_info); send_feedback(current.feedback); diff --git a/client/scenes/stream_stats.cpp b/client/scenes/stream_stats.cpp index 7877be05..5d65d327 100644 --- a/client/scenes/stream_stats.cpp +++ b/client/scenes/stream_stats.cpp @@ -109,11 +109,11 @@ void scenes::stream::accumulate_metrics(XrTime predicted_display_time, const std if (decoder_metrics.size() != active_handles.size()) decoder_metrics.resize(active_handles.size()); - auto min_encode_begin = std::numeric_limitstiming_info.encode_begin)>::max(); + auto min_encode_begin = std::numeric_limitsfeedback.encode_begin)>::max(); for (const auto & bh: active_handles) { if (bh) - min_encode_begin = std::min(min_encode_begin, bh->timing_info.encode_begin); + min_encode_begin = std::min(min_encode_begin, bh->feedback.encode_begin); } for (auto && [metrics, bh]: std::views::zip(decoder_metrics, active_handles)) @@ -125,10 +125,10 @@ void scenes::stream::accumulate_metrics(XrTime predicted_display_time, const std // clang-format off metrics[metrics_offset] = bh ? decoder_metric{ - .encode_begin = (bh->timing_info.encode_begin - min_encode_begin) * 1e-9f, - .encode_end = (bh->timing_info.encode_end - min_encode_begin) * 1e-9f, - .send_begin = (bh->timing_info.send_begin - min_encode_begin) * 1e-9f, - .send_end = (bh->timing_info.send_end - min_encode_begin) * 1e-9f, + .encode_begin = (bh->feedback.encode_begin - min_encode_begin) * 1e-9f, + .encode_end = (bh->feedback.encode_end - min_encode_begin) * 1e-9f, + .send_begin = (bh->feedback.send_begin - min_encode_begin) * 1e-9f, + .send_end = (bh->feedback.send_end - min_encode_begin) * 1e-9f, .received_first_packet = (bh->feedback.received_first_packet - min_encode_begin) * 1e-9f, .received_last_packet = (bh->feedback.received_last_packet - min_encode_begin) * 1e-9f, .sent_to_decoder = (bh->feedback.sent_to_decoder - min_encode_begin) * 1e-9f, diff --git a/common/wivrn_packets.h b/common/wivrn_packets.h index 05a521cc..5e693783 100644 --- a/common/wivrn_packets.h +++ b/common/wivrn_packets.h @@ -271,6 +271,10 @@ struct feedback uint8_t stream_index; // Timestamps + XrTime encode_begin; + XrTime encode_end; + XrTime send_begin; + XrTime send_end; XrTime received_first_packet; XrTime received_last_packet; XrTime sent_to_decoder;