diff --git a/client/decoder/android/android_decoder.cpp b/client/decoder/android/android_decoder.cpp index 707b6234..3d91df09 100644 --- a/client/decoder/android/android_decoder.cpp +++ b/client/decoder/android/android_decoder.cpp @@ -391,7 +391,6 @@ void decoder::on_image_available(AImageReader * reader) return; } - info->feedback.received_from_decoder = application::now(); assert(info->feedback.frame_index == frame_index); auto vk_data = map_hardware_buffer(image); diff --git a/client/scenes/stream.cpp b/client/scenes/stream.cpp index c1f49a36..82000b8f 100644 --- a/client/scenes/stream.cpp +++ b/client/scenes/stream.cpp @@ -34,6 +34,7 @@ #include "wivrn_packets.h" #include #include +#include #include #include @@ -244,6 +245,7 @@ void scenes::stream::push_blit_handle(shard_accumulator * decoder, std::shared_p { std::swap(i.latest_frames[j - 1], i.latest_frames[j]); } + handle->feedback.received_from_decoder = application::now(); std::swap(i.latest_frames.back(), handle); break; } @@ -276,24 +278,47 @@ std::vector scenes::stream::accumulator_images::frames() const return result; } -std::optional scenes::stream::accumulator_images::common_frame(const std::vector & sets, uint64_t preferred_frame) +std::optional scenes::stream::accumulator_images::common_frame(const std::vector & sets, XrTime display_time) { if (sets.empty()) return {}; - auto common_frames = sets[0].frames(); - for (const auto & set: sets) + thread_local std::vector common_frames; + thread_local std::vector tmp; + common_frames.clear(); + auto proj = [](const shard_accumulator::blit_handle * h) { return h ? h->feedback.frame_index : -1; }; + for (size_t i = 0; i < sets.size(); ++i) { - std::vector tmp; - auto x = set.frames(); - std::ranges::set_intersection(x, common_frames, std::back_inserter(tmp)); - common_frames = tmp; - if (common_frames.empty()) - return {}; + if (i == 0) + { + for (const auto & h: sets[i].latest_frames) + if (h) + common_frames.push_back(h.get()); + } + else + { + tmp.clear(); + std::ranges::set_intersection( + std::ranges::views::transform(sets[i].latest_frames, [](auto ptr) { return ptr.get(); }), + common_frames, + std::back_inserter(tmp), + std::ranges::less{}, + proj, + proj); + std::swap(common_frames, tmp); + } } - assert(not common_frames.empty()); - if (std::ranges::find(common_frames, preferred_frame) != common_frames.end()) - return preferred_frame; - return common_frames.back(); + if (common_frames.empty()) + return {}; + auto min = std::ranges::min_element(common_frames, + std::ranges::less{}, + [display_time](auto frame) { + if (not frame) + return std::numeric_limits::max(); + return std::abs(frame->view_info.display_time - display_time); + }); + + assert(*min); + return (*min)->feedback.frame_index; } std::shared_ptr scenes::stream::accumulator_images::frame(std::optional id) @@ -505,12 +530,9 @@ void scenes::stream::render(XrTime predicted_display_time, bool should_render) std::array fov = {views[0].fov, views[1].fov}; { std::unique_lock lock(decoder_mutex); - // Search for the most recent frame available on all decoders. + // Search for frame with desired display time on all decoders // If no such frame exists, use the latest frame for each decoder - auto common_frame = accumulator_images::common_frame(decoders, next_frame); - - if (common_frame) - next_frame = *common_frame + 1; + auto common_frame = accumulator_images::common_frame(decoders, predicted_display_time); // Blit images from the decoders for (auto & i: decoders) @@ -524,6 +546,7 @@ void scenes::stream::render(XrTime predicted_display_time, bool should_render) blit_handle->feedback.blitted = application::now(); if (blit_handle->feedback.blitted - blit_handle->feedback.received_from_decoder > 1'000'000'000) state_ = stream::state::stalled; + ++blit_handle->feedback.times_displayed; blit_handle->feedback.displayed = predicted_display_time; blit_handle->feedback.real_pose[0] = views[0].pose; blit_handle->feedback.real_pose[1] = views[1].pose; diff --git a/client/scenes/stream.h b/client/scenes/stream.h index 93b809c8..39328f95 100644 --- a/client/scenes/stream.h +++ b/client/scenes/stream.h @@ -58,7 +58,7 @@ private: // latest frames from oldest to most recent std::array, 3> latest_frames; - static std::optional common_frame(const std::vector &, uint64_t preferred_index); + static std::optional common_frame(const std::vector &, XrTime display_time); std::shared_ptr frame(std::optional id); std::vector frames() const; }; @@ -83,7 +83,6 @@ private: std::mutex decoder_mutex; std::optional video_stream_description; - uint64_t next_frame; // Preferred index for next frame std::vector decoders; // Locked by decoder_mutex vk::raii::DescriptorPool blit_descriptor_pool = nullptr; vk::raii::RenderPass blit_render_pass = nullptr; diff --git a/common/wivrn_packets.h b/common/wivrn_packets.h index 06056001..14afc719 100644 --- a/common/wivrn_packets.h +++ b/common/wivrn_packets.h @@ -206,6 +206,8 @@ struct feedback XrTime blitted; XrTime displayed; + uint8_t times_displayed; + std::array received_pose; std::array real_pose; }; diff --git a/server/driver/wivrn_pacer.cpp b/server/driver/wivrn_pacer.cpp index e3a15641..2c9f57dd 100644 --- a/server/driver/wivrn_pacer.cpp +++ b/server/driver/wivrn_pacer.cpp @@ -75,7 +75,7 @@ void wivrn_pacer::on_feedback(const xrt::drivers::wivrn::from_headset::feedback } last = feedback; - if (feedback.blitted) + if (feedback.blitted and feedback.times_displayed == 1) { auto & stream = streams[feedback.stream_index]; if (stream.times.size() < num_wait_times)