diff --git a/client/decoder/android/android_decoder.cpp b/client/decoder/android/android_decoder.cpp index 3ae6d12d..679c59dc 100644 --- a/client/decoder/android/android_decoder.cpp +++ b/client/decoder/android/android_decoder.cpp @@ -77,44 +77,42 @@ void check(media_status_t status, const char * msg) namespace wivrn::android { -void decoder::push_nals(std::span> data, int64_t timestamp, uint32_t flags) +void decoder::push_nals(std::span> data, uint64_t frame_index, bool partial) { - auto t1 = application::now(); - auto input_buffer = input_buffers.pop(); - auto t2 = application::now(); - assert(input_buffer >= 0); - - if (t2 - t1 > 1'000'000) - spdlog::warn("input_buffers.pop() took {}µs", (t2 - t1) / 1000); - - size_t size; - t1 = application::now(); - uint8_t * buffer = AMediaCodec_getInputBuffer(media_codec.get(), input_buffer, &size); - t2 = application::now(); - if (t2 - t1 > 1'000'000) - spdlog::warn("AMediaCodec_getInputBuffer() took {}µs", (t2 - t1) / 1000); - size_t data_size = 0; + if (current_input_buffer.data == nullptr) + current_input_buffer = input_buffers.pop(); + else if (current_input_buffer.frame_index != frame_index) + { + // Reuse the input buffer, discard existing data + current_input_buffer.data_size = 0; + } + current_input_buffer.frame_index = frame_index; for (const auto & sub_data: data) { - if (sub_data.size() > size) + if (current_input_buffer.data_size + sub_data.size() > current_input_buffer.capacity) { spdlog::error("data to decode is larger than decoder buffer, skipping frame"); return; } - memcpy(buffer + data_size, sub_data.data(), sub_data.size()); - data_size += sub_data.size(); + memcpy(current_input_buffer.data + current_input_buffer.data_size, sub_data.data(), sub_data.size()); + current_input_buffer.data_size += sub_data.size(); } - jobs.push([=, mc = media_codec.get()]() { - auto status = AMediaCodec_queueInputBuffer(mc, input_buffer, 0, data_size, timestamp, flags); + if (partial) + return; + + jobs.push([=, idx = current_input_buffer.idx, data_size = current_input_buffer.data_size, mc = media_codec.get()]() { + uint64_t timestamp = frame_index * 10'000; + auto status = AMediaCodec_queueInputBuffer(mc, idx, 0, data_size, timestamp, 0); if (status != AMEDIA_OK) spdlog::error("AMediaCodec_queueInputBuffer: MediaCodec error {}({})", int(status), std::string(magic_enum::enum_name(status)).c_str()); return false; }); + current_input_buffer = input_buffer{}; } decoder::decoder( @@ -226,8 +224,7 @@ void decoder::push_data(std::span> data, uint64_t frame check(AMediaCodec_start(media_codec.get()), "AMediaCodec_start"); } - uint64_t fake_timestamp_us = frame_index * 10'000; - push_nals(data, fake_timestamp_us, partial ? AMEDIACODEC_BUFFER_FLAG_PARTIAL_FRAME : 0); + push_nals(data, frame_index, partial); } void decoder::frame_completed(xrt::drivers::wivrn::from_headset::feedback & feedback, const xrt::drivers::wivrn::to_headset::video_stream_data_shard::timing_info_t & timing_info, const xrt::drivers::wivrn::to_headset::video_stream_data_shard::view_info_t & view_info) @@ -490,10 +487,16 @@ void decoder::on_media_format_changed(AMediaCodec *, void * userdata, AMediaForm { spdlog::info("Mediacodec format changed"); } -void decoder::on_media_input_available(AMediaCodec *, void * userdata, int32_t index) +void decoder::on_media_input_available(AMediaCodec * media_codec, void * userdata, int32_t index) { auto self = (decoder *)userdata; - self->input_buffers.push(index); + size_t size; + uint8_t * buffer = AMediaCodec_getInputBuffer(media_codec, index, &size); + self->input_buffers.push({ + .idx = index, + .capacity = size, + .data = buffer, + }); } void decoder::on_media_output_available(AMediaCodec * media_codec, void * userdata, int32_t index, AMediaCodecBufferInfo * bufferInfo) { diff --git a/client/decoder/android/android_decoder.h b/client/decoder/android/android_decoder.h index 179d7d5f..bc6546dd 100644 --- a/client/decoder/android/android_decoder.h +++ b/client/decoder/android/android_decoder.h @@ -102,7 +102,17 @@ private: void on_image_available(AImageReader * reader); static void on_image_available(void * context, AImageReader * reader); - utils::sync_queue input_buffers; + struct input_buffer + { + int32_t idx; + uint64_t frame_index = 0; + size_t data_size = 0; + size_t capacity = 0; + uint8_t * data = nullptr; + }; + + utils::sync_queue input_buffers; + input_buffer current_input_buffer; // Only accessed in network thread utils::sync_queue> jobs; struct frame_info @@ -119,7 +129,7 @@ private: static void on_media_input_available(AMediaCodec *, void * userdata, int32_t index); static void on_media_output_available(AMediaCodec *, void * userdata, int32_t index, AMediaCodecBufferInfo * bufferInfo); - void push_nals(std::span> data, int64_t timestamp, uint32_t flags); + void push_nals(std::span> data, uint64_t frame_index, bool partial); std::unordered_map> hardware_buffer_map; vk::raii::RenderPass renderpass = nullptr; diff --git a/client/decoder/shard_accumulator.cpp b/client/decoder/shard_accumulator.cpp index 52432545..2bd5564e 100644 --- a/client/decoder/shard_accumulator.cpp +++ b/client/decoder/shard_accumulator.cpp @@ -60,16 +60,16 @@ static bool is_complete(const shard_set & shards) return true; } -uint16_t shard_set::insert(data_shard && shard) +std::optional shard_set::insert(data_shard && shard) { - XrTime now = application::now(); if (empty()) - feedback.received_first_packet = now; - feedback.received_last_packet = now; + feedback.received_first_packet = application::now(); auto idx = shard.shard_idx; if (idx >= data.size()) data.resize(idx + 1); + if (data[idx]) + return {}; data[idx] = std::move(shard); return idx; } @@ -165,30 +165,32 @@ void shard_accumulator::try_submit_frame(std::optional shard_idx) void shard_accumulator::try_submit_frame(uint16_t shard_idx) { auto & data_shards = current.data; - // Do not submit if the frame is not complete - for (const auto & shard: data_shards) - { - if (not shard) + + for (size_t idx = 0; idx < shard_idx; ++idx) + if (not data_shards[idx]) return; + + uint16_t last_idx = shard_idx + 1; + for (size_t size = data_shards.size(); + last_idx < size and data_shards[last_idx]; + ++last_idx) + { } - if (not(data_shards.back()->flags & video_stream_data_shard::end_of_frame)) + + std::vector> payload; + payload.reserve(last_idx - shard_idx); + for (size_t idx = shard_idx; idx < last_idx; ++idx) + payload.emplace_back(data_shards[idx]->payload); + + bool frame_complete = last_idx == data_shards.size() and data_shards.back()->flags & video_stream_data_shard::end_of_frame; + decoder->push_data(payload, data_shards[shard_idx]->frame_idx, not frame_complete); + + if (not frame_complete) return; - uint64_t frame_index = data_shards[0]->frame_idx; - std::vector> payload; - payload.reserve(data_shards.size()); + current.feedback.received_last_packet = application::now(); + data_shard::timing_info_t timing_info = data_shards.back()->timing_info.value_or(data_shard::timing_info_t{}); - data_shard::timing_info_t timing_info{}; - for (const auto & shard: data_shards) - { - if (shard->timing_info) - timing_info = *shard->timing_info; - - payload.emplace_back(shard->payload); - } - decoder->push_data(payload, frame_index, false); - - auto feedback = current.feedback; if (not data_shards.front()->view_info) { spdlog::warn("first shard has no view_info"); @@ -196,13 +198,15 @@ void shard_accumulator::try_submit_frame(uint16_t shard_idx) } // Try to extract a frame - decoder->frame_completed(feedback, timing_info, *data_shards.front()->view_info); + decoder->frame_completed(current.feedback, timing_info, *data_shards.front()->view_info); advance(); } -void shard_accumulator::send_feedback(const xrt::drivers::wivrn::from_headset::feedback & feedback) +void shard_accumulator::send_feedback(xrt::drivers::wivrn::from_headset::feedback & feedback) { + if (not feedback.received_last_packet) + feedback.received_first_packet = application::now(); auto scene = weak_scene.lock(); if (scene) scene->send_feedback(feedback); diff --git a/client/decoder/shard_accumulator.h b/client/decoder/shard_accumulator.h index 7541ccde..adf08dad 100644 --- a/client/decoder/shard_accumulator.h +++ b/client/decoder/shard_accumulator.h @@ -47,7 +47,7 @@ public: void reset(uint64_t frame_index); bool empty() const; - uint16_t insert(data_shard &&); + std::optional insert(data_shard &&); xrt::drivers::wivrn::from_headset::feedback feedback{}; @@ -106,6 +106,6 @@ public: private: void try_submit_frame(std::optional shard_idx); void try_submit_frame(uint16_t shard_idx); - void send_feedback(const xrt::drivers::wivrn::from_headset::feedback & feedback); + void send_feedback(xrt::drivers::wivrn::from_headset::feedback & feedback); void advance(); };