From 768424d49cce2ea48c8d7897eed9824ce9752e52 Mon Sep 17 00:00:00 2001 From: loucass003 Date: Thu, 3 Sep 2026 12:28:41 +0200 Subject: [PATCH] Better telemetry readings --- .../components/tracker/TrackerSettings.tsx | 59 +++---- .../src/components/tracker/TrackerWifi.tsx | 15 +- gui/app/src/hooks/datafeed-config.ts | 2 +- gui/app/src/hooks/dongle-telemetry-feed.ts | 41 ++++- .../main/java/dev/slimevr/device/module.kt | 21 +-- .../java/dev/slimevr/device/packet-history.kt | 166 ++++++++++++++++++ .../main/java/dev/slimevr/device/reducer.kt | 33 +--- .../java/dev/slimevr/device/sample-history.kt | 89 ---------- .../main/java/dev/slimevr/hid/behaviours.kt | 35 ++-- .../dev/slimevr/solarxr/datafeed/datafeed.kt | 12 +- .../java/dev/slimevr/solarxr/rpc/telemetry.kt | 86 ++++----- .../main/java/dev/slimevr/udp/behaviours.kt | 43 ++--- .../dev/slimevr/udp/connection-reducer.kt | 7 +- .../dev/slimevr/device/PacketStatsTest.kt | 133 ++++++++++++++ .../java/dev/slimevr/udp/PacketLossTest.kt | 50 ++++++ 15 files changed, 512 insertions(+), 280 deletions(-) create mode 100644 server/core/src/main/java/dev/slimevr/device/packet-history.kt delete mode 100644 server/core/src/main/java/dev/slimevr/device/sample-history.kt create mode 100644 server/core/src/test/java/dev/slimevr/device/PacketStatsTest.kt create mode 100644 server/core/src/test/java/dev/slimevr/udp/PacketLossTest.kt diff --git a/gui/app/src/components/tracker/TrackerSettings.tsx b/gui/app/src/components/tracker/TrackerSettings.tsx index d88c4e092..7ac0feb27 100644 --- a/gui/app/src/components/tracker/TrackerSettings.tsx +++ b/gui/app/src/components/tracker/TrackerSettings.tsx @@ -358,37 +358,34 @@ export function TrackerSettingsPage() { {tracker?.device?.hardwareInfo?.networkProtocolVersion || '--'} - {tracker?.device?.hardwareStatus?.packetsReceived !== null && ( - <> -
- - {l10n.getString('tracker-infos-packet_loss')} - - - {( - (tracker?.device?.hardwareStatus?.packetLoss ?? 0) * 100 - ).toFixed(0)} - % - -
-
- - {l10n.getString('tracker-infos-packets_lost')} - - - {tracker?.device?.hardwareStatus?.packetsLost ?? '0'} - -
-
- - {l10n.getString('tracker-infos-packets_received')} - - - {tracker?.device?.hardwareStatus?.packetsReceived ?? '0'} - -
- - )} +
+ + {l10n.getString('tracker-infos-packet_loss')} + + + {tracker?.device?.hardwareStatus?.packetLoss != null + ? `${( + tracker.device.hardwareStatus.packetLoss * 100 + ).toFixed(0)}%` + : '--'} + +
+
+ + {l10n.getString('tracker-infos-packets_lost')} + + + {tracker?.device?.hardwareStatus?.packetsLost ?? '--'} + +
+
+ + {l10n.getString('tracker-infos-packets_received')} + + + {tracker?.device?.hardwareStatus?.packetsReceived ?? '--'} + +
{tracker && } diff --git a/gui/app/src/components/tracker/TrackerWifi.tsx b/gui/app/src/components/tracker/TrackerWifi.tsx index 3d9fc94ba..b6b2c2d3a 100644 --- a/gui/app/src/components/tracker/TrackerWifi.tsx +++ b/gui/app/src/components/tracker/TrackerWifi.tsx @@ -40,15 +40,20 @@ export function TrackerWifi({ {rssi} dBm )} - {showPacketLoss && packetsReceived != null && ( + {showPacketLoss && ( } > - {`${((packetLoss ?? 0) * 100).toFixed(0)}% (${packetsLost ?? 0} / ${packetsReceived})`} + + {packetLoss == null + ? '--' + : `${(packetLoss * 100).toFixed(0)}%${ + packetsLost != null && packetsReceived != null + ? ` (${packetsLost} / ${packetsReceived})` + : '' + }`} + )} diff --git a/gui/app/src/hooks/datafeed-config.ts b/gui/app/src/hooks/datafeed-config.ts index 9326437b5..b68504c87 100644 --- a/gui/app/src/hooks/datafeed-config.ts +++ b/gui/app/src/hooks/datafeed-config.ts @@ -71,7 +71,7 @@ export function useBonesDataFeedConfig() { boneMask.headPosition = true; boneMask.tailPosition = false; boneMask.angularVelocity = false; - boneMask.angularVelocity = false; + boneMask.linearVelocity = false; dataFeedConfig.boneMask = boneMask; dataFeedConfig.minimumTimeSinceLast = 1000 / feedMaxTps; return dataFeedConfig; diff --git a/gui/app/src/hooks/dongle-telemetry-feed.ts b/gui/app/src/hooks/dongle-telemetry-feed.ts index f6541c9e5..1d3d2173e 100644 --- a/gui/app/src/hooks/dongle-telemetry-feed.ts +++ b/gui/app/src/hooks/dongle-telemetry-feed.ts @@ -16,6 +16,17 @@ const TARGET_FPS = 20; const FRAME_INTERVAL_MS = 1000 / TARGET_FPS; // 50ms interval for smooth 20 FPS rolling const MAX_DISPLAY_POINTS = 250; // Cap rendered chart points to guarantee flat, constant CPU usage over time +/** Highest loss any tracker reports in this row, null when none of them reported one */ +function rowLossPeak(row: ChartRow): number | null { + let peak: number | null = null; + for (const key in row) { + if (!key.endsWith('_loss')) continue; + const val = row[key]; + if (val != null && (peak == null || val > peak)) peak = val; + } + return peak; +} + function downsampleChartRows( rows: ChartRow[], windowSec: number, @@ -25,16 +36,32 @@ function downsampleChartRows( const slotSec = Math.max(0.1, windowSec / maxPoints); const result: ChartRow[] = []; - let lastSlot = -1; - for (let i = 0; i < rows.length; i++) { - const row = rows[i]; - const slot = Math.floor(row.t / slotSec); - if (slot !== lastSlot) { - result.push(row); - lastSlot = slot; + // The lossiest row wins its slot, so decimation cannot swallow a spike + const flushSlot = (start: number, end: number) => { + let best = rows[start]; + let bestPeak = rowLossPeak(best); + for (let i = start + 1; i < end; i++) { + const peak = rowLossPeak(rows[i]); + if (peak != null && (bestPeak == null || peak > bestPeak)) { + best = rows[i]; + bestPeak = peak; + } + } + result.push(best); + }; + + let slotStart = 0; + let currentSlot = Math.floor(rows[0].t / slotSec); + for (let i = 1; i < rows.length; i++) { + const slot = Math.floor(rows[i].t / slotSec); + if (slot !== currentSlot) { + flushSlot(slotStart, i); + slotStart = i; + currentSlot = slot; } } + flushSlot(slotStart, rows.length); return result; } diff --git a/server/core/src/main/java/dev/slimevr/device/module.kt b/server/core/src/main/java/dev/slimevr/device/module.kt index 49671efbf..aa97cb5ce 100644 --- a/server/core/src/main/java/dev/slimevr/device/module.kt +++ b/server/core/src/main/java/dev/slimevr/device/module.kt @@ -30,15 +30,10 @@ data class DeviceState( val status: TrackerStatus, val origin: DeviceOrigin, val driverName: String? = null, - val packetsReceived: Long, - val packetsLost: Long, - val packetLossRecent: Float? = null, - val samples: List = emptyList(), ) sealed interface DeviceActions { data class Update(val transform: DeviceState.() -> DeviceState) : DeviceActions - data class PacketStats(val packetsReceived: Long, val packetsLost: Long) : DeviceActions } typealias DeviceContext = Context @@ -48,7 +43,17 @@ class Device( val context: DeviceContext, val appContext: AppContextProvider, ) { - fun getStatsForWindow(windowMs: Long, now: Long = System.currentTimeMillis()): WindowedDeviceStats = computeWindowedStats(context.state.value.samples, windowMs, now) + val packetHistory = DevicePacketHistory() + + fun recordPacketStats(received: Int, lost: Int, at: Long = System.currentTimeMillis()) { + packetHistory.record(DevicePacketSample(time = at, received = received, lost = lost)) + } + + fun recordRssi(rssi: Int, at: Long = System.currentTimeMillis()) { + packetHistory.recordRssi(DeviceRssiSample(time = at, rssi = rssi)) + } + + fun getStatsForWindow(windowMs: Long, now: Long = System.currentTimeMillis()): WindowedDeviceStats = packetHistory.statsForWindow(windowMs, now) fun startObserving() = context.observeAll(this) @@ -84,9 +89,6 @@ class Device( boardType = BoardType.UNKNOWN, firmwareVersion = null, firmwareDate = null, - packetsReceived = 0L, - packetsLost = 0L, - packetLossRecent = null, ) val context = Context.create( @@ -94,7 +96,6 @@ class Device( scope = scope, reducer = ::reduce, debugMiddleware = LoggingMiddleware( - block = setOf(DeviceActions.PacketStats::class), diffStyle = DiffStyle.MULTILINE, ), name = "Device[$address]", diff --git a/server/core/src/main/java/dev/slimevr/device/packet-history.kt b/server/core/src/main/java/dev/slimevr/device/packet-history.kt new file mode 100644 index 000000000..51c1fdba7 --- /dev/null +++ b/server/core/src/main/java/dev/slimevr/device/packet-history.kt @@ -0,0 +1,166 @@ +package dev.slimevr.device + +import kotlinx.coroutines.channels.BufferOverflow +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.SharedFlow +import kotlin.math.roundToInt + +private const val MAX_SAMPLE_RETENTION_MS = 12_000L +private const val MAX_SAMPLES = 4096 + +private const val MIN_PACKETS_FOR_LOSS = 20 + +private const val MAX_LOSS_LOOKBACK_MS = 5_000L + +/** Fewer dropped packets than this is jitter, not an outage anyone can see in the pose */ +private const val MIN_PACKETS_FOR_GAP = 3 + +/** + * Share of a sample that has to go missing before it reads as an outage. + * + * Sources deliver very different batches per sample: a datagram carries one packet, a dongle status + * report carries a second's worth. A flat count would mark a run on the first and ordinary radio + * jitter on the second. As a share, a run of three lost datagrams is most of its sample and still + * counts, while a handful lost out of two hundred does not. + */ +private const val MIN_SHARE_FOR_GAP = 0.25f + +data class DevicePacketSample( + val time: Long, + val received: Int, + val lost: Int, +) + +data class DeviceRssiSample( + val time: Long, + val rssi: Int, +) + +data class WindowedDeviceStats( + val rssiAvg: Int?, + val rssiMin: Int?, + val rssiMax: Int?, + val packetLoss: Float?, + val packetsReceived: Int?, + val packetsLost: Int?, +) + +/** + * Rolling packet telemetry for one device. + * + * This stays outside the device state, because it is too much data and changes to handle. + * Since nothing really need to watch for all these changes it is easier to have it outside the device. + */ +class DevicePacketHistory { + // Both series share one lock. Taking them separately would only add an ordering to get wrong + private val lock = Any() + + private val samples = ArrayDeque() + + private val rssiSamples = ArrayDeque() + + private val mutableLatest = MutableSharedFlow( + replay = 1, + onBufferOverflow = BufferOverflow.DROP_OLDEST, + ) + + /** One emission per packet, for consumers measuring the gap between two of them */ + val latest: SharedFlow = mutableLatest + + fun record(sample: DevicePacketSample) { + synchronized(lock) { + samples.addLast(sample) + // Age bounds the window, the count bounds a device reporting faster than expected + val cutoff = sample.time - MAX_SAMPLE_RETENTION_MS + while (samples.size > 1 && (samples.first().time < cutoff || samples.size > MAX_SAMPLES)) { + samples.removeFirst() + } + } + mutableLatest.tryEmit(sample) + } + + fun recordRssi(sample: DeviceRssiSample) { + synchronized(lock) { + if (rssiSamples.lastOrNull()?.rssi == sample.rssi) return + rssiSamples.addLast(sample) + val cutoff = sample.time - MAX_SAMPLE_RETENTION_MS + while (rssiSamples.size > 1 && (rssiSamples.first().time < cutoff || rssiSamples.size > MAX_SAMPLES)) { + rssiSamples.removeFirst() + } + } + } + + fun statsForWindow(windowMs: Long, now: Long): WindowedDeviceStats = synchronized(lock) { + computeWindowedStats(samples, rssiSamples, windowMs, now) + } +} + +/** + * Sums the packet increments falling in [windowMs] and reduces the signal strength readings over + * the same span to an average and a range. + */ +fun computeWindowedStats( + packetSamples: List, + rssiSamples: List, + windowMs: Long, + now: Long = System.currentTimeMillis(), +): WindowedDeviceStats { + val cutoff = now - windowMs + val lookbackCutoff = now - maxOf(windowMs, MAX_LOSS_LOOKBACK_MS) + var received = 0L + var lost = 0L + + for (index in packetSamples.indices.reversed()) { + val sample = packetSamples[index] + if (sample.time < lookbackCutoff) break + if (sample.time < cutoff && received + lost >= MIN_PACKETS_FOR_LOSS) break + received += sample.received + lost += sample.lost + } + + val total = received + lost + val packetLoss = if (total >= MIN_PACKETS_FOR_LOSS) lost.toFloat() / total.toFloat() else null + + var rssiMin = Int.MAX_VALUE + var rssiMax = Int.MIN_VALUE + var weightedSum = 0.0 + var weightTotal = 0L + val latestRssi = rssiSamples.lastOrNull()?.rssi + + var nextTime = now + for (index in rssiSamples.indices.reversed()) { + val sample = rssiSamples[index] + val to = minOf(nextTime, now) + nextTime = sample.time + + if (to <= cutoff) break + + val from = maxOf(sample.time, cutoff) + if (to <= from) continue + + if (sample.rssi < rssiMin) rssiMin = sample.rssi + if (sample.rssi > rssiMax) rssiMax = sample.rssi + weightedSum += sample.rssi.toDouble() * (to - from) + weightTotal += to - from + } + + val measured = weightTotal > 0 + return WindowedDeviceStats( + rssiAvg = if (measured) (weightedSum / weightTotal).roundToInt() else latestRssi, + rssiMin = if (measured) rssiMin else latestRssi, + rssiMax = if (measured) rssiMax else latestRssi, + packetLoss = packetLoss, + packetsReceived = if (packetLoss != null) received.toInt() else null, + packetsLost = if (packetLoss != null) lost.toInt() else null, + ) +} + +/** + * Whether a sample lost enough, and a large enough share of itself, to read as an outage rather + * than as the ordinary jitter of a radio link. + */ +fun isOutage(sample: DevicePacketSample): Boolean { + if (sample.lost < MIN_PACKETS_FOR_GAP) return false + val observed = sample.received + sample.lost + return observed > 0 && sample.lost.toFloat() / observed.toFloat() >= MIN_SHARE_FOR_GAP +} diff --git a/server/core/src/main/java/dev/slimevr/device/reducer.kt b/server/core/src/main/java/dev/slimevr/device/reducer.kt index 1a4ca398a..922aa89c1 100644 --- a/server/core/src/main/java/dev/slimevr/device/reducer.kt +++ b/server/core/src/main/java/dev/slimevr/device/reducer.kt @@ -1,34 +1,5 @@ package dev.slimevr.device -fun reduce(state: DeviceState, action: DeviceActions): DeviceState { - var nextState = when (action) { - is DeviceActions.Update -> action.transform(state) - - is DeviceActions.PacketStats -> state.copy( - packetsReceived = action.packetsReceived, - packetsLost = action.packetsLost, - ) - } - - if (nextState.signalStrength == null && nextState.packetsReceived == 0L && nextState.packetsLost == 0L) { - return nextState - } - - val prevSample = nextState.samples.lastOrNull() - if (prevSample == null || - prevSample.rssi != nextState.signalStrength || - prevSample.packetsReceived != nextState.packetsReceived || - prevSample.packetsLost != nextState.packetsLost - ) { - val now = System.currentTimeMillis() - val sample = DevicePacketSample( - time = now, - rssi = nextState.signalStrength, - packetsReceived = nextState.packetsReceived, - packetsLost = nextState.packetsLost, - ) - nextState = nextState.copy(samples = trimSamples(nextState.samples + sample, now)) - } - - return nextState +fun reduce(state: DeviceState, action: DeviceActions): DeviceState = when (action) { + is DeviceActions.Update -> action.transform(state) } diff --git a/server/core/src/main/java/dev/slimevr/device/sample-history.kt b/server/core/src/main/java/dev/slimevr/device/sample-history.kt deleted file mode 100644 index e875dc56a..000000000 --- a/server/core/src/main/java/dev/slimevr/device/sample-history.kt +++ /dev/null @@ -1,89 +0,0 @@ -package dev.slimevr.device - -import kotlin.math.roundToInt - -data class DevicePacketSample( - val time: Long, - val rssi: Int?, - val packetsReceived: Long?, - val packetsLost: Long?, -) - -data class WindowedDeviceStats( - val rssiAvg: Int?, - val rssiMin: Int?, - val rssiMax: Int?, - val packetLoss: Float?, -) - -private const val MAX_SAMPLE_RETENTION_MS = 5_000L -private const val MIN_SAMPLES_FOR_LOSS = 20 - -fun trimSamples(samples: List, now: Long = System.currentTimeMillis()): List { - val cutoff = now - MAX_SAMPLE_RETENTION_MS - val idx = samples.indexOfFirst { it.time >= cutoff } - return if (idx <= 0) samples else samples.subList(idx, samples.size) -} - -fun computeWindowedStats( - samples: List, - windowMs: Long = 1000L, - now: Long = System.currentTimeMillis(), -): WindowedDeviceStats { - if (samples.isEmpty()) { - return WindowedDeviceStats(null, null, null, null) - } - - val cutoff = now - windowMs - var windowSamples = samples.filter { it.time >= cutoff } - - if (windowSamples.isEmpty()) { - windowSamples = listOf(samples.last()) - } - - val rssiValues = windowSamples.mapNotNull { it.rssi } - val rssiAvg: Int? - val rssiMin: Int? - val rssiMax: Int? - - if (rssiValues.isNotEmpty()) { - rssiAvg = rssiValues.average().roundToInt() - rssiMin = rssiValues.minOrNull() - rssiMax = rssiValues.maxOrNull() - } else { - val latestRssi = samples.lastOrNull { it.rssi != null }?.rssi - rssiAvg = latestRssi - rssiMin = latestRssi - rssiMax = latestRssi - } - - var lossSamples = samples.filter { it.packetsReceived != null && it.packetsLost != null } - val recentLossSamples = lossSamples.filter { it.time >= cutoff } - if (recentLossSamples.size >= MIN_SAMPLES_FOR_LOSS) { - lossSamples = recentLossSamples - } else if (lossSamples.size > MIN_SAMPLES_FOR_LOSS) { - lossSamples = lossSamples.takeLast(MIN_SAMPLES_FOR_LOSS) - } - - val packetLoss: Float? = if (lossSamples.size >= 2) { - val first = lossSamples.first() - val last = lossSamples.last() - val dRec = last.packetsReceived!! - first.packetsReceived!! - val dLost = last.packetsLost!! - first.packetsLost!! - val total = dRec + dLost - if (total > 0) { - (dLost.toFloat() / total.toFloat()).coerceIn(0f, 1f) - } else { - 0f - } - } else { - null - } - - return WindowedDeviceStats( - rssiAvg = rssiAvg, - rssiMin = rssiMin, - rssiMax = rssiMax, - packetLoss = packetLoss, - ) -} diff --git a/server/core/src/main/java/dev/slimevr/hid/behaviours.kt b/server/core/src/main/java/dev/slimevr/hid/behaviours.kt index 88a643eea..e53207425 100644 --- a/server/core/src/main/java/dev/slimevr/hid/behaviours.kt +++ b/server/core/src/main/java/dev/slimevr/hid/behaviours.kt @@ -88,6 +88,7 @@ class HIDDeviceInfoBehaviour : HIDReceiverBehaviour { ) }, ) + device.recordRssi(packet.rssi) val tracker = receiver.getTracker(packet.hidId) ?: receiver.appContext.server.context.state.value.trackers.values @@ -141,23 +142,25 @@ class HIDRotationBehaviour : HIDReceiverBehaviour { class HIDBatteryBehaviour : HIDReceiverBehaviour { override fun observe(receiver: HIDReceiver) { receiver.packetEvents.on { packet -> - receiver.getDevice(packet.hidId)?.context?.dispatch( + val device = receiver.getDevice(packet.hidId) ?: return@on + device.context.dispatch( DeviceActions.Update { copy(batteryLevel = packet.batteryLevel, batteryVoltage = packet.batteryVoltage, signalStrength = packet.rssi) }, ) + device.recordRssi(packet.rssi) }.launchIn(receiver.context.scope) receiver.packetEvents.on { packet -> - receiver.getDevice(packet.hidId)?.context?.dispatch( - DeviceActions.Update { copy(signalStrength = packet.rssi) }, - ) + val device = receiver.getDevice(packet.hidId) ?: return@on + device.context.dispatch(DeviceActions.Update { copy(signalStrength = packet.rssi) }) + device.recordRssi(packet.rssi) }.launchIn(receiver.context.scope) receiver.packetEvents.on { packet -> - receiver.getDevice(packet.hidId)?.context?.dispatch( - DeviceActions.Update { copy(signalStrength = packet.rssi) }, - ) + val device = receiver.getDevice(packet.hidId) ?: return@on + device.context.dispatch(DeviceActions.Update { copy(signalStrength = packet.rssi) }) + device.recordRssi(packet.rssi) }.launchIn(receiver.context.scope) receiver.packetEvents.on { packet -> @@ -246,25 +249,13 @@ class HIDStatusBehaviour : HIDReceiverBehaviour { override fun observe(receiver: HIDReceiver) { receiver.packetEvents.on { packet -> val device = receiver.getDevice(packet.hidId) ?: return@on - val packetsReceived = packet.packetsReceived.toLong() - val packetsLost = packet.packetsLost.toLong() - - // Dispatched together so DeviceTelemetryBehaviour sees one atomic - // state change per packet, not two separate samples microseconds apart. if (receiver.getTracker(packet.hidId) != null) { device.context.dispatch( - DeviceActions.Update { - copy( - status = packet.status, - signalStrength = packet.rssi, - packetsReceived = packetsReceived, - packetsLost = packetsLost, - ) - }, + DeviceActions.Update { copy(status = packet.status, signalStrength = packet.rssi) }, ) - } else { - device.context.dispatch(DeviceActions.PacketStats(packetsReceived, packetsLost)) } + device.recordRssi(packet.rssi) + device.recordPacketStats(received = packet.packetsReceived, lost = packet.packetsLost) }.launchIn(receiver.context.scope) } } diff --git a/server/core/src/main/java/dev/slimevr/solarxr/datafeed/datafeed.kt b/server/core/src/main/java/dev/slimevr/solarxr/datafeed/datafeed.kt index cee1e6de1..6dd750483 100644 --- a/server/core/src/main/java/dev/slimevr/solarxr/datafeed/datafeed.kt +++ b/server/core/src/main/java/dev/slimevr/solarxr/datafeed/datafeed.kt @@ -85,14 +85,15 @@ private fun createTracker(device: DeviceState, tracker: TrackerState, trackerMas origin = tracker.origin, ) +private const val DEVICE_STATS_WINDOW_MS = 5_000L + private fun createDevice( device: Device, trackers: List, datafeedConfig: DataFeedConfig, - windowMs: Long, ): DeviceData { val deviceState = device.context.state.value - val stats = device.getStatsForWindow(windowMs) + val stats = device.getStatsForWindow(DEVICE_STATS_WINDOW_MS) val trackerMask = datafeedConfig.dataMask?.trackerData return DeviceData( @@ -105,8 +106,8 @@ private fun createDevice( rssi = stats.rssiAvg?.toShort(), rssiMin = stats.rssiMin?.toShort(), rssiMax = stats.rssiMax?.toShort(), - packetsReceived = deviceState.packetsReceived.toInt(), - packetsLost = deviceState.packetsLost.toInt(), + packetsReceived = stats.packetsReceived, + packetsLost = stats.packetsLost, packetLoss = stats.packetLoss, // TODO missing fields ), @@ -167,9 +168,8 @@ fun createDatafeedFrame( ): DataFeedMessageHeader { val serverState = server.context.state.value val trackers = serverState.trackers.values.map { it.context.state.value } - val windowMs = datafeedConfig.minimumTimeSinceLast.toLong().coerceAtLeast(50L) val devices = if (datafeedConfig.dataMask?.deviceData != null) { - serverState.devices.values.map { device -> createDevice(device, trackers, datafeedConfig, windowMs) } + serverState.devices.values.map { device -> createDevice(device, trackers, datafeedConfig) } } else { null } diff --git a/server/core/src/main/java/dev/slimevr/solarxr/rpc/telemetry.kt b/server/core/src/main/java/dev/slimevr/solarxr/rpc/telemetry.kt index 14ef791e8..49f00dc35 100644 --- a/server/core/src/main/java/dev/slimevr/solarxr/rpc/telemetry.kt +++ b/server/core/src/main/java/dev/slimevr/solarxr/rpc/telemetry.kt @@ -3,15 +3,12 @@ package dev.slimevr.solarxr.rpc import dev.slimevr.VRServer import dev.slimevr.device.Device import dev.slimevr.device.DevicePacketSample -import dev.slimevr.device.computeWindowedStats +import dev.slimevr.device.isOutage import dev.slimevr.solarxr.SolarXRBridge import dev.slimevr.solarxr.SolarXRBridgeBehaviour import kotlinx.coroutines.Job import kotlinx.coroutines.cancelAndJoin import kotlinx.coroutines.delay -import kotlinx.coroutines.flow.distinctUntilChanged -import kotlinx.coroutines.flow.filterNotNull -import kotlinx.coroutines.flow.map import kotlinx.coroutines.isActive import kotlinx.coroutines.launch import solarxr_protocol.datatypes.TrackerStatus @@ -21,25 +18,15 @@ import solarxr_protocol.rpc.TelemetryGapEvent import solarxr_protocol.rpc.TelemetryGapResponse import solarxr_protocol.rpc.TelemetrySample import solarxr_protocol.rpc.TelemetryUpdateResponse -import kotlin.math.max -private fun Device.isDeviceOk(server: VRServer): Boolean { - val deviceState = context.state.value +private fun isDeviceOk(device: Device, server: VRServer): Boolean { + val deviceState = device.context.state.value if (deviceState.status == TrackerStatus.OK) return true val deviceTrackers = server.context.state.value.trackers.values .filter { it.context.state.value.deviceId == deviceState.id } return deviceTrackers.any { it.context.state.value.status == TrackerStatus.OK } } -private fun Device.getMeasuredTps(server: VRServer): Double { - val deviceState = context.state.value - val deviceTrackers = server.context.state.value.trackers.values - .filter { it.context.state.value.deviceId == deviceState.id } - return deviceTrackers - .mapNotNull { it.context.state.value.tps.toDouble().takeIf { t -> t > 0.0 } } - .maxOrNull() ?: 100.0 -} - class TelemetryBehaviour(private val server: VRServer) : SolarXRBridgeBehaviour { override fun observe(receiver: SolarXRBridge) { var activeJobs: List = emptyList() @@ -62,24 +49,20 @@ class TelemetryBehaviour(private val server: VRServer) : SolarXRBridgeBehaviour for (id in deviceIds) { val device = server.getDevice(id.toInt()) ?: continue - val isOk = device.isDeviceOk(server) + val isOk = isDeviceOk(device, server) - val deviceSamples = device.context.state.value.samples - val stats = computeWindowedStats(deviceSamples, windowMs = 1000L, now = now) - val lastSample = deviceSamples.lastOrNull() - - val isTotalLoss = !isOk || (stats.packetLoss != null && stats.packetLoss >= 1.0f) + val stats = device.getStatsForWindow(windowMs = 1000L, now = now) samples.add( TelemetrySample( deviceId = id, time = now.toULong(), - rssi = if (isTotalLoss) null else stats.rssiAvg?.toShort(), - rssiMin = if (isTotalLoss) null else stats.rssiMin?.toShort(), - rssiMax = if (isTotalLoss) null else stats.rssiMax?.toShort(), - packetLossPct = if (!isOk) null else stats.packetLoss, - packetsLost = lastSample?.packetsLost?.toInt(), - packetsReceived = lastSample?.packetsReceived?.toInt(), + rssi = if (isOk) stats.rssiAvg?.toShort() else null, + rssiMin = if (isOk) stats.rssiMin?.toShort() else null, + rssiMax = if (isOk) stats.rssiMax?.toShort() else null, + packetLossPct = if (isOk) stats.packetLoss else null, + packetsLost = if (isOk) stats.packetsLost else null, + packetsReceived = if (isOk) stats.packetsReceived else null, ), ) } @@ -98,37 +81,28 @@ class TelemetryBehaviour(private val server: VRServer) : SolarXRBridgeBehaviour jobs.add( receiver.context.scope.launch { var prevSample: DevicePacketSample? = null - device.context.state - .map { it.samples.lastOrNull() } - .distinctUntilChanged() - .filterNotNull() + device.packetHistory.latest .collect { cur -> val p = prevSample - if (p != null && device.isDeviceOk(server)) { - val durationMs = (cur.time - p.time).coerceAtLeast(0L) - val hasPacketLoss = cur.packetsLost != null && p.packetsLost != null && cur.packetsLost > p.packetsLost - - val deviceTps = device.getMeasuredTps(server) - val expectedIntervalMs = 1000.0 / deviceTps.coerceIn(5.0, 200.0) - val thresholdMs = max(3.0 * expectedIntervalMs, 30.0) - - if (hasPacketLoss && durationMs.toDouble() >= thresholdMs) { - val lostDiff = (cur.packetsLost - p.packetsLost).toUInt() - receiver.sendRpc( - TelemetryGapResponse( - events = listOf( - TelemetryGapEvent( - deviceId = id, - time = cur.time.toULong(), - durationMs = durationMs.toUInt(), - packetsLost = lostDiff, - ), - ), - ), - ) - } - } prevSample = cur + if (p == null || !isDeviceOk(device, server)) return@collect + + if (!isOutage(cur)) return@collect + + val durationMs = (cur.time - p.time).coerceAtLeast(0L) + + receiver.sendRpc( + TelemetryGapResponse( + events = listOf( + TelemetryGapEvent( + deviceId = id, + time = cur.time.toULong(), + durationMs = durationMs.toUInt(), + packetsLost = cur.lost.toUInt(), + ), + ), + ), + ) } }, ) diff --git a/server/core/src/main/java/dev/slimevr/udp/behaviours.kt b/server/core/src/main/java/dev/slimevr/udp/behaviours.kt index be95aeb3f..5f39231ab 100644 --- a/server/core/src/main/java/dev/slimevr/udp/behaviours.kt +++ b/server/core/src/main/java/dev/slimevr/udp/behaviours.kt @@ -45,33 +45,33 @@ class PacketBehaviour : UDPConnectionBehaviour { } } +/** + * Packets missing between [last] and [num]. A number at or below the mark arrived, so it is not + * loss, whether it is a duplicate or one that overtook its neighbours. + */ +internal fun packetsLostBetween(last: Long?, num: Long): Int = when { + last == null || num <= last -> 0 + else -> (num - last - 1).coerceAtMost(Int.MAX_VALUE.toLong()).toInt() +} + class PacketLossBehaviour : UDPConnectionBehaviour { override fun observe(receiver: UDPConnection) { - var totalPacketsReceived = 0L - var acceptedPackets = 0L - var lastPacketCounterReset = System.currentTimeMillis() - var lastPacketNumber = 0L + var lastPacketNumber: Long? = null + + // A tracker restarts its counter when it reconnects + receiver.context.state + .distinctUntilChangedBy { it.lastHandshake } + .onEach { lastPacketNumber = null } + .launchIn(receiver.context.scope) receiver.packetEvents.on> { packet -> val num = packet.packetNumber ?: return@on - val now = System.currentTimeMillis() + val last = lastPacketNumber + val lost = packetsLostBetween(last, num) + // Highest number seen wins, so a late arrival cannot drag the mark backwards + lastPacketNumber = if (last == null) num else maxOf(last, num) - if (now - lastPacketCounterReset >= 10_000L) { - totalPacketsReceived = 0L - acceptedPackets = 0L - lastPacketCounterReset = now - } - - totalPacketsReceived++ - val accepted = num == 0L || num > lastPacketNumber - if (accepted) { - lastPacketNumber = num - acceptedPackets++ - } - - receiver.getDevice()?.context?.dispatch( - DeviceActions.PacketStats(packetsReceived = totalPacketsReceived, packetsLost = totalPacketsReceived - acceptedPackets), - ) + receiver.getDevice()?.recordPacketStats(received = 1, lost = lost, at = System.currentTimeMillis()) }.launchIn(receiver.context.scope) } } @@ -273,6 +273,7 @@ class DeviceStatsBehaviour : UDPConnectionBehaviour { receiver.packetEvents.onPacket { event -> val device = receiver.getDevice() ?: return@onPacket device.context.dispatch(DeviceActions.Update { copy(signalStrength = event.data.signal) }) + device.recordRssi(event.data.signal) }.launchIn(receiver.context.scope) } } diff --git a/server/core/src/main/java/dev/slimevr/udp/connection-reducer.kt b/server/core/src/main/java/dev/slimevr/udp/connection-reducer.kt index f36d4986c..67d5b7947 100644 --- a/server/core/src/main/java/dev/slimevr/udp/connection-reducer.kt +++ b/server/core/src/main/java/dev/slimevr/udp/connection-reducer.kt @@ -9,7 +9,12 @@ fun reduce(state: UDPConnectionState, action: UDPConnectionActions): UDPConnecti is UDPConnectionActions.StartPing -> state.copy(lastPing = state.lastPing.copy(startTime = action.startTime, id = action.pingId)) - is UDPConnectionActions.Handshake -> state.copy(didHandshake = true, lastHandshake = System.currentTimeMillis(), deviceId = action.deviceId) + is UDPConnectionActions.Handshake -> state.copy( + didHandshake = true, + lastHandshake = System.currentTimeMillis(), + deviceId = action.deviceId, + lastPacketNum = 0, + ) is UDPConnectionActions.AssignTracker -> state.copy(trackerIds = state.trackerIds + action.trackerId) diff --git a/server/core/src/test/java/dev/slimevr/device/PacketStatsTest.kt b/server/core/src/test/java/dev/slimevr/device/PacketStatsTest.kt new file mode 100644 index 000000000..14ca05797 --- /dev/null +++ b/server/core/src/test/java/dev/slimevr/device/PacketStatsTest.kt @@ -0,0 +1,133 @@ +package dev.slimevr.device + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNull +import kotlin.test.assertTrue + +private const val NOW = 100_000L + +private fun sample(time: Long, received: Int, lost: Int) = DevicePacketSample(time = time, received = received, lost = lost) +private fun rssi(time: Long, rssi: Int) = DeviceRssiSample(time = time, rssi = rssi) +private fun run(count: Int, received: Int, lost: Int) = (0 until count).map { index -> sample(NOW - (count - 1 - index), received, lost) } + +private fun statsOf( + packets: List = emptyList(), + rssis: List = emptyList(), + windowMs: Long = 1000L, +) = computeWindowedStats(packets, rssis, windowMs, NOW) + +class PacketStatsTest { + @Test + fun `too few packets means nothing measured`() { + // 19 arrivals, one under the gate. A tracker that is off must not read as a clean link + val stats = statsOf(packets = run(19, received = 1, lost = 0)) + assertNull(stats.packetLoss) + assertNull(stats.packetsReceived) + assertNull(stats.packetsLost) + } + + @Test + fun `loss is the fraction of packets that went missing`() { + // 100 arrivals, and one of them followed a gap of 25 + val stats = statsOf(packets = run(100, received = 1, lost = 0) + sample(NOW, received = 1, lost = 25)) + assertEquals(25f / 126f, stats.packetLoss) + assertEquals(101, stats.packetsReceived) + assertEquals(25, stats.packetsLost) + } + + @Test + fun `loss always lands inside zero to one`() { + val cases = listOf( + listOf(sample(NOW, received = 200, lost = 1)), + listOf(sample(NOW, received = 1, lost = 200)), + listOf(sample(NOW - 5, received = 0, lost = 30), sample(NOW, received = 30, lost = 0)), + run(50, received = 3, lost = 1), + ) + for (packets in cases) { + val loss = statsOf(packets = packets).packetLoss + assertTrue(loss != null && loss in 0f..1f, "loss out of range: $loss") + } + } + + @Test + fun `a window holding enough packets ignores everything older`() { + // Heavy loss two seconds back, a clean second since + val old = (0 until 40).map { sample(NOW - 2039 + it, received = 1, lost = 5) } + val stats = statsOf(packets = old + run(40, received = 1, lost = 0)) + assertEquals(0f, stats.packetLoss) + assertEquals(40, stats.packetsReceived) + } + + @Test + fun `a thin window reaches back for a slow reporter`() { + // One batched report a second and a half back, which the window alone would miss + val stats = statsOf(packets = listOf(sample(NOW - 1500, received = 190, lost = 10))) + assertEquals(0.05f, stats.packetLoss) + assertEquals(190, stats.packetsReceived) + } + + @Test + fun `the reach back stops at the lookback`() { + val packets = (0 until 40).map { sample(NOW - 20_000 + it, received = 1, lost = 1) } + assertNull(statsOf(packets = packets).packetLoss) + } + + @Test + fun `the range spans every reading in the window`() { + val rssis = listOf(rssi(NOW - 2500, -60), rssi(NOW - 2000, -40), rssi(NOW - 500, -50)) + val stats = statsOf(rssis = rssis, windowMs = 3000L) + assertEquals(-60, stats.rssiMin) + assertEquals(-40, stats.rssiMax) + } + + @Test + fun `the average weighs a reading by how long it stood`() { + // -50 held for 9 of the 10 seconds, -70 for the last one + val rssis = listOf(rssi(NOW - 10_000, -50), rssi(NOW - 1000, -70)) + val stats = statsOf(rssis = rssis, windowMs = 10_000L) + assertEquals(-52, stats.rssiAvg) + assertEquals(-70, stats.rssiMin) + assertEquals(-50, stats.rssiMax) + } + + @Test + fun `a run of lost datagrams is an outage`() { + // One datagram per sample, so a run is nearly all of what the sample saw + assertTrue(isOutage(sample(NOW, received = 1, lost = 3))) + assertTrue(isOutage(sample(NOW, received = 1, lost = 30))) + // Under the floor, a share alone would have marked these + assertFalse(isOutage(sample(NOW, received = 1, lost = 1))) + assertFalse(isOutage(sample(NOW, received = 1, lost = 2))) + } + + @Test + fun `ordinary jitter in a batched report is not an outage`() { + // A second's worth of packets losing a handful is a healthy link + assertFalse(isOutage(sample(NOW, received = 197, lost = 3))) + assertFalse(isOutage(sample(NOW, received = 190, lost = 10))) + assertFalse(isOutage(sample(NOW, received = 160, lost = 40))) + } + + @Test + fun `a batched report losing a quarter of itself is an outage`() { + assertTrue(isOutage(sample(NOW, received = 150, lost = 50))) + assertTrue(isOutage(sample(NOW, received = 0, lost = 200))) + } + + @Test + fun `a repeated reading is not stored twice`() { + val history = DevicePacketHistory() + history.recordRssi(rssi(NOW - 2000, -70)) + history.recordRssi(rssi(NOW - 1500, -70)) + history.recordRssi(rssi(NOW - 1000, -40)) + repeat(30) { index -> history.record(sample(NOW - 29 + index, received = 1, lost = 0)) } + + // Dropping the repeat leaves -70 covering the stretch up to the change, not a slice of it + val stats = history.statsForWindow(windowMs = 3000L, now = NOW) + assertEquals(-70, stats.rssiMin) + assertEquals(-40, stats.rssiMax) + assertEquals(30, stats.packetsReceived) + } +} diff --git a/server/core/src/test/java/dev/slimevr/udp/PacketLossTest.kt b/server/core/src/test/java/dev/slimevr/udp/PacketLossTest.kt new file mode 100644 index 000000000..6846f5ddf --- /dev/null +++ b/server/core/src/test/java/dev/slimevr/udp/PacketLossTest.kt @@ -0,0 +1,50 @@ +package dev.slimevr.udp + +import kotlin.test.Test +import kotlin.test.assertEquals + +/** Feeds a sequence through the gap math the way PacketLossBehaviour does, returning total loss */ +private fun lossOver(numbers: List): Int { + var last: Long? = null + var lost = 0 + for (num in numbers) { + lost += packetsLostBetween(last, num) + last = if (last == null) num else maxOf(last, num) + } + return lost +} + +class PacketLossTest { + @Test + fun `a gap counts the packets that never arrived`() { + assertEquals(0, packetsLostBetween(last = 10L, num = 11L)) + assertEquals(1, packetsLostBetween(last = 10L, num = 12L)) + assertEquals(9, packetsLostBetween(last = 10L, num = 20L)) + } + + @Test + fun `gaps across a sequence add up`() { + // First packet is not a gap, then 5 after 1 misses 3, then 10 after 6 misses 3 + assertEquals(6, lossOver(listOf(1L, 5L, 6L, 10L))) + assertEquals(0, lossOver((1L..100L).toList())) + } + + @Test + fun `a duplicate or a late packet is not loss`() { + assertEquals(0, packetsLostBetween(last = 10L, num = 10L)) + assertEquals(0, packetsLostBetween(last = 20L, num = 15L)) + // The gap was already charged when 20 arrived, so replaying 15 adds nothing + assertEquals(9, lossOver(listOf(10L, 20L, 15L, 21L))) + } + + @Test + fun `a long dropout reports every packet it missed`() { + // Six seconds of silence at 100 TPS, counted in full rather than written off + assertEquals(599, packetsLostBetween(last = 5_000L, num = 5_600L)) + } + + @Test + fun `a nonsense packet number cannot produce a negative count`() { + assertEquals(Int.MAX_VALUE, packetsLostBetween(last = 0L, num = Long.MAX_VALUE)) + } +}