Fix/wedged sync recovery (#1926)

<!-- Please read
https://github.com/SableClient/Sable/blob/dev/CONTRIBUTING.md before
submitting your pull request -->

### Description

<!-- Please include a summary of the change. Please also include
relevant motivation and context. List any dependencies that are required
for this change. -->

Fixes #

#### Type of change

- [ ] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to not work as expected)
- [ ] This change requires a documentation update

### Checklist:

- [ ] My code follows the style guidelines of this project
- [ ] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have made corresponding changes to the documentation
- [ ] My changes generate no new warnings

### AI disclosure:

- [ ] Partially AI assisted (clarify which code was AI assisted and
briefly explain what it does).
- [ ] Fully AI generated (explain what all the generated code does in
moderate detail).

<!-- Write any explanation required here, but do not generate the
explanation using AI!! You must prove you understand what the code in
this PR does. -->
This commit is contained in:
Erwan Leboucher authored and GitHub committed 2026-08-23 15:02:23 +02:00
commit be5258eec2
10 files changed
+260 -21

No files matched your search

+1 -2
View File
@@ -2,7 +2,6 @@ name: Cloudflare Worker Preview Deploy
on:
pull_request:
types: [labeled]
paths:
- 'src/**'
- 'index.html'
@@ -30,7 +29,7 @@ concurrency:
jobs:
deploy:
if: (github.event_name == 'pull_request' && github.event.label.name == 'deploy-preview' && github.event.pull_request.head.repo.full_name == github.repository && github.head_ref != 'release') || github.event_name == 'push'
if: (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.head_ref != 'release') || github.event_name == 'push'
runs-on: ubuntu-latest
permissions:
contents: read
@@ -23,6 +23,7 @@ import {
parseUnifiedPushMessage,
} from './UnifiedPushMessageListener';
import { addPluginListener, invoke } from '@tauri-apps/api/core';
import { getSlidingSyncManager } from '$client/initMatrix';
import type { PushTransportConfig } from './NotificationTransport';
import { getTauriNotificationsApi, isMobileTauri } from './TauriNotificationsApiClient';
import {
@@ -1063,7 +1064,9 @@ export async function listenForUnifiedPushMessages(getSettings: () => Notificati
const listener = await addPluginListener('notifications', 'push-message', (data: unknown) => {
const notification = parseUnifiedPushMessage(data);
if (notification) dispatch(notification);
if (!notification) return;
getSlidingSyncManager(getSettings().mx)?.resumeForPush();
dispatch(notification);
});
try {
@@ -100,6 +100,27 @@ describe('useBackgroundSyncPause', () => {
expect(resume).toHaveBeenCalled();
});
it('resumes on window focus when visibilitychange was missed', () => {
setVisibility('hidden');
renderHook(() => useBackgroundSyncPause({ clientRunning: true } as never));
resume.mockClear();
setVisibility('visible');
window.dispatchEvent(new Event('focus'));
expect(resume).toHaveBeenCalled();
});
it('does not resume on a focus event while still hidden', () => {
setVisibility('hidden');
renderHook(() => useBackgroundSyncPause({ clientRunning: true } as never));
resume.mockClear();
window.dispatchEvent(new Event('focus'));
expect(resume).not.toHaveBeenCalled();
});
it('keeps polling when a browser tab is hidden', () => {
mockIsMobileTauri.value = false;
setVisibility('visible');
+8
View File
@@ -28,13 +28,21 @@ export const useBackgroundSyncPause = (mx: MatrixClient | undefined): void => {
else manager.resume();
};
// Nothing re-arms a paused transport, so a missed `visible` event strands it until a
// restart. `focus` also fires in the Android webview.
const resumeIfVisible = () => {
if (document.visibilityState !== 'hidden') resume();
};
document.addEventListener('visibilitychange', applyVisibility);
window.addEventListener('focus', resumeIfVisible);
const unlisten = isTauri() ? listen(TauriEvent.WINDOW_RESUMED, resume) : undefined;
applyVisibility();
return () => {
document.removeEventListener('visibilitychange', applyVisibility);
window.removeEventListener('focus', resumeIfVisible);
unlisten?.then((off) => off());
resume();
};
+61 -7
View File
@@ -1,16 +1,19 @@
import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest';
import { renderHook } from '@testing-library/react';
const { nudgeReconnect, setOnline, listenOff, listen, mockIsTauri } = vi.hoisted(() => ({
nudgeReconnect: vi.fn<(mx: unknown, reason: string, opts?: { force?: boolean }) => boolean>(),
setOnline: vi.fn<() => void>(),
listenOff: vi.fn<() => void>(),
listen: vi.fn<(_event: string, _cb: () => void) => Promise<() => void>>(),
mockIsTauri: { value: false },
}));
const { nudgeReconnect, abortClassicSyncPoll, setOnline, listenOff, listen, mockIsTauri } =
vi.hoisted(() => ({
nudgeReconnect: vi.fn<(mx: unknown, reason: string, opts?: { force?: boolean }) => boolean>(),
abortClassicSyncPoll: vi.fn<(mx: unknown) => boolean>(),
setOnline: vi.fn<() => void>(),
listenOff: vi.fn<() => void>(),
listen: vi.fn<(_event: string, _cb: () => void) => Promise<() => void>>(),
mockIsTauri: { value: false },
}));
vi.mock('$client/reconnect', () => ({
nudgeReconnect,
abortClassicSyncPoll,
}));
vi.mock('@tanstack/react-query', () => ({
@@ -43,6 +46,7 @@ describe('useNetworkRecovery', () => {
vi.useFakeTimers();
vi.setSystemTime(0);
nudgeReconnect.mockReset().mockReturnValue(true);
abortClassicSyncPoll.mockReset().mockReturnValue(true);
setOnline.mockReset();
listenOff.mockReset();
listen.mockReset().mockResolvedValue(listenOff);
@@ -172,6 +176,56 @@ describe('useNetworkRecovery', () => {
expect(nudgeReconnect).toHaveBeenCalledTimes(2);
});
describe('wedged classic sync escalation', () => {
it('aborts the poll after three consecutive nudges that could not bite', () => {
nudgeReconnect.mockReturnValue(false);
renderHook(() => useNetworkRecovery({ clientRunning: true } as never));
vi.advanceTimersByTime(80_000);
vi.advanceTimersByTime(10_000);
expect(abortClassicSyncPoll).not.toHaveBeenCalled();
vi.advanceTimersByTime(10_000);
expect(abortClassicSyncPoll).toHaveBeenCalledOnce();
});
it('never escalates while nudges keep biting', () => {
renderHook(() => useNetworkRecovery({ clientRunning: true } as never));
vi.advanceTimersByTime(200_000);
expect(nudgeReconnect).toHaveBeenCalled();
expect(abortClassicSyncPoll).not.toHaveBeenCalled();
});
it('forgets dead nudges once a Sync arrives', () => {
nudgeReconnect.mockReturnValue(false);
renderHook(() => useNetworkRecovery({ clientRunning: true } as never));
vi.advanceTimersByTime(80_000);
vi.advanceTimersByTime(10_000);
if (syncCallback) syncCallback();
vi.advanceTimersByTime(80_000);
vi.advanceTimersByTime(10_000);
expect(abortClassicSyncPoll).not.toHaveBeenCalled();
vi.advanceTimersByTime(10_000);
expect(abortClassicSyncPoll).toHaveBeenCalledOnce();
});
it('resets the stall clock so one wedge escalates once', () => {
nudgeReconnect.mockReturnValue(false);
renderHook(() => useNetworkRecovery({ clientRunning: true } as never));
vi.advanceTimersByTime(100_000);
expect(abortClassicSyncPoll).toHaveBeenCalledOnce();
vi.advanceTimersByTime(10_000);
expect(abortClassicSyncPoll).toHaveBeenCalledOnce();
});
});
describe('foreground nudge verification', () => {
it('retries once past the throttle when no Sync follows the nudge', () => {
renderHook(() => useNetworkRecovery({ clientRunning: true } as never));
+21 -3
View File
@@ -4,7 +4,7 @@ import { TauriEvent, listen } from '@tauri-apps/api/event';
import { isTauri } from '@tauri-apps/api/core';
import type { MatrixClient } from '$types/matrix-sdk';
import type { NudgeReason } from '$client/reconnect';
import { nudgeReconnect } from '$client/reconnect';
import { abortClassicSyncPoll, nudgeReconnect } from '$client/reconnect';
import { useSyncState } from './useSyncState';
// Sync fires on every poll response: at most 45s apart on sliding, 30s classic.
@@ -14,10 +14,13 @@ const STALL_CHECK_INTERVAL_MS = 10_000;
const VISIBLE_STALE_MS = 30_000;
// After a foreground nudge, verify a Sync arrived within this window; retry once otherwise.
const VERIFY_AFTER_NUDGE_MS = 3_000;
// Dead stalled nudges before aborting; spares a warm resume whose poll survived.
const WEDGED_NUDGE_ATTEMPTS = 3;
export const useNetworkRecovery = (mx: MatrixClient | undefined): void => {
const lastSyncAtRef = useRef(Date.now());
const verifyTimerRef = useRef<number | undefined>(undefined);
const deadNudgesRef = useRef(0);
const cancelVerify = useCallback(() => {
if (verifyTimerRef.current === undefined) return;
@@ -38,6 +41,7 @@ export const useNetworkRecovery = (mx: MatrixClient | undefined): void => {
mx,
useCallback(() => {
lastSyncAtRef.current = Date.now();
deadNudgesRef.current = 0;
cancelVerify();
}, [cancelVerify])
);
@@ -78,8 +82,22 @@ export const useNetworkRecovery = (mx: MatrixClient | undefined): void => {
document.addEventListener('visibilitychange', onVisible);
const stallCheck = window.setInterval(() => {
if (Date.now() - lastSyncAtRef.current < SYNC_STALL_MS) return;
if (nudge('stalled')) lastSyncAtRef.current = Date.now();
if (Date.now() - lastSyncAtRef.current < SYNC_STALL_MS) {
deadNudgesRef.current = 0;
return;
}
if (nudge('stalled')) {
lastSyncAtRef.current = Date.now();
deadNudgesRef.current = 0;
return;
}
// Throttled counts too: no request went out, and the 10s tick clears the 3s throttle.
deadNudgesRef.current += 1;
if (deadNudgesRef.current < WEDGED_NUDGE_ATTEMPTS) return;
deadNudgesRef.current = 0;
// Give the keepalive poke its own stall window before escalating again.
lastSyncAtRef.current = Date.now();
abortClassicSyncPoll(mx);
}, STALL_CHECK_INTERVAL_MS);
const unlisten = isTauri()
+27 -2
View File
@@ -18,14 +18,14 @@ vi.mock('$utils/debugLogger', () => ({
}),
}));
import { nudgeReconnect } from './reconnect';
import { abortClassicSyncPoll, nudgeReconnect } from './reconnect';
function stubMx(
overrides: Partial<{ clientRunning: boolean; retryImmediately: ReturnType<typeof vi.fn> }> = {}
) {
return {
clientRunning: overrides.clientRunning ?? true,
retryImmediately: overrides.retryImmediately ?? vi.fn<() => boolean>(),
retryImmediately: overrides.retryImmediately ?? vi.fn<() => boolean>().mockReturnValue(true),
};
}
@@ -59,6 +59,14 @@ describe('nudgeReconnect', () => {
expect(slidingSyncResend).not.toHaveBeenCalled();
});
it('reports false when classic retryImmediately() could not bite', () => {
getSlidingSyncManager.mockReturnValue(undefined);
const mx = stubMx({ retryImmediately: vi.fn<() => boolean>().mockReturnValue(false) });
expect(nudgeReconnect(mx as never, 'resumed')).toBe(false);
expect(mx.retryImmediately).toHaveBeenCalledOnce();
});
it('returns false and does nothing when client is not running', () => {
getSlidingSyncManager.mockReturnValue({ slidingSync: { resend: slidingSyncResend } } as never);
const mx = stubMx({ clientRunning: false });
@@ -109,3 +117,20 @@ describe('nudgeReconnect', () => {
expect(slidingSyncResend).toHaveBeenCalledTimes(2);
});
});
describe('abortClassicSyncPoll', () => {
it('aborts the in-flight poll and swaps in a usable controller', () => {
const first = new AbortController();
const mx = { syncApi: { abortController: first } };
expect(abortClassicSyncPoll(mx as never)).toBe(true);
expect(first.signal.aborted).toBe(true);
expect(mx.syncApi.abortController).not.toBe(first);
expect(mx.syncApi.abortController.signal.aborted).toBe(false);
});
it('reports false when the transport holds no abort controller', () => {
expect(abortClassicSyncPoll({ syncApi: {} } as never)).toBe(false);
expect(abortClassicSyncPoll({} as never)).toBe(false);
});
});
+18 -4
View File
@@ -11,9 +11,8 @@ const lastNudgeAt = new WeakMap<MatrixClient, number>();
export type NudgeReason = 'online' | 'visible' | 'resumed' | 'stalled' | 'retry';
/**
* Sliding sync aborts the long-poll and re-requests with no backoff sleep.
* Classic has no public abort, so this only collapses an existing keepalive
* backoff — a hung classic poll still waits out its 110s localTimeout.
* Sliding sync aborts the long-poll and re-requests with no backoff sleep. Classic returns
* false unless the SDK already entered keepalive backoff, which a hung poll never reaches.
*/
export const nudgeReconnect = (
mx: MatrixClient,
@@ -28,15 +27,30 @@ export const nudgeReconnect = (
lastNudgeAt.set(mx, now);
const slidingSync = getSlidingSyncManager(mx)?.slidingSync;
let nudged: boolean;
if (slidingSync) {
slidingSync.resend();
nudged = true;
} else {
mx.retryImmediately();
nudged = mx.retryImmediately();
}
debugLog.info('network', `Nudged sync transport (${reason})`, {
reason,
transport: slidingSync ? 'sliding' : 'classic',
nudged,
});
return nudged;
};
type ClassicSyncApi = { abortController?: AbortController };
export const abortClassicSyncPoll = (mx: MatrixClient): boolean => {
const syncApi = (mx as unknown as { syncApi?: ClassicSyncApi }).syncApi;
if (!syncApi?.abortController) return false;
syncApi.abortController.abort();
syncApi.abortController = new AbortController();
debugLog.warn('network', 'Aborted a wedged classic sync poll');
return true;
};
+66
View File
@@ -1794,6 +1794,72 @@ describe('SlidingSyncManager pause/resume', () => {
await expect(manager.waitForResume()).resolves.toBeUndefined();
});
it('resumeForPush() lifts the pause and parks once to-device comes back empty', () => {
const manager = makeManager(makeMockMx());
manager.attach();
manager.pause();
expect(manager.resumeForPush()).toBe(true);
expect(manager.isPaused()).toBe(false);
fireLifecycle(SlidingSyncState.Complete, { extensions: { to_device: { events: [] } } });
expect(manager.isPaused()).toBe(true);
});
it('keeps draining while to-device still carries events', () => {
const manager = makeManager(makeMockMx());
manager.attach();
manager.pause();
manager.resumeForPush();
const withKeys = { extensions: { to_device: { events: [{ type: 'm.room.key' }] } } };
fireLifecycle(SlidingSyncState.Complete, withKeys);
expect(manager.isPaused()).toBe(false);
fireLifecycle(SlidingSyncState.Complete, { extensions: { to_device: { events: [] } } });
expect(manager.isPaused()).toBe(true);
});
it('parks a never-draining queue once the poll budget runs out', () => {
const MAX_PUSH_DRAIN_POLLS = 5;
const manager = makeManager(makeMockMx());
manager.attach();
manager.pause();
manager.resumeForPush();
const withKeys = { extensions: { to_device: { events: [{ type: 'm.room.key' }] } } };
for (let i = 0; i < MAX_PUSH_DRAIN_POLLS; i += 1) {
expect(manager.isPaused()).toBe(false);
fireLifecycle(SlidingSyncState.Complete, withKeys);
}
expect(manager.isPaused()).toBe(true);
});
it('lets a real resume outrank an in-flight drain', () => {
const manager = makeManager(makeMockMx());
manager.attach();
manager.pause();
manager.resumeForPush();
manager.resume();
fireLifecycle(SlidingSyncState.Complete, { extensions: { to_device: { events: [] } } });
expect(manager.isPaused()).toBe(false);
});
it('resumeForPush() is a no-op when sync is already running', () => {
const manager = makeManager(makeMockMx());
manager.attach();
expect(manager.resumeForPush()).toBe(false);
fireLifecycle(SlidingSyncState.Complete, { extensions: { to_device: { events: [] } } });
expect(manager.isPaused()).toBe(false);
});
it('silences the poll watchdog while paused', () => {
vi.useFakeTimers();
try {
+33 -2
View File
@@ -49,6 +49,9 @@ const DEFAULT_POLL_TIMEOUT_MS = 45000;
const SDK_CLIENT_TIMEOUT_BUFFER_MS = 10_000;
const POLL_DEADLINE_MARGIN_MS = 20_000;
// The SDK's to_device extension takes 100 events per response, so a backlog needs several.
const MAX_PUSH_DRAIN_POLLS = 5;
const ACTIVE_ROOM_SUBSCRIPTION_KEY = 'active_room';
const CALL_ROOM_SUBSCRIPTION_KEY = 'call_room';
const SIDEBAR_ROOM_SUBSCRIPTION_KEY = 'sidebar_room';
@@ -702,6 +705,8 @@ export class SlidingSyncManager {
private paused = false;
private pushDrainPollsLeft = 0;
private readonly resumeWaiters = new Set<() => void>();
/** Span covering the period from attach() to the first successful complete cycle. */
@@ -803,6 +808,7 @@ export class SlidingSyncManager {
this.roomDataAwaitingSyncCompletion.clear();
this.syncCount += 1;
this.settlePushDrain(resp);
// A subscription that saw no room data is settled once a full cycle has
// completed after the one it was requested in: the server had nothing to
@@ -1004,6 +1010,7 @@ export class SlidingSyncManager {
* instead.
*/
public pause(): void {
this.pushDrainPollsLeft = 0;
if (this.paused || this.disposed) return;
this.paused = true;
globalThis.clearTimeout(this.pollWatchdogTimer);
@@ -1012,14 +1019,37 @@ export class SlidingSyncManager {
debugLog.info('sync', 'Sliding sync paused');
}
public resume(): void {
if (!this.paused) return;
private liftPause(): void {
this.paused = false;
this.releaseResumeWaiters();
this.armPollWatchdog();
}
public resume(): void {
this.pushDrainPollsLeft = 0;
if (!this.paused) return;
this.liftPause();
debugLog.info('sync', 'Sliding sync resumed');
}
/** Poll on while backgrounded, then park: to-device only arrives over a live `/sync`. */
public resumeForPush(): boolean {
if (this.disposed || !this.paused) return false;
this.pushDrainPollsLeft = MAX_PUSH_DRAIN_POLLS;
this.liftPause();
debugLog.info('sync', 'Sliding sync resumed to drain to-device after a push');
return true;
}
private settlePushDrain(resp: MSC3575SlidingSyncResponse): void {
if (this.pushDrainPollsLeft === 0) return;
const toDevice = resp.extensions?.to_device as { events?: unknown[] } | undefined;
const drained = (toDevice?.events?.length ?? 0) === 0;
this.pushDrainPollsLeft -= 1;
if (!drained && this.pushDrainPollsLeft > 0) return;
this.pause();
}
public isPaused(): boolean {
return this.paused;
}
@@ -1067,6 +1097,7 @@ export class SlidingSyncManager {
this.disposed = true;
this.paused = false;
this.pushDrainPollsLeft = 0;
this.releaseResumeWaiters();
globalThis.clearTimeout(this.pollWatchdogTimer);
this.pollWatchdogTimer = undefined;