Fix sync cache recovery and push diagnostics (#2065)

<!-- Please read
https://github.com/SableClient/Sable/blob/dev/CONTRIBUTING.md before
submitting your pull request -->

### Description

<!-- Please include a summary of the change. Please also include
relevant motivation and context. List any dependencies that are required
for this change. -->

Fixes #

#### Type of change

- [ ] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to not work as expected)
- [ ] This change requires a documentation update

### Checklist:

- [ ] My code follows the style guidelines of this project
- [ ] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have made corresponding changes to the documentation
- [ ] My changes generate no new warnings

### AI disclosure:

- [ ] Partially AI assisted (clarify which code was AI assisted and
briefly explain what it does).
- [ ] Fully AI generated (explain what all the generated code does in
moderate detail).

<!-- Write any explanation required here, but do not generate the
explanation using AI!! You must prove you understand what the code in
this PR does. -->
This commit is contained in:
Erwan Leboucher authored and GitHub committed 2026-09-08 12:04:34 +02:00
commit 337b70114c
6 files changed
+247 -29

No files matched your search

+1
View File
@@ -26,6 +26,7 @@ runs:
id: setup-mise
uses: jdx/mise-action@9e7f7633ff6f6d6048a9418a68d48f288f50eb14 # v4.2.3
with:
version: '2026.9.2'
cache: true
- name: Cache Rust toolchain (workaround jdx/mise-action#215)
+2
View File
@@ -32,6 +32,7 @@ jobs:
- name: Setup mise
uses: jdx/mise-action@c2a87611a18de5b3828c5652fe268e992400cb5c # v4.3.0
with:
version: '2026.9.2'
cache: true
- name: Check Rust formatting
@@ -68,6 +69,7 @@ jobs:
- name: Setup mise
uses: jdx/mise-action@c2a87611a18de5b3828c5652fe268e992400cb5c # v4.3.0
with:
version: '2026.9.2'
cache: true
- name: Cache Rust build
@@ -437,6 +437,41 @@ describe('UnifiedPushNotifications', () => {
});
});
it('retries an immediate missing-room-key failure until the key arrives', async () => {
vi.useFakeTimers();
try {
matrixClient.getRoom.mockReturnValue(makeRoom());
let attempts = 0;
matrixClient.getCrypto.mockReturnValue({
decryptEvent: vi
.fn<() => Promise<Record<string, unknown>>>()
.mockImplementation(async () => {
attempts += 1;
if (attempts === 1) throw new Error('MissingRoomKey');
return {
clearEvent: {
type: 'm.room.message',
content: { body: 'retried message' },
},
};
}),
});
await listenAndPush(encryptedPush('$missing-key-retry:example.com'));
await vi.waitFor(() => expect(notificationsApi.sendNotification).toHaveBeenCalledOnce());
expect(attempts).toBe(1);
await vi.advanceTimersByTimeAsync(1000);
await vi.waitFor(() => expect(notificationsApi.sendNotification).toHaveBeenCalledTimes(2));
expect(notificationsApi.sendNotification.mock.calls[1]?.[0]).toMatchObject({
body: 'You: retried message',
silent: true,
});
} finally {
vi.useRealTimers();
}
});
it('deduplicates rich encrypted pushes before decryption', async () => {
matrixClient.getRoom.mockReturnValue(makeRoom());
@@ -3,6 +3,8 @@ import {
type MatrixClient,
MatrixEvent,
MatrixEventEvent,
type CryptoApi,
type CryptoBackend,
} from '$types/matrix-sdk';
import { EventType } from 'matrix-js-sdk/lib/@types/event';
import {
@@ -454,9 +456,7 @@ async function resolvePreviewEvent(
try {
const evt = await mx.fetchRoomEvent(roomId, eventId);
const mEvent = new MatrixEvent(evt);
if (mEvent.isEncrypted()) {
await mx.decryptEventIfNeeded(mEvent);
}
if (mEvent.isEncrypted()) await mx.decryptEventIfNeeded(mEvent).catch(() => undefined);
return mEvent;
} catch (error) {
unifiedPushLog.warn(
@@ -494,30 +494,52 @@ function holdsPlaintext(event: MatrixEvent): boolean {
);
}
/**
* Runs `apply` as soon as `event` holds plaintext: right away when it is already
* decrypted, or later once the Megolm key arrives — a backgrounded app routinely
* receives the push before the to-device key, and the SDK retries decryption on
* its own when the key lands.
*/
function whenDecrypted(event: MatrixEvent, apply: () => Promise<void>): void {
const DECRYPT_RETRY_DELAYS_MS = [1000, 2000, 5000, 10_000, 30_000, 60_000] as const;
const supportsEventDecryption = (crypto: CryptoApi | undefined): crypto is CryptoBackend =>
!!crypto && 'decryptEvent' in crypto && typeof crypto.decryptEvent === 'function';
function whenDecrypted(event: MatrixEvent, apply: () => Promise<void>, mx: MatrixClient): void {
if (holdsPlaintext(event)) {
void apply();
return;
}
let retryIndex = 0;
let retryTimer: ReturnType<typeof setTimeout> | undefined;
let finished = false;
const retryDeadline = Date.now() + ENCRYPTED_PREVIEW_RETRY_WINDOW_MS;
const finish = () => {
if (finished) return;
finished = true;
event.off(MatrixEventEvent.Decrypted, onDecrypted);
clearTimeout(retryTimer);
};
const onDecrypted = () => {
if (!holdsPlaintext(event)) return;
event.off(MatrixEventEvent.Decrypted, onDecrypted);
clearTimeout(retryWindowTimer);
finish();
void apply();
};
event.on(MatrixEventEvent.Decrypted, onDecrypted);
const retryWindowTimer = setTimeout(() => {
event.off(MatrixEventEvent.Decrypted, onDecrypted);
const retry = () => {
if (finished) return;
const remaining = retryDeadline - Date.now();
if (remaining > 0) {
const crypto = mx.getCrypto();
if (supportsEventDecryption(crypto)) {
void event.attemptDecryption(crypto, { isRetry: true }).catch(() => undefined);
}
const delay = DECRYPT_RETRY_DELAYS_MS[retryIndex] ?? 60_000;
retryTimer = setTimeout(retry, Math.min(delay, remaining));
retryIndex += 1;
return;
}
finish();
unifiedPushLog.warn('notification', 'Encrypted preview never decrypted within retry window');
}, ENCRYPTED_PREVIEW_RETRY_WINDOW_MS);
};
retryTimer = setTimeout(retry, DECRYPT_RETRY_DELAYS_MS[0]);
retryIndex += 1;
}
const roomNotifId = (userId: string, roomId: string) => hashCode(`${userId}\u0000${roomId}`);
@@ -919,16 +941,17 @@ function scheduleEncryptedPreviewEnrichment(
};
const fallBackToSdkDecryption = (): void => {
whenDecrypted(decrypted, () =>
applyDecryptedPreview({
content: decrypted.getContent(),
eventType: decrypted.getType(),
sender: decrypted.getSender(),
})
whenDecrypted(
decrypted,
() =>
applyDecryptedPreview({
content: decrypted.getContent(),
eventType: decrypted.getType(),
sender: decrypted.getSender(),
}),
initialSettings.mx
);
void initialSettings.mx.decryptEventIfNeeded(decrypted).catch(() => {
unifiedPushLog.warn('notification', 'Encrypted preview decryption failed');
});
void initialSettings.mx.decryptEventIfNeeded(decrypted).catch(() => undefined);
};
// The engine reads the crypto store directly, so it answers without waiting on the SDK
@@ -1138,7 +1161,7 @@ async function handleMinimalPushPayload(
}
});
whenDecrypted(fetched, applyFetchedPreview);
whenDecrypted(fetched, applyFetchedPreview, settings.mx);
})
.catch((error) => {
unifiedPushLog.warn(
+110 -2
View File
@@ -1,6 +1,7 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
import 'fake-indexeddb/auto';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import type { ClientEvent, MatrixClient } from '$types/matrix-sdk';
import { SyncState } from '$types/matrix-sdk';
import { createClient, IndexedDBStore, SyncState } from '$types/matrix-sdk';
import type { Session } from '$state/sessions';
import type * as PlatformModule from '$utils/platform';
import { ACTIVE_SESSION_KEY, MATRIX_SESSIONS_KEY } from '$state/sessions';
@@ -24,8 +25,115 @@ import {
releaseCryptoStore,
resolvePollTimeoutMs,
supportsSlidingSync,
startupSyncStore,
} from './initMatrix';
const failSyncRead = (message: string, persistent = false, name = 'UnknownError') => {
const original = IDBObjectStore.prototype.openCursor;
let failures = 0;
vi.spyOn(IDBObjectStore.prototype, 'openCursor').mockImplementation(function (
this: IDBObjectStore,
...args
) {
if (this.name !== 'sync' || (!persistent && failures > 0)) {
return original.apply(this, args);
}
failures += 1;
const request: IDBRequest<IDBCursorWithValue | null> = new IDBRequest();
Object.defineProperty(request, 'error', { value: new DOMException(message, name) });
queueMicrotask(() => request.onerror?.(new Event('error')));
return request;
});
return () => failures;
};
describe('startupSyncStore', () => {
let mx: MatrixClient;
let dbName: string;
beforeEach(async () => {
dbName = `sync-test-${crypto.randomUUID()}`;
mx = createClient({
baseUrl: 'https://example.org',
userId: '@alice:example.org',
accessToken: 'session-token',
store: new IndexedDBStore({ indexedDB, localStorage, dbName }),
});
await mx.store.startup();
await mx.store.setSyncData({
next_batch: 'cached-token',
rooms: { join: {}, invite: {}, leave: {}, knock: {} },
account_data: { events: [] },
});
await mx.store.save(true);
await mx.store.saveToDeviceBatches([
{ eventType: 'm.test', txnId: 'pending-message', batch: [] },
]);
await mx.store.destroy();
mx.store = new IndexedDBStore({ indexedDB, localStorage, dbName });
});
afterEach(async () => {
vi.restoreAllMocks();
await mx.store.destroy();
await mx.store.deleteAllData();
});
it.each([
['Failed to read large IndexedDB value', 'UnknownError'],
["Attempt to iterate a cursor that doesn't exist", 'UnknownError'],
[
'Data lost due to missing file. Affected record should be considered irrecoverable',
'NotReadableError',
],
])('recovers from %s without losing the session or pending messages', async (message, name) => {
failSyncRead(message, false, name);
const deleteDatabase = vi.spyOn(indexedDB, 'deleteDatabase');
await expect(startupSyncStore(mx, dbName)).resolves.toBeUndefined();
expect(await mx.store.getSavedSync()).toBeNull();
expect(await mx.store.getOldestToDeviceBatch()).toMatchObject({ txnId: 'pending-message' });
expect(mx.getAccessToken()).toBe('session-token');
expect(deleteDatabase).not.toHaveBeenCalled();
});
it('keeps a readable sync snapshot', async () => {
await startupSyncStore(mx, dbName);
expect(await mx.store.getSavedSyncToken()).toBe('cached-token');
});
it('stops after one recovery attempt if the sync cache remains unreadable', async () => {
const failures = failSyncRead('Failed to read large IndexedDB value', true);
await expect(startupSyncStore(mx, dbName)).rejects.toThrow('selectQuery failed for sync');
expect(failures()).toBe(2);
});
it('does not clear data for unrelated startup failures', async () => {
const error = new DOMException('Storage access denied', 'SecurityError');
vi.spyOn(mx.store, 'startup').mockRejectedValueOnce(error);
const clear = vi.spyOn(IDBObjectStore.prototype, 'clear');
await expect(startupSyncStore(mx, dbName)).rejects.toBe(error);
expect(clear).not.toHaveBeenCalled();
});
it('surfaces a failed cache repair without retrying or deleting databases', async () => {
const failures = failSyncRead('Failed to read large IndexedDB value');
const original = IDBObjectStore.prototype.clear;
vi.spyOn(IDBObjectStore.prototype, 'clear').mockImplementation(function (this: IDBObjectStore) {
const request = original.call(this);
this.transaction.abort();
return request;
});
const deleteDatabase = vi.spyOn(indexedDB, 'deleteDatabase');
await expect(startupSyncStore(mx, dbName)).rejects.toThrow('Failed to clear sync snapshot');
expect(failures()).toBe(1);
expect(deleteDatabase).not.toHaveBeenCalled();
});
});
describe('installSlidingSyncRequestPatch', () => {
it('invalidates device lists once per pos-less run, not once per request', async () => {
const markAllTrackedUsersAsDirty = vi.fn<() => Promise<void>>(async () => undefined);
+51 -2
View File
@@ -422,6 +422,53 @@ type ClientInitializationResult =
| { ok: true; mx: MatrixClient }
| { ok: false; error: unknown; phase: 'sync_store' | 'rust_crypto' };
export const startupSyncStore = async (mx: MatrixClient, dbName: string): Promise<void> => {
try {
await mx.store.startup();
return;
} catch (error) {
await mx.store.destroy();
if (!(error instanceof Error) || error.message !== 'selectQuery failed for sync') throw error;
debugLog.warn('sync', 'Rebuilding unreadable sync snapshot', { error });
}
await new Promise<void>((resolve, reject) => {
const request = global.indexedDB.open(`matrix-js-sdk:${dbName}`);
request.addEventListener('error', () => reject(request.error));
request.addEventListener('upgradeneeded', () => request.transaction?.abort());
request.addEventListener('success', () => {
const db = request.result;
try {
const transaction = db.transaction('sync', 'readwrite');
transaction.addEventListener('complete', () => {
db.close();
resolve();
});
transaction.addEventListener('abort', () => {
db.close();
reject(transaction.error ?? new Error('Failed to clear sync snapshot'));
});
transaction.objectStore('sync').clear();
} catch (error) {
db.close();
reject(error);
}
});
});
mx.store = new IndexedDBStore({
indexedDB: global.indexedDB,
localStorage: global.localStorage,
dbName,
});
try {
await mx.store.startup();
} catch (error) {
await mx.store.destroy();
throw error;
}
};
const initializeClient = async (
session: Session,
cryptoDatabasePrefix: string
@@ -432,14 +479,16 @@ const initializeClient = async (
} catch (error) {
return { ok: false, error, phase: 'sync_store' };
}
const { mx, indexedDBStore } = builtClient;
const { mx } = builtClient;
void primeVersionsFromCache(mx, session.baseUrl, session.userId).then((primed) => {
if (primed) void revalidateVersionsCache(mx, session.baseUrl, session.userId);
else void cacheVersionsFromClient(mx, session.baseUrl, session.userId);
});
const syncStorePromise = measureStartupPhase('sync_store', () => indexedDBStore.startup());
const syncStorePromise = measureStartupPhase('sync_store', () =>
startupSyncStore(mx, getSessionStoreName(session).sync)
);
const cryptoPromise = measureStartupPhase('rust_crypto', async () => {
let nativeEngine: boolean;
try {