mirror of
https://github.com/SableClient/Sable.git
synced 2026-11-20 17:11:17 -08:00
Fix sync cache recovery and push diagnostics (#2065)
<!-- Please read https://github.com/SableClient/Sable/blob/dev/CONTRIBUTING.md before submitting your pull request --> ### Description <!-- Please include a summary of the change. Please also include relevant motivation and context. List any dependencies that are required for this change. --> Fixes # #### Type of change - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [ ] This change requires a documentation update ### Checklist: - [ ] My code follows the style guidelines of this project - [ ] I have performed a self-review of my own code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [ ] My changes generate no new warnings ### AI disclosure: - [ ] Partially AI assisted (clarify which code was AI assisted and briefly explain what it does). - [ ] Fully AI generated (explain what all the generated code does in moderate detail). <!-- Write any explanation required here, but do not generate the explanation using AI!! You must prove you understand what the code in this PR does. -->
This commit is contained in:
6 files changed
+247
-29
No files matched your search
@@ -26,6 +26,7 @@ runs:
|
||||
id: setup-mise
|
||||
uses: jdx/mise-action@9e7f7633ff6f6d6048a9418a68d48f288f50eb14 # v4.2.3
|
||||
with:
|
||||
version: '2026.9.2'
|
||||
cache: true
|
||||
|
||||
- name: Cache Rust toolchain (workaround jdx/mise-action#215)
|
||||
|
||||
@@ -32,6 +32,7 @@ jobs:
|
||||
- name: Setup mise
|
||||
uses: jdx/mise-action@c2a87611a18de5b3828c5652fe268e992400cb5c # v4.3.0
|
||||
with:
|
||||
version: '2026.9.2'
|
||||
cache: true
|
||||
|
||||
- name: Check Rust formatting
|
||||
@@ -68,6 +69,7 @@ jobs:
|
||||
- name: Setup mise
|
||||
uses: jdx/mise-action@c2a87611a18de5b3828c5652fe268e992400cb5c # v4.3.0
|
||||
with:
|
||||
version: '2026.9.2'
|
||||
cache: true
|
||||
|
||||
- name: Cache Rust build
|
||||
|
||||
@@ -437,6 +437,41 @@ describe('UnifiedPushNotifications', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('retries an immediate missing-room-key failure until the key arrives', async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
matrixClient.getRoom.mockReturnValue(makeRoom());
|
||||
let attempts = 0;
|
||||
matrixClient.getCrypto.mockReturnValue({
|
||||
decryptEvent: vi
|
||||
.fn<() => Promise<Record<string, unknown>>>()
|
||||
.mockImplementation(async () => {
|
||||
attempts += 1;
|
||||
if (attempts === 1) throw new Error('MissingRoomKey');
|
||||
return {
|
||||
clearEvent: {
|
||||
type: 'm.room.message',
|
||||
content: { body: 'retried message' },
|
||||
},
|
||||
};
|
||||
}),
|
||||
});
|
||||
|
||||
await listenAndPush(encryptedPush('$missing-key-retry:example.com'));
|
||||
await vi.waitFor(() => expect(notificationsApi.sendNotification).toHaveBeenCalledOnce());
|
||||
expect(attempts).toBe(1);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
await vi.waitFor(() => expect(notificationsApi.sendNotification).toHaveBeenCalledTimes(2));
|
||||
expect(notificationsApi.sendNotification.mock.calls[1]?.[0]).toMatchObject({
|
||||
body: 'You: retried message',
|
||||
silent: true,
|
||||
});
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('deduplicates rich encrypted pushes before decryption', async () => {
|
||||
matrixClient.getRoom.mockReturnValue(makeRoom());
|
||||
|
||||
|
||||
@@ -3,6 +3,8 @@ import {
|
||||
type MatrixClient,
|
||||
MatrixEvent,
|
||||
MatrixEventEvent,
|
||||
type CryptoApi,
|
||||
type CryptoBackend,
|
||||
} from '$types/matrix-sdk';
|
||||
import { EventType } from 'matrix-js-sdk/lib/@types/event';
|
||||
import {
|
||||
@@ -454,9 +456,7 @@ async function resolvePreviewEvent(
|
||||
try {
|
||||
const evt = await mx.fetchRoomEvent(roomId, eventId);
|
||||
const mEvent = new MatrixEvent(evt);
|
||||
if (mEvent.isEncrypted()) {
|
||||
await mx.decryptEventIfNeeded(mEvent);
|
||||
}
|
||||
if (mEvent.isEncrypted()) await mx.decryptEventIfNeeded(mEvent).catch(() => undefined);
|
||||
return mEvent;
|
||||
} catch (error) {
|
||||
unifiedPushLog.warn(
|
||||
@@ -494,30 +494,52 @@ function holdsPlaintext(event: MatrixEvent): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs `apply` as soon as `event` holds plaintext: right away when it is already
|
||||
* decrypted, or later once the Megolm key arrives — a backgrounded app routinely
|
||||
* receives the push before the to-device key, and the SDK retries decryption on
|
||||
* its own when the key lands.
|
||||
*/
|
||||
function whenDecrypted(event: MatrixEvent, apply: () => Promise<void>): void {
|
||||
const DECRYPT_RETRY_DELAYS_MS = [1000, 2000, 5000, 10_000, 30_000, 60_000] as const;
|
||||
|
||||
const supportsEventDecryption = (crypto: CryptoApi | undefined): crypto is CryptoBackend =>
|
||||
!!crypto && 'decryptEvent' in crypto && typeof crypto.decryptEvent === 'function';
|
||||
|
||||
function whenDecrypted(event: MatrixEvent, apply: () => Promise<void>, mx: MatrixClient): void {
|
||||
if (holdsPlaintext(event)) {
|
||||
void apply();
|
||||
return;
|
||||
}
|
||||
|
||||
let retryIndex = 0;
|
||||
let retryTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
let finished = false;
|
||||
const retryDeadline = Date.now() + ENCRYPTED_PREVIEW_RETRY_WINDOW_MS;
|
||||
const finish = () => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
event.off(MatrixEventEvent.Decrypted, onDecrypted);
|
||||
clearTimeout(retryTimer);
|
||||
};
|
||||
const onDecrypted = () => {
|
||||
if (!holdsPlaintext(event)) return;
|
||||
event.off(MatrixEventEvent.Decrypted, onDecrypted);
|
||||
clearTimeout(retryWindowTimer);
|
||||
finish();
|
||||
void apply();
|
||||
};
|
||||
|
||||
event.on(MatrixEventEvent.Decrypted, onDecrypted);
|
||||
const retryWindowTimer = setTimeout(() => {
|
||||
event.off(MatrixEventEvent.Decrypted, onDecrypted);
|
||||
const retry = () => {
|
||||
if (finished) return;
|
||||
const remaining = retryDeadline - Date.now();
|
||||
if (remaining > 0) {
|
||||
const crypto = mx.getCrypto();
|
||||
if (supportsEventDecryption(crypto)) {
|
||||
void event.attemptDecryption(crypto, { isRetry: true }).catch(() => undefined);
|
||||
}
|
||||
const delay = DECRYPT_RETRY_DELAYS_MS[retryIndex] ?? 60_000;
|
||||
retryTimer = setTimeout(retry, Math.min(delay, remaining));
|
||||
retryIndex += 1;
|
||||
return;
|
||||
}
|
||||
finish();
|
||||
unifiedPushLog.warn('notification', 'Encrypted preview never decrypted within retry window');
|
||||
}, ENCRYPTED_PREVIEW_RETRY_WINDOW_MS);
|
||||
};
|
||||
retryTimer = setTimeout(retry, DECRYPT_RETRY_DELAYS_MS[0]);
|
||||
retryIndex += 1;
|
||||
}
|
||||
|
||||
const roomNotifId = (userId: string, roomId: string) => hashCode(`${userId}\u0000${roomId}`);
|
||||
@@ -919,16 +941,17 @@ function scheduleEncryptedPreviewEnrichment(
|
||||
};
|
||||
|
||||
const fallBackToSdkDecryption = (): void => {
|
||||
whenDecrypted(decrypted, () =>
|
||||
applyDecryptedPreview({
|
||||
content: decrypted.getContent(),
|
||||
eventType: decrypted.getType(),
|
||||
sender: decrypted.getSender(),
|
||||
})
|
||||
whenDecrypted(
|
||||
decrypted,
|
||||
() =>
|
||||
applyDecryptedPreview({
|
||||
content: decrypted.getContent(),
|
||||
eventType: decrypted.getType(),
|
||||
sender: decrypted.getSender(),
|
||||
}),
|
||||
initialSettings.mx
|
||||
);
|
||||
void initialSettings.mx.decryptEventIfNeeded(decrypted).catch(() => {
|
||||
unifiedPushLog.warn('notification', 'Encrypted preview decryption failed');
|
||||
});
|
||||
void initialSettings.mx.decryptEventIfNeeded(decrypted).catch(() => undefined);
|
||||
};
|
||||
|
||||
// The engine reads the crypto store directly, so it answers without waiting on the SDK
|
||||
@@ -1138,7 +1161,7 @@ async function handleMinimalPushPayload(
|
||||
}
|
||||
});
|
||||
|
||||
whenDecrypted(fetched, applyFetchedPreview);
|
||||
whenDecrypted(fetched, applyFetchedPreview, settings.mx);
|
||||
})
|
||||
.catch((error) => {
|
||||
unifiedPushLog.warn(
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import 'fake-indexeddb/auto';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import type { ClientEvent, MatrixClient } from '$types/matrix-sdk';
|
||||
import { SyncState } from '$types/matrix-sdk';
|
||||
import { createClient, IndexedDBStore, SyncState } from '$types/matrix-sdk';
|
||||
import type { Session } from '$state/sessions';
|
||||
import type * as PlatformModule from '$utils/platform';
|
||||
import { ACTIVE_SESSION_KEY, MATRIX_SESSIONS_KEY } from '$state/sessions';
|
||||
@@ -24,8 +25,115 @@ import {
|
||||
releaseCryptoStore,
|
||||
resolvePollTimeoutMs,
|
||||
supportsSlidingSync,
|
||||
startupSyncStore,
|
||||
} from './initMatrix';
|
||||
|
||||
const failSyncRead = (message: string, persistent = false, name = 'UnknownError') => {
|
||||
const original = IDBObjectStore.prototype.openCursor;
|
||||
let failures = 0;
|
||||
vi.spyOn(IDBObjectStore.prototype, 'openCursor').mockImplementation(function (
|
||||
this: IDBObjectStore,
|
||||
...args
|
||||
) {
|
||||
if (this.name !== 'sync' || (!persistent && failures > 0)) {
|
||||
return original.apply(this, args);
|
||||
}
|
||||
failures += 1;
|
||||
const request: IDBRequest<IDBCursorWithValue | null> = new IDBRequest();
|
||||
Object.defineProperty(request, 'error', { value: new DOMException(message, name) });
|
||||
queueMicrotask(() => request.onerror?.(new Event('error')));
|
||||
return request;
|
||||
});
|
||||
return () => failures;
|
||||
};
|
||||
|
||||
describe('startupSyncStore', () => {
|
||||
let mx: MatrixClient;
|
||||
let dbName: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
dbName = `sync-test-${crypto.randomUUID()}`;
|
||||
mx = createClient({
|
||||
baseUrl: 'https://example.org',
|
||||
userId: '@alice:example.org',
|
||||
accessToken: 'session-token',
|
||||
store: new IndexedDBStore({ indexedDB, localStorage, dbName }),
|
||||
});
|
||||
await mx.store.startup();
|
||||
await mx.store.setSyncData({
|
||||
next_batch: 'cached-token',
|
||||
rooms: { join: {}, invite: {}, leave: {}, knock: {} },
|
||||
account_data: { events: [] },
|
||||
});
|
||||
await mx.store.save(true);
|
||||
await mx.store.saveToDeviceBatches([
|
||||
{ eventType: 'm.test', txnId: 'pending-message', batch: [] },
|
||||
]);
|
||||
await mx.store.destroy();
|
||||
mx.store = new IndexedDBStore({ indexedDB, localStorage, dbName });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
await mx.store.destroy();
|
||||
await mx.store.deleteAllData();
|
||||
});
|
||||
|
||||
it.each([
|
||||
['Failed to read large IndexedDB value', 'UnknownError'],
|
||||
["Attempt to iterate a cursor that doesn't exist", 'UnknownError'],
|
||||
[
|
||||
'Data lost due to missing file. Affected record should be considered irrecoverable',
|
||||
'NotReadableError',
|
||||
],
|
||||
])('recovers from %s without losing the session or pending messages', async (message, name) => {
|
||||
failSyncRead(message, false, name);
|
||||
const deleteDatabase = vi.spyOn(indexedDB, 'deleteDatabase');
|
||||
|
||||
await expect(startupSyncStore(mx, dbName)).resolves.toBeUndefined();
|
||||
|
||||
expect(await mx.store.getSavedSync()).toBeNull();
|
||||
expect(await mx.store.getOldestToDeviceBatch()).toMatchObject({ txnId: 'pending-message' });
|
||||
expect(mx.getAccessToken()).toBe('session-token');
|
||||
expect(deleteDatabase).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('keeps a readable sync snapshot', async () => {
|
||||
await startupSyncStore(mx, dbName);
|
||||
expect(await mx.store.getSavedSyncToken()).toBe('cached-token');
|
||||
});
|
||||
|
||||
it('stops after one recovery attempt if the sync cache remains unreadable', async () => {
|
||||
const failures = failSyncRead('Failed to read large IndexedDB value', true);
|
||||
await expect(startupSyncStore(mx, dbName)).rejects.toThrow('selectQuery failed for sync');
|
||||
expect(failures()).toBe(2);
|
||||
});
|
||||
|
||||
it('does not clear data for unrelated startup failures', async () => {
|
||||
const error = new DOMException('Storage access denied', 'SecurityError');
|
||||
vi.spyOn(mx.store, 'startup').mockRejectedValueOnce(error);
|
||||
const clear = vi.spyOn(IDBObjectStore.prototype, 'clear');
|
||||
|
||||
await expect(startupSyncStore(mx, dbName)).rejects.toBe(error);
|
||||
expect(clear).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('surfaces a failed cache repair without retrying or deleting databases', async () => {
|
||||
const failures = failSyncRead('Failed to read large IndexedDB value');
|
||||
const original = IDBObjectStore.prototype.clear;
|
||||
vi.spyOn(IDBObjectStore.prototype, 'clear').mockImplementation(function (this: IDBObjectStore) {
|
||||
const request = original.call(this);
|
||||
this.transaction.abort();
|
||||
return request;
|
||||
});
|
||||
const deleteDatabase = vi.spyOn(indexedDB, 'deleteDatabase');
|
||||
|
||||
await expect(startupSyncStore(mx, dbName)).rejects.toThrow('Failed to clear sync snapshot');
|
||||
expect(failures()).toBe(1);
|
||||
expect(deleteDatabase).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('installSlidingSyncRequestPatch', () => {
|
||||
it('invalidates device lists once per pos-less run, not once per request', async () => {
|
||||
const markAllTrackedUsersAsDirty = vi.fn<() => Promise<void>>(async () => undefined);
|
||||
|
||||
@@ -422,6 +422,53 @@ type ClientInitializationResult =
|
||||
| { ok: true; mx: MatrixClient }
|
||||
| { ok: false; error: unknown; phase: 'sync_store' | 'rust_crypto' };
|
||||
|
||||
export const startupSyncStore = async (mx: MatrixClient, dbName: string): Promise<void> => {
|
||||
try {
|
||||
await mx.store.startup();
|
||||
return;
|
||||
} catch (error) {
|
||||
await mx.store.destroy();
|
||||
if (!(error instanceof Error) || error.message !== 'selectQuery failed for sync') throw error;
|
||||
debugLog.warn('sync', 'Rebuilding unreadable sync snapshot', { error });
|
||||
}
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const request = global.indexedDB.open(`matrix-js-sdk:${dbName}`);
|
||||
request.addEventListener('error', () => reject(request.error));
|
||||
request.addEventListener('upgradeneeded', () => request.transaction?.abort());
|
||||
request.addEventListener('success', () => {
|
||||
const db = request.result;
|
||||
try {
|
||||
const transaction = db.transaction('sync', 'readwrite');
|
||||
transaction.addEventListener('complete', () => {
|
||||
db.close();
|
||||
resolve();
|
||||
});
|
||||
transaction.addEventListener('abort', () => {
|
||||
db.close();
|
||||
reject(transaction.error ?? new Error('Failed to clear sync snapshot'));
|
||||
});
|
||||
transaction.objectStore('sync').clear();
|
||||
} catch (error) {
|
||||
db.close();
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
mx.store = new IndexedDBStore({
|
||||
indexedDB: global.indexedDB,
|
||||
localStorage: global.localStorage,
|
||||
dbName,
|
||||
});
|
||||
try {
|
||||
await mx.store.startup();
|
||||
} catch (error) {
|
||||
await mx.store.destroy();
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
const initializeClient = async (
|
||||
session: Session,
|
||||
cryptoDatabasePrefix: string
|
||||
@@ -432,14 +479,16 @@ const initializeClient = async (
|
||||
} catch (error) {
|
||||
return { ok: false, error, phase: 'sync_store' };
|
||||
}
|
||||
const { mx, indexedDBStore } = builtClient;
|
||||
const { mx } = builtClient;
|
||||
|
||||
void primeVersionsFromCache(mx, session.baseUrl, session.userId).then((primed) => {
|
||||
if (primed) void revalidateVersionsCache(mx, session.baseUrl, session.userId);
|
||||
else void cacheVersionsFromClient(mx, session.baseUrl, session.userId);
|
||||
});
|
||||
|
||||
const syncStorePromise = measureStartupPhase('sync_store', () => indexedDBStore.startup());
|
||||
const syncStorePromise = measureStartupPhase('sync_store', () =>
|
||||
startupSyncStore(mx, getSessionStoreName(session).sync)
|
||||
);
|
||||
const cryptoPromise = measureStartupPhase('rust_crypto', async () => {
|
||||
let nativeEngine: boolean;
|
||||
try {
|
||||
|
||||
Reference in new issue
Block a user