diff --git a/package.json b/package.json index 198dd6b1a..e0b429072 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ "@choochmeque/tauri-plugin-sharekit-api": "0.4.0-rc.5", "@fontsource-variable/nunito": "5.2.7", "@fontsource/space-mono": "5.2.9", + "@livekit/components-react": "2.9.23", "@lottiefiles/dotlottie-react": "^0.12.0", "@noble/hashes": "^2.2.0", "@phosphor-icons/react": "^2.1.10", @@ -97,6 +98,7 @@ "leaflet": "1.9.4", "linkify-react": "^4.3.3", "linkifyjs": "^4.3.3", + "livekit-client": "2.21.0", "marked": "^18.0.5", "matrix-js-sdk": "42.0.0", "matrix-widget-api": "^1.17.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2183f02c6..199eb7f6f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -43,6 +43,9 @@ importers: '@fontsource/space-mono': specifier: 5.2.9 version: 5.2.9 + '@livekit/components-react': + specifier: 2.9.23 + version: 2.9.23(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(tslib@2.8.1) '@lottiefiles/dotlottie-react': specifier: ^0.12.0 version: 0.12.3(react@18.3.1) @@ -196,6 +199,9 @@ importers: linkifyjs: specifier: ^4.3.3 version: 4.3.3 + livekit-client: + specifier: 2.21.0 + version: 2.21.0(@types/dom-mediacapture-record@1.0.22) marked: specifier: ^18.0.5 version: 18.0.5 @@ -955,6 +961,9 @@ packages: resolution: {integrity: sha512-ctxtJ/eA+t+6q2++vj5j7FYX3nRu311q1wfYH3xjlLOsczhlhxAg2FWNUXhpGvAw3BWo1xBcvOV6/YLc2r5FJw==} hasBin: true + '@bufbuild/protobuf@1.10.1': + resolution: {integrity: sha512-wJ8ReQbHxsAfXhrf9ixl0aYbZorRuOWpBNzm8pL8ftmSxQx/wnJD5Eg861NwJU/czy2VXFIebCeZnZrI9rktIQ==} + '@choochmeque/tauri-plugin-notifications-api@0.5.0-rc.11': resolution: {integrity: sha512-TSuf8HQ4Ayp9IKKB5v2yTdgKMa/I0bqbcACj0IZ46CSpzn/HKner/BKG7rM8tGOarEcyBd/VXnDhK5cJA/YMtw==} @@ -1404,6 +1413,15 @@ packages: '@noble/hashes': optional: true + '@floating-ui/core@1.8.0': + resolution: {integrity: sha512-0CIZ5itps/8x7BG8dEIhs53BvCUH2PCoogtakwRTut+Arm58sJooJ0AuZhLw2HJYIR5cMLNPBSS728sPho2khQ==} + + '@floating-ui/dom@1.7.6': + resolution: {integrity: sha512-9gZSAI5XM36880PPMm//9dfiEngYoC6Am2izES1FF406YFsjvyBMmeJ2g4SAju3xWwtuynNRFL2s9hgxpLI5SQ==} + + '@floating-ui/utils@0.2.12': + resolution: {integrity: sha512-HpCo8tmWzLVad5s2d19EhAz5zqrrQ6s69qd6moPMQvkOuSwDT1YgRfWSVuc4ennqrgv3OHppiOGMQ7oC13yIww==} + '@fontsource-variable/nunito@5.2.7': resolution: {integrity: sha512-2N8QhatkyKgSUbAGZO2FYLioxA32+RyI1EplVLawbpkGjUeui9Qg9VMrpkCaik1ydjFjfLV+kzQ0cGEsMrMenQ==} @@ -1625,6 +1643,32 @@ packages: '@kwsites/file-exists@1.1.1': resolution: {integrity: sha512-m9/5YGR18lIwxSFDwfE3oA7bWuq9kdau6ugN4H2rJeyhFQZcG9AgSHkQtSD15a8WvTgfz9aikZMrKPHvbpqFiw==} + '@livekit/components-core@0.12.14': + resolution: {integrity: sha512-6OKP/1Ok2fCZewDLKd3SzaTb7KvfZl/6hjggI+TgUdhp0Y7HBg3+tHA7YmhZRc0BO8wIyVy4VgTPn7qkLHt2nQ==} + engines: {node: '>=18'} + peerDependencies: + livekit-client: ^2.20.1 + tslib: ^2.6.2 + + '@livekit/components-react@2.9.23': + resolution: {integrity: sha512-clO+0g/u3YBpuOvnAjUSAJBH/o7w+RpUAseswjiSML9rHrXlTUhwBNm8LPk+qN5GOU3A6lzsNnFYVpUHUBVOkg==} + engines: {node: '>=18'} + peerDependencies: + '@livekit/krisp-noise-filter': ^0.2.12 || ^0.3.0 || ^0.4.0 + livekit-client: ^2.20.1 + react: '>=18' + react-dom: '>=18' + tslib: ^2.6.2 + peerDependenciesMeta: + '@livekit/krisp-noise-filter': + optional: true + + '@livekit/mutex@1.1.1': + resolution: {integrity: sha512-EsshAucklmpuUAfkABPxJNhzj9v2sG7JuzFDL4ML1oJQSV14sqrpTYnsaOudMAw9yOaW53NU3QQTlUQoRs4czw==} + + '@livekit/protocol@1.50.4': + resolution: {integrity: sha512-L1uggNQAqyY21smQY8AllyOYbcv9Me9TaxwuLytL1R8ck9nbYPmQLNwEDi3pOFGAMa5F8I2nUi2Jc59W5awxlA==} + '@lottiefiles/dotlottie-react@0.12.3': resolution: {integrity: sha512-b0k0Lakj2hmhyIkwJSpKySh6xoelpgWRaijyrCb6fraOCnzuitKglVTtYs0VWf72rk+2aSmC/0IK8j8wruVfOw==} peerDependencies: @@ -3104,6 +3148,9 @@ packages: '@types/dockerode@4.0.1': resolution: {integrity: sha512-cmUpB+dPN955PxBEuXE3f6lKO1hHiIGYJA46IVF3BJpNsZGvtBDcRnlrHYHtOH/B6vtDOyl2kZ2ShAu3mgc27Q==} + '@types/dom-mediacapture-record@1.0.22': + resolution: {integrity: sha512-mUMZLK3NvwRLcAAT9qmcK+9p7tpU2FHdDsntR3YI4+GY88XrgG4XiE7u1Q2LAN2/FZOz/tdMDC3GQCR4T8nFuw==} + '@types/estree@0.0.39': resolution: {integrity: sha512-EYNwp3bU+98cpU4lAWYYL7Zz+2gryWH1qbdDTidVd6hkiR6weksdbMadyXKXNPEkQFhXM+hVO9ZygomHXp+AIw==} @@ -4349,6 +4396,9 @@ packages: resolution: {integrity: sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==} hasBin: true + jose@6.2.4: + resolution: {integrity: sha512-N8acGzVsQy6M/fjFcxtysNc4Q379TcM5dM/qKkNtsHFji88yANnXTr7BLeP75iPnFwBfQzM/jg2BZ9+HZrHCZA==} + jotai@2.20.1: resolution: {integrity: sha512-dnuKfU/GLi8B28RRMjQ3AfoN7kfzP8o41+AX2FmITZqEMY8PHnjABq+VkEooomLwYaGjda+pgy0yFSjaHX/ZPg==} engines: {node: '>=12.20.0'} @@ -4519,6 +4569,11 @@ packages: linkifyjs@4.3.3: resolution: {integrity: sha512-P8aEP5U/D1/IlTY2OeYsErdwh9bGuLE30NcXtKEjgdHcahveQoQwM2yZNsioQHsWFz0P7KKudisbrzCgR0sDHg==} + livekit-client@2.21.0: + resolution: {integrity: sha512-RBUhPkV/sl1nzl8lokVlK5uATPwn0AlsudCBZXissw/kDl9yz8ac4pNJ43iPpMVoHOeBYH/BZ3vUC1adqa/zFQ==} + peerDependencies: + '@types/dom-mediacapture-record': ^1 + locate-path@6.0.0: resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} engines: {node: '>=10'} @@ -4535,6 +4590,10 @@ packages: lodash@4.18.1: resolution: {integrity: sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==} + loglevel@1.9.1: + resolution: {integrity: sha512-hP3I3kCrDIMuRwAwHltphhDM1r8i55H33GgqjXbrisuJhF4kRhW1dNuxsRklp4bXl8DSdLaNLuiL4A/LWRfxvg==} + engines: {node: '>= 0.6.0'} + loglevel@1.9.2: resolution: {integrity: sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg==} engines: {node: '>= 0.6.0'} @@ -5064,6 +5123,9 @@ packages: engines: {node: '>=18.0.0', npm: '>=8.0.0'} hasBin: true + rxjs@7.8.2: + resolution: {integrity: sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==} + safe-array-concat@1.1.4: resolution: {integrity: sha512-wtZlHyOje6OZTGqAoaDKxFkgRtkF9CnHAVnCHKfuj200wAgL+bSJhdsCD2l0Qx/2ekEXjPWcyKkfGb5CPboslg==} engines: {node: '>=0.4'} @@ -5095,10 +5157,17 @@ packages: scroll-into-view-if-needed@3.1.0: resolution: {integrity: sha512-49oNpRjWRvnU8NyGVmUaYG4jtTkNonFZI86MmGRDqBphEK2EXT9gdEUoQPZhuBM8yWHxCWbobltqYO5M4XrUvQ==} + sdp-transform@2.15.0: + resolution: {integrity: sha512-KrOH82c/W+GYQ0LHqtr3caRpM3ITglq3ljGUIb8LTki7ByacJZ9z+piSGiwZDsRyhQbYBOBJgr2k6X4BZXi3Kw==} + hasBin: true + sdp-transform@3.0.0: resolution: {integrity: sha512-gfYVRGxjHkGF2NPeUWHw5u6T/KGFtS5/drPms73gaSuMaVHKCY3lpLnGDfswVQO0kddeePoti09AwhYP4zA8dQ==} hasBin: true + sdp@3.2.2: + resolution: {integrity: sha512-xZocWwfyp4hkbN4hLWxMjmv2Q8aNa9MhmOZ7L9aCZPT+dZsgRr6wZRrSYE3HTdyk/2pZKPSgqI7ns7Een1xMSA==} + semver@6.3.1: resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} hasBin: true @@ -5447,6 +5516,9 @@ packages: resolution: {integrity: sha512-phPGCwqr2+Qo0fwniCE8e4pKnGu/yFb5nD5Y8bf0EEeiI5GklnACYA9GFy/DrAeRrKHXvHn+1SUsOWgJp6RO+g==} engines: {node: '>= 0.4'} + typed-emitter@2.1.0: + resolution: {integrity: sha512-g/KzbYKbH5C2vPkaXGu8DJlHrGKHLsM25Zg9WuC9pMGfuvT+X25tZQWo5fK1BjBm8+UrVE9LDCvaY0CQk+fXDA==} + typescript@5.9.3: resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} engines: {node: '>=14.17'} @@ -5525,6 +5597,12 @@ packages: peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + usehooks-ts@3.1.1: + resolution: {integrity: sha512-I4diPp9Cq6ieSUH2wu+fDAVQO43xwtulo+fKEidHUwZPnYImbtkTjzIJYcDcJqxgmX31GVqNFURodvcgHcW0pA==} + engines: {node: '>=16.15.0'} + peerDependencies: + react: ^16.8.0 || ^17 || ^18 || ^19 || ^19.0.0-rc + util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} @@ -5735,6 +5813,10 @@ packages: resolution: {integrity: sha512-BMhLD/Sw+GbJC21C/UgyaZX41nPt8bUTg+jWyDeg7e7YN4xOM05YPSIXceACnXVtqyEw/LMClUQMtMZ+PGGpqQ==} engines: {node: '>=20'} + webrtc-adapter@9.0.6: + resolution: {integrity: sha512-CHbl2ZQbxx164IgWRgzJno4hWtM4tFbRam1QfI3Yxhs3w/DvqluVxVWeXs3oL5/fbGkSNLKo0Ty5MgUWceNhog==} + engines: {node: '>=6.0.0', npm: '>=3.10.0'} + whatwg-mimetype@5.0.0: resolution: {integrity: sha512-sXcNcHOC51uPGF0P/D4NVtrkjSU2fNsm9iog4ZvZJsL3rjoDAzXZhkm2MWt1y+PUdggKAYVoMAIYcs78wJ51Cw==} engines: {node: '>=20'} @@ -6654,6 +6736,8 @@ snapshots: dependencies: css-tree: 3.2.1 + '@bufbuild/protobuf@1.10.1': {} + '@choochmeque/tauri-plugin-notifications-api@0.5.0-rc.11': dependencies: '@tauri-apps/api': 2.11.1 @@ -6935,6 +7019,17 @@ snapshots: optionalDependencies: '@noble/hashes': 2.2.0 + '@floating-ui/core@1.8.0': + dependencies: + '@floating-ui/utils': 0.2.12 + + '@floating-ui/dom@1.7.6': + dependencies: + '@floating-ui/core': 1.8.0 + '@floating-ui/utils': 0.2.12 + + '@floating-ui/utils@0.2.12': {} + '@fontsource-variable/nunito@5.2.7': {} '@fontsource/space-mono@5.2.9': {} @@ -7116,6 +7211,32 @@ snapshots: transitivePeerDependencies: - supports-color + '@livekit/components-core@0.12.14(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))(tslib@2.8.1)': + dependencies: + '@floating-ui/dom': 1.7.6 + livekit-client: 2.21.0(@types/dom-mediacapture-record@1.0.22) + loglevel: 1.9.1 + rxjs: 7.8.2 + tslib: 2.8.1 + + '@livekit/components-react@2.9.23(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))(react-dom@18.3.1(react@18.3.1))(react@18.3.1)(tslib@2.8.1)': + dependencies: + '@livekit/components-core': 0.12.14(livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22))(tslib@2.8.1) + clsx: 2.1.1 + events: 3.3.0 + jose: 6.2.4 + livekit-client: 2.21.0(@types/dom-mediacapture-record@1.0.22) + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + tslib: 2.8.1 + usehooks-ts: 3.1.1(react@18.3.1) + + '@livekit/mutex@1.1.1': {} + + '@livekit/protocol@1.50.4': + dependencies: + '@bufbuild/protobuf': 1.10.1 + '@lottiefiles/dotlottie-react@0.12.3(react@18.3.1)': dependencies: '@lottiefiles/dotlottie-web': 0.40.1 @@ -8171,6 +8292,8 @@ snapshots: '@types/node': 24.13.2 '@types/ssh2': 1.15.5 + '@types/dom-mediacapture-record@1.0.22': {} + '@types/estree@0.0.39': {} '@types/estree@1.0.9': {} @@ -9583,6 +9706,8 @@ snapshots: jiti@2.7.0: {} + jose@6.2.4: {} + jotai@2.20.1(@babel/core@7.29.7)(@babel/template@7.29.7)(@types/react@18.3.31)(react@18.3.1): optionalDependencies: '@babel/core': 7.29.7 @@ -9728,6 +9853,19 @@ snapshots: linkifyjs@4.3.3: {} + livekit-client@2.21.0(@types/dom-mediacapture-record@1.0.22): + dependencies: + '@livekit/mutex': 1.1.1 + '@livekit/protocol': 1.50.4 + '@types/dom-mediacapture-record': 1.0.22 + events: 3.3.0 + jose: 6.2.4 + loglevel: 1.9.2 + sdp-transform: 2.15.0 + tslib: 2.8.1 + typed-emitter: 2.1.0 + webrtc-adapter: 9.0.6 + locate-path@6.0.0: dependencies: p-locate: 5.0.0 @@ -9740,6 +9878,8 @@ snapshots: lodash@4.18.1: {} + loglevel@1.9.1: {} + loglevel@1.9.2: {} long@5.3.2: {} @@ -10395,6 +10535,10 @@ snapshots: '@rollup/rollup-win32-x64-msvc': 4.62.0 fsevents: 2.3.3 + rxjs@7.8.2: + dependencies: + tslib: 2.8.1 + safe-array-concat@1.1.4: dependencies: call-bind: 1.0.9 @@ -10432,8 +10576,12 @@ snapshots: dependencies: compute-scroll-into-view: 3.1.1 + sdp-transform@2.15.0: {} + sdp-transform@3.0.0: {} + sdp@3.2.2: {} + semver@6.3.1: {} semver@7.8.4: {} @@ -10914,6 +11062,10 @@ snapshots: possible-typed-array-names: 1.1.0 reflect.getprototypeof: 1.0.10 + typed-emitter@2.1.0: + optionalDependencies: + rxjs: 7.8.2 + typescript@5.9.3: {} ua-is-frozen@0.1.2: {} @@ -10976,6 +11128,11 @@ snapshots: dependencies: react: 18.3.1 + usehooks-ts@3.1.1(react@18.3.1): + dependencies: + lodash.debounce: 4.0.8 + react: 18.3.1 + util-deprecate@1.0.2: {} uuid@10.0.0: {} @@ -11129,6 +11286,10 @@ snapshots: webidl-conversions@8.0.1: {} + webrtc-adapter@9.0.6: + dependencies: + sdp: 3.2.2 + whatwg-mimetype@5.0.0: {} whatwg-url@16.0.1(@noble/hashes@2.2.0): diff --git a/src-tauri/plugins/tauri-plugin-call-lifecycle/Cargo.toml b/src-tauri/plugins/tauri-plugin-call-lifecycle/Cargo.toml index c9a2e5836..bcc180939 100644 --- a/src-tauri/plugins/tauri-plugin-call-lifecycle/Cargo.toml +++ b/src-tauri/plugins/tauri-plugin-call-lifecycle/Cargo.toml @@ -20,6 +20,7 @@ tauri-plugin = { version = "2.6.3", features = ["build"] } [dev-dependencies] serde_json = "1.0" +tauri = { version = "2.11.3", default-features = false, features = ["test"] } [target.'cfg(target_os = "linux")'.dependencies] ashpd = { version = "0.13", features = ["tokio", "screencast"] } diff --git a/src-tauri/plugins/tauri-plugin-call-lifecycle/android/src/main/AndroidManifest.xml b/src-tauri/plugins/tauri-plugin-call-lifecycle/android/src/main/AndroidManifest.xml index 20073832a..442af058e 100644 --- a/src-tauri/plugins/tauri-plugin-call-lifecycle/android/src/main/AndroidManifest.xml +++ b/src-tauri/plugins/tauri-plugin-call-lifecycle/android/src/main/AndroidManifest.xml @@ -5,6 +5,9 @@ + + + @@ -12,5 +15,9 @@ android:name="io.livekit.android.room.track.screencapture.ScreenCaptureService" android:exported="false" android:foregroundServiceType="mediaProjection" /> + diff --git a/src-tauri/plugins/tauri-plugin-call-lifecycle/android/src/main/java/app/tauri/call_lifecycle/CallLifecycleForegroundService.kt b/src-tauri/plugins/tauri-plugin-call-lifecycle/android/src/main/java/app/tauri/call_lifecycle/CallLifecycleForegroundService.kt new file mode 100644 index 000000000..6fcee8c6b --- /dev/null +++ b/src-tauri/plugins/tauri-plugin-call-lifecycle/android/src/main/java/app/tauri/call_lifecycle/CallLifecycleForegroundService.kt @@ -0,0 +1,103 @@ +package app.tauri.call_lifecycle + +import android.app.Notification +import android.app.NotificationChannel +import android.app.NotificationManager +import android.app.Service +import android.content.Intent +import android.content.pm.ServiceInfo +import android.os.Build +import android.os.IBinder +import androidx.core.app.NotificationCompat + +/** + * Owns only the Android foreground-service notification for a JS-owned room. + * The WebView and LiveKit JS remain the owners of the room and its tracks. + */ +class CallLifecycleForegroundService : Service() { + /** Positive acknowledgement of foreground promotion for the owning plugin. */ + interface StartListener { + fun onServiceStarted() + fun onServiceStartFailed() + } + + override fun onCreate() { + super.onCreate() + createNotificationChannel() + } + + override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int { + val wantsMicrophone = intent?.getBooleanExtra(EXTRA_MICROPHONE, false) ?: false + val wantsPlayback = intent?.getBooleanExtra(EXTRA_PLAYBACK, false) ?: false + val started = try { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + var types = 0 + if (wantsMicrophone) types = types or ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE + if (wantsPlayback) types = types or ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PLAYBACK + if (types != 0) { + startForeground(NOTIFICATION_ID, buildNotification(), types) + } else { + startForeground(NOTIFICATION_ID, buildNotification()) + } + } else { + startForeground(NOTIFICATION_ID, buildNotification()) + } + true + } catch (_: SecurityException) { + false + } catch (_: IllegalArgumentException) { + false + } catch (_: IllegalStateException) { + false + } + if (started) { + activeStartListener?.onServiceStarted() + } else { + stopSelfResult(startId) + activeStartListener?.onServiceStartFailed() + } + return START_NOT_STICKY + } + + override fun onBind(intent: Intent?): IBinder? = null + + override fun onDestroy() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { + stopForeground(STOP_FOREGROUND_REMOVE) + } else { + @Suppress("DEPRECATION") + stopForeground(true) + } + super.onDestroy() + } + + private fun createNotificationChannel() { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) return + val channel = NotificationChannel( + CHANNEL_ID, + "Call audio", + NotificationManager.IMPORTANCE_LOW, + ) + getSystemService(NotificationManager::class.java).createNotificationChannel(channel) + } + + private fun buildNotification(): Notification = NotificationCompat.Builder(this, CHANNEL_ID) + .setSmallIcon(android.R.drawable.ic_dialog_info) + .setContentTitle("Call in progress") + .setContentText("Microphone and call audio are active") + .setCategory(Notification.CATEGORY_CALL) + .setOngoing(true) + .setOnlyAlertOnce(true) + .build() + + companion object { + const val EXTRA_MICROPHONE = "app.tauri.call_lifecycle.extra.MICROPHONE" + const val EXTRA_PLAYBACK = "app.tauri.call_lifecycle.extra.PLAYBACK" + + @Volatile + var activeStartListener: StartListener? = null + + private const val CHANNEL_ID = "call_lifecycle_audio" + private const val NOTIFICATION_ID = 1396 + } +} diff --git a/src-tauri/plugins/tauri-plugin-call-lifecycle/android/src/main/java/app/tauri/call_lifecycle/CallLifecyclePlugin.kt b/src-tauri/plugins/tauri-plugin-call-lifecycle/android/src/main/java/app/tauri/call_lifecycle/CallLifecyclePlugin.kt index b856efe4d..1472aaef6 100644 --- a/src-tauri/plugins/tauri-plugin-call-lifecycle/android/src/main/java/app/tauri/call_lifecycle/CallLifecyclePlugin.kt +++ b/src-tauri/plugins/tauri-plugin-call-lifecycle/android/src/main/java/app/tauri/call_lifecycle/CallLifecyclePlugin.kt @@ -4,7 +4,14 @@ import android.app.Activity import android.content.Context import android.content.Intent import android.content.pm.PackageManager +import android.media.AudioAttributes +import android.media.AudioDeviceCallback +import android.media.AudioDeviceInfo +import android.media.AudioFocusRequest import android.media.AudioManager +import android.os.Build +import android.os.Handler +import android.os.Looper import android.os.SystemClock import androidx.activity.result.ActivityResult import androidx.core.content.ContextCompat @@ -49,6 +56,27 @@ internal class DisconnectArgs { var connectionId: String = "" } +@InvokeArg +internal class SetMediaEnabledArgs { + var operationId: String = "" + var connectionId: String = "" + var kind: String = "" + var enabled: Boolean = false +} + +@InvokeArg +internal class PlatformLifecycleStartArgs { + var sessionId: String = "" + var microphone: Boolean = false + var playback: Boolean = false + lateinit var channel: Channel +} + +@InvokeArg +internal class PlatformLifecycleStopArgs { + var sessionId: String = "" +} + private data class PendingConnect( val invoke: Invoke, val args: ConnectArgs, @@ -56,6 +84,12 @@ private data class PendingConnect( var settled: Boolean = false, ) +private data class PendingMedia( + val invoke: Invoke, + val args: SetMediaEnabledArgs, + val generation: Long, +) + @TauriPlugin( permissions = [ Permission( @@ -80,10 +114,30 @@ class CallLifecyclePlugin(private val activity: Activity) : Plugin(activity) { private var activeEvents: Channel? = null private var pendingPermission: PendingConnect? = null private var pendingProjection: PendingConnect? = null + private var pendingMediaPermission: PendingMedia? = null + private var pendingMediaProjection: PendingMedia? = null private var projectionData: Intent? = null private var previousAudioMode: Int? = null private var previousSpeakerphoneState: Boolean? = null private val lastEventAt = mutableMapOf() + private val lastPlatformEventAt = mutableMapOf() + private var platformSessionId: String? = null + private var platformRevision = 0L + private var platformState = PLATFORM_IDLE + private var platformChannel: Channel? = null + private var platformFocus = PLATFORM_FOCUS_NONE + private var platformRoute = PLATFORM_ROUTE_UNKNOWN + private var platformMicrophone = false + private var platformPlayback = false + private var platformFocusRequest: AudioFocusRequest? = null + private var platformAudioManager: AudioManager? = null + private var platformAudioCallback: AudioDeviceCallback? = null + private var pendingPlatformStart: PendingPlatformStart? = null + + private data class PendingPlatformStart( + val invoke: Invoke, + val args: PlatformLifecycleStartArgs, + ) @Command fun connect(invoke: Invoke) { @@ -97,7 +151,7 @@ class CallLifecyclePlugin(private val activity: Activity) : Plugin(activity) { invoke.resolve(result(args)) return } - if (room != null || pendingPermission != null || pendingProjection != null) { + if (isNativeRoomModeActive() || isPlatformLifecycleActive()) { invoke.reject("call connection failed", "connect_failed") return } @@ -120,6 +174,91 @@ class CallLifecyclePlugin(private val activity: Activity) : Plugin(activity) { requestProjectionOrConnect(pending) } + @Command + fun getPlatformLifecycleCapabilities(invoke: Invoke) { + invoke.resolve( + JSObject() + .put("platform", "android") + .put("microphone", true) + .put("audioPlayback", true) + .put("audioFocus", true) + .put("routeEvents", true) + .put("foregroundService", true) + .put("backgroundJavascript", false) + .put("camera", false) + .put("screenShare", false), + ) + } + + @Command + fun startPlatformLifecycle(invoke: Invoke) { + val args = runCatching { invoke.parseArgs(PlatformLifecycleStartArgs::class.java) }.getOrNull() + if (args == null || args.sessionId.isBlank() || !hasChannel(args)) { + invoke.reject("platform lifecycle start failed", "invalid_request") + return + } + + if (isNativeRoomModeActive()) { + invoke.reject("platform lifecycle is busy", "busy") + return + } + if (platformSessionId != null && platformSessionId != args.sessionId) { + invoke.reject("platform lifecycle is busy", "busy") + return + } + if (platformSessionId == args.sessionId) { + invoke.resolve(platformLifecycleState()) + return + } + if (!isActivityVisible()) { + invoke.reject("platform lifecycle requires a visible activity", "not_visible") + return + } + + platformSessionId = args.sessionId + platformChannel = args.channel + platformMicrophone = args.microphone + platformPlayback = args.playback + platformState = PLATFORM_STARTING + bumpPlatformRevision() + + if (args.microphone && !hasPermission(android.Manifest.permission.RECORD_AUDIO)) { + val pending = PendingPlatformStart(invoke, args) + pendingPlatformStart = pending + try { + requestPermissionForAliases(arrayOf("microphone"), invoke, "permissionResult") + } catch (_: Exception) { + pendingPlatformStart = null + failPlatformStart(pending, "permission_denied") + } + return + } + beginPlatformLifecycle(invoke, args) + } + + @Command + fun stopPlatformLifecycle(invoke: Invoke) { + val args = runCatching { invoke.parseArgs(PlatformLifecycleStopArgs::class.java) }.getOrNull() + if (args == null || args.sessionId.isBlank()) { + invoke.reject("platform lifecycle stop failed", "invalid_request") + return + } + if (platformSessionId != args.sessionId) { + // A stale stop must never tear down a replacement session. + invoke.resolve(platformLifecycleState()) + return + } + + pendingPlatformStart = null + releasePlatformLifecycle() + invoke.resolve(platformLifecycleState()) + } + + @Command + fun getPlatformLifecycleState(invoke: Invoke) { + invoke.resolve(platformLifecycleState()) + } + @Command fun disconnect(invoke: Invoke) { val args = runCatching { invoke.parseArgs(DisconnectArgs::class.java) }.getOrNull() @@ -161,8 +300,118 @@ class CallLifecyclePlugin(private val activity: Activity) : Plugin(activity) { } } + @Command + fun setMediaEnabled(invoke: Invoke) { + val args = runCatching { invoke.parseArgs(SetMediaEnabledArgs::class.java) }.getOrNull() + val code = mediaCode(args?.kind) + if (args == null || args.operationId.isBlank() || args.connectionId.isBlank() || code == null) { + invoke.reject(errorMessage(code ?: "connect_failed"), code ?: "connect_failed") + return + } + if (activeOperationId != args.operationId || activeConnectionId != args.connectionId || room == null) { + invoke.reject(errorMessage(code), code) + return + } + + val currentGeneration = generation + if (!args.enabled) { + applyMediaToggle(args, currentGeneration, null, invoke) + return + } + + val missingAlias = when (args.kind) { + "microphone" -> if (hasPermission(android.Manifest.permission.RECORD_AUDIO)) null else "microphone" + "camera" -> if (hasPermission(android.Manifest.permission.CAMERA)) null else "camera" + else -> null + } + if (missingAlias != null) { + pendingMediaPermission = PendingMedia(invoke, args, currentGeneration) + requestPermissionForAliases(arrayOf(missingAlias), invoke, "permissionResult") + return + } + if (args.kind == "screen_share") { + pendingMediaProjection = PendingMedia(invoke, args, currentGeneration) + val manager = activity.getSystemService(Context.MEDIA_PROJECTION_SERVICE) as android.media.projection.MediaProjectionManager + startActivityForResult(invoke, manager.createScreenCaptureIntent(), "projectionResult") + return + } + applyMediaToggle(args, currentGeneration, null, invoke) + } + + private fun applyMediaToggle( + args: SetMediaEnabledArgs, + expectedGeneration: Long, + projection: Intent?, + invoke: Invoke, + ) { + val code = mediaCode(args.kind) ?: "connect_failed" + scope.launch { + val current = room + if (current == null || expectedGeneration != generation) { + invoke.reject(errorMessage(code), code) + return@launch + } + val ok = try { + when (args.kind) { + "microphone" -> current.localParticipant.setMicrophoneEnabled(args.enabled) + "camera" -> current.localParticipant.setCameraEnabled(args.enabled) + else -> { + if (args.enabled) { + val data = projection ?: throw NativeFailure("screen_share_failed") + current.localParticipant.setScreenShareEnabled(true, ScreenCaptureParams(mediaProjectionPermissionResultData = data)) + } else { + current.localParticipant.setScreenShareEnabled(false) + } + } + } + } catch (_: Exception) { + false + } + if (expectedGeneration != generation || room !== current) { + invoke.reject(errorMessage(code), code) + return@launch + } + if (ok == true) { + invoke.resolve( + JSObject() + .put("operationId", args.operationId) + .put("connectionId", args.connectionId), + ) + } else { + invoke.reject(errorMessage(code), code) + } + } + } + @PermissionCallback private fun permissionResult(invoke: Invoke) { + run { + val pending = pendingPlatformStart ?: return@run + if (pending.invoke.id != invoke.id) return@run + pendingPlatformStart = null + if (hasPermission(android.Manifest.permission.RECORD_AUDIO)) { + beginPlatformLifecycle(pending.invoke, pending.args) + } else { + failPlatformStart(pending, "permission_denied") + } + return + } + run { + val pending = pendingMediaPermission ?: return@run + if (pending.invoke.id != invoke.id) return@run + pendingMediaPermission = null + val granted = when (pending.args.kind) { + "microphone" -> hasPermission(android.Manifest.permission.RECORD_AUDIO) + else -> hasPermission(android.Manifest.permission.CAMERA) + } + if (granted) { + applyMediaToggle(pending.args, pending.generation, null, pending.invoke) + } else { + val code = mediaCode(pending.args.kind) ?: "connect_failed" + pending.invoke.reject(errorMessage(code), code) + } + return + } val pending = pendingPermission ?: return if (pending.invoke.id != invoke.id) return pendingPermission = null @@ -178,6 +427,17 @@ class CallLifecyclePlugin(private val activity: Activity) : Plugin(activity) { @ActivityCallback private fun projectionResult(invoke: Invoke, result: ActivityResult) { + run { + val pending = pendingMediaProjection ?: return@run + if (pending.invoke.id != invoke.id) return@run + pendingMediaProjection = null + if (result.resultCode != Activity.RESULT_OK || result.data == null) { + pending.invoke.reject(errorMessage("screen_share_failed"), "screen_share_failed") + } else { + applyMediaToggle(pending.args, pending.generation, result.data, pending.invoke) + } + return + } val pending = pendingProjection ?: return if (pending.invoke.id != invoke.id) return pendingProjection = null @@ -190,6 +450,8 @@ class CallLifecyclePlugin(private val activity: Activity) : Plugin(activity) { } override fun onDestroy(activity: androidx.appcompat.app.AppCompatActivity) { + pendingPlatformStart = null + releasePlatformLifecycle() roomEvents?.cancel() connectionJob?.cancel() val closeGeneration = generation + 1 @@ -308,6 +570,278 @@ class CallLifecyclePlugin(private val activity: Activity) : Plugin(activity) { } } + private fun beginPlatformLifecycle(invoke: Invoke, args: PlatformLifecycleStartArgs) { + if (platformSessionId != args.sessionId || !isActivityVisible()) { + failPlatformStart(PendingPlatformStart(invoke, args), "not_visible") + return + } + val audioManager = activity.getSystemService(Context.AUDIO_SERVICE) as AudioManager + platformAudioManager = audioManager + try { + if (args.microphone) { + if (previousAudioMode == null) previousAudioMode = audioManager.mode + audioManager.mode = AudioManager.MODE_IN_COMMUNICATION + if (!requestPlatformAudioFocus(audioManager)) { + failPlatformStart(PendingPlatformStart(invoke, args), "audio_focus_failed") + return + } + } + // The platform lifecycle only reports active once the foreground + // service positively acknowledges its promotion. + registerServiceStartAck(PendingPlatformStart(invoke, args)) + startPlatformService(args) + } catch (_: SecurityException) { + CallLifecycleForegroundService.activeStartListener = null + failPlatformStart(PendingPlatformStart(invoke, args), "service_start_failed") + } catch (_: IllegalStateException) { + CallLifecycleForegroundService.activeStartListener = null + failPlatformStart(PendingPlatformStart(invoke, args), "service_start_failed") + } catch (_: RuntimeException) { + CallLifecycleForegroundService.activeStartListener = null + failPlatformStart(PendingPlatformStart(invoke, args), "service_start_failed") + } + } + + private fun registerServiceStartAck(pending: PendingPlatformStart) { + val mainHandler = Handler(Looper.getMainLooper()) + val sessionId = pending.args.sessionId + CallLifecycleForegroundService.activeStartListener = + object : CallLifecycleForegroundService.StartListener { + override fun onServiceStarted() { + mainHandler.post { completeServiceStart(pending, failed = false) } + } + + override fun onServiceStartFailed() { + mainHandler.post { completeServiceStart(pending, failed = true) } + } + } + mainHandler.postDelayed({ completeServiceStartTimeout(pending, sessionId) }, SERVICE_START_TIMEOUT_MS) + } + + private fun completeServiceStart(pending: PendingPlatformStart, failed: Boolean) { + CallLifecycleForegroundService.activeStartListener = null + if (platformSessionId != pending.args.sessionId || platformState != PLATFORM_STARTING) return + if (failed) { + failPlatformStart(pending, "service_start_failed") + return + } + val audioManager = platformAudioManager ?: return + registerPlatformAudioRoutes(audioManager) + platformState = PLATFORM_ACTIVE + bumpPlatformRevision() + pending.invoke.resolve(platformLifecycleState()) + if (pending.args.microphone) emitPlatformFocus(PLATFORM_FOCUS_GAINED) + emitPlatformRoute(platformRoute) + } + + private fun completeServiceStartTimeout(pending: PendingPlatformStart, sessionId: String) { + if (platformSessionId != sessionId || platformState != PLATFORM_STARTING) return + CallLifecycleForegroundService.activeStartListener = null + failPlatformStart(pending, "service_start_failed") + } + + private fun requestPlatformAudioFocus(audioManager: AudioManager): Boolean { + val attributes = AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION) + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .build() + return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + val request = AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN) + .setAudioAttributes(attributes) + .setOnAudioFocusChangeListener(platformFocusListener, Handler(Looper.getMainLooper())) + .setWillPauseWhenDucked(false) + .build() + platformFocusRequest = request + audioManager.requestAudioFocus(request) == AudioManager.AUDIOFOCUS_REQUEST_GRANTED + } else { + @Suppress("DEPRECATION") + audioManager.requestAudioFocus( + platformFocusListener, + AudioManager.STREAM_VOICE_CALL, + AudioManager.AUDIOFOCUS_GAIN, + ) == AudioManager.AUDIOFOCUS_REQUEST_GRANTED + } + } + + private fun startPlatformService(args: PlatformLifecycleStartArgs) { + val intent = Intent(activity, CallLifecycleForegroundService::class.java) + .putExtra(CallLifecycleForegroundService.EXTRA_MICROPHONE, args.microphone) + .putExtra(CallLifecycleForegroundService.EXTRA_PLAYBACK, args.playback) + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + ContextCompat.startForegroundService(activity, intent) + } else { + @Suppress("DEPRECATION") + activity.startService(intent) + } + } + + private fun registerPlatformAudioRoutes(audioManager: AudioManager) { + platformRoute = currentPlatformRoute(audioManager) + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) return + val callback = object : AudioDeviceCallback() { + override fun onAudioDevicesAdded(addedDevices: Array) { + handlePlatformRouteChanged() + } + + override fun onAudioDevicesRemoved(removedDevices: Array) { + handlePlatformRouteChanged() + } + } + platformAudioCallback = callback + audioManager.registerAudioDeviceCallback(callback, Handler(Looper.getMainLooper())) + } + + private fun handlePlatformRouteChanged() { + val audioManager = platformAudioManager ?: return + val route = currentPlatformRoute(audioManager) + if (route != platformRoute) { + platformRoute = route + emitPlatformRoute(route) + } + } + + private fun currentPlatformRoute(audioManager: AudioManager): String { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + val types = audioManager.getDevices(AudioManager.GET_DEVICES_OUTPUTS).map { it.type }.toSet() + return when { + types.any { it == AudioDeviceInfo.TYPE_BLUETOOTH_A2DP || it == AudioDeviceInfo.TYPE_BLUETOOTH_SCO || + (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S && it == AudioDeviceInfo.TYPE_BLE_HEADSET) } -> + PLATFORM_ROUTE_BLUETOOTH + types.any { it == AudioDeviceInfo.TYPE_WIRED_HEADSET || it == AudioDeviceInfo.TYPE_WIRED_HEADPHONES } -> + PLATFORM_ROUTE_WIRED + types.any { it == AudioDeviceInfo.TYPE_USB_DEVICE || it == AudioDeviceInfo.TYPE_USB_HEADSET } -> + PLATFORM_ROUTE_USB + types.any { it == AudioDeviceInfo.TYPE_BUILTIN_EARPIECE } -> PLATFORM_ROUTE_EARPIECE + types.any { it == AudioDeviceInfo.TYPE_BUILTIN_SPEAKER } -> PLATFORM_ROUTE_SPEAKER + else -> PLATFORM_ROUTE_UNKNOWN + } + } + @Suppress("DEPRECATION") + return when { + audioManager.isBluetoothA2dpOn || audioManager.isBluetoothScoOn -> PLATFORM_ROUTE_BLUETOOTH + audioManager.isWiredHeadsetOn -> PLATFORM_ROUTE_WIRED + audioManager.isSpeakerphoneOn -> PLATFORM_ROUTE_SPEAKER + else -> PLATFORM_ROUTE_UNKNOWN + } + } + + private fun emitPlatformFocus(focus: String) { + platformFocus = focus + emitPlatformEvent("focus", focus = focus) + } + + private fun emitPlatformRoute(route: String) { + emitPlatformEvent("route", route = route) + } + + private fun emitPlatformEvent( + event: String, + focus: String? = null, + route: String? = null, + code: String? = null, + ) { + val channel = platformChannel ?: return + val now = SystemClock.elapsedRealtime() + val previous = lastPlatformEventAt[event] + if (event != "failure" && previous != null && now - previous < PLATFORM_EVENT_INTERVAL_MS) return + lastPlatformEventAt[event] = now + bumpPlatformRevision() + val payload = JSObject() + .put("sessionId", platformSessionId) + .put("revision", platformRevision) + .put("event", event) + focus?.let { payload.put("focus", it) } + route?.let { payload.put("route", it) } + code?.let { payload.put("code", it) } + channel.send(payload) + } + + private fun emitPlatformFailure(code: String) { + val safeCode = if (code in PLATFORM_FAILURE_CODES) code else "service_start_failed" + emitPlatformEvent("failure", code = safeCode) + } + + private fun failPlatformStart(pending: PendingPlatformStart, code: String) { + emitPlatformFailure(code) + pending.invoke.reject("platform lifecycle start failed", if (code in PLATFORM_FAILURE_CODES) code else "service_start_failed") + releasePlatformLifecycle() + } + + private fun releasePlatformLifecycle() { + CallLifecycleForegroundService.activeStartListener = null + platformAudioCallback?.let { callback -> + platformAudioManager?.let { manager -> + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + manager.unregisterAudioDeviceCallback(callback) + } + } + } + platformAudioCallback = null + platformAudioManager?.let { manager -> + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) { + platformFocusRequest?.let { manager.abandonAudioFocusRequest(it) } + } else { + @Suppress("DEPRECATION") + manager.abandonAudioFocus(platformFocusListener) + } + } + platformFocusRequest = null + platformAudioManager = null + activity.stopService(Intent(activity, CallLifecycleForegroundService::class.java)) + restoreAudio() + val hadState = platformSessionId != null || platformState != PLATFORM_IDLE + platformSessionId = null + platformChannel = null + platformFocus = PLATFORM_FOCUS_NONE + platformRoute = PLATFORM_ROUTE_UNKNOWN + platformMicrophone = false + platformPlayback = false + lastPlatformEventAt.clear() + platformState = PLATFORM_IDLE + if (hadState) bumpPlatformRevision() + } + + private fun platformLifecycleState(): JSObject = JSObject() + .put("sessionId", platformSessionId) + .put("revision", platformRevision) + .put("state", platformState) + .put("focus", platformFocus) + .put("route", platformRoute) + .put("microphone", platformMicrophone) + .put("playback", platformPlayback) + .put("backgroundJavascript", false) + + private fun bumpPlatformRevision() { + platformRevision = if (platformRevision == Long.MAX_VALUE) 1L else platformRevision + 1L + } + + private fun hasChannel(args: PlatformLifecycleStartArgs): Boolean = + runCatching { args.channel }.isSuccess + + private fun isNativeRoomModeActive(): Boolean = + room != null || activeConnect != null || pendingPermission != null || pendingProjection != null + + private fun isPlatformLifecycleActive(): Boolean = platformSessionId != null + + private fun isActivityVisible(): Boolean = + !activity.isFinishing && + (Build.VERSION.SDK_INT < Build.VERSION_CODES.JELLY_BEAN_MR1 || !activity.isDestroyed) && + activity.window.decorView.isShown && + activity.hasWindowFocus() + + private val platformFocusListener = AudioManager.OnAudioFocusChangeListener { change -> + val update = { + when (change) { + AudioManager.AUDIOFOCUS_GAIN -> emitPlatformFocus(PLATFORM_FOCUS_GAINED) + AudioManager.AUDIOFOCUS_LOSS_TRANSIENT_CAN_DUCK -> emitPlatformFocus(PLATFORM_FOCUS_DUCKED) + AudioManager.AUDIOFOCUS_LOSS, + AudioManager.AUDIOFOCUS_LOSS_TRANSIENT -> emitPlatformFocus(PLATFORM_FOCUS_LOST) + else -> Unit + } + } + if (Looper.myLooper() == Looper.getMainLooper()) update() else activity.runOnUiThread(update) + } + private suspend fun closeRoom(closeGeneration: Long, emitResult: Boolean = false): Boolean { if (closeGeneration != generation) return false val current = room ?: return false @@ -414,6 +948,13 @@ class CallLifecyclePlugin(private val activity: Activity) : Plugin(activity) { previousSpeakerphoneState = null } + private fun mediaCode(kind: String?): String? = when (kind) { + "microphone" -> "audio_failed" + "camera" -> "camera_failed" + "screen_share" -> "screen_share_failed" + else -> null + } + private fun errorMessage(code: String): String = when (code) { "audio_failed" -> "native audio failed" "camera_failed" -> "native camera failed" @@ -433,6 +974,29 @@ class CallLifecyclePlugin(private val activity: Activity) : Plugin(activity) { companion object { private const val EVENT_INTERVAL_MS = 1_000L + private const val PLATFORM_EVENT_INTERVAL_MS = 1_000L + private const val SERVICE_START_TIMEOUT_MS = 1_500L + private const val PLATFORM_IDLE = "idle" + private const val PLATFORM_STARTING = "starting" + private const val PLATFORM_ACTIVE = "active" + private const val PLATFORM_FOCUS_NONE = "none" + private const val PLATFORM_FOCUS_GAINED = "gained" + private const val PLATFORM_FOCUS_LOST = "lost" + private const val PLATFORM_FOCUS_DUCKED = "ducked" + private const val PLATFORM_ROUTE_UNKNOWN = "unknown" + private const val PLATFORM_ROUTE_EARPIECE = "earpiece" + private const val PLATFORM_ROUTE_SPEAKER = "speaker" + private const val PLATFORM_ROUTE_WIRED = "wired" + private const val PLATFORM_ROUTE_BLUETOOTH = "bluetooth" + private const val PLATFORM_ROUTE_USB = "usb" + private val PLATFORM_FAILURE_CODES = setOf( + "invalid_request", + "busy", + "not_visible", + "permission_denied", + "audio_focus_failed", + "service_start_failed", + ) private val ERROR_CODES = setOf( "connect_failed", "audio_failed", diff --git a/src-tauri/plugins/tauri-plugin-call-lifecycle/build.rs b/src-tauri/plugins/tauri-plugin-call-lifecycle/build.rs index 2b5e2b380..87571ee23 100644 --- a/src-tauri/plugins/tauri-plugin-call-lifecycle/build.rs +++ b/src-tauri/plugins/tauri-plugin-call-lifecycle/build.rs @@ -1,4 +1,13 @@ -const COMMANDS: &[&str] = &["connect", "disconnect", "get_state"]; +const COMMANDS: &[&str] = &[ + "connect", + "disconnect", + "set_media_enabled", + "get_state", + "getPlatformCallCapabilities", + "startPlatformCallLifecycle", + "stopPlatformCallLifecycle", + "getPlatformCallState", +]; fn main() { tauri_plugin::Builder::new(COMMANDS) diff --git a/src-tauri/plugins/tauri-plugin-call-lifecycle/dist-js/index.cjs b/src-tauri/plugins/tauri-plugin-call-lifecycle/dist-js/index.cjs index 6828d9846..4a30fae74 100644 --- a/src-tauri/plugins/tauri-plugin-call-lifecycle/dist-js/index.cjs +++ b/src-tauri/plugins/tauri-plugin-call-lifecycle/dist-js/index.cjs @@ -1,17 +1,48 @@ 'use strict'; var core = require('@tauri-apps/api/core'); +var event = require('@tauri-apps/api/event'); +const PLATFORM_CALL_EVENT = 'plugin:call-lifecycle://platform-event'; async function connect(request) { return await core.invoke('plugin:call-lifecycle|connect', { payload: request }); } async function disconnect(request) { return await core.invoke('plugin:call-lifecycle|disconnect', { payload: request }); } +async function setMediaEnabled(request) { + return await core.invoke('plugin:call-lifecycle|set_media_enabled', { payload: request }); +} async function getState() { return await core.invoke('plugin:call-lifecycle|get_state'); } +async function getPlatformCallCapabilities() { + return await core.invoke('plugin:call-lifecycle|getPlatformCallCapabilities'); +} +async function startPlatformCallLifecycle(request) { + return await core.invoke('plugin:call-lifecycle|startPlatformCallLifecycle', { + payload: request, + }); +} +async function stopPlatformCallLifecycle(request) { + return await core.invoke('plugin:call-lifecycle|stopPlatformCallLifecycle', { + payload: request, + }); +} +async function getPlatformCallState() { + return await core.invoke('plugin:call-lifecycle|getPlatformCallState'); +} +async function listenPlatformCallEvent(handler) { + return await event.listen(PLATFORM_CALL_EVENT, ({ payload }) => handler(payload)); +} +exports.PLATFORM_CALL_EVENT = PLATFORM_CALL_EVENT; exports.connect = connect; exports.disconnect = disconnect; +exports.getPlatformCallCapabilities = getPlatformCallCapabilities; +exports.getPlatformCallState = getPlatformCallState; exports.getState = getState; +exports.listenPlatformCallEvent = listenPlatformCallEvent; +exports.setMediaEnabled = setMediaEnabled; +exports.startPlatformCallLifecycle = startPlatformCallLifecycle; +exports.stopPlatformCallLifecycle = stopPlatformCallLifecycle; diff --git a/src-tauri/plugins/tauri-plugin-call-lifecycle/dist-js/index.d.ts b/src-tauri/plugins/tauri-plugin-call-lifecycle/dist-js/index.d.ts index 4bd666584..28345b2d2 100644 --- a/src-tauri/plugins/tauri-plugin-call-lifecycle/dist-js/index.d.ts +++ b/src-tauri/plugins/tauri-plugin-call-lifecycle/dist-js/index.d.ts @@ -1,3 +1,4 @@ +import { type UnlistenFn } from '@tauri-apps/api/event'; export interface ConnectRequest { connectionId: string; serverUrl: string; @@ -10,10 +11,28 @@ export interface DisconnectRequest { connectionId: string; } export type ConnectionState = 'idle' | 'connecting' | 'connected' | 'reconnecting' | 'disconnecting'; +export type MediaKind = 'microphone' | 'camera' | 'screen_share'; +export interface SetMediaEnabledRequest { + connectionId: string; + kind: MediaKind; + enabled: boolean; +} +export interface MediaState { + microphone: boolean; + camera: boolean; + screenShare: boolean; +} +export interface MediaCapabilities { + microphone: boolean; + camera: boolean; + screenShare: boolean; +} export interface CallState { revision: number; state: ConnectionState; connectionId: string | null; + media: MediaState; + capabilities: MediaCapabilities; } export interface CallLifecycleError { revision: number; @@ -21,6 +40,57 @@ export interface CallLifecycleError { message: string; connectionId: string | null; } +export type PlatformCallStateKind = 'idle' | 'starting' | 'active' | 'stopping'; +export type PlatformCallRoute = 'earpiece' | 'speaker' | 'wired' | 'bluetooth' | 'unknown'; +export type PlatformCallInterruption = 'began' | 'ended'; +export type PlatformCallFailureCode = 'unsupported' | 'permission_denied' | 'audio_unavailable' | 'start_failed' | 'stop_failed' | 'stale_session' | 'busy'; +export interface PlatformCallCapabilities { + supported: boolean; + microphone: boolean; + playback: boolean; +} +export interface StartPlatformCallLifecycleRequest { + sessionId: string; + microphone: boolean; + playback: boolean; +} +export interface StopPlatformCallLifecycleRequest { + sessionId: string; +} +export interface PlatformCallState { + revision: number; + state: PlatformCallStateKind; + sessionId: string | null; + microphone: boolean; + playback: boolean; + capabilities: PlatformCallCapabilities; +} +export type PlatformCallEventKind = { + type: 'focus_changed'; + focused: boolean; +} | { + type: 'route_changed'; + route: PlatformCallRoute; +} | { + type: 'interrupted'; + state: PlatformCallInterruption; +} | { + type: 'media_reset'; +} | { + type: 'failed'; + code: PlatformCallFailureCode; +}; +export type PlatformCallEvent = { + revision: number; + sessionId: string; +} & PlatformCallEventKind; +export declare const PLATFORM_CALL_EVENT = "plugin:call-lifecycle://platform-event"; export declare function connect(request: ConnectRequest): Promise; export declare function disconnect(request: DisconnectRequest): Promise; +export declare function setMediaEnabled(request: SetMediaEnabledRequest): Promise; export declare function getState(): Promise; +export declare function getPlatformCallCapabilities(): Promise; +export declare function startPlatformCallLifecycle(request: StartPlatformCallLifecycleRequest): Promise; +export declare function stopPlatformCallLifecycle(request: StopPlatformCallLifecycleRequest): Promise; +export declare function getPlatformCallState(): Promise; +export declare function listenPlatformCallEvent(handler: (event: PlatformCallEvent) => void): Promise; diff --git a/src-tauri/plugins/tauri-plugin-call-lifecycle/dist-js/index.js b/src-tauri/plugins/tauri-plugin-call-lifecycle/dist-js/index.js index 109d467e1..603654712 100644 --- a/src-tauri/plugins/tauri-plugin-call-lifecycle/dist-js/index.js +++ b/src-tauri/plugins/tauri-plugin-call-lifecycle/dist-js/index.js @@ -1,13 +1,37 @@ import { invoke } from '@tauri-apps/api/core'; +import { listen } from '@tauri-apps/api/event'; +const PLATFORM_CALL_EVENT = 'plugin:call-lifecycle://platform-event'; async function connect(request) { return await invoke('plugin:call-lifecycle|connect', { payload: request }); } async function disconnect(request) { return await invoke('plugin:call-lifecycle|disconnect', { payload: request }); } +async function setMediaEnabled(request) { + return await invoke('plugin:call-lifecycle|set_media_enabled', { payload: request }); +} async function getState() { return await invoke('plugin:call-lifecycle|get_state'); } +async function getPlatformCallCapabilities() { + return await invoke('plugin:call-lifecycle|getPlatformCallCapabilities'); +} +async function startPlatformCallLifecycle(request) { + return await invoke('plugin:call-lifecycle|startPlatformCallLifecycle', { + payload: request, + }); +} +async function stopPlatformCallLifecycle(request) { + return await invoke('plugin:call-lifecycle|stopPlatformCallLifecycle', { + payload: request, + }); +} +async function getPlatformCallState() { + return await invoke('plugin:call-lifecycle|getPlatformCallState'); +} +async function listenPlatformCallEvent(handler) { + return await listen(PLATFORM_CALL_EVENT, ({ payload }) => handler(payload)); +} -export { connect, disconnect, getState }; +export { PLATFORM_CALL_EVENT, connect, disconnect, getPlatformCallCapabilities, getPlatformCallState, getState, listenPlatformCallEvent, setMediaEnabled, startPlatformCallLifecycle, stopPlatformCallLifecycle }; diff --git a/src-tauri/plugins/tauri-plugin-call-lifecycle/guest-js/index.ts b/src-tauri/plugins/tauri-plugin-call-lifecycle/guest-js/index.ts index 85b5f879d..b1c42787d 100644 --- a/src-tauri/plugins/tauri-plugin-call-lifecycle/guest-js/index.ts +++ b/src-tauri/plugins/tauri-plugin-call-lifecycle/guest-js/index.ts @@ -1,16 +1,17 @@ -import { invoke } from '@tauri-apps/api/core' +import { invoke } from '@tauri-apps/api/core'; +import { listen, type UnlistenFn } from '@tauri-apps/api/event'; export interface ConnectRequest { - connectionId: string - serverUrl: string - participantToken: string - audio?: boolean - video?: boolean - screenShare?: boolean + connectionId: string; + serverUrl: string; + participantToken: string; + audio?: boolean; + video?: boolean; + screenShare?: boolean; } export interface DisconnectRequest { - connectionId: string + connectionId: string; } export type ConnectionState = @@ -18,29 +19,138 @@ export type ConnectionState = | 'connecting' | 'connected' | 'reconnecting' - | 'disconnecting' + | 'disconnecting'; + +export type MediaKind = 'microphone' | 'camera' | 'screen_share'; + +export interface SetMediaEnabledRequest { + connectionId: string; + kind: MediaKind; + enabled: boolean; +} + +export interface MediaState { + microphone: boolean; + camera: boolean; + screenShare: boolean; +} + +export interface MediaCapabilities { + microphone: boolean; + camera: boolean; + screenShare: boolean; +} export interface CallState { - revision: number - state: ConnectionState - connectionId: string | null + revision: number; + state: ConnectionState; + connectionId: string | null; + media: MediaState; + capabilities: MediaCapabilities; } export interface CallLifecycleError { - revision: number - code: string - message: string - connectionId: string | null + revision: number; + code: string; + message: string; + connectionId: string | null; } +export type PlatformCallStateKind = 'idle' | 'starting' | 'active' | 'stopping'; +export type PlatformCallRoute = 'earpiece' | 'speaker' | 'wired' | 'bluetooth' | 'unknown'; +export type PlatformCallInterruption = 'began' | 'ended'; +export type PlatformCallFailureCode = + | 'unsupported' + | 'permission_denied' + | 'audio_unavailable' + | 'start_failed' + | 'stop_failed' + | 'stale_session' + | 'busy'; + +export interface PlatformCallCapabilities { + supported: boolean; + microphone: boolean; + playback: boolean; +} + +export interface StartPlatformCallLifecycleRequest { + sessionId: string; + microphone: boolean; + playback: boolean; +} + +export interface StopPlatformCallLifecycleRequest { + sessionId: string; +} + +export interface PlatformCallState { + revision: number; + state: PlatformCallStateKind; + sessionId: string | null; + microphone: boolean; + playback: boolean; + capabilities: PlatformCallCapabilities; +} + +export type PlatformCallEventKind = + | { type: 'focus_changed'; focused: boolean } + | { type: 'route_changed'; route: PlatformCallRoute } + | { type: 'interrupted'; state: PlatformCallInterruption } + | { type: 'media_reset' } + | { type: 'failed'; code: PlatformCallFailureCode }; + +export type PlatformCallEvent = { + revision: number; + sessionId: string; +} & PlatformCallEventKind; + +export const PLATFORM_CALL_EVENT = 'plugin:call-lifecycle://platform-event'; + export async function connect(request: ConnectRequest): Promise { - return await invoke('plugin:call-lifecycle|connect', { payload: request }) + return await invoke('plugin:call-lifecycle|connect', { payload: request }); } export async function disconnect(request: DisconnectRequest): Promise { - return await invoke('plugin:call-lifecycle|disconnect', { payload: request }) + return await invoke('plugin:call-lifecycle|disconnect', { payload: request }); +} + +export async function setMediaEnabled(request: SetMediaEnabledRequest): Promise { + return await invoke('plugin:call-lifecycle|set_media_enabled', { payload: request }); } export async function getState(): Promise { - return await invoke('plugin:call-lifecycle|get_state') + return await invoke('plugin:call-lifecycle|get_state'); +} + +export async function getPlatformCallCapabilities(): Promise { + return await invoke( + 'plugin:call-lifecycle|getPlatformCallCapabilities' + ); +} + +export async function startPlatformCallLifecycle( + request: StartPlatformCallLifecycleRequest +): Promise { + return await invoke('plugin:call-lifecycle|startPlatformCallLifecycle', { + payload: request, + }); +} + +export async function stopPlatformCallLifecycle( + request: StopPlatformCallLifecycleRequest +): Promise { + return await invoke('plugin:call-lifecycle|stopPlatformCallLifecycle', { + payload: request, + }); +} + +export async function getPlatformCallState(): Promise { + return await invoke('plugin:call-lifecycle|getPlatformCallState'); +} + +export async function listenPlatformCallEvent( + handler: (event: PlatformCallEvent) => void +): Promise { + return await listen(PLATFORM_CALL_EVENT, ({ payload }) => handler(payload)); } diff --git a/src-tauri/plugins/tauri-plugin-call-lifecycle/ios/Sources/CallLifecyclePlugin.swift b/src-tauri/plugins/tauri-plugin-call-lifecycle/ios/Sources/CallLifecyclePlugin.swift index 56a572437..1d23d6a8b 100644 --- a/src-tauri/plugins/tauri-plugin-call-lifecycle/ios/Sources/CallLifecyclePlugin.swift +++ b/src-tauri/plugins/tauri-plugin-call-lifecycle/ios/Sources/CallLifecyclePlugin.swift @@ -18,11 +18,61 @@ private struct DisconnectArgs: Decodable { let connectionId: String } +private enum MediaKind: String, Decodable { + case microphone + case camera + case screenShare = "screen_share" + + var failureCode: FailureCode { + switch self { + case .microphone: return .audioFailed + case .camera: return .cameraFailed + case .screenShare: return .screenShareFailed + } + } +} + +private struct SetMediaEnabledArgs: Decodable { + let operationId: String + let connectionId: String + let kind: MediaKind + let enabled: Bool +} + +private struct PlatformStartArgs: Decodable { + let sessionId: String + let microphone: Bool + let playback: Bool + let channel: Channel +} + +private struct PlatformStopArgs: Decodable { + let sessionId: String +} + private struct ConnectResponse: Encodable { let operationId: String let connectionId: String } +private enum PlatformStateKind: String, Encodable { + case idle + case active +} + +private struct PlatformStateResponse: Encodable { + let sessionId: String? + let revision: UInt64 + let state: PlatformStateKind + let microphone: Bool + let playback: Bool +} + +private struct PlatformCapabilitiesResponse: Encodable { + let microphone: Bool + let backgroundAudio: Bool +} + private enum EventKind: String, Encodable { case reconnecting case reconnected @@ -39,6 +89,51 @@ private enum FailureCode: String, Encodable { case closeFailed = "close_failed" } +private enum PlatformEventKind: String, Encodable { + case focus + case route + case interruption + case mediaServicesReset = "media_services_reset" + case failure +} + +private enum PlatformFocus: String, Encodable { + case active + case lost + case regained +} + +private enum PlatformInterruption: String, Encodable { + case began + case ended +} + +private enum PlatformRoute: String, Encodable { + case speaker + case receiver + case bluetooth + case wired + case other + case none +} + +private enum PlatformFailureCode: String, Encodable { + case busy + case permissionDenied = "permission_denied" + case audioSessionFailed = "audio_session_failed" + case stopFailed = "stop_failed" +} + +private struct PlatformEvent: Encodable { + let sessionId: String + let revision: UInt64 + let event: PlatformEventKind + let focus: PlatformFocus? + let route: PlatformRoute? + let interruption: PlatformInterruption? + let code: PlatformFailureCode? +} + private struct ControlEvent: Encodable { let operationId: String let connectionId: String @@ -71,6 +166,22 @@ final class CallLifecyclePlugin: Plugin, RoomDelegate { private var eventChannel: Channel? private var generation: UInt64 = 0 private var terminalGenerations = Set() + private var platformSessionId: String? + private var platformEventChannel: Channel? + private var platformMicrophone = false + private var platformPlayback = false + private var platformRevision: UInt64 = 0 + private var platformObservers: [NSObjectProtocol] = [] + + deinit { + removePlatformObservers() + if platformSessionId != nil { + try? AVAudioSession.sharedInstance().setActive( + false, + options: .notifyOthersOnDeactivation + ) + } + } @objc public func connect(_ invoke: Invoke) throws { let args = try invoke.parseArgs(ConnectArgs.self) @@ -86,6 +197,48 @@ final class CallLifecyclePlugin: Plugin, RoomDelegate { } } + @objc public func setMediaEnabled(_ invoke: Invoke) throws { + let args = try invoke.parseArgs(SetMediaEnabledArgs.self) + Task { @MainActor [weak self] in + await self?.setMediaEnabled(args, invoke: invoke) + } + } + + @objc public func capabilities(_ invoke: Invoke) throws { + invoke.resolve( + PlatformCapabilitiesResponse( + microphone: true, + backgroundAudio: true + ) + ) + } + + @objc public func start(_ invoke: Invoke) throws { + guard let args = try? invoke.parseArgs(PlatformStartArgs.self) else { + invoke.reject("Platform audio session start failed.", code: PlatformFailureCode.audioSessionFailed.rawValue) + return + } + Task { @MainActor [weak self] in + await self?.start(args, invoke: invoke) + } + } + + @objc public func stop(_ invoke: Invoke) throws { + guard let args = try? invoke.parseArgs(PlatformStopArgs.self) else { + invoke.reject("Platform audio session stop failed.", code: PlatformFailureCode.stopFailed.rawValue) + return + } + Task { @MainActor [weak self] in + await self?.stop(args, invoke: invoke) + } + } + + @objc public func getState(_ invoke: Invoke) throws { + Task { @MainActor [weak self] in + self?.resolvePlatformState(invoke) + } + } + @MainActor private func connect(_ args: ConnectArgs, invoke: Invoke) async { generation &+= 1 @@ -175,6 +328,112 @@ final class CallLifecyclePlugin: Plugin, RoomDelegate { invoke.resolve(ConnectResponse(operationId: args.operationId, connectionId: args.connectionId)) } + @MainActor + private func start(_ args: PlatformStartArgs, invoke: Invoke) async { + if room != nil { + rejectPlatformStart( + sessionId: args.sessionId, + channel: args.channel, + code: .busy, + invoke: invoke + ) + return + } + + if platformSessionId == args.sessionId { + invoke.resolve(platformState()) + return + } + + if platformSessionId != nil { + rejectPlatformStart( + sessionId: args.sessionId, + channel: args.channel, + code: .busy, + invoke: invoke + ) + return + } + + do { + if args.microphone { + try await requestPlatformMicrophonePermission() + } + try configurePlatformAudioSession(microphone: args.microphone, playback: args.playback) + } catch SetupError.denied { + rejectPlatformStart( + sessionId: args.sessionId, + channel: args.channel, + code: .permissionDenied, + invoke: invoke + ) + return + } catch { + try? AVAudioSession.sharedInstance().setActive( + false, + options: .notifyOthersOnDeactivation + ) + rejectPlatformStart( + sessionId: args.sessionId, + channel: args.channel, + code: .audioSessionFailed, + invoke: invoke + ) + return + } + + platformSessionId = args.sessionId + platformEventChannel = args.channel + platformMicrophone = args.microphone + platformPlayback = args.playback + platformRevision &+= 1 + installPlatformObservers() + emitPlatformEvent(.focus, focus: .active) + invoke.resolve(platformState()) + } + + @MainActor + private func stop(_ args: PlatformStopArgs, invoke: Invoke) async { + guard platformSessionId == args.sessionId, + let activeSessionId = platformSessionId, + let activeChannel = platformEventChannel + else { + // A stale stop is intentionally a no-op. In particular, it cannot + // deactivate a newer session that reused this plugin instance. + invoke.resolve(platformState()) + return + } + + removePlatformObservers() + platformSessionId = nil + platformEventChannel = nil + platformMicrophone = false + platformPlayback = false + platformRevision &+= 1 + + do { + try AVAudioSession.sharedInstance().setActive( + false, + options: .notifyOthersOnDeactivation + ) + invoke.resolve(platformState()) + } catch { + sendPlatformEvent( + .failure, + sessionId: activeSessionId, + channel: activeChannel, + revision: platformRevision, + code: .stopFailed + ) + invoke.reject("Platform audio session stop failed.", code: PlatformFailureCode.stopFailed.rawValue) + } + } + + @MainActor + private func resolvePlatformState(_ invoke: Invoke) { + invoke.resolve(platformState()) + } + @MainActor private func disconnect(_ args: DisconnectArgs, invoke: Invoke) async { guard let activeRoom = room, @@ -205,6 +464,48 @@ final class CallLifecyclePlugin: Plugin, RoomDelegate { invoke.resolve(ConnectResponse(operationId: args.operationId, connectionId: args.connectionId)) } + @MainActor + private func setMediaEnabled(_ args: SetMediaEnabledArgs, invoke: Invoke) async { + let code = args.kind.failureCode + guard let activeRoom = room, + operationId == args.operationId, + connectionId == args.connectionId + else { + invoke.reject("Native call media toggle failed.", code: code.rawValue) + return + } + + // ReplayKit broadcast support is not implemented; screen share is + // explicitly unsupported on iOS. + guard args.kind != .screenShare else { + invoke.reject("Screen share is not supported on iOS.", code: code.rawValue) + return + } + + do { + switch args.kind { + case .microphone: + if args.enabled { + try await requestMicrophoneAndConfigureAudio() + } + try await activeRoom.localParticipant.setMicrophone(enabled: args.enabled) + case .camera: + if args.enabled { + guard await requestCameraPermission() else { throw SetupError.denied } + } + try await activeRoom.localParticipant.setCamera(enabled: args.enabled) + case .screenShare: + throw SetupError.denied + } + } catch { + invoke.reject("Native call media toggle failed.", code: code.rawValue) + return + } + + invoke.resolve( + ConnectResponse(operationId: args.operationId, connectionId: args.connectionId)) + } + @MainActor private func fail( _ failure: SetupFailure, @@ -263,6 +564,251 @@ final class CallLifecyclePlugin: Plugin, RoomDelegate { try session.setActive(true) } + @MainActor + private func requestPlatformMicrophonePermission() async throws { + let session = AVAudioSession.sharedInstance() + if session.recordPermission == .undetermined { + let granted = await withCheckedContinuation { continuation in + session.requestRecordPermission { granted in + continuation.resume(returning: granted) + } + } + guard granted else { throw SetupError.denied } + } else { + guard session.recordPermission == .granted else { throw SetupError.denied } + } + } + + @MainActor + private func configurePlatformAudioSession(microphone: Bool, playback: Bool) throws { + let session = AVAudioSession.sharedInstance() + if microphone { + // Recording requires playAndRecord, which also covers playback. + try session.setCategory( + .playAndRecord, + mode: .voiceChat, + options: [.allowBluetooth, .allowBluetoothA2DP, .defaultToSpeaker] + ) + } else if playback { + try session.setCategory(.playback) + } else { + try session.setCategory(.ambient) + } + try session.setActive(true) + } + + @MainActor + private func installPlatformObservers() { + let center = NotificationCenter.default + let session = AVAudioSession.sharedInstance() + platformObservers = [ + center.addObserver( + forName: AVAudioSession.interruptionNotification, + object: session, + queue: .main + ) { [weak self] notification in + Task { @MainActor [weak self] in + self?.handlePlatformInterruption(notification) + } + }, + center.addObserver( + forName: AVAudioSession.routeChangeNotification, + object: session, + queue: .main + ) { [weak self] _ in + Task { @MainActor [weak self] in + self?.handlePlatformRouteChange() + } + }, + center.addObserver( + forName: AVAudioSession.mediaServicesWereResetNotification, + object: session, + queue: .main + ) { [weak self] _ in + Task { @MainActor [weak self] in + await self?.handlePlatformMediaServicesReset() + } + }, + ] + } + + private func removePlatformObservers() { + let center = NotificationCenter.default + platformObservers.forEach { center.removeObserver($0) } + platformObservers.removeAll() + } + + @MainActor + private func handlePlatformInterruption(_ notification: Notification) { + guard platformSessionId != nil else { return } + let value = notification.userInfo?[AVAudioSessionInterruptionTypeKey] as? UInt, + let type = value.flatMap(AVAudioSession.InterruptionType.init(rawValue:)) + else { return } + + switch type { + case .began: + emitPlatformEvent(.focus, focus: .lost) + emitPlatformEvent(.interruption, interruption: .began) + case .ended: + let optionsValue = notification.userInfo?[AVAudioSessionInterruptionOptionKey] as? UInt ?? 0 + let options = AVAudioSession.InterruptionOptions(rawValue: optionsValue) + if options.contains(.shouldResume) { + do { + try AVAudioSession.sharedInstance().setActive(true) + emitPlatformEvent(.interruption, interruption: .ended) + emitPlatformEvent(.focus, focus: .regained) + } catch { + failPlatformSession(.audioSessionFailed) + } + } else { + // No resume hint: keep the session inactive and report the bounded + // interruption/focus state so JS decides whether to restart. + emitPlatformEvent(.interruption, interruption: .ended) + emitPlatformEvent(.focus, focus: .lost) + } + @unknown default: + break + } + } + + @MainActor + private func handlePlatformRouteChange() { + guard platformSessionId != nil else { return } + emitPlatformEvent(.route, route: currentPlatformRoute()) + } + + @MainActor + private func handlePlatformMediaServicesReset() async { + guard platformSessionId != nil else { return } + emitPlatformEvent(.mediaServicesReset) + do { + try configurePlatformAudioSession( + microphone: platformMicrophone, + playback: platformPlayback + ) + } catch { + failPlatformSession(.audioSessionFailed) + } + } + + @MainActor + private func failPlatformSession(_ code: PlatformFailureCode) { + guard let activeSessionId = platformSessionId, + let activeChannel = platformEventChannel + else { return } + + platformRevision &+= 1 + sendPlatformEvent( + .failure, + sessionId: activeSessionId, + channel: activeChannel, + revision: platformRevision, + code: code + ) + removePlatformObservers() + platformSessionId = nil + platformEventChannel = nil + platformMicrophone = false + platformPlayback = false + try? AVAudioSession.sharedInstance().setActive( + false, + options: .notifyOthersOnDeactivation + ) + } + + @MainActor + private func rejectPlatformStart( + sessionId: String, + channel: Channel, + code: PlatformFailureCode, + invoke: Invoke + ) { + platformRevision &+= 1 + sendPlatformEvent( + .failure, + sessionId: sessionId, + channel: channel, + revision: platformRevision, + code: code + ) + invoke.reject("Platform audio session start failed.", code: code.rawValue) + } + + @MainActor + private func platformState() -> PlatformStateResponse { + PlatformStateResponse( + sessionId: platformSessionId, + revision: platformRevision, + state: platformSessionId == nil ? .idle : .active, + microphone: platformSessionId == nil ? false : platformMicrophone, + playback: platformSessionId == nil ? false : platformPlayback + ) + } + + @MainActor + private func emitPlatformEvent( + _ event: PlatformEventKind, + focus: PlatformFocus? = nil, + route: PlatformRoute? = nil, + interruption: PlatformInterruption? = nil + ) { + guard let sessionId = platformSessionId, + let channel = platformEventChannel + else { return } + platformRevision &+= 1 + sendPlatformEvent( + event, + sessionId: sessionId, + channel: channel, + revision: platformRevision, + focus: focus, + route: route, + interruption: interruption + ) + } + + private func sendPlatformEvent( + _ event: PlatformEventKind, + sessionId: String, + channel: Channel, + revision: UInt64, + focus: PlatformFocus? = nil, + route: PlatformRoute? = nil, + interruption: PlatformInterruption? = nil, + code: PlatformFailureCode? = nil + ) { + try? channel.send( + PlatformEvent( + sessionId: sessionId, + revision: revision, + event: event, + focus: focus, + route: route, + interruption: interruption, + code: code + ) + ) + } + + @MainActor + private func currentPlatformRoute() -> PlatformRoute { + guard let output = AVAudioSession.sharedInstance().currentRoute.outputs.first else { + return .none + } + switch output.portType { + case .builtInSpeaker: + return .speaker + case .builtInReceiver: + return .receiver + case .bluetoothA2DP, .bluetoothHFP, .bluetoothLE: + return .bluetooth + case .headphones, .headsetMic, .lineOut: + return .wired + default: + return .other + } + } + @MainActor private func requestCameraPermission() async -> Bool { switch AVCaptureDevice.authorizationStatus(for: .video) { diff --git a/src-tauri/plugins/tauri-plugin-call-lifecycle/permissions/autogenerated/commands/getPlatformCallCapabilities.toml b/src-tauri/plugins/tauri-plugin-call-lifecycle/permissions/autogenerated/commands/getPlatformCallCapabilities.toml new file mode 100644 index 000000000..46d1aca39 --- /dev/null +++ b/src-tauri/plugins/tauri-plugin-call-lifecycle/permissions/autogenerated/commands/getPlatformCallCapabilities.toml @@ -0,0 +1,13 @@ +# Automatically generated - DO NOT EDIT! + +"$schema" = "../../schemas/schema.json" + +[[permission]] +identifier = "allow-getPlatformCallCapabilities" +description = "Enables the getPlatformCallCapabilities command without any pre-configured scope." +commands.allow = ["getPlatformCallCapabilities"] + +[[permission]] +identifier = "deny-getPlatformCallCapabilities" +description = "Denies the getPlatformCallCapabilities command without any pre-configured scope." +commands.deny = ["getPlatformCallCapabilities"] diff --git a/src-tauri/plugins/tauri-plugin-call-lifecycle/permissions/autogenerated/commands/getPlatformCallState.toml b/src-tauri/plugins/tauri-plugin-call-lifecycle/permissions/autogenerated/commands/getPlatformCallState.toml new file mode 100644 index 000000000..93cd8437a --- /dev/null +++ b/src-tauri/plugins/tauri-plugin-call-lifecycle/permissions/autogenerated/commands/getPlatformCallState.toml @@ -0,0 +1,13 @@ +# Automatically generated - DO NOT EDIT! + +"$schema" = "../../schemas/schema.json" + +[[permission]] +identifier = "allow-getPlatformCallState" +description = "Enables the getPlatformCallState command without any pre-configured scope." +commands.allow = ["getPlatformCallState"] + +[[permission]] +identifier = "deny-getPlatformCallState" +description = "Denies the getPlatformCallState command without any pre-configured scope." +commands.deny = ["getPlatformCallState"] diff --git a/src-tauri/plugins/tauri-plugin-call-lifecycle/permissions/autogenerated/commands/set_media_enabled.toml b/src-tauri/plugins/tauri-plugin-call-lifecycle/permissions/autogenerated/commands/set_media_enabled.toml new file mode 100644 index 000000000..98d0d76a6 --- /dev/null +++ b/src-tauri/plugins/tauri-plugin-call-lifecycle/permissions/autogenerated/commands/set_media_enabled.toml @@ -0,0 +1,13 @@ +# Automatically generated - DO NOT EDIT! + +"$schema" = "../../schemas/schema.json" + +[[permission]] +identifier = "allow-set-media-enabled" +description = "Enables the set_media_enabled command without any pre-configured scope." +commands.allow = ["set_media_enabled"] + +[[permission]] +identifier = "deny-set-media-enabled" +description = "Denies the set_media_enabled command without any pre-configured scope." +commands.deny = ["set_media_enabled"] diff --git a/src-tauri/plugins/tauri-plugin-call-lifecycle/permissions/autogenerated/commands/startPlatformCallLifecycle.toml b/src-tauri/plugins/tauri-plugin-call-lifecycle/permissions/autogenerated/commands/startPlatformCallLifecycle.toml new file mode 100644 index 000000000..042ed7ab8 --- /dev/null +++ b/src-tauri/plugins/tauri-plugin-call-lifecycle/permissions/autogenerated/commands/startPlatformCallLifecycle.toml @@ -0,0 +1,13 @@ +# Automatically generated - DO NOT EDIT! + +"$schema" = "../../schemas/schema.json" + +[[permission]] +identifier = "allow-startPlatformCallLifecycle" +description = "Enables the startPlatformCallLifecycle command without any pre-configured scope." +commands.allow = ["startPlatformCallLifecycle"] + +[[permission]] +identifier = "deny-startPlatformCallLifecycle" +description = "Denies the startPlatformCallLifecycle command without any pre-configured scope." +commands.deny = ["startPlatformCallLifecycle"] diff --git a/src-tauri/plugins/tauri-plugin-call-lifecycle/permissions/autogenerated/commands/stopPlatformCallLifecycle.toml b/src-tauri/plugins/tauri-plugin-call-lifecycle/permissions/autogenerated/commands/stopPlatformCallLifecycle.toml new file mode 100644 index 000000000..4e8b712d7 --- /dev/null +++ b/src-tauri/plugins/tauri-plugin-call-lifecycle/permissions/autogenerated/commands/stopPlatformCallLifecycle.toml @@ -0,0 +1,13 @@ +# Automatically generated - DO NOT EDIT! + +"$schema" = "../../schemas/schema.json" + +[[permission]] +identifier = "allow-stopPlatformCallLifecycle" +description = "Enables the stopPlatformCallLifecycle command without any pre-configured scope." +commands.allow = ["stopPlatformCallLifecycle"] + +[[permission]] +identifier = "deny-stopPlatformCallLifecycle" +description = "Denies the stopPlatformCallLifecycle command without any pre-configured scope." +commands.deny = ["stopPlatformCallLifecycle"] diff --git a/src-tauri/plugins/tauri-plugin-call-lifecycle/permissions/autogenerated/reference.md b/src-tauri/plugins/tauri-plugin-call-lifecycle/permissions/autogenerated/reference.md index 1ea007202..718d1c16d 100644 --- a/src-tauri/plugins/tauri-plugin-call-lifecycle/permissions/autogenerated/reference.md +++ b/src-tauri/plugins/tauri-plugin-call-lifecycle/permissions/autogenerated/reference.md @@ -6,7 +6,12 @@ Default permissions for the plugin - `allow-connect` - `allow-disconnect` +- `allow-set-media-enabled` - `allow-get-state` +- `allow-getPlatformCallCapabilities` +- `allow-startPlatformCallLifecycle` +- `allow-stopPlatformCallLifecycle` +- `allow-getPlatformCallState` ## Permission Table @@ -72,6 +77,58 @@ Denies the disconnect command without any pre-configured scope. +`call-lifecycle:allow-getPlatformCallCapabilities` + + + + +Enables the getPlatformCallCapabilities command without any pre-configured scope. + + + + + + + +`call-lifecycle:deny-getPlatformCallCapabilities` + + + + +Denies the getPlatformCallCapabilities command without any pre-configured scope. + + + + + + + +`call-lifecycle:allow-getPlatformCallState` + + + + +Enables the getPlatformCallState command without any pre-configured scope. + + + + + + + +`call-lifecycle:deny-getPlatformCallState` + + + + +Denies the getPlatformCallState command without any pre-configured scope. + + + + + + + `call-lifecycle:allow-get-state` @@ -92,6 +149,84 @@ Enables the get_state command without any pre-configured scope. Denies the get_state command without any pre-configured scope. + + + + + + +`call-lifecycle:allow-set-media-enabled` + + + + +Enables the set_media_enabled command without any pre-configured scope. + + + + + + + +`call-lifecycle:deny-set-media-enabled` + + + + +Denies the set_media_enabled command without any pre-configured scope. + + + + + + + +`call-lifecycle:allow-startPlatformCallLifecycle` + + + + +Enables the startPlatformCallLifecycle command without any pre-configured scope. + + + + + + + +`call-lifecycle:deny-startPlatformCallLifecycle` + + + + +Denies the startPlatformCallLifecycle command without any pre-configured scope. + + + + + + + +`call-lifecycle:allow-stopPlatformCallLifecycle` + + + + +Enables the stopPlatformCallLifecycle command without any pre-configured scope. + + + + + + + +`call-lifecycle:deny-stopPlatformCallLifecycle` + + + + +Denies the stopPlatformCallLifecycle command without any pre-configured scope. + diff --git a/src-tauri/plugins/tauri-plugin-call-lifecycle/permissions/default.toml b/src-tauri/plugins/tauri-plugin-call-lifecycle/permissions/default.toml index ada76b365..5d9ad298f 100644 --- a/src-tauri/plugins/tauri-plugin-call-lifecycle/permissions/default.toml +++ b/src-tauri/plugins/tauri-plugin-call-lifecycle/permissions/default.toml @@ -1,3 +1,12 @@ [default] description = "Default permissions for the plugin" -permissions = ["allow-connect", "allow-disconnect", "allow-get-state"] +permissions = [ + "allow-connect", + "allow-disconnect", + "allow-set-media-enabled", + "allow-get-state", + "allow-getPlatformCallCapabilities", + "allow-startPlatformCallLifecycle", + "allow-stopPlatformCallLifecycle", + "allow-getPlatformCallState", +] diff --git a/src-tauri/plugins/tauri-plugin-call-lifecycle/permissions/schemas/schema.json b/src-tauri/plugins/tauri-plugin-call-lifecycle/permissions/schemas/schema.json index 3561320b3..27dcd64df 100644 --- a/src-tauri/plugins/tauri-plugin-call-lifecycle/permissions/schemas/schema.json +++ b/src-tauri/plugins/tauri-plugin-call-lifecycle/permissions/schemas/schema.json @@ -318,6 +318,30 @@ "const": "deny-disconnect", "markdownDescription": "Denies the disconnect command without any pre-configured scope." }, + { + "description": "Enables the getPlatformCallCapabilities command without any pre-configured scope.", + "type": "string", + "const": "allow-getPlatformCallCapabilities", + "markdownDescription": "Enables the getPlatformCallCapabilities command without any pre-configured scope." + }, + { + "description": "Denies the getPlatformCallCapabilities command without any pre-configured scope.", + "type": "string", + "const": "deny-getPlatformCallCapabilities", + "markdownDescription": "Denies the getPlatformCallCapabilities command without any pre-configured scope." + }, + { + "description": "Enables the getPlatformCallState command without any pre-configured scope.", + "type": "string", + "const": "allow-getPlatformCallState", + "markdownDescription": "Enables the getPlatformCallState command without any pre-configured scope." + }, + { + "description": "Denies the getPlatformCallState command without any pre-configured scope.", + "type": "string", + "const": "deny-getPlatformCallState", + "markdownDescription": "Denies the getPlatformCallState command without any pre-configured scope." + }, { "description": "Enables the get_state command without any pre-configured scope.", "type": "string", @@ -331,10 +355,46 @@ "markdownDescription": "Denies the get_state command without any pre-configured scope." }, { - "description": "Default permissions for the plugin\n#### This default permission set includes:\n\n- `allow-connect`\n- `allow-disconnect`\n- `allow-get-state`", + "description": "Enables the set_media_enabled command without any pre-configured scope.", + "type": "string", + "const": "allow-set-media-enabled", + "markdownDescription": "Enables the set_media_enabled command without any pre-configured scope." + }, + { + "description": "Denies the set_media_enabled command without any pre-configured scope.", + "type": "string", + "const": "deny-set-media-enabled", + "markdownDescription": "Denies the set_media_enabled command without any pre-configured scope." + }, + { + "description": "Enables the startPlatformCallLifecycle command without any pre-configured scope.", + "type": "string", + "const": "allow-startPlatformCallLifecycle", + "markdownDescription": "Enables the startPlatformCallLifecycle command without any pre-configured scope." + }, + { + "description": "Denies the startPlatformCallLifecycle command without any pre-configured scope.", + "type": "string", + "const": "deny-startPlatformCallLifecycle", + "markdownDescription": "Denies the startPlatformCallLifecycle command without any pre-configured scope." + }, + { + "description": "Enables the stopPlatformCallLifecycle command without any pre-configured scope.", + "type": "string", + "const": "allow-stopPlatformCallLifecycle", + "markdownDescription": "Enables the stopPlatformCallLifecycle command without any pre-configured scope." + }, + { + "description": "Denies the stopPlatformCallLifecycle command without any pre-configured scope.", + "type": "string", + "const": "deny-stopPlatformCallLifecycle", + "markdownDescription": "Denies the stopPlatformCallLifecycle command without any pre-configured scope." + }, + { + "description": "Default permissions for the plugin\n#### This default permission set includes:\n\n- `allow-connect`\n- `allow-disconnect`\n- `allow-set-media-enabled`\n- `allow-get-state`\n- `allow-getPlatformCallCapabilities`\n- `allow-startPlatformCallLifecycle`\n- `allow-stopPlatformCallLifecycle`\n- `allow-getPlatformCallState`", "type": "string", "const": "default", - "markdownDescription": "Default permissions for the plugin\n#### This default permission set includes:\n\n- `allow-connect`\n- `allow-disconnect`\n- `allow-get-state`" + "markdownDescription": "Default permissions for the plugin\n#### This default permission set includes:\n\n- `allow-connect`\n- `allow-disconnect`\n- `allow-set-media-enabled`\n- `allow-get-state`\n- `allow-getPlatformCallCapabilities`\n- `allow-startPlatformCallLifecycle`\n- `allow-stopPlatformCallLifecycle`\n- `allow-getPlatformCallState`" } ] } diff --git a/src-tauri/plugins/tauri-plugin-call-lifecycle/src/actor.rs b/src-tauri/plugins/tauri-plugin-call-lifecycle/src/actor.rs index 3136c5eab..2ceb9407c 100644 --- a/src-tauri/plugins/tauri-plugin-call-lifecycle/src/actor.rs +++ b/src-tauri/plugins/tauri-plugin-call-lifecycle/src/actor.rs @@ -10,6 +10,7 @@ use tokio::sync::{mpsc, oneshot}; #[cfg(mobile)] use crate::mobile::{ MobileBackend, NativeConnectRequest, NativeControlEvent, NativeControlEventKind, + NativeStartPlatformCallLifecycleRequest, NativeStopPlatformCallLifecycleRequest, }; #[cfg(test)] @@ -21,6 +22,13 @@ use std::sync::{ use crate::error::{Error, Result}; use crate::models::{ CallLifecycleError, CallState, ConnectRequest, ConnectionState, DisconnectRequest, + MediaCapabilities, MediaKind, MediaState, PlatformCallCapabilities, PlatformCallState, + PlatformCallStateKind, SetMediaEnabledRequest, StartPlatformCallLifecycleRequest, + StopPlatformCallLifecycleRequest, +}; +#[cfg(mobile)] +use crate::models::{ + NativePlatformCallEvent, NativePlatformStartFields, PlatformCallEvent, PlatformCallEventKind, }; #[cfg(target_os = "linux")] @@ -66,6 +74,8 @@ enum ScreenShareFailureEvent { pub(crate) const STATE_EVENT: &str = "plugin:call-lifecycle://state"; pub(crate) const ERROR_EVENT: &str = "plugin:call-lifecycle://error"; +#[cfg(mobile)] +pub(crate) const PLATFORM_CALL_EVENT: &str = "plugin:call-lifecycle://platform-event"; #[cfg(not(any(target_os = "android", target_os = "ios")))] type RoomEvents = mpsc::UnboundedReceiver; @@ -117,7 +127,18 @@ impl OwnedRoom { pub(crate) enum Command { Connect(ConnectRequest, oneshot::Sender>), Disconnect(DisconnectRequest, oneshot::Sender>), + SetMediaEnabled(SetMediaEnabledRequest, oneshot::Sender>), GetState(oneshot::Sender), + GetPlatformCallCapabilities(oneshot::Sender>), + StartPlatformCallLifecycle( + StartPlatformCallLifecycleRequest, + oneshot::Sender>, + ), + StopPlatformCallLifecycle( + StopPlatformCallLifecycleRequest, + oneshot::Sender>, + ), + GetPlatformCallState(oneshot::Sender>), } enum InternalMessage { @@ -126,6 +147,8 @@ enum InternalMessage { operation_id: String, room: Option, }, + #[cfg(mobile)] + PlatformCallEvent(NativePlatformCallEvent), } struct PendingConnection { @@ -210,6 +233,15 @@ impl CallLifecycle { result.await.map_err(|_| Error::ActorUnavailable)? } + pub async fn set_media_enabled(&self, request: SetMediaEnabledRequest) -> Result { + let (response, result) = oneshot::channel(); + self.commands + .send(Command::SetMediaEnabled(request, response)) + .await + .map_err(|_| Error::ActorUnavailable)?; + result.await.map_err(|_| Error::ActorUnavailable)? + } + pub async fn get_state(&self) -> Result { let (response, result) = oneshot::channel(); self.commands @@ -218,6 +250,48 @@ impl CallLifecycle { .map_err(|_| Error::ActorUnavailable)?; result.await.map_err(|_| Error::ActorUnavailable) } + + pub async fn get_platform_call_capabilities(&self) -> Result { + let (response, result) = oneshot::channel(); + self.commands + .send(Command::GetPlatformCallCapabilities(response)) + .await + .map_err(|_| Error::ActorUnavailable)?; + result.await.map_err(|_| Error::ActorUnavailable)? + } + + pub async fn start_platform_call_lifecycle( + &self, + request: StartPlatformCallLifecycleRequest, + ) -> Result { + let (response, result) = oneshot::channel(); + self.commands + .send(Command::StartPlatformCallLifecycle(request, response)) + .await + .map_err(|_| Error::ActorUnavailable)?; + result.await.map_err(|_| Error::ActorUnavailable)? + } + + pub async fn stop_platform_call_lifecycle( + &self, + request: StopPlatformCallLifecycleRequest, + ) -> Result { + let (response, result) = oneshot::channel(); + self.commands + .send(Command::StopPlatformCallLifecycle(request, response)) + .await + .map_err(|_| Error::ActorUnavailable)?; + result.await.map_err(|_| Error::ActorUnavailable)? + } + + pub async fn get_platform_call_state(&self) -> Result { + let (response, result) = oneshot::channel(); + self.commands + .send(Command::GetPlatformCallState(response)) + .await + .map_err(|_| Error::ActorUnavailable)?; + result.await.map_err(|_| Error::ActorUnavailable)? + } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -245,11 +319,20 @@ fn event_matches_generation( active_operation_id == event_operation_id && active_connection_id == event_connection_id } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum MediaToggleDecision { + Apply, + Idempotent, + Stale, + Unsupported, +} + #[derive(Debug, Clone)] struct StateMachine { state: ConnectionState, revision: u64, connection_id: Option, + media: MediaState, } impl Default for StateMachine { @@ -258,6 +341,7 @@ impl Default for StateMachine { state: ConnectionState::Idle, revision: 0, connection_id: None, + media: MediaState::default(), } } } @@ -268,6 +352,8 @@ impl StateMachine { revision: self.revision, state: self.state, connection_id: self.connection_id.clone(), + media: self.media, + capabilities: MediaCapabilities::current(), } } @@ -300,6 +386,165 @@ impl StateMachine { self.revision += 1; self.state = state; self.connection_id = connection_id; + if !matches!( + state, + ConnectionState::Connected | ConnectionState::Reconnecting + ) { + self.media = MediaState::default(); + } + } + + fn confirm_media(&mut self, kind: MediaKind, enabled: bool) { + self.revision += 1; + self.media.set(kind, enabled); + } + + fn media_decision(&self, request: &SetMediaEnabledRequest) -> MediaToggleDecision { + if self.connection_id.as_deref() != Some(request.connection_id.as_str()) { + return MediaToggleDecision::Stale; + } + if !matches!( + self.state, + ConnectionState::Connected | ConnectionState::Reconnecting + ) { + return MediaToggleDecision::Stale; + } + if !MediaCapabilities::current().supports(request.kind) { + return MediaToggleDecision::Unsupported; + } + if self.media.enabled(request.kind) == request.enabled { + return MediaToggleDecision::Idempotent; + } + MediaToggleDecision::Apply + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PlatformStartDecision { + Start, + Idempotent, + Busy, + Unsupported, + InvalidSession, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PlatformStopDecision { + Stop, + Idempotent, + Stale, + Unsupported, + InvalidSession, +} + +#[derive(Debug, Clone)] +struct PlatformStateMachine { + revision: u64, + state: PlatformCallStateKind, + session_id: Option, + microphone: bool, + playback: bool, + capabilities: PlatformCallCapabilities, + last_session_id: Option, +} + +impl Default for PlatformStateMachine { + fn default() -> Self { + Self { + revision: 0, + state: PlatformCallStateKind::Idle, + session_id: None, + microphone: false, + playback: false, + capabilities: PlatformCallCapabilities::current(), + last_session_id: None, + } + } +} + +impl PlatformStateMachine { + fn snapshot(&self) -> PlatformCallState { + PlatformCallState { + revision: self.revision, + state: self.state, + session_id: self.session_id.clone(), + microphone: self.microphone, + playback: self.playback, + capabilities: self.capabilities, + } + } + + fn start_decision(&self, request: &StartPlatformCallLifecycleRequest) -> PlatformStartDecision { + if !self.capabilities.supported { + return PlatformStartDecision::Unsupported; + } + if request.session_id.is_empty() { + return PlatformStartDecision::InvalidSession; + } + if (request.microphone && !self.capabilities.microphone) + || (request.playback && !self.capabilities.playback) + { + return PlatformStartDecision::Unsupported; + } + match self.session_id.as_deref() { + None => PlatformStartDecision::Start, + Some(session_id) + if session_id == request.session_id + && self.microphone == request.microphone + && self.playback == request.playback => + { + PlatformStartDecision::Idempotent + } + Some(_) => PlatformStartDecision::Busy, + } + } + + fn stop_decision(&self, session_id: &str) -> PlatformStopDecision { + if !self.capabilities.supported { + return PlatformStopDecision::Unsupported; + } + if session_id.is_empty() { + return PlatformStopDecision::InvalidSession; + } + match self.session_id.as_deref() { + Some(current) if current == session_id => PlatformStopDecision::Stop, + Some(_) => PlatformStopDecision::Stale, + None if self.last_session_id.as_deref() == Some(session_id) => { + PlatformStopDecision::Idempotent + } + None => PlatformStopDecision::Stale, + } + } + + fn activate(&mut self, request: &StartPlatformCallLifecycleRequest) { + self.revision += 1; + self.state = PlatformCallStateKind::Active; + self.session_id = Some(request.session_id.clone()); + self.last_session_id = Some(request.session_id.clone()); + self.microphone = request.microphone; + self.playback = request.playback; + } + + fn stop(&mut self) { + self.revision += 1; + self.state = PlatformCallStateKind::Idle; + self.session_id = None; + self.microphone = false; + self.playback = false; + } + + #[cfg(mobile)] + fn next_event_revision(&mut self) -> u64 { + self.revision += 1; + self.revision + } + + #[cfg(mobile)] + fn fail(&mut self) { + self.state = PlatformCallStateKind::Idle; + self.session_id = None; + self.microphone = false; + self.playback = false; } } @@ -319,6 +564,9 @@ struct Actor { audio_failure: Option>, camera_failure: Option>, screen_share_failure: Option>, + platform: PlatformStateMachine, + #[cfg(mobile)] + platform_events_task: Option>, } async fn run_actor( @@ -341,6 +589,9 @@ async fn run_actor( audio_failure: None, camera_failure: None, screen_share_failure: None, + platform: PlatformStateMachine::default(), + #[cfg(mobile)] + platform_events_task: None, }; actor.emit_state(); @@ -491,9 +742,26 @@ impl Actor { Command::Disconnect(request, response) => { self.handle_disconnect(request, response).await } + Command::SetMediaEnabled(request, response) => { + self.handle_set_media_enabled(request, response).await + } Command::GetState(response) => { let _ = response.send(self.machine.snapshot()); } + Command::GetPlatformCallCapabilities(response) => { + self.handle_get_platform_call_capabilities(response).await + } + Command::StartPlatformCallLifecycle(request, response) => { + self.handle_start_platform_call_lifecycle(request, response) + .await + } + Command::StopPlatformCallLifecycle(request, response) => { + self.handle_stop_platform_call_lifecycle(request, response) + .await + } + Command::GetPlatformCallState(response) => { + let _ = response.send(Ok(self.platform.snapshot())); + } } } @@ -502,6 +770,14 @@ impl Actor { request: ConnectRequest, response: oneshot::Sender>, ) { + // Mutual exclusion with the platform lifecycle: a native room cannot + // start while a platform call session owns the audio session. + if self.platform.session_id.is_some() { + let error = Error::Busy; + self.emit_error(&error, Some(request.connection_id)); + let _ = response.send(Err(error)); + return; + } match self.machine.connect_decision(&request.connection_id) { ConnectDecision::Same => { let _ = response.send(Ok(self.machine.snapshot())); @@ -649,8 +925,211 @@ impl Actor { } } + async fn handle_set_media_enabled( + &mut self, + request: SetMediaEnabledRequest, + response: oneshot::Sender>, + ) { + match self.machine.media_decision(&request) { + MediaToggleDecision::Idempotent => { + let _ = response.send(Ok(self.machine.snapshot())); + } + MediaToggleDecision::Stale => { + let error = Error::StaleConnection; + self.emit_error(&error, Some(request.connection_id)); + let _ = response.send(Err(error)); + } + MediaToggleDecision::Unsupported => { + let error = Error::MediaUnsupported; + self.emit_error(&error, Some(request.connection_id)); + let _ = response.send(Err(error)); + } + MediaToggleDecision::Apply => self.apply_media_toggle(request, response).await, + } + } + + async fn apply_media_toggle( + &mut self, + request: SetMediaEnabledRequest, + response: oneshot::Sender>, + ) { + let connection_id = request.connection_id.clone(); + let Some(active) = self.active.as_mut() else { + let error = Error::StaleConnection; + self.emit_error(&error, Some(connection_id)); + let _ = response.send(Err(error)); + return; + }; + let enabled = request.enabled; + let outcome = match request.kind { + MediaKind::Microphone => { + toggle_microphone(active, &mut self.audio_failure, enabled).await + } + MediaKind::Camera => toggle_camera(active, &mut self.camera_failure, enabled).await, + MediaKind::ScreenShare => { + toggle_screen_share(active, &mut self.screen_share_failure, enabled, &self.app) + .await + } + }; + match outcome { + Ok(()) => { + self.machine.confirm_media(request.kind, request.enabled); + self.emit_state(); + let _ = response.send(Ok(self.machine.snapshot())); + } + Err(error) => { + self.emit_error(&error, Some(connection_id)); + let _ = response.send(Err(error)); + } + } + } + + async fn handle_get_platform_call_capabilities( + &mut self, + response: oneshot::Sender>, + ) { + #[cfg(mobile)] + let result = self.mobile.get_platform_call_capabilities().await; + #[cfg(not(mobile))] + let result = Ok(PlatformCallCapabilities::current()); + + match result { + Ok(capabilities) => { + self.platform.capabilities = capabilities; + let _ = response.send(Ok(capabilities)); + } + Err(error) => { + let _ = response.send(Err(error)); + } + } + } + + async fn handle_start_platform_call_lifecycle( + &mut self, + request: StartPlatformCallLifecycleRequest, + response: oneshot::Sender>, + ) { + match self.platform.start_decision(&request) { + PlatformStartDecision::Idempotent => { + let _ = response.send(Ok(self.platform.snapshot())); + } + PlatformStartDecision::Unsupported => { + let _ = response.send(Err(Error::PlatformCallUnsupported)); + } + PlatformStartDecision::Busy => { + let _ = response.send(Err(Error::PlatformCallBusy)); + } + PlatformStartDecision::InvalidSession => { + let _ = response.send(Err(Error::PlatformCallStaleSession)); + } + PlatformStartDecision::Start => { + // The native-room mode and the platform lifecycle are mutually + // exclusive: both platforms own a single audio session. + if self.pending.is_some() || self.active.is_some() { + let error = Error::PlatformCallBusy; + self.emit_error(&error, None); + let _ = response.send(Err(error)); + return; + } + + #[cfg(mobile)] + let result = { + let (events_sender, mut events) = mpsc::channel(32); + let channel = MobileBackend::platform_event_channel(events_sender); + let internal_tx = self.internal_tx.clone(); + let forwarder = async_runtime::spawn(async move { + while let Some(event) = events.recv().await { + let _ = internal_tx.send(InternalMessage::PlatformCallEvent(event)); + } + }); + let result = self + .mobile + .start_platform_call_lifecycle(NativeStartPlatformCallLifecycleRequest { + fields: NativePlatformStartFields { + session_id: &request.session_id, + microphone: request.microphone, + playback: request.playback, + }, + channel, + }) + .await; + if result.is_err() { + forwarder.abort(); + let _ = forwarder.await; + } else { + if let Some(stale) = self.platform_events_task.take() { + stale.abort(); + } + self.platform_events_task = Some(forwarder); + } + result + }; + #[cfg(not(mobile))] + let result: Result<()> = Err(Error::PlatformCallUnsupported); + + match result { + Ok(()) => { + self.platform.activate(&request); + let _ = response.send(Ok(self.platform.snapshot())); + } + Err(error) => { + let _ = response.send(Err(error)); + } + } + } + } + } + + async fn handle_stop_platform_call_lifecycle( + &mut self, + request: StopPlatformCallLifecycleRequest, + response: oneshot::Sender>, + ) { + match self.platform.stop_decision(&request.session_id) { + PlatformStopDecision::Idempotent => { + let _ = response.send(Ok(self.platform.snapshot())); + } + PlatformStopDecision::Unsupported => { + let _ = response.send(Err(Error::PlatformCallUnsupported)); + } + PlatformStopDecision::Stale | PlatformStopDecision::InvalidSession => { + let _ = response.send(Err(Error::PlatformCallStaleSession)); + } + PlatformStopDecision::Stop => { + #[cfg(mobile)] + let result = self + .mobile + .stop_platform_call_lifecycle(NativeStopPlatformCallLifecycleRequest { + session_id: &request.session_id, + }) + .await; + #[cfg(not(mobile))] + let result: Result<()> = Err(Error::PlatformCallUnsupported); + + match result { + Ok(()) => { + #[cfg(mobile)] + if let Some(task) = self.platform_events_task.take() { + task.abort(); + let _ = task.await; + } + self.platform.stop(); + let _ = response.send(Ok(self.platform.snapshot())); + } + Err(error) => { + let _ = response.send(Err(error)); + } + } + } + } + } + async fn handle_internal(&mut self, message: InternalMessage) { match message { + #[cfg(mobile)] + InternalMessage::PlatformCallEvent(event) => { + self.handle_platform_call_event(event); + } InternalMessage::ConnectFinished { connection_id, operation_id, @@ -787,6 +1266,18 @@ impl Actor { self.screen_share_failure = screen_share_failure; } + #[allow(unused_mut)] + let mut media = MediaState::default(); + #[cfg(target_os = "linux")] + { + media.microphone = audio.is_some(); + } + #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] + { + media.camera = camera.is_some(); + media.screen_share = screen_share.is_some(); + } + self.active = Some(ActiveConnection { connection_id: connection_id.clone(), operation_id, @@ -804,12 +1295,36 @@ impl Actor { #[cfg(test)] screen_share_shutdown_marker: None, }); + // Media is only confirmed once the sessions above have started. + self.machine.media = media; self.transition(ConnectionState::Connected, Some(connection_id)); let _ = pending.response.send(Ok(self.machine.snapshot())); } } } + #[cfg(mobile)] + fn handle_platform_call_event(&mut self, event: NativePlatformCallEvent) { + if self.platform.session_id.as_deref() != Some(event.session_id.as_str()) { + return; + } + let Some(kind) = event.to_kind() else { + return; + }; + if matches!(kind, PlatformCallEventKind::Failed { .. }) { + self.platform.fail(); + if let Some(task) = self.platform_events_task.take() { + task.abort(); + } + } + let payload = PlatformCallEvent { + revision: self.platform.next_event_revision(), + session_id: event.session_id, + kind, + }; + let _ = self.app.emit(PLATFORM_CALL_EVENT, payload); + } + async fn handle_audio_failure(&mut self) { #[cfg(not(target_os = "linux"))] { @@ -980,6 +1495,11 @@ impl Actor { } async fn cleanup(&mut self) { + #[cfg(mobile)] + if let Some(task) = self.platform_events_task.take() { + task.abort(); + let _ = task.await; + } if let Some(pending) = self.pending.take() { pending.task.abort(); let _ = pending.task.await; @@ -995,6 +1515,176 @@ impl Actor { } } +#[cfg(target_os = "linux")] +async fn toggle_microphone( + active: &mut ActiveConnection, + audio_failure: &mut Option>, + enabled: bool, +) -> Result<()> { + if enabled { + let result = match &active.room { + OwnedRoom::LiveKit(room) => AudioSession::start(room).await, + #[cfg(test)] + OwnedRoom::Fake(_) => Err("audio requires a live room".to_owned()), + #[cfg(test)] + OwnedRoom::FakeOrdered(_) => Err("audio requires a live room".to_owned()), + }; + match result { + Ok((audio, failure_rx)) => { + active.audio = Some(audio); + *audio_failure = Some(failure_rx); + Ok(()) + } + Err(_) => Err(Error::AudioFailed), + } + } else { + audio_failure.take(); + if let Some(mut audio) = active.audio.take() { + audio.shutdown().await; + } + Ok(()) + } +} + +#[cfg(all( + not(target_os = "linux"), + not(any(target_os = "android", target_os = "ios")) +))] +async fn toggle_microphone( + active: &mut ActiveConnection, + _audio_failure: &mut Option>, + enabled: bool, +) -> Result<()> { + let _ = (active, enabled); + Err(Error::MediaUnsupported) +} + +#[cfg(mobile)] +async fn toggle_microphone( + active: &mut ActiveConnection, + _audio_failure: &mut Option>, + enabled: bool, +) -> Result<()> { + active + .room + .set_media_enabled( + &active.operation_id, + &active.connection_id, + MediaKind::Microphone, + enabled, + ) + .await +} + +#[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] +async fn toggle_camera( + active: &mut ActiveConnection, + camera_failure: &mut Option>, + enabled: bool, +) -> Result<()> { + if enabled { + let result = match &active.room { + OwnedRoom::LiveKit(room) => CameraSession::start(room).await, + #[cfg(test)] + OwnedRoom::Fake(_) => Err("camera requires a live room".to_owned()), + #[cfg(test)] + OwnedRoom::FakeOrdered(_) => Err("camera requires a live room".to_owned()), + }; + match result { + Ok((camera, failure_rx)) => { + active.camera = Some(camera); + *camera_failure = Some(failure_rx); + Ok(()) + } + Err(_) => Err(Error::CameraFailed), + } + } else { + camera_failure.take(); + if let Some(mut camera) = active.camera.take() { + camera.shutdown().await; + } + Ok(()) + } +} + +#[cfg(mobile)] +async fn toggle_camera( + active: &mut ActiveConnection, + _camera_failure: &mut Option>, + enabled: bool, +) -> Result<()> { + active + .room + .set_media_enabled( + &active.operation_id, + &active.connection_id, + MediaKind::Camera, + enabled, + ) + .await +} + +#[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] +async fn toggle_screen_share( + active: &mut ActiveConnection, + screen_share_failure: &mut Option>, + enabled: bool, + app: &AppHandle, +) -> Result<()> { + if enabled { + let result = match &active.room { + OwnedRoom::LiveKit(room) => { + #[cfg(target_os = "windows")] + { + ScreenShareSession::start(app, room).await + } + #[cfg(not(target_os = "windows"))] + { + let _ = app; + ScreenShareSession::start(room).await + } + } + #[cfg(test)] + OwnedRoom::Fake(_) => Err("screen share requires a live room".to_owned()), + #[cfg(test)] + OwnedRoom::FakeOrdered(_) => Err("screen share requires a live room".to_owned()), + }; + match result { + Ok((screen_share, failure_rx)) => { + active.screen_share = Some(screen_share); + *screen_share_failure = Some(failure_rx); + Ok(()) + } + Err(_) => Err(Error::ScreenShareFailed), + } + } else { + let _ = app; + screen_share_failure.take(); + if let Some(mut screen_share) = active.screen_share.take() { + screen_share.shutdown().await; + } + Ok(()) + } +} + +#[cfg(mobile)] +async fn toggle_screen_share( + active: &mut ActiveConnection, + _screen_share_failure: &mut Option>, + enabled: bool, + _app: &AppHandle, +) -> Result<()> { + active + .room + .set_media_enabled( + &active.operation_id, + &active.connection_id, + MediaKind::ScreenShare, + enabled, + ) + .await +} + #[cfg(not(any(target_os = "android", target_os = "ios")))] async fn close_active_connection(active: Option) -> std::result::Result<(), ()> { if let Some(active) = active { @@ -1060,14 +1750,22 @@ async fn close_queued_rooms(internal_rx: &mut mpsc::UnboundedReceiver SetMediaEnabledRequest { + SetMediaEnabledRequest { + connection_id: connection_id.into(), + kind, + enabled, + } + } + + #[cfg(target_os = "linux")] + fn actor_with_camera( + app: tauri::AppHandle, + camera: Option, + media: MediaState, + ) -> super::Actor { + let (_commands_tx, commands) = tokio::sync::mpsc::channel(1); + let (internal_tx, internal_rx) = tokio::sync::mpsc::unbounded_channel(); + let (_events_tx, events) = tokio::sync::mpsc::unbounded_channel(); + + super::Actor { + app, + commands, + internal_tx, + internal_rx, + machine: super::StateMachine { + state: ConnectionState::Connected, + revision: 1, + connection_id: Some("one".into()), + media, + }, + pending: None, + active: Some(super::ActiveConnection { + connection_id: "one".into(), + operation_id: "one:1".into(), + room: super::OwnedRoom::Fake(Arc::new(AtomicUsize::new(0))), + events, + audio: None, + camera, + screen_share: None, + screen_share_shutdown_marker: None, + }), + audio_failure: None, + camera_failure: None, + screen_share_failure: None, + platform: super::PlatformStateMachine::default(), + #[cfg(mobile)] + platform_events_task: None, + } + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn failed_media_start_preserves_confirmed_state_without_emitting_state() { + let app = tauri::test::mock_app(); + let state_events = Arc::new(AtomicUsize::new(0)); + let listener_events = state_events.clone(); + let _listener = app.listen(super::STATE_EVENT, move |_| { + listener_events.fetch_add(1, Ordering::SeqCst); + }); + let mut actor = actor_with_camera(app.handle().clone(), None, MediaState::default()); + let before = actor.machine.snapshot(); + let (response, result) = tokio::sync::oneshot::channel(); + + actor + .handle_set_media_enabled(media_request("one", MediaKind::Camera, true), response) + .await; + + assert!(matches!(result.await, Ok(Err(Error::CameraFailed)))); + assert_eq!(actor.machine.snapshot(), before); + assert_eq!(state_events.load(Ordering::SeqCst), 0); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn successful_media_stop_emits_confirmed_snapshot_once() { + let app = tauri::test::mock_app(); + let state_events = Arc::new(Mutex::new(Vec::::new())); + let listener_events = state_events.clone(); + let _listener = app.listen(super::STATE_EVENT, move |event| { + listener_events + .lock() + .unwrap() + .push(serde_json::from_str(event.payload()).unwrap()); + }); + let mut actor = actor_with_camera( + app.handle().clone(), + Some(crate::camera::CameraSession::without_devices(None)), + MediaState { + camera: true, + ..Default::default() + }, + ); + let (response, result) = tokio::sync::oneshot::channel(); + + actor + .handle_set_media_enabled(media_request("one", MediaKind::Camera, false), response) + .await; + + let state = result.await.unwrap().unwrap(); + assert_eq!(state.revision, 2); + assert!(!state.media.camera); + assert_eq!(actor.machine.snapshot(), state); + assert_eq!( + &*state_events.lock().unwrap(), + &[serde_json::to_value(state).unwrap()] + ); + } + + #[test] + fn media_toggle_is_stale_when_idle_in_progress_or_for_another_connection() { + let mut machine = StateMachine::default(); + assert_eq!( + machine.media_decision(&media_request("one", MediaKind::Camera, true)), + MediaToggleDecision::Stale + ); + machine.transition(ConnectionState::Connecting, Some("one".into())); + assert_eq!( + machine.media_decision(&media_request("one", MediaKind::Camera, true)), + MediaToggleDecision::Stale + ); + machine.transition(ConnectionState::Connected, Some("one".into())); + assert_eq!( + machine.media_decision(&media_request("two", MediaKind::Camera, true)), + MediaToggleDecision::Stale + ); + assert!(machine.snapshot().connection_id.is_some()); + } + + #[test] + fn media_toggle_respects_truthful_capabilities() { + let mut machine = StateMachine::default(); + machine.transition(ConnectionState::Connected, Some("one".into())); + let capabilities = MediaCapabilities::current(); + let kinds = [ + (MediaKind::Microphone, capabilities.microphone), + (MediaKind::Camera, capabilities.camera), + (MediaKind::ScreenShare, capabilities.screen_share), + ]; + for (kind, supported) in kinds { + let expected = if supported { + MediaToggleDecision::Apply + } else { + MediaToggleDecision::Unsupported + }; + assert_eq!( + machine.media_decision(&media_request("one", kind, true)), + expected, + "unexpected decision for {kind:?}" + ); + } + #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] + { + assert!(capabilities.camera); + assert!(capabilities.screen_share); + } + assert_eq!(capabilities.microphone, cfg!(target_os = "linux")); + } + + #[test] + fn media_toggle_is_idempotent_without_a_revision_bump() { + let mut machine = StateMachine::default(); + machine.transition(ConnectionState::Connected, Some("one".into())); + machine.confirm_media(MediaKind::Camera, true); + let revision = machine.snapshot().revision; + assert_eq!( + machine.media_decision(&media_request("one", MediaKind::Camera, true)), + MediaToggleDecision::Idempotent + ); + assert_eq!(machine.snapshot().revision, revision); + assert_eq!( + machine.media_decision(&media_request("one", MediaKind::Camera, false)), + MediaToggleDecision::Apply + ); + assert_eq!(machine.snapshot().revision, revision); + } + + #[test] + fn confirmed_media_bumps_revision_and_resets_outside_connected_states() { + let mut machine = StateMachine::default(); + machine.transition(ConnectionState::Connecting, Some("one".into())); + assert_eq!(machine.snapshot().media, Default::default()); + + machine.transition(ConnectionState::Connected, Some("one".into())); + machine.confirm_media(MediaKind::ScreenShare, true); + let snapshot = machine.snapshot(); + assert!(snapshot.media.screen_share); + assert_eq!(snapshot.revision, 3); + + machine.transition(ConnectionState::Reconnecting, Some("one".into())); + assert!(machine.snapshot().media.screen_share); + + machine.transition(ConnectionState::Disconnecting, Some("one".into())); + assert_eq!(machine.snapshot().media, Default::default()); + machine.transition(ConnectionState::Idle, None); + let snapshot = machine.snapshot(); + assert_eq!(snapshot.media, Default::default()); + assert!(snapshot.connection_id.is_none()); + } + + #[test] + fn media_toggle_reconnecting_keeps_confirmed_state_and_stays_togglable() { + let mut machine = StateMachine::default(); + machine.transition(ConnectionState::Connected, Some("one".into())); + machine.confirm_media(MediaKind::Camera, true); + machine.transition(ConnectionState::Reconnecting, Some("one".into())); + assert!(machine.snapshot().media.camera); + if MediaCapabilities::current().camera { + assert_eq!( + machine.media_decision(&media_request("one", MediaKind::Camera, false)), + MediaToggleDecision::Apply + ); + } + } + + #[test] + fn set_media_enabled_request_and_call_state_have_stable_wire_shapes() { + let request: SetMediaEnabledRequest = serde_json::from_value(serde_json::json!({ + "connectionId": "one", + "kind": "screen_share", + "enabled": true + })) + .unwrap(); + assert_eq!(request.kind, MediaKind::ScreenShare); + assert!(request.enabled); + + let state = serde_json::to_value(StateMachine::default().snapshot()).unwrap(); + assert_eq!( + state["media"], + serde_json::json!({ + "microphone": false, + "camera": false, + "screenShare": false + }) + ); + assert!(state["capabilities"].is_object()); + + let error = serde_json::to_value(Error::MediaUnsupported).unwrap(); + assert_eq!( + error, + serde_json::json!({ + "code": "media_unsupported", + "message": "media kind is not supported on this platform" + }) + ); + } + + #[test] + fn platform_lifecycle_has_opaque_session_and_idempotent_stop_semantics() { + let mut machine = PlatformStateMachine::default(); + assert_eq!( + machine.start_decision(&StartPlatformCallLifecycleRequest { + session_id: "session".into(), + microphone: true, + playback: true, + }), + PlatformStartDecision::Unsupported + ); + + machine.capabilities = PlatformCallCapabilities { + supported: true, + microphone: true, + playback: true, + }; + let request = StartPlatformCallLifecycleRequest { + session_id: "opaque-session".into(), + microphone: true, + playback: true, + }; + assert_eq!( + machine.start_decision(&request), + PlatformStartDecision::Start + ); + machine.activate(&request); + assert_eq!( + machine.start_decision(&request), + PlatformStartDecision::Idempotent + ); + assert_eq!( + machine.start_decision(&StartPlatformCallLifecycleRequest { + session_id: "another-session".into(), + microphone: true, + playback: true, + }), + PlatformStartDecision::Busy + ); + assert_eq!( + machine.stop_decision("another-session"), + PlatformStopDecision::Stale + ); + assert_eq!( + machine.stop_decision("opaque-session"), + PlatformStopDecision::Stop + ); + machine.stop(); + assert_eq!( + machine.stop_decision("opaque-session"), + PlatformStopDecision::Idempotent + ); + assert_eq!(machine.snapshot().session_id, None); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn platform_start_is_busy_while_native_room_is_active() { + let app = tauri::test::mock_app(); + let mut actor = actor_with_camera(app.handle().clone(), None, MediaState::default()); + actor.platform.capabilities = PlatformCallCapabilities { + supported: true, + microphone: true, + playback: true, + }; + let revision_before = actor.platform.snapshot().revision; + let (response, result) = tokio::sync::oneshot::channel(); + + actor + .handle_start_platform_call_lifecycle( + StartPlatformCallLifecycleRequest { + session_id: "session".into(), + microphone: true, + playback: true, + }, + response, + ) + .await; + + assert!(matches!(result.await, Ok(Err(Error::PlatformCallBusy)))); + assert_eq!(actor.platform.snapshot().revision, revision_before); + assert_eq!( + actor.platform.snapshot().state, + super::PlatformCallStateKind::Idle + ); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn connect_is_busy_while_platform_call_session_is_active() { + let app = tauri::test::mock_app(); + let mut actor = actor_with_camera(app.handle().clone(), None, MediaState::default()); + // No active native room: clear it so only the platform session remains. + let connection = actor.active.take(); + let _ = super::close_active_connection(connection).await; + actor.transition(ConnectionState::Idle, None); + actor.platform.activate(&StartPlatformCallLifecycleRequest { + session_id: "session".into(), + microphone: true, + playback: true, + }); + let (response, result) = tokio::sync::oneshot::channel(); + + actor + .handle_connect( + ConnectRequest { + connection_id: "one".into(), + server_url: "wss://livekit.example".into(), + participant_token: SecretString::from("secret-jwt"), + audio: false, + video: false, + screen_share: false, + }, + response, + ) + .await; + + assert!(matches!(result.await, Ok(Err(Error::Busy)))); + assert_eq!(actor.machine.snapshot().state, ConnectionState::Idle); + assert_eq!( + actor.platform.snapshot().session_id.as_deref(), + Some("session") + ); + } + + #[test] + fn platform_stop_protects_against_replacement_session() { + let mut machine = PlatformStateMachine::default(); + machine.capabilities = PlatformCallCapabilities { + supported: true, + microphone: true, + playback: true, + }; + let first = StartPlatformCallLifecycleRequest { + session_id: "first".into(), + microphone: true, + playback: true, + }; + machine.activate(&first); + machine.stop(); + machine.activate(&StartPlatformCallLifecycleRequest { + session_id: "second".into(), + microphone: true, + playback: true, + }); + + // A stop targeted at the replaced session must not touch the active one. + assert_eq!(machine.stop_decision("first"), PlatformStopDecision::Stale); + assert_eq!(machine.stop_decision("second"), PlatformStopDecision::Stop); + machine.stop(); + assert_eq!( + machine.stop_decision("never-seen"), + PlatformStopDecision::Stale + ); + } + + #[test] + fn platform_contract_serializes_only_bounded_wire_values() { + let event = serde_json::to_value(crate::models::PlatformCallEvent { + revision: 4, + session_id: "opaque-session".into(), + kind: crate::models::PlatformCallEventKind::RouteChanged { + route: crate::models::PlatformCallRoute::Bluetooth, + }, + }) + .unwrap(); + assert_eq!( + event, + serde_json::json!({ + "revision": 4, + "sessionId": "opaque-session", + "type": "route_changed", + "route": "bluetooth" + }) + ); + assert!( + serde_json::from_value::( + serde_json::json!({ "type": "failed", "code": "start_failed" }) + ) + .is_ok() + ); + assert!( + serde_json::from_value::( + serde_json::json!({ "type": "failed", "message": "raw native error" }) + ) + .is_err() + ); + } } diff --git a/src-tauri/plugins/tauri-plugin-call-lifecycle/src/commands.rs b/src-tauri/plugins/tauri-plugin-call-lifecycle/src/commands.rs index 64aed22ae..ade2d9cb7 100644 --- a/src-tauri/plugins/tauri-plugin-call-lifecycle/src/commands.rs +++ b/src-tauri/plugins/tauri-plugin-call-lifecycle/src/commands.rs @@ -20,7 +20,53 @@ pub(crate) async fn disconnect( app.call_lifecycle().disconnect(payload).await } +#[command] +pub(crate) async fn set_media_enabled( + app: AppHandle, + payload: SetMediaEnabledRequest, +) -> Result { + app.call_lifecycle().set_media_enabled(payload).await +} + #[command] pub(crate) async fn get_state(app: AppHandle) -> Result { app.call_lifecycle().get_state().await } + +#[allow(non_snake_case)] +#[command] +pub(crate) async fn getPlatformCallCapabilities( + app: AppHandle, +) -> Result { + app.call_lifecycle().get_platform_call_capabilities().await +} + +#[allow(non_snake_case)] +#[command] +pub(crate) async fn startPlatformCallLifecycle( + app: AppHandle, + payload: StartPlatformCallLifecycleRequest, +) -> Result { + app.call_lifecycle() + .start_platform_call_lifecycle(payload) + .await +} + +#[allow(non_snake_case)] +#[command] +pub(crate) async fn stopPlatformCallLifecycle( + app: AppHandle, + payload: StopPlatformCallLifecycleRequest, +) -> Result { + app.call_lifecycle() + .stop_platform_call_lifecycle(payload) + .await +} + +#[allow(non_snake_case)] +#[command] +pub(crate) async fn getPlatformCallState( + app: AppHandle, +) -> Result { + app.call_lifecycle().get_platform_call_state().await +} diff --git a/src-tauri/plugins/tauri-plugin-call-lifecycle/src/error.rs b/src-tauri/plugins/tauri-plugin-call-lifecycle/src/error.rs index c27415936..d99cf363e 100644 --- a/src-tauri/plugins/tauri-plugin-call-lifecycle/src/error.rs +++ b/src-tauri/plugins/tauri-plugin-call-lifecycle/src/error.rs @@ -22,6 +22,18 @@ pub enum Error { CameraFailed, #[error("native screen share failed")] ScreenShareFailed, + #[error("media kind is not supported on this platform")] + MediaUnsupported, + #[error("platform call lifecycle is not supported on this platform")] + PlatformCallUnsupported, + #[error("platform call lifecycle is already active")] + PlatformCallBusy, + #[error("platform call session does not match the active session")] + PlatformCallStaleSession, + #[error("platform call lifecycle failed to start")] + PlatformCallStartFailed, + #[error("platform call lifecycle failed to stop")] + PlatformCallStopFailed, } impl Serialize for Error { @@ -48,6 +60,12 @@ impl Error { Self::VideoFailed => "video_failed", Self::CameraFailed => "camera_failed", Self::ScreenShareFailed => "screen_share_failed", + Self::MediaUnsupported => "media_unsupported", + Self::PlatformCallUnsupported => "platform_call_unsupported", + Self::PlatformCallBusy => "platform_call_busy", + Self::PlatformCallStaleSession => "platform_call_stale_session", + Self::PlatformCallStartFailed => "platform_call_start_failed", + Self::PlatformCallStopFailed => "platform_call_stop_failed", } } @@ -62,6 +80,16 @@ impl Error { Self::VideoFailed => "native video failed", Self::CameraFailed => "native camera failed", Self::ScreenShareFailed => "native screen share failed", + Self::MediaUnsupported => "media kind is not supported on this platform", + Self::PlatformCallUnsupported => { + "platform call lifecycle is not supported on this platform" + } + Self::PlatformCallBusy => "platform call lifecycle is already active", + Self::PlatformCallStaleSession => { + "platform call session does not match the active session" + } + Self::PlatformCallStartFailed => "platform call lifecycle failed to start", + Self::PlatformCallStopFailed => "platform call lifecycle failed to stop", } } } diff --git a/src-tauri/plugins/tauri-plugin-call-lifecycle/src/lib.rs b/src-tauri/plugins/tauri-plugin-call-lifecycle/src/lib.rs index f2325f348..c0a71f160 100644 --- a/src-tauri/plugins/tauri-plugin-call-lifecycle/src/lib.rs +++ b/src-tauri/plugins/tauri-plugin-call-lifecycle/src/lib.rs @@ -47,7 +47,12 @@ pub fn init() -> TauriPlugin { .invoke_handler(tauri::generate_handler![ commands::connect, commands::disconnect, - commands::get_state + commands::set_media_enabled, + commands::get_state, + commands::getPlatformCallCapabilities, + commands::startPlatformCallLifecycle, + commands::stopPlatformCallLifecycle, + commands::getPlatformCallState ]) .setup(|app, api| { #[cfg(mobile)] diff --git a/src-tauri/plugins/tauri-plugin-call-lifecycle/src/mobile.rs b/src-tauri/plugins/tauri-plugin-call-lifecycle/src/mobile.rs index b0e49d209..a6eec96af 100644 --- a/src-tauri/plugins/tauri-plugin-call-lifecycle/src/mobile.rs +++ b/src-tauri/plugins/tauri-plugin-call-lifecycle/src/mobile.rs @@ -6,7 +6,13 @@ use tauri::{ }; use tokio::sync::mpsc; -use crate::{error::Error, models::DisconnectRequest}; +use crate::{ + error::Error, + models::{ + DisconnectRequest, MediaKind, NativePlatformCallEvent, NativePlatformStartFields, + PlatformCallCapabilities, + }, +}; #[cfg(target_os = "android")] const PLUGIN_IDENTIFIER: &str = "app.tauri.call_lifecycle"; @@ -53,6 +59,56 @@ pub(crate) struct NativeControlEvent { pub kind: NativeControlEventKind, } +#[cfg(target_os = "android")] +mod platform_commands { + pub(super) const CAPABILITIES: &str = "getPlatformLifecycleCapabilities"; + pub(super) const START: &str = "startPlatformLifecycle"; + pub(super) const STOP: &str = "stopPlatformLifecycle"; +} + +#[cfg(target_os = "ios")] +mod platform_commands { + pub(super) const CAPABILITIES: &str = "capabilities"; + pub(super) const START: &str = "start"; + pub(super) const STOP: &str = "stop"; +} + +/// Mirrors the Android `getPlatformLifecycleCapabilities` response. +#[cfg(target_os = "android")] +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct NativePlatformCapabilities { + #[serde(default)] + microphone: bool, + #[serde(default)] + audio_playback: bool, +} + +/// Mirrors the iOS `capabilities` response. +#[cfg(target_os = "ios")] +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct NativePlatformCapabilities { + #[serde(default)] + microphone: bool, + #[serde(default)] + background_audio: bool, +} + +impl From for PlatformCallCapabilities { + fn from(native: NativePlatformCapabilities) -> Self { + // A native response is proof the platform lifecycle exists. + Self { + supported: true, + microphone: native.microphone, + #[cfg(target_os = "android")] + playback: native.audio_playback, + #[cfg(target_os = "ios")] + playback: native.background_audio, + } + } +} + #[derive(Debug, Clone, Deserialize)] #[serde(tag = "event", rename_all = "snake_case")] pub(crate) enum NativeControlEventKind { @@ -103,6 +159,17 @@ impl MobileBackend { }) } + pub(crate) fn platform_event_channel( + sender: mpsc::Sender, + ) -> Channel { + Channel::new(move |body: InvokeResponseBody| { + if let Ok(event) = body.deserialize::() { + let _ = sender.try_send(event); + } + Ok(()) + }) + } + pub(crate) async fn connect( &self, request: NativeConnectRequest<'_>, @@ -128,6 +195,63 @@ impl MobileBackend { .map(|_| ()) .map_err(|_| Error::CloseFailed) } + + pub(crate) async fn set_media_enabled( + &self, + operation_id: &str, + connection_id: &str, + kind: MediaKind, + enabled: bool, + ) -> crate::Result<()> { + let payload = NativeSetMediaEnabledRequest { + operation_id, + connection_id, + kind, + enabled, + }; + self.handle + .run_mobile_plugin_async::("setMediaEnabled", payload) + .await + .map(|_| ()) + .map_err(|_| match kind { + MediaKind::Microphone => Error::AudioFailed, + MediaKind::Camera => Error::CameraFailed, + MediaKind::ScreenShare => Error::ScreenShareFailed, + }) + } + + pub(crate) async fn get_platform_call_capabilities( + &self, + ) -> crate::Result { + let native: NativePlatformCapabilities = self + .handle + .run_mobile_plugin_async(platform_commands::CAPABILITIES, ()) + .await + .map_err(|_| Error::PlatformCallUnsupported)?; + Ok(native.into()) + } + + pub(crate) async fn start_platform_call_lifecycle( + &self, + request: NativeStartPlatformCallLifecycleRequest<'_>, + ) -> crate::Result<()> { + self.handle + .run_mobile_plugin_async::(platform_commands::START, request) + .await + .map(|_| ()) + .map_err(|_| Error::PlatformCallStartFailed) + } + + pub(crate) async fn stop_platform_call_lifecycle( + &self, + request: NativeStopPlatformCallLifecycleRequest<'_>, + ) -> crate::Result<()> { + self.handle + .run_mobile_plugin_async::(platform_commands::STOP, request) + .await + .map(|_| ()) + .map_err(|_| Error::PlatformCallStopFailed) + } } #[derive(Debug, Serialize)] @@ -137,6 +261,32 @@ struct NativeDisconnectRequest<'a> { connection_id: &'a str, } +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct NativeSetMediaEnabledRequest<'a> { + operation_id: &'a str, + connection_id: &'a str, + kind: MediaKind, + enabled: bool, +} + +/// Native start payload. The natives accept `{ sessionId, channel }` and +/// tolerate the extra `microphone`/`playback` flags, which the bridge forwards +/// so later stages do not need to re-extend the payload. +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct NativeStartPlatformCallLifecycleRequest<'a> { + #[serde(flatten)] + pub fields: NativePlatformStartFields<'a>, + pub channel: Channel, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct NativeStopPlatformCallLifecycleRequest<'a> { + pub session_id: &'a str, +} + pub fn init( _app: &AppHandle, api: PluginApi, diff --git a/src-tauri/plugins/tauri-plugin-call-lifecycle/src/models.rs b/src-tauri/plugins/tauri-plugin-call-lifecycle/src/models.rs index 0527d2cae..c823540ec 100644 --- a/src-tauri/plugins/tauri-plugin-call-lifecycle/src/models.rs +++ b/src-tauri/plugins/tauri-plugin-call-lifecycle/src/models.rs @@ -21,6 +21,438 @@ pub struct DisconnectRequest { pub connection_id: String, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PlatformCallStateKind { + Idle, + Starting, + Active, + Stopping, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PlatformCallRoute { + Earpiece, + Speaker, + Wired, + Bluetooth, + Unknown, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PlatformCallInterruption { + Began, + Ended, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PlatformCallFailureCode { + Unsupported, + PermissionDenied, + AudioUnavailable, + StartFailed, + StopFailed, + StaleSession, + Busy, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum PlatformCallEventKind { + FocusChanged { focused: bool }, + RouteChanged { route: PlatformCallRoute }, + Interrupted { state: PlatformCallInterruption }, + MediaReset, + Failed { code: PlatformCallFailureCode }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PlatformCallEvent { + pub revision: u64, + pub session_id: String, + #[serde(flatten)] + pub kind: PlatformCallEventKind, +} + +/// Raw platform lifecycle event as emitted by the Android/iOS native plugins. +/// +/// Native fields stay as strings at the boundary; the bridge maps them into the +/// bounded guest enums below, dropping unknown values instead of forwarding raw +/// native errors or device names. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NativePlatformCallEvent { + pub session_id: String, + #[serde(default)] + #[allow(dead_code)] + pub revision: Option, + pub event: String, + #[serde(default)] + pub focus: Option, + #[serde(default)] + pub route: Option, + #[serde(default)] + pub interruption: Option, + #[serde(default)] + pub code: Option, +} + +impl NativePlatformCallEvent { + pub fn to_kind(&self) -> Option { + match self.event.as_str() { + // Android reports "gained"|"lost"|"ducked"; iOS reports "active"|"lost"|"regained". + "focus" => Some(PlatformCallEventKind::FocusChanged { + focused: matches!( + self.focus.as_deref(), + Some("gained" | "active" | "regained") + ), + }), + // Union of Android (earpiece|speaker|wired|bluetooth|usb|unknown) + // and iOS (receiver|speaker|bluetooth|wired|other|none) routes. + "route" => Some(PlatformCallEventKind::RouteChanged { + route: match self.route.as_deref() { + Some("earpiece" | "receiver") => PlatformCallRoute::Earpiece, + Some("speaker") => PlatformCallRoute::Speaker, + Some("wired") => PlatformCallRoute::Wired, + Some("bluetooth") => PlatformCallRoute::Bluetooth, + _ => PlatformCallRoute::Unknown, + }, + }), + "interruption" => Some(PlatformCallEventKind::Interrupted { + state: match self.interruption.as_deref() { + Some("began") => PlatformCallInterruption::Began, + _ => PlatformCallInterruption::Ended, + }, + }), + "media_services_reset" => Some(PlatformCallEventKind::MediaReset), + // Android: invalid_request|busy|not_visible|permission_denied| + // audio_focus_failed|service_start_failed. iOS: busy| + // permission_denied|audio_session_failed|stop_failed. + "failure" => Some(PlatformCallEventKind::Failed { + code: match self.code.as_deref() { + Some("busy") => PlatformCallFailureCode::Busy, + Some("permission_denied") => PlatformCallFailureCode::PermissionDenied, + Some("audio_focus_failed" | "audio_session_failed") => { + PlatformCallFailureCode::AudioUnavailable + } + Some("stop_failed") => PlatformCallFailureCode::StopFailed, + _ => PlatformCallFailureCode::StartFailed, + }, + }), + _ => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn native_event(event: &str) -> NativePlatformCallEvent { + NativePlatformCallEvent { + session_id: "session".into(), + revision: None, + event: event.into(), + focus: None, + route: None, + interruption: None, + code: None, + } + } + + #[test] + fn native_focus_maps_to_bounded_focused_flag() { + for (raw, focused) in [ + ("gained", true), + ("active", true), + ("regained", true), + ("lost", false), + ("ducked", false), + ] { + let mut event = native_event("focus"); + event.focus = Some(raw.into()); + assert_eq!( + event.to_kind(), + Some(PlatformCallEventKind::FocusChanged { focused }), + "focus {raw}" + ); + } + assert_eq!( + native_event("focus").to_kind(), + Some(PlatformCallEventKind::FocusChanged { focused: false }) + ); + } + + #[test] + fn native_route_maps_to_bounded_route_without_device_names() { + for (raw, route) in [ + ("earpiece", PlatformCallRoute::Earpiece), + ("receiver", PlatformCallRoute::Earpiece), + ("speaker", PlatformCallRoute::Speaker), + ("wired", PlatformCallRoute::Wired), + ("bluetooth", PlatformCallRoute::Bluetooth), + ("usb", PlatformCallRoute::Unknown), + ("other", PlatformCallRoute::Unknown), + ("none", PlatformCallRoute::Unknown), + ("bose-quietcomfort-35", PlatformCallRoute::Unknown), + ] { + let mut event = native_event("route"); + event.route = Some(raw.into()); + assert_eq!( + event.to_kind(), + Some(PlatformCallEventKind::RouteChanged { route }), + "route {raw}" + ); + } + } + + #[test] + fn native_interruption_and_reset_map_to_bounded_variants() { + let mut began = native_event("interruption"); + began.interruption = Some("began".into()); + assert_eq!( + began.to_kind(), + Some(PlatformCallEventKind::Interrupted { + state: PlatformCallInterruption::Began + }) + ); + let mut ended = native_event("interruption"); + ended.interruption = Some("ended".into()); + assert_eq!( + ended.to_kind(), + Some(PlatformCallEventKind::Interrupted { + state: PlatformCallInterruption::Ended + }) + ); + assert_eq!( + native_event("media_services_reset").to_kind(), + Some(PlatformCallEventKind::MediaReset) + ); + } + + #[test] + fn native_failure_codes_map_to_safe_codes() { + for (raw, code) in [ + ("busy", PlatformCallFailureCode::Busy), + ( + "permission_denied", + PlatformCallFailureCode::PermissionDenied, + ), + ( + "audio_focus_failed", + PlatformCallFailureCode::AudioUnavailable, + ), + ( + "audio_session_failed", + PlatformCallFailureCode::AudioUnavailable, + ), + ("stop_failed", PlatformCallFailureCode::StopFailed), + ("invalid_request", PlatformCallFailureCode::StartFailed), + ("not_visible", PlatformCallFailureCode::StartFailed), + ("service_start_failed", PlatformCallFailureCode::StartFailed), + ("Device is muted", PlatformCallFailureCode::StartFailed), + ] { + let mut event = native_event("failure"); + event.code = Some(raw.into()); + assert_eq!( + event.to_kind(), + Some(PlatformCallEventKind::Failed { code }), + "code {raw}" + ); + } + } + + #[test] + fn unknown_native_event_is_dropped() { + assert_eq!(native_event("unknown_thing").to_kind(), None); + } + + #[test] + fn native_start_fields_serialize_as_camel_case() { + let fields = NativePlatformStartFields { + session_id: "opaque-session", + microphone: true, + playback: false, + }; + assert_eq!( + serde_json::to_value(fields).unwrap(), + serde_json::json!({ + "sessionId": "opaque-session", + "microphone": true, + "playback": false + }) + ); + } + + #[test] + fn native_event_deserializes_android_and_ios_payloads() { + let android: NativePlatformCallEvent = serde_json::from_value(serde_json::json!({ + "sessionId": "s", + "revision": 3, + "event": "route", + "route": "usb" + })) + .unwrap(); + assert_eq!( + android.to_kind(), + Some(PlatformCallEventKind::RouteChanged { + route: PlatformCallRoute::Unknown + }) + ); + + let ios: NativePlatformCallEvent = serde_json::from_value(serde_json::json!({ + "sessionId": "s", + "revision": 3, + "event": "interruption", + "interruption": "began" + })) + .unwrap(); + assert_eq!( + ios.to_kind(), + Some(PlatformCallEventKind::Interrupted { + state: PlatformCallInterruption::Began + }) + ); + } +} + +/// The media flags forwarded to the native platform-lifecycle plugins. +/// +/// Kept in `models.rs` (not `mobile.rs`, which only compiles on mobile +/// targets) so the wire shape is regression-tested on the host. +#[derive(Debug, Clone, Copy, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct NativePlatformStartFields<'a> { + pub session_id: &'a str, + pub microphone: bool, + pub playback: bool, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StartPlatformCallLifecycleRequest { + pub session_id: String, + pub microphone: bool, + pub playback: bool, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StopPlatformCallLifecycleRequest { + pub session_id: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PlatformCallCapabilities { + pub supported: bool, + pub microphone: bool, + pub playback: bool, +} + +impl PlatformCallCapabilities { + pub fn current() -> Self { + let supported = cfg!(any(target_os = "android", target_os = "ios")); + Self { + supported, + microphone: supported, + playback: supported, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PlatformCallState { + pub revision: u64, + pub state: PlatformCallStateKind, + pub session_id: Option, + pub microphone: bool, + pub playback: bool, + pub capabilities: PlatformCallCapabilities, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MediaKind { + Microphone, + Camera, + ScreenShare, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SetMediaEnabledRequest { + pub connection_id: String, + pub kind: MediaKind, + pub enabled: bool, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct MediaState { + pub microphone: bool, + pub camera: bool, + pub screen_share: bool, +} + +impl MediaState { + pub fn enabled(&self, kind: MediaKind) -> bool { + match kind { + MediaKind::Microphone => self.microphone, + MediaKind::Camera => self.camera, + MediaKind::ScreenShare => self.screen_share, + } + } + + pub fn set(&mut self, kind: MediaKind, enabled: bool) { + match kind { + MediaKind::Microphone => self.microphone = enabled, + MediaKind::Camera => self.camera = enabled, + MediaKind::ScreenShare => self.screen_share = enabled, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct MediaCapabilities { + pub microphone: bool, + pub camera: bool, + pub screen_share: bool, +} + +impl MediaCapabilities { + pub fn current() -> Self { + let desktop = cfg!(any( + target_os = "linux", + target_os = "macos", + target_os = "windows" + )); + Self { + microphone: cfg!(any( + target_os = "linux", + target_os = "android", + target_os = "ios" + )), + camera: desktop || cfg!(any(target_os = "android", target_os = "ios")), + screen_share: desktop || cfg!(target_os = "android"), + } + } + + pub fn supports(&self, kind: MediaKind) -> bool { + match kind { + MediaKind::Microphone => self.microphone, + MediaKind::Camera => self.camera, + MediaKind::ScreenShare => self.screen_share, + } + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] #[serde(rename_all = "snake_case")] pub enum ConnectionState { @@ -37,6 +469,8 @@ pub struct CallState { pub revision: u64, pub state: ConnectionState, pub connection_id: Option, + pub media: MediaState, + pub capabilities: MediaCapabilities, } #[derive(Debug, Clone, Serialize)] diff --git a/src/app/features/call/CallView.test.tsx b/src/app/features/call/CallView.test.tsx new file mode 100644 index 000000000..7d55d11eb --- /dev/null +++ b/src/app/features/call/CallView.test.tsx @@ -0,0 +1,388 @@ +import { act, render, renderHook, screen } from '@testing-library/react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { NativeCallSession } from '$state/nativeCall'; +import { getDebugLogger, type LogEntry } from '$utils/debugLogger'; +import type * as Platform from '$utils/platform'; +import { NATIVE_CALL_PROBE_STORAGE_KEY } from './nativeCallProbe'; +import { + getNativeCallDiagnostics, + serializeNativeCallDiagnostics, + useNativeCallDiagnostics, + useNativeCallProbeSession, + NativeMediaControls, + LivekitJsCallProbe, +} from './CallView'; + +const setMediaEnabled = vi.hoisted(() => vi.fn<(...args: unknown[]) => Promise>()); + +vi.mock('$plugins/call/callLifecycle', () => ({ + getState: vi.fn<() => Promise>(), + onState: vi.fn<(handler: unknown) => Promise>(), + setMediaEnabled, +})); + +const desktopTauri = vi.hoisted(() => vi.fn<() => boolean>()); +const mobileTauri = vi.hoisted(() => vi.fn<() => boolean>()); + +vi.mock('$utils/platform', async (importOriginal) => ({ + ...(await importOriginal()), + isDesktopTauri: desktopTauri, + isMobileTauri: mobileTauri, +})); + +const makeSession = (): NativeCallSession => ({ + roomId: '!room:example.org', + connectionId: 'connection-id', + lifecycle: 'connected', + hangup: vi.fn<() => Promise>().mockResolvedValue(), +}); + +const mediaSnapshot = { + connectionId: 'connection-id', + media: { microphone: true, camera: false, screenShare: false }, + capabilities: { microphone: true, camera: true, screenShare: false }, +}; + +describe('native media controls', () => { + beforeEach(() => { + setMediaEnabled.mockReset(); + setMediaEnabled.mockResolvedValue({}); + }); + + it('shows only advertised controls and reflects confirmed state', () => { + render(); + + expect(screen.getByRole('button', { name: 'Turn off microphone' })).toBeEnabled(); + expect(screen.getByRole('button', { name: 'Start camera' })).toBeEnabled(); + expect(screen.queryByRole('button', { name: 'Start screen sharing' })).not.toBeInTheDocument(); + }); + + it('sends the requested media kind and state, then blocks duplicate requests', async () => { + let resolveRequest!: (value: unknown) => void; + setMediaEnabled.mockReturnValue( + new Promise((resolve) => { + resolveRequest = resolve; + }) + ); + render(); + + const camera = screen.getByRole('button', { name: 'Start camera' }); + await act(async () => camera.click()); + expect(setMediaEnabled).toHaveBeenCalledWith({ + connectionId: 'connection-id', + kind: 'camera', + enabled: true, + }); + expect(camera).toBeDisabled(); + expect(screen.getByRole('button', { name: 'Turn off microphone' })).toBeDisabled(); + + await act(async () => resolveRequest({})); + }); + + it('shows a recoverable error when the native command fails', async () => { + setMediaEnabled.mockRejectedValueOnce(new Error('native failure')); + render(); + + const camera = screen.getByRole('button', { name: 'Start camera' }); + await act(async () => camera.click()); + + expect(screen.getByText('Couldn’t change camera.')).toBeInTheDocument(); + expect(camera).toBeEnabled(); + expect(screen.getByRole('button', { name: 'Turn off microphone' })).toBeEnabled(); + }); + + it('shows the safe failure code and records a sanitized diagnostic on rejection', async () => { + const debugLogger = getDebugLogger(); + debugLogger.clear(); + debugLogger.setEnabled(true); + setMediaEnabled.mockRejectedValueOnce({ + code: 'camera_failed', + message: 'native camera failed', + }); + render(); + + const camera = screen.getByRole('button', { name: 'Start camera' }); + await act(async () => camera.click()); + + expect(screen.getByText('Couldn’t change camera (camera_failed).')).toBeInTheDocument(); + expect(camera).toBeEnabled(); + expect(getNativeCallDiagnostics(debugLogger.getLogs())).toContainEqual({ + stage: 'media control', + code: 'camera_failed', + errorName: 'UnknownError', + errorMessage: 'native camera failed', + }); + }); + + it('supports media_unsupported and never logs raw invoke details', async () => { + const debugLogger = getDebugLogger(); + debugLogger.clear(); + debugLogger.setEnabled(true); + setMediaEnabled.mockRejectedValueOnce({ + code: 'media_unsupported', + message: 'media kind is not supported on this platform', + }); + const { rerender } = render( + + ); + + await act(async () => screen.getByRole('button', { name: 'Start camera' }).click()); + expect(screen.getByText('Couldn’t change camera (media_unsupported).')).toBeInTheDocument(); + + setMediaEnabled.mockRejectedValueOnce({ + code: 'https://evil.example/token?access_token=code-secret', + message: 'invoke failed at https://evil.example/track?token=message-secret', + }); + rerender(); + await act(async () => screen.getByRole('button', { name: 'Start camera' }).click()); + + expect(screen.getByText('Couldn’t change camera.')).toBeInTheDocument(); + const payload = serializeNativeCallDiagnostics(getNativeCallDiagnostics(debugLogger.getLogs())); + expect(payload).not.toContain('evil.example'); + expect(payload).not.toContain('secret'); + }); +}); + +describe('LiveKit JS connection probe', () => { + it('renders connection and E2EE readiness without media controls', () => { + render( + {}} /> + ); + + expect(screen.getByText('LiveKit JS connection probe')).toBeInTheDocument(); + expect(screen.getByText('Connection: Connected')).toBeInTheDocument(); + expect(screen.getByText('E2EE readiness:')).toHaveTextContent('Ready'); + expect( + screen.getByText('Connection-only experiment · media is not published') + ).toBeInTheDocument(); + expect( + screen.queryByRole('button', { name: /microphone|camera|screen/i }) + ).not.toBeInTheDocument(); + }); + + it('uses safe status and setup failure text', () => { + render( + {}} + /> + ); + + expect(screen.getByText('Connection: Connection failed')).toBeInTheDocument(); + expect(screen.getByText('E2EE readiness:')).toHaveTextContent('Unavailable'); + expect(screen.getByText('LiveKit JS connection setup failed.')).toBeInTheDocument(); + expect(screen.queryByText(/token|url|secret|error:/i)).not.toBeInTheDocument(); + }); + + it('shows unavailable E2EE status and calls hangup from End', () => { + const onHangup = vi.fn<() => void>(); + render( + + ); + + expect(screen.getByText('E2EE readiness:')).toHaveTextContent('Unavailable on this device'); + screen.getByRole('button', { name: 'End' }).click(); + expect(onHangup).toHaveBeenCalledOnce(); + }); +}); + +describe('native call probe session gate', () => { + beforeEach(() => { + desktopTauri.mockReturnValue(true); + mobileTauri.mockReturnValue(false); + localStorage.clear(); + vi.stubEnv('VITE_ENABLE_NATIVE_CALL_PROBE', 'false'); + }); + + it('hangs up a native session once when native calls are disabled', () => { + const session = makeSession(); + const { rerender } = renderHook( + ({ nativeCallsEnabled }) => useNativeCallProbeSession(session, nativeCallsEnabled), + { initialProps: { nativeCallsEnabled: false } } + ); + + expect(session.hangup).toHaveBeenCalledTimes(1); + + rerender({ nativeCallsEnabled: false }); + expect(session.hangup).toHaveBeenCalledTimes(1); + }); + + it('keeps the native session when a developer override enables the probe', () => { + localStorage.setItem(NATIVE_CALL_PROBE_STORAGE_KEY, '1'); + const session = makeSession(); + + renderHook(() => useNativeCallProbeSession(session, false)); + + expect(session.hangup).not.toHaveBeenCalled(); + }); +}); + +describe('native call diagnostics', () => { + const debugLogger = getDebugLogger(); + + beforeEach(() => { + debugLogger.clear(); + debugLogger.setEnabled(true); + }); + + it('filters controller call logs and keeps only sanitized diagnostic fields', () => { + const logs = [ + { + timestamp: 1, + level: 'error', + category: 'call', + namespace: 'nativeCallController', + message: 'Native call setup failed', + data: { + stage: 'connecting', + errorName: 'Error', + errorMessage: 'redacted', + cause: { errorName: 'UnknownError', errorMessage: 'redacted', token: 'secret' }, + roomId: '!secret:example.org', + }, + }, + { + timestamp: 2, + level: 'error', + category: 'call', + namespace: 'nativeCallController', + message: 'Native call setup failed', + data: { stage: 'authorizing', errorName: 'Error', errorMessage: 'redacted' }, + }, + { + timestamp: 3, + level: 'error', + category: 'network', + namespace: 'nativeCallController', + message: 'Ignored', + data: { stage: 'ignored' }, + }, + ] satisfies LogEntry[]; + + expect(getNativeCallDiagnostics(logs)).toEqual([ + { stage: 'authorizing', errorName: 'Error', errorMessage: 'redacted' }, + { + stage: 'connecting', + errorName: 'Error', + errorMessage: 'redacted', + cause: { errorName: 'UnknownError', errorMessage: 'redacted' }, + }, + ]); + }); + + it('updates when a matching logger entry arrives', () => { + const { result } = renderHook(() => useNativeCallDiagnostics()); + + act(() => { + debugLogger.log('error', 'call', 'nativeCallController', 'Native call setup failed', { + stage: 'authorizing', + errorName: 'Error', + errorMessage: 'redacted', + }); + }); + + expect(result.current).toEqual([ + { stage: 'authorizing', errorName: 'Error', errorMessage: 'redacted' }, + ]); + }); + + it('copies only the safe diagnostic payload', () => { + const diagnostics = getNativeCallDiagnostics([ + { + timestamp: 1, + level: 'error', + category: 'call', + namespace: 'nativeCallController', + message: 'Native call setup failed', + data: { + code: 'connect_failed', + stage: 'joining the call', + errorName: 'Error', + errorMessage: 'redacted', + roomId: '!secret:example.org', + token: 'secret-token', + }, + }, + ] satisfies LogEntry[]); + + const payload = serializeNativeCallDiagnostics(diagnostics); + + expect(JSON.parse(payload)).toEqual([ + { + code: 'connect_failed', + stage: 'joining the call', + errorName: 'Error', + errorMessage: 'redacted', + }, + ]); + expect(payload).not.toContain('secret'); + }); + + it('redacts unallowlisted diagnostic values in fields and nested causes', () => { + const diagnostics = getNativeCallDiagnostics([ + { + timestamp: 1, + level: 'error', + category: 'call', + namespace: 'nativeCallController', + message: 'Native call setup failed', + data: { + stage: 'https://evil.example/token?access_token=stage-secret', + errorName: 'RawNativeException', + errorMessage: 'https://evil.example/token?access_token=message-secret', + cause: { + errorName: 'RawCauseException token-secret', + errorMessage: 'raw cause exception https://evil.example/cause-secret', + }, + }, + }, + ] satisfies LogEntry[]); + const payload = serializeNativeCallDiagnostics(diagnostics); + + expect(diagnostics).toEqual([ + { + stage: 'redacted', + errorName: 'redacted', + errorMessage: 'redacted', + cause: { errorName: 'redacted', errorMessage: 'redacted' }, + }, + ]); + expect(payload).not.toContain('evil.example'); + expect(payload).not.toContain('secret'); + expect(payload).not.toContain('RawNativeException'); + expect(payload).not.toContain('RawCauseException'); + }); + + it('redacts an unallowlisted lifecycle code in UI-derived diagnostics and copy payload', () => { + const diagnostics = getNativeCallDiagnostics([ + { + timestamp: 1, + level: 'error', + category: 'call', + namespace: 'nativeCallController', + message: 'Native call connection failed', + data: { + code: 'https://evil.example/token?access_token=code-secret', + stage: 'connecting', + errorName: 'Error', + errorMessage: 'redacted', + }, + }, + ] satisfies LogEntry[]); + const payload = serializeNativeCallDiagnostics(diagnostics); + + expect(diagnostics).toEqual([ + { + code: 'redacted', + stage: 'connecting', + errorName: 'Error', + errorMessage: 'redacted', + }, + ]); + expect(payload).not.toContain('evil.example'); + expect(payload).not.toContain('code-secret'); + }); +}); diff --git a/src/app/features/call/CallView.tsx b/src/app/features/call/CallView.tsx index 434948bcf..5427a77e3 100644 --- a/src/app/features/call/CallView.tsx +++ b/src/app/features/call/CallView.tsx @@ -1,5 +1,18 @@ import { useCallback, useEffect, useRef, useState, type RefObject } from 'react'; -import { Badge, Box, Button, color, Header, Scroll, Text, toRem } from 'folds'; +import { + Badge, + Box, + Button, + color, + Header, + IconButton, + Scroll, + Spinner, + Text, + toRem, + Tooltip, + TooltipProvider, +} from 'folds'; import { useAtomValue } from 'jotai'; import { ContainerColor } from '$styles/ContainerColor.css'; import { useRoom } from '$hooks/useRoom'; @@ -13,7 +26,259 @@ import { CallMemberRenderer } from './CallMemberCard'; import { PrescreenControls } from './PrescreenControls'; import { callEmbedAtom, callEmbedStartErrorAtom } from '$state/callEmbed'; import { canJoinCall } from './callStartCapabilities'; +import type { NativeCallSession } from '$state/nativeCall'; import { nativeCallAtom } from '$state/nativeCall'; +import type { LivekitJsCallSession } from '$state/livekitJsCall'; +import { livekitJsCallAtom } from '$state/livekitJsCall'; +import { isNativeCallProbeEnabled } from './nativeCallProbe'; +import { settingsAtom } from '$state/settings'; +import { useSetting } from '$state/hooks/settings'; +import { createDebugLogger, getDebugLogger, type LogEntry } from '$utils/debugLogger'; +import { copyToClipboard } from '$utils/dom'; +import { + Microphone, + MicrophoneSlash, + ScreenShare, + VideoCamera, + VideoCameraSlash, + sizedIcon, +} from '$components/icons/phosphor'; +import { getState, onState, setMediaEnabled, type MediaKind } from '$plugins/call/callLifecycle'; +import { canRenderLivekitJsMediaTest, LivekitJsMediaTestSurface } from './LivekitJsMediaTest'; + +type NativeMediaSnapshot = { + connectionId: string; + media: { + microphone: boolean; + camera: boolean; + screenShare: boolean; + }; + capabilities: { + microphone: boolean; + camera: boolean; + screenShare: boolean; + }; +}; + +type NativeMediaKind = MediaKind; + +const nativeMediaDebug = createDebugLogger('nativeCallController'); + +const allowlistedErrorField = ( + error: unknown, + property: 'code' | 'name' | 'message', + allowlist: Set +): string | undefined => { + const value = isRecord(error) ? error[property] : undefined; + return typeof value === 'string' && allowlist.has(value) ? value : undefined; +}; + +// Records a bounded, allowlisted diagnostic for a failed media toggle and +// returns the safe failure code (if known). Raw invoke messages, URLs, and +// tokens are never logged. +const logMediaControlFailure = (error: unknown): string | undefined => { + const code = allowlistedErrorField(error, 'code', nativeCallCodes); + nativeMediaDebug.error('call', 'Native media control failed', { + stage: 'media control', + code: code ?? 'unknown', + errorName: allowlistedErrorField(error, 'name', nativeCallErrorNames) ?? 'UnknownError', + errorMessage: allowlistedErrorField(error, 'message', nativeCallErrorMessages) ?? 'redacted', + }); + return code === 'unknown' ? undefined : code; +}; + +const toNativeMediaSnapshot = (state: unknown): NativeMediaSnapshot | undefined => { + if (!isRecord(state) || !isRecord(state.media) || !isRecord(state.capabilities)) return undefined; + if (typeof state.connectionId !== 'string') return undefined; + + const media = state.media; + const capabilities = state.capabilities; + if ( + typeof media.microphone !== 'boolean' || + typeof media.camera !== 'boolean' || + typeof media.screenShare !== 'boolean' || + typeof capabilities.microphone !== 'boolean' || + typeof capabilities.camera !== 'boolean' || + typeof capabilities.screenShare !== 'boolean' + ) { + return undefined; + } + + return { + connectionId: state.connectionId, + media: { + microphone: media.microphone, + camera: media.camera, + screenShare: media.screenShare, + }, + capabilities: { + microphone: capabilities.microphone, + camera: capabilities.camera, + screenShare: capabilities.screenShare, + }, + }; +}; + +export function NativeMediaControls({ + connectionId, + snapshot, +}: { + connectionId: string; + snapshot: NativeMediaSnapshot | undefined; +}) { + const [pending, setPending] = useState(); + const [error, setError] = useState(); + + const toggle = useCallback( + async (kind: NativeMediaKind, enabled: boolean) => { + if (pending || !snapshot) return; + setPending(kind); + setError(undefined); + try { + await setMediaEnabled({ + connectionId, + kind, + enabled, + }); + } catch (toggleError) { + const code = logMediaControlFailure(toggleError); + setError( + `Couldn’t change ${kind === 'screen_share' ? 'screen sharing' : kind}${ + code ? ` (${code})` : '' + }.` + ); + } finally { + setPending(undefined); + } + }, + [connectionId, pending, snapshot] + ); + + if (!snapshot) return null; + + const controls: Array<{ + kind: NativeMediaKind; + enabled: boolean; + supported: boolean; + label: string; + icon: typeof Microphone; + }> = [ + { + kind: 'microphone', + enabled: snapshot.media.microphone, + supported: snapshot.capabilities.microphone, + label: snapshot.media.microphone ? 'Turn off microphone' : 'Turn on microphone', + icon: snapshot.media.microphone ? Microphone : MicrophoneSlash, + }, + { + kind: 'camera', + enabled: snapshot.media.camera, + supported: snapshot.capabilities.camera, + label: snapshot.media.camera ? 'Stop camera' : 'Start camera', + icon: snapshot.media.camera ? VideoCamera : VideoCameraSlash, + }, + { + kind: 'screen_share', + enabled: snapshot.media.screenShare, + supported: snapshot.capabilities.screenShare, + label: snapshot.media.screenShare ? 'Stop screen sharing' : 'Start screen sharing', + icon: ScreenShare, + }, + ]; + + return ( + + + {controls.map(({ kind, enabled, supported, label, icon }) => { + if (!supported) return null; + const busy = pending === kind; + return ( + + {label} + + } + > + {(anchorRef) => ( + void toggle(kind, !enabled)} + > + {busy ? ( + + ) : ( + sizedIcon(icon, '100', { + filled: kind === 'microphone' ? !enabled : enabled, + }) + )} + + )} + + ); + })} + + {error && ( + + {error} + + )} + + ); +} + +function useNativeMediaSnapshot( + connectionId: string, + active: boolean +): NativeMediaSnapshot | undefined { + const [snapshot, setSnapshot] = useState(); + + useEffect(() => { + if (!active) { + setSnapshot(undefined); + return undefined; + } + let current = true; + void (async () => { + try { + const next = toNativeMediaSnapshot(await getState()); + if (current && next?.connectionId === connectionId) setSnapshot(next); + } catch { + // The probe can still receive a state event if the initial read fails. + } + })(); + const unlisten = onState((state) => { + const next = toNativeMediaSnapshot(state); + if (next?.connectionId === connectionId) setSnapshot(next); + }); + return () => { + current = false; + void (async () => { + (await unlisten)(); + })(); + }; + }, [active, connectionId]); + + return snapshot; +} function LivekitServerMissingMessage() { return ( @@ -83,24 +348,58 @@ function WidgetPreparationErrorMessage({ message }: { message: string }) { function NativeCallProbe({ lifecycle, + connectionId, error, onHangup, }: { lifecycle: string; + connectionId: string; error?: string; onHangup: () => void; }) { + const diagnostics = useNativeCallDiagnostics(); + const mediaSnapshot = useNativeMediaSnapshot(connectionId, lifecycle !== 'error'); + return ( Native call: {lifecycle} - Media is not implemented. + Experimental native call + {error && ( {error} )} + + Native diagnostics + + {diagnostics.length > 0 ? ( + diagnostics.map((diagnostic) => ( + + {diagnostic.code && `${diagnostic.code} · `} + {diagnostic.stage} · {diagnostic.errorName}: {diagnostic.errorMessage} + {diagnostic.cause && + ` (cause: ${diagnostic.cause.errorName}: ${diagnostic.cause.errorMessage})`} + + )) + ) : ( + No native call diagnostics yet. + )} + + + + + ); +} + +export type NativeCallDiagnostic = { + code?: string; + stage?: string; + errorName?: string; + errorMessage?: string; + cause?: { + errorName?: string; + errorMessage?: string; + }; +}; + +const isRecord = (value: unknown): value is Record => + typeof value === 'object' && value !== null; + +const safeString = (value: unknown): string | undefined => + typeof value === 'string' ? value : undefined; + +const nativeCallStages = new Set([ + 'joining the call', + 'no transport', + 'authorizing', + 'connecting', + 'media control', +]); + +const nativeCallCodes = new Set([ + 'actor_unavailable', + 'audio_failed', + 'busy', + 'camera_failed', + 'close_failed', + 'connect_failed', + 'media_unsupported', + 'screen_share_failed', + 'stale_connection', + 'video_failed', + 'unknown', +]); + +const nativeCallErrorNames = new Set([ + 'AggregateError', + 'AbortError', + 'DataError', + 'DOMException', + 'Error', + 'EvalError', + 'InvalidStateError', + 'NetworkError', + 'NotAllowedError', + 'NotSupportedError', + 'OperationError', + 'QuotaExceededError', + 'RangeError', + 'ReferenceError', + 'SecurityError', + 'SyntaxError', + 'TimeoutError', + 'TypeError', + 'URIError', + 'UnknownError', +]); + +const nativeCallErrorMessages = new Set([ + 'MatrixRTC device unavailable', + 'MatrixRTC membership listener setup failed', + 'MatrixRTC membership publication failed', + 'MatrixRTC membership wait cancelled', + 'MatrixRTC slot was not assigned', + 'No LiveKit transport available', + 'media kind is not supported on this platform', + 'native audio failed', + 'native camera failed', + 'native screen share failed', + 'native video failed', + 'redacted', +]); + +const allowlistedString = (value: unknown, allowlist: Set): string | undefined => { + const string = safeString(value); + if (string === undefined) return undefined; + return allowlist.has(string) ? string : 'redacted'; +}; + +const safeCause = (value: unknown): NativeCallDiagnostic['cause'] | undefined => { + if (!isRecord(value)) return undefined; + const errorName = allowlistedString(value.errorName, nativeCallErrorNames); + const errorMessage = allowlistedString(value.errorMessage, nativeCallErrorMessages); + if (!errorName && !errorMessage) return undefined; + return { ...(errorName ? { errorName } : {}), ...(errorMessage ? { errorMessage } : {}) }; +}; + +export const getNativeCallDiagnostics = (logs: readonly LogEntry[]): NativeCallDiagnostic[] => + logs + .filter((entry) => entry.category === 'call' && entry.namespace === 'nativeCallController') + .reduce((sorted, entry) => { + const index = sorted.findIndex((candidate) => candidate.timestamp < entry.timestamp); + if (index === -1) sorted.push(entry); + else sorted.splice(index, 0, entry); + return sorted; + }, []) + .map((entry) => { + const data = isRecord(entry.data) ? entry.data : {}; + const code = allowlistedString(data.code, nativeCallCodes); + const stage = allowlistedString(data.stage, nativeCallStages); + const errorName = allowlistedString(data.errorName, nativeCallErrorNames); + const errorMessage = allowlistedString(data.errorMessage, nativeCallErrorMessages); + const cause = safeCause(data.cause); + return { + ...(code ? { code } : {}), + ...(stage ? { stage } : {}), + ...(errorName ? { errorName } : {}), + ...(errorMessage ? { errorMessage } : {}), + ...(cause ? { cause } : {}), + }; + }); + +export const serializeNativeCallDiagnostics = ( + diagnostics: readonly NativeCallDiagnostic[] +): string => JSON.stringify(diagnostics, null, 2); + +export function useNativeCallDiagnostics(): NativeCallDiagnostic[] { + const [diagnostics, setDiagnostics] = useState(() => + getNativeCallDiagnostics(getDebugLogger().getLogs()) + ); + + useEffect(() => { + const debugLogger = getDebugLogger(); + const update = (entry?: LogEntry): void => { + if (entry && (entry.category !== 'call' || entry.namespace !== 'nativeCallController')) { + return; + } + setDiagnostics(getNativeCallDiagnostics(debugLogger.getLogs())); + }; + return debugLogger.addListener(update); + }, []); + + return diagnostics; +} + function CallPrescreen() { const room = useRoom(); const callEmbed = useAtomValue(callEmbedAtom); @@ -183,6 +690,29 @@ function CallJoined({ containerRef }: CallJoinedProps) { ); } +export function useNativeCallProbeSession( + nativeCallForRoom: NativeCallSession | undefined, + nativeCallsEnabled: boolean +): boolean { + const nativeCallProbeEnabled = isNativeCallProbeEnabled(nativeCallsEnabled); + const hungUpSession = useRef(undefined); + + useEffect(() => { + if ( + !nativeCallForRoom || + nativeCallProbeEnabled || + hungUpSession.current === nativeCallForRoom + ) { + return; + } + + hungUpSession.current = nativeCallForRoom; + void nativeCallForRoom.hangup(); + }, [nativeCallForRoom, nativeCallProbeEnabled]); + + return nativeCallProbeEnabled; +} + interface CallViewProps { resizable?: boolean; } @@ -199,9 +729,19 @@ export function CallView({ resizable }: CallViewProps) { const callEmbed = useCallEmbed(); const callJoined = useCallJoined(callEmbed); const nativeCall = useAtomValue(nativeCallAtom); + const livekitJsCall = useAtomValue(livekitJsCallAtom); + const [nativeCallsEnabled] = useSetting(settingsAtom, 'nativeCallsEnabled'); + const [livekitJsMediaTestEnabled] = useSetting(settingsAtom, 'livekitJsMediaTestEnabled'); - const nativeCallForRoom = nativeCall?.roomId === room.roomId ? nativeCall : undefined; - const currentJoined = !nativeCallForRoom && callEmbed?.roomId === room.roomId && callJoined; + const nativeCallSessionForRoom = nativeCall?.roomId === room.roomId ? nativeCall : undefined; + const nativeCallProbeEnabled = useNativeCallProbeSession( + nativeCallSessionForRoom, + nativeCallsEnabled + ); + const nativeCallForRoom = nativeCallProbeEnabled ? nativeCallSessionForRoom : undefined; + const livekitJsCallForRoom = livekitJsCall?.roomId === room.roomId ? livekitJsCall : undefined; + const currentJoined = + !nativeCallForRoom && !livekitJsCallForRoom && callEmbed?.roomId === room.roomId && callJoined; const [heightRatio, setHeightRatio] = useState(isMobile ? 0.3 : 0.72); const [availableHeight, setAvailableHeight] = useState(0); @@ -316,13 +856,25 @@ export function CallView({ resizable }: CallViewProps) { /> )} - {!currentJoined && !nativeCallForRoom && } + {!currentJoined && !nativeCallForRoom && !livekitJsCallForRoom && } {nativeCallForRoom ? ( void nativeCallForRoom.hangup()} /> + ) : canRenderLivekitJsMediaTest(livekitJsMediaTestEnabled, livekitJsCallForRoom) ? ( + void livekitJsCallForRoom.hangup()} + /> + ) : livekitJsCallForRoom ? ( + void livekitJsCallForRoom.hangup()} + /> ) : ( )} diff --git a/src/app/features/call/LivekitJsMediaTest.test.tsx b/src/app/features/call/LivekitJsMediaTest.test.tsx new file mode 100644 index 000000000..a9fbd53a8 --- /dev/null +++ b/src/app/features/call/LivekitJsMediaTest.test.tsx @@ -0,0 +1,176 @@ +import { type Context, type ReactNode, useContext } from 'react'; +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { Room } from 'livekit-client'; +import type { LivekitJsCallSession } from '$state/livekitJsCall'; +import { canRenderLivekitJsMediaTest, LivekitJsMediaTestSurface } from './LivekitJsMediaTest'; + +const mocks = vi.hoisted(() => ({ + mockRoomContext: undefined as unknown as Context, + mockUseTracks: vi.fn<() => readonly unknown[]>(() => []), + mockObservedRoom: vi.fn<(room: Room) => void>(), + mockUseLocalParticipant: vi.fn< + () => { + localParticipant: { identity: string }; + isMicrophoneEnabled: boolean; + isCameraEnabled: boolean; + isScreenShareEnabled: boolean; + } + >(), +})); + +vi.mock('@livekit/components-react', async () => { + const { createContext } = await import('react'); + mocks.mockRoomContext = createContext(undefined); + return { + ParticipantTile: ({ children }: { children: ReactNode }) =>
{children}
, + RoomAudioRenderer: () =>
, + RoomContext: mocks.mockRoomContext, + Track: { Source: { Camera: 'camera', ScreenShare: 'screen_share' } }, + useLocalParticipant: mocks.mockUseLocalParticipant, + useTracks: mocks.mockUseTracks, + VideoTrack: () =>
, + }; +}); + +vi.mock('$components/icons/phosphor', () => ({ + Microphone: 'microphone', + MicrophoneSlash: 'microphone-slash', + ScreenShare: 'screen-share', + VideoCamera: 'video-camera', + VideoCameraSlash: 'video-camera-slash', + sizedIcon: () => null, +})); + +vi.mock('folds', () => ({ + Box: ({ children }: { children: ReactNode }) =>
{children}
, + Button: ({ children, onClick }: { children: ReactNode; onClick: () => void }) => ( + + ), + color: { Critical: { Main: 'red' } }, + IconButton: ({ + children, + onClick, + disabled, + 'aria-label': ariaLabel, + }: { + children: ReactNode; + onClick: () => void; + disabled?: boolean; + 'aria-label': string; + }) => ( + + ), + Text: ({ children }: { children: ReactNode }) => {children}, +})); + +const room = { + connect: vi.fn<() => Promise>(), + disconnect: vi.fn<() => Promise>(), +} as unknown as Room; + +const makeSession = (overrides: Partial = {}): LivekitJsCallSession => ({ + roomId: '!room:example.org', + lifecycle: 'active', + failure: null, + room, + media: { + setMicrophoneEnabled: vi.fn<() => Promise>().mockResolvedValue(), + setCameraEnabled: vi.fn<() => Promise>().mockResolvedValue(), + setScreenShareEnabled: vi.fn<() => Promise>().mockResolvedValue(), + }, + hangup: vi.fn<() => Promise>().mockResolvedValue(), + ...overrides, +}); + +beforeEach(() => { + mocks.mockUseTracks.mockReset(); + mocks.mockUseTracks.mockReturnValue([]); + mocks.mockObservedRoom.mockReset(); + mocks.mockUseLocalParticipant.mockReset(); + mocks.mockUseLocalParticipant.mockImplementation(() => { + const activeRoom = useContext(mocks.mockRoomContext); + if (activeRoom) mocks.mockObservedRoom(activeRoom); + return { + localParticipant: { identity: 'local-user' }, + isMicrophoneEnabled: false, + isCameraEnabled: false, + isScreenShareEnabled: false, + }; + }); +}); + +describe('LiveKit JS manual media test', () => { + it('renders only for an enabled active session with the controller bridge', () => { + const session = makeSession(); + expect(canRenderLivekitJsMediaTest(true, session)).toBe(true); + expect(canRenderLivekitJsMediaTest(false, session)).toBe(false); + expect(canRenderLivekitJsMediaTest(true, { ...session, lifecycle: 'joining-matrix' })).toBe( + false + ); + expect(canRenderLivekitJsMediaTest(true, { ...session, media: undefined })).toBe(false); + }); + + it('provides the controller-owned Room and does not connect or capture on render', () => { + const session = makeSession(); + render( + {}} /> + ); + + expect(mocks.mockObservedRoom).toHaveBeenCalledWith(room); + expect(room.connect).not.toHaveBeenCalled(); + expect(room.disconnect).not.toHaveBeenCalled(); + expect(session.media!.setMicrophoneEnabled).not.toHaveBeenCalled(); + expect(session.media!.setCameraEnabled).not.toHaveBeenCalled(); + expect(session.media!.setScreenShareEnabled).not.toHaveBeenCalled(); + expect(screen.getByText(/manual local media test only/i)).toBeInTheDocument(); + expect(screen.getByTestId('room-audio-renderer')).toBeInTheDocument(); + }); + + it('calls the narrow facade directly from manual controls', async () => { + const session = makeSession(); + render( + {}} /> + ); + + fireEvent.click(screen.getByRole('button', { name: 'Turn on microphone' })); + await waitFor(() => expect(session.media!.setMicrophoneEnabled).toHaveBeenCalledWith(true)); + fireEvent.click(screen.getByRole('button', { name: 'Start camera' })); + await waitFor(() => expect(session.media!.setCameraEnabled).toHaveBeenCalledWith(true)); + fireEvent.click(screen.getByRole('button', { name: 'Start screen sharing' })); + + await waitFor(() => expect(session.media!.setScreenShareEnabled).toHaveBeenCalledWith(true)); + }); + + it('shows safe controller failure text without exposing raw errors', async () => { + const session = makeSession(); + session.media!.setCameraEnabled = vi + .fn<() => Promise>() + .mockRejectedValue({ code: 'e2ee-key-not-ready', secret: 'do-not-show' }); + render( + {}} /> + ); + + fireEvent.click(screen.getByRole('button', { name: 'Start camera' })); + + expect(await screen.findByText('Waiting for encrypted media keys.')).toBeInTheDocument(); + expect(screen.queryByText(/do-not-show|secret/i)).not.toBeInTheDocument(); + }); + + it('keeps End as the only teardown action', () => { + const onHangup = vi.fn<() => void>(); + const session = makeSession(); + render( + + ); + + fireEvent.click(screen.getByRole('button', { name: 'End' })); + + expect(onHangup).toHaveBeenCalledOnce(); + expect(room.disconnect).not.toHaveBeenCalled(); + }); +}); diff --git a/src/app/features/call/LivekitJsMediaTest.tsx b/src/app/features/call/LivekitJsMediaTest.tsx new file mode 100644 index 000000000..b09abd9e5 --- /dev/null +++ b/src/app/features/call/LivekitJsMediaTest.tsx @@ -0,0 +1,227 @@ +import { useState } from 'react'; +import { + Microphone, + MicrophoneSlash, + ScreenShare, + VideoCamera, + VideoCameraSlash, + sizedIcon, +} from '$components/icons/phosphor'; +import { Box, Button, color, IconButton, Text } from 'folds'; +import { + ParticipantTile, + RoomAudioRenderer, + RoomContext, + useLocalParticipant, + useTracks, + VideoTrack, +} from '@livekit/components-react'; +import { Track, type Room } from 'livekit-client'; +import type { LivekitJsCallSession } from '$state/livekitJsCall'; +import type { LivekitJsMediaFacade, LivekitJsMediaFailure } from './livekitJsController'; + +export const canRenderLivekitJsMediaTest = ( + enabled: boolean, + session: LivekitJsCallSession | undefined +): session is LivekitJsCallSession & { room: Room; media: LivekitJsMediaFacade } => + enabled && + session?.lifecycle === 'active' && + session.room !== undefined && + session.media !== undefined; + +const mediaFailureMessages: Record = { + 'media-test-disabled': 'Manual media test is disabled.', + 'e2ee-unsupported': 'Encrypted media is unavailable.', + 'e2ee-key-not-ready': 'Waiting for encrypted media keys.', + 'e2ee-key-failed': 'Encrypted media setup failed.', + 'room-not-active': 'The LiveKit room is not active.', + 'media-operation-failed': 'The media operation failed.', +}; + +const getMediaFailureMessage = (error: unknown): string => { + if ( + typeof error === 'object' && + error !== null && + 'code' in error && + typeof error.code === 'string' && + error.code in mediaFailureMessages + ) { + return mediaFailureMessages[error.code as LivekitJsMediaFailure]; + } + return 'The media operation failed.'; +}; + +function VideoTiles({ localIdentity }: { localIdentity: string }) { + const tracks = useTracks([Track.Source.Camera, Track.Source.ScreenShare], { + onlySubscribed: false, + }); + + if (tracks.length === 0) { + return No camera or screen-share tracks are active.; + } + + return ( + + {tracks.map((track) => ( + + + + {track.participant.identity === localIdentity ? 'You' : track.participant.identity} + + + ))} + + ); +} + +function MediaControl({ + label, + enabled, + pending, + icon, + onClick, +}: { + label: string; + enabled: boolean; + pending: boolean; + icon: typeof Microphone; + onClick: () => void; +}) { + return ( + + {sizedIcon(icon, '100', { filled: enabled })} + + ); +} + +export function LivekitJsMediaTestSurface({ + room, + media, + onHangup, +}: { + room: Room; + media: LivekitJsMediaFacade; + onHangup: () => void; +}) { + return ( + + + + ); +} + +function LivekitJsMediaTestContent({ + media, + onHangup, +}: { + media: LivekitJsMediaFacade; + onHangup: () => void; +}) { + const { localParticipant, isMicrophoneEnabled, isCameraEnabled, isScreenShareEnabled } = + useLocalParticipant(); + const [pending, setPending] = useState<'microphone' | 'camera' | 'screen' | undefined>(); + const [error, setError] = useState(); + + const runMediaAction = async ( + kind: 'microphone' | 'camera' | 'screen', + action: () => Promise + ): Promise => { + if (pending) return; + setPending(kind); + setError(undefined); + try { + await action(); + } catch (actionError) { + setError(getMediaFailureMessage(actionError)); + } finally { + setPending(undefined); + } + }; + + return ( + + + LiveKit JS manual media test + + Manual local media test only · encrypted media is required · not release-ready + + + + + void runMediaAction('microphone', () => + media.setMicrophoneEnabled(!isMicrophoneEnabled) + ) + } + /> + + void runMediaAction('camera', () => media.setCameraEnabled(!isCameraEnabled)) + } + /> + + void runMediaAction('screen', () => media.setScreenShareEnabled(!isScreenShareEnabled)) + } + /> + + {error && ( + + {error} + + )} + + + ); +} diff --git a/src/app/features/call/callStartSelection.test.ts b/src/app/features/call/callStartSelection.test.ts new file mode 100644 index 000000000..eeb520ea2 --- /dev/null +++ b/src/app/features/call/callStartSelection.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from 'vitest'; +import { selectCallStartOwner } from './callStartSelection'; + +describe('selectCallStartOwner', () => { + it('prioritizes the enabled supported LiveKit JS probe', () => { + expect(selectCallStartOwner({ livekitJsProbeEnabled: true, nativeProbeEnabled: true })).toBe( + 'livekit-js' + ); + }); + + it('preserves native then Element Call fallback priority', () => { + expect(selectCallStartOwner({ livekitJsProbeEnabled: false, nativeProbeEnabled: true })).toBe( + 'native' + ); + expect(selectCallStartOwner({ livekitJsProbeEnabled: false, nativeProbeEnabled: false })).toBe( + 'element' + ); + }); +}); diff --git a/src/app/features/call/callStartSelection.ts b/src/app/features/call/callStartSelection.ts new file mode 100644 index 000000000..e40889687 --- /dev/null +++ b/src/app/features/call/callStartSelection.ts @@ -0,0 +1,13 @@ +export type CallStartOwner = 'livekit-js' | 'native' | 'element'; + +export const selectCallStartOwner = ({ + livekitJsProbeEnabled, + nativeProbeEnabled, +}: { + livekitJsProbeEnabled: boolean; + nativeProbeEnabled: boolean; +}): CallStartOwner => { + if (livekitJsProbeEnabled) return 'livekit-js'; + if (nativeProbeEnabled) return 'native'; + return 'element'; +}; diff --git a/src/app/features/call/livekitJsCallProbe.test.ts b/src/app/features/call/livekitJsCallProbe.test.ts new file mode 100644 index 000000000..b7038c5e6 --- /dev/null +++ b/src/app/features/call/livekitJsCallProbe.test.ts @@ -0,0 +1,20 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const desktopTauri = vi.hoisted(() => vi.fn<() => boolean>()); + +vi.mock('$utils/platform', () => ({ isDesktopTauri: desktopTauri })); + +import { isLivekitJsCallProbeEnabled } from './livekitJsCallProbe'; + +beforeEach(() => desktopTauri.mockReset()); + +describe('isLivekitJsCallProbeEnabled', () => { + it('requires both the persisted setting and supported desktop Tauri runtime', () => { + desktopTauri.mockReturnValue(true); + expect(isLivekitJsCallProbeEnabled(false)).toBe(false); + expect(isLivekitJsCallProbeEnabled(true)).toBe(true); + + desktopTauri.mockReturnValue(false); + expect(isLivekitJsCallProbeEnabled(true)).toBe(false); + }); +}); diff --git a/src/app/features/call/livekitJsCallProbe.ts b/src/app/features/call/livekitJsCallProbe.ts new file mode 100644 index 000000000..2ef9076ad --- /dev/null +++ b/src/app/features/call/livekitJsCallProbe.ts @@ -0,0 +1,4 @@ +import { isDesktopTauri } from '$utils/platform'; + +export const isLivekitJsCallProbeEnabled = (enabled = false): boolean => + enabled && isDesktopTauri(); diff --git a/src/app/features/call/livekitJsController.test.ts b/src/app/features/call/livekitJsController.test.ts new file mode 100644 index 000000000..acd3fd3fe --- /dev/null +++ b/src/app/features/call/livekitJsController.test.ts @@ -0,0 +1,917 @@ +import { beforeEach, describe, expect, it, vi, type Mock } from 'vitest'; +import { + MatrixRTCSessionEvent, + type CallMembership, + type MatrixRTCSession, +} from '$types/matrix-sdk'; +import type { MatrixClient, Room as MatrixRoom } from '$types/matrix-sdk'; +import type { Room as LivekitRoom, RoomOptions } from 'livekit-client'; +import { + createLivekitJsController, + type LivekitJsControllerDependencies, + type LivekitJsPlatformBridge, + type LivekitJsPlatformStartRequest, +} from './livekitJsController'; +import type { PlatformCallEvent } from '$plugins/call/platformCallLifecycle'; +import type { + LivekitMatrixKeyProvider, + LivekitMatrixKeyProviderState, +} from './livekitMatrixKeyProvider'; +import { resetCallOwnerForTests } from '$state/callOwner'; + +const transport = { type: 'livekit' as const, livekit_service_url: 'https://sfu.example' }; +const room = { roomId: '!room:example.org' } as MatrixRoom; +const membership = { + userId: '@alice:example.org', + deviceId: 'DEVICE', + rtcBackendIdentity: 'local-backend-identity', +} as CallMembership; + +const deferred = (): { + promise: Promise; + resolve: (value: T) => void; +} => { + let resolve!: (value: T) => void; + const promise = new Promise((promiseResolve) => { + resolve = promiseResolve; + }); + return { promise, resolve }; +}; + +type SessionHandler = (...args: unknown[]) => void; + +type TestSession = MatrixRTCSession & { + handlers: Map; +}; + +const makeSession = (order: string[] = []): TestSession => { + const handlers = new Map(); + const session = { + handlers, + memberships: [] as CallMembership[], + slotId: 'm.call#real-slot', + on: vi.fn<(...args: unknown[]) => void>().mockImplementation((event, handler) => { + handlers.set(event as MatrixRTCSessionEvent, handler as SessionHandler); + }), + removeListener: vi.fn<(...args: unknown[]) => void>().mockImplementation((event, handler) => { + if (handlers.get(event as MatrixRTCSessionEvent) === handler) { + handlers.delete(event as MatrixRTCSessionEvent); + } + }), + joinRTCSession: vi + .fn<(...args: unknown[]) => void>() + .mockImplementation(() => order.push('join')), + leaveRoomSession: vi.fn().mockImplementation(async () => { + order.push('leave'); + return true; + }), + } as unknown as TestSession; + return session; +}; + +const emitOwnMembership = (session: TestSession): void => { + session.memberships = [membership]; + session.handlers.get(MatrixRTCSessionEvent.MembershipsChanged)?.([], [membership]); +}; + +type FakeProvider = { + state: LivekitMatrixKeyProviderState; + attach: Mock<(session: MatrixRTCSession) => void>; + detach: Mock<() => void>; + setLocalOutboundIdentity: Mock<(identity: string | undefined) => void>; + getKeyState: Mock<() => LivekitMatrixKeyProviderState>; + subscribe: Mock< + (listener: (state: Readonly) => void) => () => void + >; +}; + +const makeProvider = (state: Partial = {}): FakeProvider => { + const provider = { + state: { + ready: false, + localOutboundIdentity: null, + keyIndex: null, + lastImportFailure: null, + ...state, + }, + attach: vi.fn<(session: MatrixRTCSession) => void>(), + detach: vi.fn<() => void>(), + setLocalOutboundIdentity: vi.fn<(identity: string | undefined) => void>(), + getKeyState: vi.fn<() => LivekitMatrixKeyProviderState>(), + subscribe: + vi.fn<(listener: (state: Readonly) => void) => () => void>(), + } as FakeProvider; + provider.getKeyState.mockImplementation(() => provider.state); + provider.subscribe.mockImplementation((listener) => { + listener(provider.state); + return () => undefined; + }); + return provider; +}; + +const makeClient = (session: MatrixRTCSession): MatrixClient => + ({ + getDeviceId: () => 'DEVICE', + getSafeUserId: () => '@alice:example.org', + matrixRTC: { getRoomSession: () => session }, + }) as unknown as MatrixClient; + +const makeDependencies = ( + session: TestSession, + order: string[], + provider = makeProvider() +): { + dependencies: LivekitJsControllerDependencies; + provider: FakeProvider; + roomOptions: { value?: RoomOptions }; + livekitRoom: { + connect: Mock<(url: string, token: string, options?: unknown) => Promise>; + disconnect: Mock<() => Promise>; + localParticipant: { + setMicrophoneEnabled: Mock; + setCameraEnabled: Mock; + setScreenShareEnabled: Mock; + }; + }; + mediaParticipant: { + setMicrophoneEnabled: Mock; + setCameraEnabled: Mock; + setScreenShareEnabled: Mock; + }; +} => { + const worker = { terminate: vi.fn<() => void>() } as unknown as Worker; + provider.attach.mockImplementation(() => order.push('attach')); + provider.detach.mockImplementation(() => order.push('detach')); + const mediaParticipant = { + setMicrophoneEnabled: vi + .fn() + .mockResolvedValue(undefined), + setCameraEnabled: vi + .fn() + .mockResolvedValue(undefined), + setScreenShareEnabled: vi + .fn() + .mockResolvedValue(undefined), + }; + const livekitRoom = { + localParticipant: mediaParticipant, + connect: vi + .fn<(url: string, token: string, options?: unknown) => Promise>() + .mockImplementation(async () => { + order.push('connect'); + }), + disconnect: vi.fn<() => Promise>().mockImplementation(async () => { + order.push('disconnect'); + }), + }; + const roomOptions: { value?: RoomOptions } = {}; + return { + provider, + roomOptions, + livekitRoom, + mediaParticipant, + dependencies: { + createKeyProvider: () => provider as unknown as LivekitMatrixKeyProvider, + isE2EESupported: () => true, + createWorker: () => { + order.push('worker'); + return worker; + }, + createRoom: (options) => { + order.push('room'); + roomOptions.value = options; + return livekitRoom; + }, + getPreferredTransport: async () => { + order.push('transport'); + return transport; + }, + provisionToken: async () => { + order.push('provision'); + return { url: 'wss://livekit.example', jwt: 'jwt' }; + }, + }, + }; +}; + +const connectToActive = async ( + controller: ReturnType, + session: TestSession +): Promise => { + const connectPromise = controller.connect({ mx: makeClient(session), room }); + await vi.waitFor(() => + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipsChanged, + expect.any(Function) + ) + ); + emitOwnMembership(session); + await connectPromise; +}; + +type FakePlatform = { + bridge: LivekitJsPlatformBridge; + getCapabilities: Mock<() => Promise>; + start: Mock<(request: LivekitJsPlatformStartRequest) => Promise>; + stop: Mock<(request: { sessionId: string }) => Promise>; + emit: (event: PlatformCallEvent) => void; +}; + +const makePlatform = (supported: boolean, order: string[]): FakePlatform => { + let handler: ((event: PlatformCallEvent) => void) | undefined; + const getCapabilities = vi.fn<() => Promise>().mockImplementation(async () => { + order.push('platform-caps'); + return { supported, microphone: supported, playback: supported }; + }); + const start = vi + .fn<(request: LivekitJsPlatformStartRequest) => Promise>() + .mockImplementation(async (request) => { + order.push(`platform-start:${request.microphone}:${request.playback}`); + return { sessionId: request.sessionId }; + }); + const stop = vi + .fn<(request: { sessionId: string }) => Promise>() + .mockImplementation(async () => { + order.push('platform-stop'); + return {}; + }); + const bridge: LivekitJsPlatformBridge = { + getCapabilities: getCapabilities as LivekitJsPlatformBridge['getCapabilities'], + start: start as LivekitJsPlatformBridge['start'], + stop: stop as LivekitJsPlatformBridge['stop'], + onEvent: async (nextHandler) => { + handler = nextHandler; + return () => { + order.push('platform-unlisten'); + }; + }, + }; + return { bridge, getCapabilities, start, stop, emit: (event) => handler?.(event) }; +}; + +describe('livekit JS controller', () => { + beforeEach(() => resetCallOwnerForTests()); + + it('attaches E2EE before joining and provisions before connecting one no-media Room', async () => { + const order: string[] = []; + const session = makeSession(order); + const { dependencies, provider, roomOptions, mediaParticipant, livekitRoom } = makeDependencies( + session, + order + ); + const controller = createLivekitJsController(dependencies); + + await connectToActive(controller, session); + + expect(order).toEqual([ + 'attach', + 'transport', + 'join', + 'provision', + 'worker', + 'room', + 'connect', + ]); + expect(provider.attach).toHaveBeenCalledBefore(session.joinRTCSession as Mock); + expect(session.joinRTCSession).toHaveBeenCalledWith( + { userId: '@alice:example.org', deviceId: 'DEVICE', memberId: '@alice:example.org:DEVICE' }, + [transport], + undefined, + { callIntent: 'audio', notificationType: 'notification', manageMediaKeys: true } + ); + expect(roomOptions.value?.encryption).toEqual({ + keyProvider: provider, + worker: expect.anything(), + }); + expect(mediaParticipant.setMicrophoneEnabled).not.toHaveBeenCalled(); + expect(mediaParticipant.setCameraEnabled).not.toHaveBeenCalled(); + expect(mediaParticipant.setScreenShareEnabled).not.toHaveBeenCalled(); + expect(controller.getState().room).toBe(livekitRoom); + expect(controller.getState().media).toBeUndefined(); + expect(controller.getState().lifecycle).toBe('active'); + }); + + it('refuses unsupported E2EE without joining or creating a Room', async () => { + const session = makeSession(); + const { dependencies } = makeDependencies(session, []); + dependencies.isE2EESupported = () => false; + const controller = createLivekitJsController(dependencies); + + await controller.connect({ mx: makeClient(session), room }); + + expect(session.joinRTCSession).not.toHaveBeenCalled(); + expect(controller.getState()).toMatchObject({ + lifecycle: 'failed', + failure: 'e2ee-unsupported', + }); + }); + + it('refuses a provider import failure without connecting LiveKit', async () => { + const session = makeSession(); + const provider = makeProvider({ lastImportFailure: 'import-failed' }); + const { dependencies } = makeDependencies(session, [], provider); + const controller = createLivekitJsController(dependencies); + + await controller.connect({ mx: makeClient(session), room }); + + expect(session.joinRTCSession).not.toHaveBeenCalled(); + expect(controller.getState()).toMatchObject({ + lifecycle: 'failed', + failure: 'e2ee-import-failed', + }); + }); + + it('disconnects LiveKit before leaving MatrixRTC and cleans up idempotently', async () => { + const order: string[] = []; + const session = makeSession(order); + const { dependencies, provider } = makeDependencies(session, order); + const controller = createLivekitJsController(dependencies); + await connectToActive(controller, session); + + await Promise.all([controller.disconnect(), controller.disconnect()]); + + expect(order.indexOf('disconnect')).toBeGreaterThan(-1); + expect(order.indexOf('detach')).toBeGreaterThan(order.indexOf('disconnect')); + expect(order.indexOf('disconnect')).toBeLessThan(order.indexOf('leave')); + expect(order.indexOf('detach')).toBeLessThan(order.indexOf('leave')); + expect(provider.detach).toHaveBeenCalledOnce(); + expect(controller.getState().room).toBeUndefined(); + expect(controller.getState().media).toBeUndefined(); + expect(controller.getState().lifecycle).toBe('idle'); + }); + + it('detaches once when setup fails after MatrixRTC join', async () => { + const session = makeSession(); + const { dependencies, provider } = makeDependencies(session, []); + dependencies.provisionToken = vi + .fn>() + .mockRejectedValue(new Error('provision failed')); + const controller = createLivekitJsController(dependencies); + const connectPromise = controller.connect({ mx: makeClient(session), room }); + + await vi.waitFor(() => + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipsChanged, + expect.any(Function) + ) + ); + emitOwnMembership(session); + await connectPromise; + + expect(provider.detach).toHaveBeenCalledOnce(); + expect(session.leaveRoomSession).toHaveBeenCalledOnce(); + expect(controller.getState().room).toBeUndefined(); + expect(controller.getState().media).toBeUndefined(); + expect(controller.getState().lifecycle).toBe('failed'); + }); + + it('cancels during provisioning without creating a Room or becoming active', async () => { + const session = makeSession(); + const pendingProvision = deferred<{ url: string; jwt: string }>(); + const { dependencies, provider, roomOptions } = makeDependencies(session, []); + dependencies.provisionToken = vi + .fn>() + .mockImplementation(() => pendingProvision.promise); + const controller = createLivekitJsController(dependencies); + const connectPromise = controller.connect({ mx: makeClient(session), room }); + + await vi.waitFor(() => + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipsChanged, + expect.any(Function) + ) + ); + emitOwnMembership(session); + await vi.waitFor(() => expect(dependencies.provisionToken).toHaveBeenCalledOnce()); + const disconnectPromise = controller.disconnect(); + pendingProvision.resolve({ url: 'wss://livekit.example', jwt: 'jwt' }); + await Promise.all([connectPromise, disconnectPromise]); + + expect(roomOptions.value).toBeUndefined(); + expect(provider.detach).toHaveBeenCalledOnce(); + expect(session.leaveRoomSession).toHaveBeenCalledOnce(); + expect(controller.getState().room).toBeUndefined(); + expect(controller.getState().media).toBeUndefined(); + expect(controller.getState().lifecycle).toBe('idle'); + }); + + it('cancels during Room.connect and cleans up its stale room before leaving', async () => { + const session = makeSession(); + const pendingConnect = deferred(); + const { dependencies, provider } = makeDependencies(session, []); + const livekitRoom = { + connect: vi + .fn<(url: string, token: string, options?: unknown) => Promise>() + .mockImplementation(() => pendingConnect.promise), + disconnect: vi.fn<() => Promise>().mockResolvedValue(undefined), + }; + dependencies.createRoom = () => livekitRoom; + const controller = createLivekitJsController(dependencies); + const connectPromise = controller.connect({ mx: makeClient(session), room }); + + await vi.waitFor(() => + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipsChanged, + expect.any(Function) + ) + ); + emitOwnMembership(session); + await vi.waitFor(() => expect(livekitRoom.connect).toHaveBeenCalledOnce()); + const disconnectPromise = controller.disconnect(); + pendingConnect.resolve(); + await Promise.all([connectPromise, disconnectPromise]); + + expect(livekitRoom.disconnect).toHaveBeenCalledOnce(); + expect(provider.detach).toHaveBeenCalledOnce(); + expect(session.leaveRoomSession).toHaveBeenCalledOnce(); + expect(controller.getState().lifecycle).toBe('idle'); + expect(controller.getState().lifecycle).not.toBe('active'); + }); + + it('rejects duplicate setup and cancellation leaves started MatrixRTC membership', async () => { + const session = makeSession(); + const { dependencies, provider } = makeDependencies(session, []); + const controller = createLivekitJsController(dependencies); + const firstConnect = controller.connect({ mx: makeClient(session), room }); + await vi.waitFor(() => + expect(session.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.MembershipsChanged, + expect.any(Function) + ) + ); + + await expect(controller.connect({ mx: makeClient(session), room })).rejects.toThrow( + 'already in use' + ); + await controller.disconnect(); + await firstConnect; + + expect(session.leaveRoomSession).toHaveBeenCalledWith(5000); + expect(provider.detach).toHaveBeenCalledOnce(); + expect(controller.getState().lifecycle).toBe('idle'); + }); + + it('does not expose media publication methods', () => { + const session = makeSession(); + const { dependencies } = makeDependencies(session, []); + const controller = createLivekitJsController(dependencies); + + expect(Object.keys(controller).toSorted()).toEqual([ + 'connect', + 'disconnect', + 'getState', + 'subscribe', + ]); + expect('publishTrack' in controller).toBe(false); + expect('setMicrophoneEnabled' in controller).toBe(false); + expect('setCameraEnabled' in controller).toBe(false); + expect('setScreenShareEnabled' in controller).toBe(false); + }); + + it('disconnects the old mode before replacing it with a manual-media controller', async () => { + const firstSession = makeSession(); + const first = makeDependencies(firstSession, []); + const firstController = createLivekitJsController(first.dependencies); + await connectToActive(firstController, firstSession); + + const replacementSession = makeSession(); + const replacement = makeDependencies(replacementSession, [], makeProvider({ ready: true })); + const replacementController = createLivekitJsController(replacement.dependencies, { + manualMediaTest: true, + }); + + await firstController.disconnect(); + expect(firstController.getState().room).toBeUndefined(); + await connectToActive(replacementController, replacementSession); + + expect(replacementController.getState().media).toBeDefined(); + }); + + it('publishes media only through the explicit manual-media-test controller', async () => { + const session = makeSession(); + const provider = makeProvider({ + ready: true, + localOutboundIdentity: 'local-backend-identity', + keyIndex: 1, + }); + const { dependencies, mediaParticipant, livekitRoom } = makeDependencies(session, [], provider); + // No platform bridge configured: use an explicit unsupported (desktop) bridge. + dependencies.platformBridge = makePlatform(false, []).bridge; + const controller = createLivekitJsController(dependencies, { manualMediaTest: true }); + + await connectToActive(controller, session); + await controller.setMicrophoneEnabled(true); + await controller.setCameraEnabled(true); + await controller.setScreenShareEnabled(true); + + expect(mediaParticipant.setMicrophoneEnabled).toHaveBeenCalledWith(true); + expect(mediaParticipant.setCameraEnabled).toHaveBeenCalledWith(true); + expect(mediaParticipant.setScreenShareEnabled).toHaveBeenCalledWith(true); + expect(controller.getState().room).toBe(livekitRoom); + expect(controller.getState().media?.setMicrophoneEnabled).toBeDefined(); + }); + + it('refuses manual media when the local key is not ready or has failed', async () => { + const session = makeSession(); + const provider = makeProvider(); + const { dependencies } = makeDependencies(session, [], provider); + const controller = createLivekitJsController(dependencies, { manualMediaTest: true }); + + await connectToActive(controller, session); + await expect(controller.setMicrophoneEnabled(true)).rejects.toMatchObject({ + code: 'e2ee-key-not-ready', + }); + + provider.state.lastImportFailure = 'import-failed'; + await expect(controller.setCameraEnabled(true)).rejects.toMatchObject({ + code: 'e2ee-key-failed', + }); + }); + + it('refuses manual media when runtime E2EE support is unavailable', async () => { + const session = makeSession(); + const { dependencies } = makeDependencies(session, []); + dependencies.isE2EESupported = () => false; + const controller = createLivekitJsController(dependencies, { manualMediaTest: true }); + + await expect(controller.setScreenShareEnabled(true)).rejects.toMatchObject({ + code: 'e2ee-unsupported', + }); + expect(controller.getState().mediaFailure).toBe('e2ee-unsupported'); + }); + + it('treats unsupported desktop platform capabilities as a no-op', async () => { + const order: string[] = []; + const session = makeSession(order); + const platform = makePlatform(false, order); + const provider = makeProvider({ + ready: true, + localOutboundIdentity: 'local-backend-identity', + keyIndex: 1, + }); + const { dependencies, mediaParticipant } = makeDependencies(session, order, provider); + dependencies.platformBridge = platform.bridge; + const controller = createLivekitJsController(dependencies, { manualMediaTest: true }); + + await connectToActive(controller, session); + await controller.setMicrophoneEnabled(true); + + expect(platform.getCapabilities).toHaveBeenCalledOnce(); + expect(platform.start).not.toHaveBeenCalled(); + expect(platform.stop).not.toHaveBeenCalled(); + expect(mediaParticipant.setMicrophoneEnabled).toHaveBeenCalledWith(true); + expect(controller.getState().platform).toBeUndefined(); + }); + + it('starts the platform lifecycle before enabling the local microphone', async () => { + const order: string[] = []; + const session = makeSession(order); + const platform = makePlatform(true, order); + const provider = makeProvider({ + ready: true, + localOutboundIdentity: 'local-backend-identity', + keyIndex: 1, + }); + const { dependencies, mediaParticipant } = makeDependencies(session, order, provider); + dependencies.platformBridge = platform.bridge; + dependencies.createPlatformSessionId = () => 'opaque-session-1'; + mediaParticipant.setMicrophoneEnabled.mockImplementation(async () => { + order.push('mic'); + return undefined; + }); + const controller = createLivekitJsController(dependencies, { manualMediaTest: true }); + + await connectToActive(controller, session); + expect(platform.start).not.toHaveBeenCalled(); + await controller.setMicrophoneEnabled(true); + + expect(order.indexOf('platform-caps')).toBeGreaterThan(order.indexOf('connect')); + expect(order.indexOf('platform-start:true:true')).toBeGreaterThan(order.indexOf('connect')); + expect(order.indexOf('platform-start:true:true')).toBeLessThan(order.indexOf('mic')); + expect(platform.start).toHaveBeenCalledWith({ + sessionId: 'opaque-session-1', + microphone: true, + playback: true, + }); + expect(controller.getState().platform?.active).toBe(true); + }); + + it('stops the platform lifecycle after Room disconnect and before MatrixRTC leave', async () => { + const order: string[] = []; + const session = makeSession(order); + const platform = makePlatform(true, order); + const provider = makeProvider({ + ready: true, + localOutboundIdentity: 'local-backend-identity', + keyIndex: 1, + }); + const { dependencies, mediaParticipant } = makeDependencies(session, order, provider); + dependencies.platformBridge = platform.bridge; + mediaParticipant.setMicrophoneEnabled.mockImplementation(async () => { + order.push('mic'); + return undefined; + }); + const controller = createLivekitJsController(dependencies, { manualMediaTest: true }); + await connectToActive(controller, session); + await controller.setMicrophoneEnabled(true); + + await controller.disconnect(); + + expect(order.lastIndexOf('mic')).toBeLessThan(order.indexOf('disconnect')); + expect(order.indexOf('disconnect')).toBeLessThan(order.indexOf('platform-stop')); + expect(order.indexOf('platform-stop')).toBeLessThan(order.indexOf('detach')); + expect(order.indexOf('platform-stop')).toBeLessThan(order.indexOf('leave')); + expect(platform.stop).toHaveBeenCalledOnce(); + expect(controller.getState().platform).toBeUndefined(); + expect(controller.getState().lifecycle).toBe('idle'); + }); + + it('refuses the media operation and skips platform stop when platform start fails', async () => { + const order: string[] = []; + const session = makeSession(order); + const platform = makePlatform(true, order); + platform.start.mockRejectedValue(new Error('native start failed')); + const provider = makeProvider({ + ready: true, + localOutboundIdentity: 'local-backend-identity', + keyIndex: 1, + }); + const { dependencies, mediaParticipant } = makeDependencies(session, order, provider); + dependencies.platformBridge = platform.bridge; + const controller = createLivekitJsController(dependencies, { manualMediaTest: true }); + await connectToActive(controller, session); + + await expect(controller.setMicrophoneEnabled(true)).rejects.toMatchObject({ + code: 'platform-lifecycle-failed', + }); + + expect(mediaParticipant.setMicrophoneEnabled).not.toHaveBeenCalled(); + expect(controller.getState().mediaFailure).toBe('platform-lifecycle-failed'); + expect(controller.getState().platform?.active).toBe(false); + + await controller.disconnect(); + expect(platform.stop).not.toHaveBeenCalled(); + expect(provider.detach).toHaveBeenCalledOnce(); + }); + + it('restarts platform lifecycle with the same session when flags change', async () => { + const order: string[] = []; + const session = makeSession(order); + const platform = makePlatform(true, order); + const provider = makeProvider({ + ready: true, + localOutboundIdentity: 'local-backend-identity', + keyIndex: 1, + }); + const { dependencies } = makeDependencies(session, order, provider); + dependencies.platformBridge = platform.bridge; + dependencies.createPlatformSessionId = () => 'opaque-session-1'; + const controller = createLivekitJsController(dependencies, { manualMediaTest: true }); + await connectToActive(controller, session); + + await controller.setCameraEnabled(true); + expect(platform.start).toHaveBeenLastCalledWith({ + sessionId: 'opaque-session-1', + microphone: false, + playback: true, + }); + expect(platform.stop).not.toHaveBeenCalled(); + + await controller.setMicrophoneEnabled(true); + expect(platform.stop).toHaveBeenCalledWith({ sessionId: 'opaque-session-1' }); + expect(platform.start).toHaveBeenLastCalledWith({ + sessionId: 'opaque-session-1', + microphone: true, + playback: true, + }); + expect(platform.start).toHaveBeenCalledTimes(2); + + const stopsBefore = platform.stop.mock.calls.length; + await controller.setMicrophoneEnabled(true); + expect(platform.stop.mock.calls.length).toBe(stopsBefore); + expect(platform.start).toHaveBeenCalledTimes(2); + }); + + it('filters stale session and revision platform events', async () => { + const order: string[] = []; + const session = makeSession(order); + const platform = makePlatform(true, order); + const provider = makeProvider({ + ready: true, + localOutboundIdentity: 'local-backend-identity', + keyIndex: 1, + }); + const { dependencies } = makeDependencies(session, order, provider); + dependencies.platformBridge = platform.bridge; + dependencies.createPlatformSessionId = () => 'opaque-session-1'; + const controller = createLivekitJsController(dependencies, { manualMediaTest: true }); + await connectToActive(controller, session); + await controller.setCameraEnabled(true); + + platform.emit({ revision: 2, sessionId: 'other-session', type: 'media_reset' }); + platform.emit({ revision: 3, sessionId: 'opaque-session-1', type: 'media_reset' }); + platform.emit({ + revision: 3, + sessionId: 'opaque-session-1', + type: 'focus_changed', + focused: false, + }); + platform.emit({ + revision: 4, + sessionId: 'opaque-session-1', + type: 'focus_changed', + focused: false, + }); + await vi.waitFor(() => expect(controller.getState().platform?.focused).toBe(false)); + + const snapshot = controller.getState().platform; + expect(snapshot?.mediaReset).toBe(true); + expect(snapshot?.focused).toBe(false); + expect(snapshot?.active).toBe(true); + expect(controller.getState().mediaFailure).toBeNull(); + }); + + it('refuses manual media when the platform capability lookup fails', async () => { + const order: string[] = []; + const session = makeSession(order); + const platform = makePlatform(false, order); + platform.getCapabilities.mockReset(); + platform.getCapabilities.mockRejectedValue(new Error('capability bridge unavailable')); + const provider = makeProvider({ + ready: true, + localOutboundIdentity: 'local-backend-identity', + keyIndex: 1, + }); + const { dependencies, mediaParticipant } = makeDependencies(session, order, provider); + dependencies.platformBridge = platform.bridge; + const controller = createLivekitJsController(dependencies, { manualMediaTest: true }); + await connectToActive(controller, session); + + await expect(controller.setMicrophoneEnabled(true)).rejects.toMatchObject({ + code: 'platform-lifecycle-failed', + }); + + expect(mediaParticipant.setMicrophoneEnabled).not.toHaveBeenCalled(); + expect(platform.start).not.toHaveBeenCalled(); + expect(controller.getState().mediaFailure).toBe('platform-lifecycle-failed'); + expect(controller.getState().platform).toBeUndefined(); + }); + + it.each([ + ['malformed supported', { supported: 'yes', microphone: true, playback: true }], + ['missing microphone', { supported: true, playback: true }], + ['malformed microphone', { supported: true, microphone: 1, playback: true }], + ['missing playback', { supported: true, microphone: true }], + ['malformed playback', { supported: true, microphone: true, playback: null }], + ['missing everything', {}], + ['a null result', null], + ['an undefined result', undefined], + ])( + 'refuses manual media without calling media methods on %s capability result', + async (_label, capabilities) => { + const order: string[] = []; + const session = makeSession(order); + const platform = makePlatform(false, order); + platform.getCapabilities.mockReset(); + platform.getCapabilities.mockResolvedValue(capabilities); + const provider = makeProvider({ + ready: true, + localOutboundIdentity: 'local-backend-identity', + keyIndex: 1, + }); + const { dependencies, mediaParticipant } = makeDependencies(session, order, provider); + dependencies.platformBridge = platform.bridge; + const controller = createLivekitJsController(dependencies, { manualMediaTest: true }); + await connectToActive(controller, session); + + await expect(controller.setMicrophoneEnabled(true)).rejects.toMatchObject({ + code: 'platform-lifecycle-failed', + }); + await expect(controller.setCameraEnabled(true)).rejects.toMatchObject({ + code: 'platform-lifecycle-failed', + }); + + expect(mediaParticipant.setMicrophoneEnabled).not.toHaveBeenCalled(); + expect(mediaParticipant.setCameraEnabled).not.toHaveBeenCalled(); + expect(mediaParticipant.setScreenShareEnabled).not.toHaveBeenCalled(); + expect(platform.start).not.toHaveBeenCalled(); + expect(controller.getState().mediaFailure).toBe('platform-lifecycle-failed'); + expect(controller.getState().platform).toBeUndefined(); + } + ); + + it('disables the LiveKit microphone track before downgrading the platform lifecycle', async () => { + const order: string[] = []; + const session = makeSession(order); + const platform = makePlatform(true, order); + const provider = makeProvider({ + ready: true, + localOutboundIdentity: 'local-backend-identity', + keyIndex: 1, + }); + const { dependencies, mediaParticipant } = makeDependencies(session, order, provider); + dependencies.platformBridge = platform.bridge; + dependencies.createPlatformSessionId = () => 'opaque-session-1'; + mediaParticipant.setMicrophoneEnabled.mockImplementation(async (enabled) => { + order.push(enabled ? 'mic-on' : 'mic-off'); + return undefined; + }); + const controller = createLivekitJsController(dependencies, { manualMediaTest: true }); + await connectToActive(controller, session); + await controller.setMicrophoneEnabled(true); + expect(platform.start).toHaveBeenLastCalledWith({ + sessionId: 'opaque-session-1', + microphone: true, + playback: true, + }); + + await controller.setMicrophoneEnabled(false); + + expect(order.indexOf('mic-on')).toBeLessThan(order.indexOf('mic-off')); + expect(order.indexOf('mic-off')).toBeLessThan(order.indexOf('platform-stop')); + expect(order.indexOf('platform-stop')).toBeLessThan(order.indexOf('platform-start:false:true')); + expect(platform.start).toHaveBeenLastCalledWith({ + sessionId: 'opaque-session-1', + microphone: false, + playback: true, + }); + expect(controller.getState().platform?.active).toBe(true); + expect(controller.getState().mediaFailure).toBeNull(); + }); + + it('retains the microphone lifecycle when the JS microphone disable fails', async () => { + const order: string[] = []; + const session = makeSession(order); + const platform = makePlatform(true, order); + const provider = makeProvider({ + ready: true, + localOutboundIdentity: 'local-backend-identity', + keyIndex: 1, + }); + const { dependencies, mediaParticipant } = makeDependencies(session, order, provider); + dependencies.platformBridge = platform.bridge; + dependencies.createPlatformSessionId = () => 'opaque-session-1'; + mediaParticipant.setMicrophoneEnabled.mockImplementation(async (enabled) => { + order.push(enabled ? 'mic-on' : 'mic-off'); + return undefined; + }); + const controller = createLivekitJsController(dependencies, { manualMediaTest: true }); + await connectToActive(controller, session); + await controller.setMicrophoneEnabled(true); + + mediaParticipant.setMicrophoneEnabled.mockRejectedValueOnce(new Error('track stuck')); + await expect(controller.setMicrophoneEnabled(false)).rejects.toMatchObject({ + code: 'media-operation-failed', + }); + + expect(platform.stop).not.toHaveBeenCalled(); + expect(platform.start).toHaveBeenCalledTimes(1); + expect(controller.getState().mediaFailure).toBe('media-operation-failed'); + expect(controller.getState().platform?.active).toBe(true); + + // A later media operation must not downgrade the microphone lifecycle. + await controller.setCameraEnabled(true); + expect(platform.stop).not.toHaveBeenCalled(); + expect(platform.start).toHaveBeenCalledTimes(1); + + // Teardown still stops the retained microphone lifecycle. + await controller.disconnect(); + expect(platform.stop).toHaveBeenCalledWith({ sessionId: 'opaque-session-1' }); + expect(order.lastIndexOf('platform-stop')).toBeLessThan(order.indexOf('leave')); + expect(controller.getState().lifecycle).toBe('idle'); + }); + + it('stops manual media safely on a platform failure event', async () => { + const order: string[] = []; + const session = makeSession(order); + const platform = makePlatform(true, order); + const provider = makeProvider({ + ready: true, + localOutboundIdentity: 'local-backend-identity', + keyIndex: 1, + }); + const { dependencies, mediaParticipant } = makeDependencies(session, order, provider); + dependencies.platformBridge = platform.bridge; + dependencies.createPlatformSessionId = () => 'opaque-session-1'; + const controller = createLivekitJsController(dependencies, { manualMediaTest: true }); + await connectToActive(controller, session); + await controller.setMicrophoneEnabled(true); + + platform.emit({ + revision: 1, + sessionId: 'opaque-session-1', + type: 'failed', + code: 'audio_unavailable', + }); + await vi.waitFor(() => + expect(controller.getState().mediaFailure).toBe('platform-lifecycle-failed') + ); + + expect(controller.getState().platform).toMatchObject({ + active: false, + failure: 'audio_unavailable', + }); + expect(mediaParticipant.setMicrophoneEnabled).toHaveBeenLastCalledWith(false); + expect(controller.getState().lifecycle).toBe('active'); + }); +}); diff --git a/src/app/features/call/livekitJsController.ts b/src/app/features/call/livekitJsController.ts new file mode 100644 index 000000000..02b441148 --- /dev/null +++ b/src/app/features/call/livekitJsController.ts @@ -0,0 +1,738 @@ +import { Room as LivekitRoom, type RoomOptions } from 'livekit-client'; +import type { AutoDiscoveryInfo } from '../../cs-api'; +import type { MatrixClient, MatrixRTCSession, Room as MatrixRoom } from '$types/matrix-sdk'; +import { + disconnectLivekitThenLeaveMatrixRTC, + joinAndProvisionMatrixRTC, +} from './matrixRtcCallLifecycle'; +import { + LivekitMatrixKeyProvider, + type LivekitMatrixKeyProviderState, + isLivekitE2EESupported, +} from './livekitMatrixKeyProvider'; +import { getPreferredLivekitTransport, provisionLivekitToken } from './livekitProvisioning'; +import { acquireCallOwner, type CallOwnerLease } from '$state/callOwner'; +import { + getPlatformCapabilities as getPluginPlatformCapabilities, + onPlatformCallEvent, + startPlatformLifecycle, + stopPlatformLifecycle, + type PlatformCallCapabilities, + type PlatformCallEvent, + type PlatformCallFailureCode, + type PlatformCallRoute, +} from '$plugins/call/platformCallLifecycle'; + +export type LivekitJsControllerLifecycle = + | 'idle' + | 'joining-matrix' + | 'provisioning' + | 'connecting-livekit' + | 'active' + | 'stopping' + | 'failed'; + +export type LivekitJsControllerFailure = 'e2ee-unsupported' | 'e2ee-import-failed' | 'setup-failed'; + +export type LivekitJsMediaFailure = + | 'media-test-disabled' + | 'e2ee-unsupported' + | 'e2ee-key-not-ready' + | 'e2ee-key-failed' + | 'room-not-active' + | 'platform-lifecycle-failed' + | 'media-operation-failed'; + +export class LivekitJsMediaError extends Error { + public constructor(public readonly code: LivekitJsMediaFailure) { + super(`LiveKit JS media test refused: ${code}`); + this.name = 'LivekitJsMediaError'; + } +} + +export type LivekitJsMediaFacade = { + setMicrophoneEnabled: (enabled: boolean) => Promise; + setCameraEnabled: (enabled: boolean) => Promise; + setScreenShareEnabled: (enabled: boolean) => Promise; +}; + +export type LivekitJsPlatformStartRequest = { + sessionId: string; + microphone: boolean; + playback: boolean; +}; + +export type LivekitJsPlatformBridge = { + getCapabilities: () => Promise; + start: (request: LivekitJsPlatformStartRequest) => Promise; + stop: (request: { sessionId: string }) => Promise; + onEvent: (handler: (event: PlatformCallEvent) => void) => Promise<() => void>; +}; + +export type LivekitJsPlatformState = { + active: boolean; + focused: boolean; + route: PlatformCallRoute | null; + interrupted: boolean; + mediaReset: boolean; + failure: PlatformCallFailureCode | null; +}; + +export type LivekitJsControllerState = { + lifecycle: LivekitJsControllerLifecycle; + failure: LivekitJsControllerFailure | null; + mediaFailure: LivekitJsMediaFailure | null; + room?: LivekitRoom; + media?: LivekitJsMediaFacade; + platform?: LivekitJsPlatformState; + e2ee: Readonly; +}; + +export type LivekitJsControllerStateListener = (state: Readonly) => void; + +export type LivekitJsConnectOptions = { + mx: MatrixClient; + room: MatrixRoom; + discovery?: Pick; + callIntent?: 'audio' | 'video'; + dm?: boolean; + ongoing?: boolean; +}; + +type LivekitLocalParticipantLike = Pick< + LivekitRoom['localParticipant'], + 'setMicrophoneEnabled' | 'setCameraEnabled' | 'setScreenShareEnabled' +>; + +type LivekitRoomLike = Pick & { + localParticipant?: LivekitLocalParticipantLike; +}; + +type ManualMediaMethods = LivekitJsMediaFacade; + +type LivekitJsControllerBase = { + connect: (options: LivekitJsConnectOptions) => Promise; + disconnect: () => Promise; + getState: () => Readonly; + subscribe: (listener: LivekitJsControllerStateListener) => () => void; +}; + +type LivekitJsManualController = LivekitJsControllerBase & ManualMediaMethods; + +export type LivekitJsControllerOptions = { + manualMediaTest?: boolean; +}; + +export type LivekitJsControllerDependencies = { + createRoom?: (options: RoomOptions) => LivekitRoomLike; + createWorker?: () => Worker; + createKeyProvider?: () => LivekitMatrixKeyProvider; + isE2EESupported?: () => boolean; + getPreferredTransport?: typeof getPreferredLivekitTransport; + provisionToken?: typeof provisionLivekitToken; + platformBridge?: LivekitJsPlatformBridge; + createPlatformSessionId?: () => string; +}; + +type ControllerRecord = { + session: MatrixRTCSession; + provider: LivekitMatrixKeyProvider; + worker?: Worker; + room?: LivekitRoomLike; + matrixJoinStarted: boolean; + providerDetached: boolean; + ownerLease: CallOwnerLease; + cancelled: boolean; + e2eeFailure: boolean; + cancelMembershipWait?: () => void; + removeKeyStateListener?: () => void; + platformSessionId?: string; + platformCaps?: Promise; + platformActive: boolean; + platformFlags: { microphone: boolean; playback: boolean }; + platformDesired: { microphone: boolean; playback: boolean }; + platformRevision: number; + platformUnlisten?: () => void; + platformEventOp?: Promise; + mediaPublished: boolean; + cleanupPromise?: Promise; + resourcesReady: Promise; + resolveResources: () => void; +}; + +const initialE2EEState: LivekitMatrixKeyProviderState = { + ready: false, + localOutboundIdentity: null, + keyIndex: null, + lastImportFailure: null, +}; + +const defaultCreateRoom = (options: RoomOptions): LivekitRoomLike => new LivekitRoom(options); + +const defaultCreateWorker = (): Worker => + new Worker(new URL('livekit-client/e2ee-worker', import.meta.url), { type: 'module' }); + +const defaultPlatformBridge: LivekitJsPlatformBridge = { + getCapabilities: getPluginPlatformCapabilities, + start: startPlatformLifecycle, + stop: stopPlatformLifecycle, + onEvent: onPlatformCallEvent, +}; + +let platformSessionCounter = 0; +const defaultCreatePlatformSessionId = (): string => { + platformSessionCounter += 1; + const random = + typeof crypto !== 'undefined' && typeof crypto.randomUUID === 'function' + ? crypto.randomUUID() + : `${Date.now().toString(36)}-${platformSessionCounter.toString(36)}`; + return `livekit-js-platform-${random}`; +}; + +export function createLivekitJsController( + dependencies: LivekitJsControllerDependencies | undefined, + options: { manualMediaTest: true } +): LivekitJsManualController; +export function createLivekitJsController( + dependencies?: LivekitJsControllerDependencies, + options?: LivekitJsControllerOptions +): LivekitJsControllerBase; +export function createLivekitJsController( + dependencies: LivekitJsControllerDependencies = {}, + controllerOptions: LivekitJsControllerOptions = {} +) { + const createRoom = dependencies.createRoom ?? defaultCreateRoom; + const createWorker = dependencies.createWorker ?? defaultCreateWorker; + const createKeyProvider = + dependencies.createKeyProvider ?? (() => new LivekitMatrixKeyProvider()); + const supportsE2EE = dependencies.isE2EESupported ?? isLivekitE2EESupported; + const getPreferredTransport = dependencies.getPreferredTransport ?? getPreferredLivekitTransport; + const provisionToken = dependencies.provisionToken ?? provisionLivekitToken; + const platformBridge = dependencies.platformBridge ?? defaultPlatformBridge; + const createPlatformSessionId = + dependencies.createPlatformSessionId ?? defaultCreatePlatformSessionId; + const manualMediaTest = controllerOptions.manualMediaTest === true; + + let state: LivekitJsControllerState = { + lifecycle: 'idle', + failure: null, + mediaFailure: null, + room: undefined, + media: undefined, + e2ee: initialE2EEState, + }; + let record: ControllerRecord | undefined; + let operation: Promise | undefined; + const listeners = new Set(); + + const publish = (changes: Partial): void => { + state = { ...state, ...changes }; + const snapshot = { ...state, e2ee: { ...state.e2ee } }; + listeners.forEach((listener) => { + try { + listener(snapshot); + } catch { + // A state observer must not interrupt lifecycle cleanup. + } + }); + }; + + const cleanup = async ( + current: ControllerRecord, + result: 'idle' | 'failed', + failure: LivekitJsControllerFailure | null + ): Promise => { + if (current.cleanupPromise) { + await current.cleanupPromise; + return; + } + + current.cancelled = true; + current.cancelMembershipWait?.(); + current.cancelMembershipWait = undefined; + publish({ lifecycle: 'stopping' }); + const detachProvider = (): void => { + if (current.providerDetached) return; + current.providerDetached = true; + try { + current.provider.detach(); + } catch {} + }; + current.cleanupPromise = (async () => { + if (!current.room && !current.worker) await current.resourcesReady; + const stopRoom = async (): Promise => { + if (current.mediaPublished && current.room?.localParticipant) { + try { + await current.room.localParticipant.setMicrophoneEnabled(false); + await current.room.localParticipant.setCameraEnabled(false); + await current.room.localParticipant.setScreenShareEnabled(false); + } catch { + // Best-effort unpublish; disconnect clears any remaining tracks. + } + } + await current.room?.disconnect(); + }; + const stopPlatform = async (): Promise => { + if (current.platformEventOp) { + try { + await current.platformEventOp; + } catch { + // Event settling must not block platform stop. + } + } + if (current.platformActive && current.platformSessionId) { + try { + await platformBridge.stop({ sessionId: current.platformSessionId }); + } catch { + // Platform stop failures never block room/provider cleanup. + } + current.platformActive = false; + } + current.platformUnlisten?.(); + current.platformUnlisten = undefined; + publish({ platform: undefined }); + }; + if (current.matrixJoinStarted) { + await disconnectLivekitThenLeaveMatrixRTC(async () => { + try { + await stopRoom(); + } finally { + await stopPlatform(); + detachProvider(); + } + }, current.session); + } else { + try { + await stopRoom(); + } catch { + // Cleanup continues even when a setup room rejects disconnect. + } finally { + await stopPlatform(); + detachProvider(); + } + } + current.worker?.terminate(); + current.removeKeyStateListener?.(); + current.removeKeyStateListener = undefined; + current.ownerLease.release(); + if (record === current) record = undefined; + publish({ + lifecycle: result, + failure, + mediaFailure: result === 'idle' ? null : state.mediaFailure, + room: undefined, + media: undefined, + ...(result === 'idle' ? { e2ee: initialE2EEState } : {}), + }); + })(); + await current.cleanupPromise; + }; + + const setup = async ( + current: ControllerRecord, + connectOptions: LivekitJsConnectOptions + ): Promise => { + let failure: LivekitJsControllerFailure | null = null; + try { + publish({ lifecycle: 'joining-matrix', failure: null }); + if (!supportsE2EE()) { + failure = 'e2ee-unsupported'; + } else { + current.provider.attach(current.session); + if (current.e2eeFailure || current.provider.getKeyState().lastImportFailure) { + failure = 'e2ee-import-failed'; + } else { + const joined = await joinAndProvisionMatrixRTC({ + mx: connectOptions.mx, + room: connectOptions.room, + session: current.session, + discovery: connectOptions.discovery, + getPreferredTransport, + provisionToken, + callIntent: connectOptions.callIntent ?? 'audio', + ...(connectOptions.ongoing + ? {} + : { notificationType: connectOptions.dm ? 'ring' : 'notification' }), + manageMediaKeys: true, + isCancelled: () => current.cancelled, + onStage: (stage) => publish({ lifecycle: stage }), + onMembershipWait: (cancel) => { + current.cancelMembershipWait = cancel; + }, + onJoinStarted: () => { + current.matrixJoinStarted = true; + }, + }); + + current.provider.setLocalOutboundIdentity(joined.ownMembership?.rtcBackendIdentity); + if (current.e2eeFailure || current.provider.getKeyState().lastImportFailure) { + failure = 'e2ee-import-failed'; + } else if (!current.cancelled) { + publish({ lifecycle: 'connecting-livekit' }); + current.worker = createWorker(); + current.room = createRoom({ + encryption: { + keyProvider: current.provider, + worker: current.worker, + }, + }); + await current.room.connect(joined.provisioned.url, joined.provisioned.jwt); + if (current.e2eeFailure) failure = 'e2ee-import-failed'; + } + } + } + } catch { + failure = current.e2eeFailure ? 'e2ee-import-failed' : 'setup-failed'; + } finally { + current.resolveResources(); + } + + if (failure) { + await cleanup(current, 'failed', failure); + } else if (current.cancelled) { + await cleanup(current, 'idle', null); + } else { + publish({ + lifecycle: 'active', + failure: null, + room: current.room as LivekitRoom, + media: manualMediaTest + ? { + setMicrophoneEnabled, + setCameraEnabled, + setScreenShareEnabled, + } + : undefined, + }); + } + }; + + const requireMediaParticipant = (): LivekitLocalParticipantLike => { + if (!manualMediaTest) throw new LivekitJsMediaError('media-test-disabled'); + if (!supportsE2EE()) throw new LivekitJsMediaError('e2ee-unsupported'); + if (state.lifecycle !== 'active' || !record?.room?.localParticipant) { + throw new LivekitJsMediaError('room-not-active'); + } + + const keyState = record.provider.getKeyState(); + if (keyState.lastImportFailure) throw new LivekitJsMediaError('e2ee-key-failed'); + if (!keyState.ready || !keyState.localOutboundIdentity) { + throw new LivekitJsMediaError('e2ee-key-not-ready'); + } + return record.room.localParticipant; + }; + + const publishPlatform = (current: ControllerRecord, patch: Partial) => { + if (record !== current) return; + const base: LivekitJsPlatformState = state.platform ?? { + active: current.platformActive, + focused: true, + route: null, + interrupted: false, + mediaReset: false, + failure: null, + }; + publish({ platform: { ...base, ...patch } }); + }; + + const getPlatformCapabilities = ( + current: ControllerRecord + ): Promise => { + if (!current.platformCaps) { + current.platformCaps = platformBridge.getCapabilities(); + } + return current.platformCaps; + }; + + const applyPlatformEvent = async ( + current: ControllerRecord, + event: PlatformCallEvent + ): Promise => { + if (current.cancelled || event.sessionId !== current.platformSessionId) return; + if (event.revision <= current.platformRevision) return; + current.platformRevision = event.revision; + switch (event.type) { + case 'focus_changed': + publishPlatform(current, { focused: event.focused }); + break; + case 'route_changed': + publishPlatform(current, { route: event.route }); + break; + case 'interrupted': + publishPlatform(current, { interrupted: event.state === 'began' }); + break; + case 'media_reset': + publishPlatform(current, { mediaReset: true }); + break; + case 'failed': { + // Native code never controls tracks; JS fails the manual test safely. + current.platformActive = false; + publishPlatform(current, { active: false, failure: event.code }); + publish({ mediaFailure: 'platform-lifecycle-failed' }); + const participant = current.room?.localParticipant; + if (participant && current.mediaPublished) { + try { + await participant.setMicrophoneEnabled(false); + await participant.setCameraEnabled(false); + await participant.setScreenShareEnabled(false); + } catch { + // Disconnect cleanup unpublishes any remaining tracks. + } + } + break; + } + default: + break; + } + }; + + const ensurePlatformListener = async (current: ControllerRecord): Promise => { + if (current.platformUnlisten) return; + const unlisten = await platformBridge.onEvent((event) => { + current.platformEventOp = applyPlatformEvent(current, event).finally(() => { + current.platformEventOp = undefined; + }); + }); + if (current.cancelled || !current.platformActive) { + unlisten(); + return; + } + current.platformUnlisten = unlisten; + }; + + const ensurePlatformLifecycle = async (current: ControllerRecord): Promise => { + let capabilities: PlatformCallCapabilities; + try { + capabilities = await getPlatformCapabilities(current); + } catch { + // A rejected capability request is not the desktop no-op case. + throw new LivekitJsMediaError('platform-lifecycle-failed'); + } + if ( + !capabilities || + typeof capabilities.supported !== 'boolean' || + typeof capabilities.microphone !== 'boolean' || + typeof capabilities.playback !== 'boolean' + ) { + // A malformed capability result fails closed like a rejection. + throw new LivekitJsMediaError('platform-lifecycle-failed'); + } + if (!capabilities.supported || current.cancelled) return; + + const desired = { ...current.platformDesired }; + if ( + current.platformActive && + current.platformFlags.microphone === desired.microphone && + current.platformFlags.playback === desired.playback + ) { + return; + } + + try { + if (current.platformActive && current.platformSessionId) { + await platformBridge.stop({ sessionId: current.platformSessionId }); + current.platformActive = false; + } + if (!current.platformSessionId) current.platformSessionId = createPlatformSessionId(); + await platformBridge.start({ sessionId: current.platformSessionId, ...desired }); + current.platformFlags = desired; + current.platformActive = true; + publishPlatform(current, { active: true, failure: null }); + await ensurePlatformListener(current); + } catch { + current.platformActive = false; + publishPlatform(current, { active: false }); + throw new LivekitJsMediaError('platform-lifecycle-failed'); + } + }; + + const runMediaAction = async ( + action: (participant: LivekitLocalParticipantLike) => Promise, + platformOrder: 'platform-first' | 'platform-after' = 'platform-first' + ): Promise => { + let current: ControllerRecord; + let participant: LivekitLocalParticipantLike; + try { + participant = requireMediaParticipant(); + // requireMediaParticipant guarantees an active record; narrow for TS. + if (!record) throw new LivekitJsMediaError('room-not-active'); + current = record; + if (platformOrder === 'platform-first') await ensurePlatformLifecycle(current); + } catch (error) { + if (error instanceof LivekitJsMediaError) publish({ mediaFailure: error.code }); + throw error; + } + + try { + await action(participant); + } catch { + const error = new LivekitJsMediaError('media-operation-failed'); + publish({ mediaFailure: error.code }); + throw error; + } + if (platformOrder === 'platform-after') { + try { + await ensurePlatformLifecycle(current); + } catch (error) { + if (error instanceof LivekitJsMediaError) publish({ mediaFailure: error.code }); + throw error; + } + } + publish({ mediaFailure: null }); + }; + + const markMediaPublished = (promise: Promise, published: boolean): Promise => + promise.then(() => { + if (published && record) record.mediaPublished = true; + }); + + const setMicrophoneEnabled = (enabled: boolean): Promise => { + if (record) record.platformDesired.microphone = enabled; + const promise = markMediaPublished( + // A microphone disable must unpublish the JS track before the platform + // lifecycle is stopped or downgraded. + runMediaAction( + (participant) => participant.setMicrophoneEnabled(enabled), + enabled ? 'platform-first' : 'platform-after' + ), + enabled + ); + if (enabled) return promise; + return promise.catch((error) => { + // The JS track failed to disable, so the microphone may still be + // live: retain the microphone platform lifecycle. + if ( + record && + error instanceof LivekitJsMediaError && + error.code === 'media-operation-failed' + ) { + record.platformDesired.microphone = true; + } + throw error; + }); + }; + + const setCameraEnabled = (enabled: boolean): Promise => + markMediaPublished( + runMediaAction((participant) => participant.setCameraEnabled(enabled)), + enabled + ); + + const setScreenShareEnabled = (enabled: boolean): Promise => + markMediaPublished( + runMediaAction((participant) => participant.setScreenShareEnabled(enabled)), + enabled + ); + + const connect = (connectOptions: LivekitJsConnectOptions): Promise => { + if (state.lifecycle === 'failed' && !record && !operation) { + publish({ + lifecycle: 'idle', + failure: null, + mediaFailure: null, + room: undefined, + media: undefined, + e2ee: initialE2EEState, + }); + } + if (record || operation || state.lifecycle !== 'idle') { + return Promise.reject(new Error('LiveKit JS call controller is already in use')); + } + + const ownerLease = acquireCallOwner('livekit-js', connectOptions.room.roomId); + if (!ownerLease) { + publish({ + lifecycle: 'failed', + failure: 'setup-failed', + mediaFailure: null, + room: undefined, + media: undefined, + }); + return Promise.resolve(); + } + + let resolveResources!: () => void; + const resourcesReady = new Promise((resolve) => { + resolveResources = resolve; + }); + let session: MatrixRTCSession; + let provider: LivekitMatrixKeyProvider; + try { + session = connectOptions.mx.matrixRTC.getRoomSession(connectOptions.room); + provider = createKeyProvider(); + } catch { + ownerLease.release(); + publish({ + lifecycle: 'failed', + failure: 'setup-failed', + mediaFailure: null, + room: undefined, + media: undefined, + }); + return Promise.resolve(); + } + const current: ControllerRecord = { + session, + provider, + matrixJoinStarted: false, + cancelled: false, + e2eeFailure: false, + providerDetached: false, + ownerLease, + resourcesReady, + resolveResources, + platformActive: false, + platformFlags: { microphone: false, playback: false }, + platformDesired: { microphone: false, playback: true }, + platformRevision: -1, + mediaPublished: false, + }; + record = current; + current.removeKeyStateListener = current.provider.subscribe((e2ee) => { + publish({ e2ee }); + if (e2ee.lastImportFailure) { + current.e2eeFailure = true; + current.cancelMembershipWait?.(); + if (current.room) void cleanup(current, 'failed', 'e2ee-import-failed'); + } + }); + + operation = setup(current, connectOptions).finally(() => { + operation = undefined; + }); + return operation; + }; + + const disconnect = async (): Promise => { + if (!record) { + if (state.lifecycle === 'failed') { + publish({ lifecycle: 'idle', failure: null, mediaFailure: null }); + } + return; + } + await cleanup(record, 'idle', null); + await operation; + }; + + const controller: LivekitJsControllerBase = { + connect, + disconnect, + getState: (): Readonly => ({ + ...state, + e2ee: { ...state.e2ee }, + }), + subscribe: (listener: LivekitJsControllerStateListener): (() => void) => { + listeners.add(listener); + listener({ ...state, e2ee: { ...state.e2ee } }); + return () => listeners.delete(listener); + }, + }; + + if (!manualMediaTest) return controller; + return { + ...controller, + setMicrophoneEnabled, + setCameraEnabled, + setScreenShareEnabled, + }; +} diff --git a/src/app/features/call/livekitMatrixKeyProvider.test.ts b/src/app/features/call/livekitMatrixKeyProvider.test.ts new file mode 100644 index 000000000..3d482e163 --- /dev/null +++ b/src/app/features/call/livekitMatrixKeyProvider.test.ts @@ -0,0 +1,299 @@ +import { beforeEach, describe, expect, it, vi, type Mock } from 'vitest'; +import { MatrixRTCSessionEvent } from '$types/matrix-sdk'; +import { LivekitMatrixKeyProvider, isLivekitE2EESupported } from './livekitMatrixKeyProvider'; + +type EncryptionKeyChangedHandler = ( + key: Uint8Array, + keyIndex: number, + membershipParts: unknown, + rtcBackendIdentity: string +) => void; + +type SessionOn = (event: MatrixRTCSessionEvent, handler: EncryptionKeyChangedHandler) => void; + +type TestSession = { + on: Mock; + off: Mock; + reemitEncryptionKeys: Mock<() => void>; +}; + +const deferred = (): { + promise: Promise; + resolve: (value: T) => void; + reject: (reason?: unknown) => void; +} => { + let resolve!: (value: T) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((promiseResolve, promiseReject) => { + resolve = promiseResolve; + reject = promiseReject; + }); + return { promise, resolve, reject }; +}; + +const session = (): TestSession => ({ + on: vi.fn(), + off: vi.fn(), + reemitEncryptionKeys: vi.fn<() => void>(), +}); + +const handlerFor = (testSession: TestSession): EncryptionKeyChangedHandler => + testSession.on.mock.calls.find( + ([event]) => event === MatrixRTCSessionEvent.EncryptionKeyChanged + )?.[1] as EncryptionKeyChangedHandler; + +describe('LivekitMatrixKeyProvider', () => { + const importedKey = { imported: true } as unknown as CryptoKey; + const importKey = vi.fn().mockResolvedValue(importedKey); + + beforeEach(() => { + importKey.mockClear(); + vi.stubGlobal('crypto', { subtle: { importKey } }); + }); + + it('attaches, re-emits tracked keys, and forwards HKDF material with identity and index', async () => { + const testSession = session(); + const provider = new LivekitMatrixKeyProvider(); + const onSetEncryptionKey = vi.spyOn( + provider as unknown as { + onSetEncryptionKey: (key: CryptoKey, identity: string, index: number) => void; + }, + 'onSetEncryptionKey' + ); + const matrixKey = new Uint8Array([1, 2, 3, 4]); + provider.setLocalOutboundIdentity('hashed-member'); + + expect(provider.getOptions()).toMatchObject({ + ratchetWindowSize: 10, + keyringSize: 256, + sharedKey: false, + }); + + provider.attach(testSession as never); + + expect(testSession.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.EncryptionKeyChanged, + expect.any(Function) + ); + expect(testSession.reemitEncryptionKeys).toHaveBeenCalledOnce(); + + handlerFor(testSession)(matrixKey, 7, { userId: '@alice:example.org' }, 'hashed-member'); + await vi.waitFor(() => expect(onSetEncryptionKey).toHaveBeenCalledOnce()); + + expect(importKey).toHaveBeenCalledWith('raw', matrixKey, 'HKDF', false, [ + 'deriveBits', + 'deriveKey', + ]); + expect(onSetEncryptionKey).toHaveBeenCalledWith(importedKey, 'hashed-member', 7); + expect(provider.getKeyState()).toEqual({ + ready: true, + localOutboundIdentity: 'hashed-member', + keyIndex: 7, + lastImportFailure: null, + }); + }); + + it('removes the old session listener before attaching a new one', () => { + const oldSession = session(); + const newSession = session(); + const provider = new LivekitMatrixKeyProvider(); + + provider.attach(oldSession as never); + const oldHandler = handlerFor(oldSession); + provider.attach(newSession as never); + + expect(oldSession.off).toHaveBeenCalledWith( + MatrixRTCSessionEvent.EncryptionKeyChanged, + oldHandler + ); + expect(newSession.on).toHaveBeenCalledWith( + MatrixRTCSessionEvent.EncryptionKeyChanged, + oldHandler + ); + }); + + it('detaches the exact listener and does not expose raw key material', async () => { + const testSession = session(); + const provider = new LivekitMatrixKeyProvider(); + const rawKey = new Uint8Array([9, 8, 7]); + + provider.attach(testSession as never); + const handler = handlerFor(testSession); + handler(rawKey, 3, {}, 'member'); + await vi.waitFor(() => expect(provider.getKeys()).toHaveLength(1)); + provider.detach(); + + expect(testSession.off).toHaveBeenCalledWith( + MatrixRTCSessionEvent.EncryptionKeyChanged, + handler + ); + expect(provider.getKeys()[0]?.key).toBe(importedKey); + expect(provider.getKeys()[0]?.key).not.toBe(rawKey); + }); + + it('does not forward an import that completes after detach', async () => { + const testSession = session(); + const pendingImport = deferred(); + importKey.mockImplementationOnce(() => pendingImport.promise); + const provider = new LivekitMatrixKeyProvider(); + const onSetEncryptionKey = vi.spyOn( + provider as unknown as { + onSetEncryptionKey: (key: CryptoKey, identity: string, index: number) => void; + }, + 'onSetEncryptionKey' + ); + + provider.attach(testSession as never); + handlerFor(testSession)(new Uint8Array([1]), 1, {}, 'member'); + await vi.waitFor(() => expect(importKey).toHaveBeenCalledOnce()); + provider.detach(); + pendingImport.resolve(importedKey); + await Promise.resolve(); + await Promise.resolve(); + + expect(onSetEncryptionKey).not.toHaveBeenCalled(); + }); + + it('does not forward an import that completes after attaching a replacement session', async () => { + const oldSession = session(); + const newSession = session(); + const pendingImport = deferred(); + importKey.mockImplementationOnce(() => pendingImport.promise); + const provider = new LivekitMatrixKeyProvider(); + const onSetEncryptionKey = vi.spyOn( + provider as unknown as { + onSetEncryptionKey: (key: CryptoKey, identity: string, index: number) => void; + }, + 'onSetEncryptionKey' + ); + + provider.attach(oldSession as never); + handlerFor(oldSession)(new Uint8Array([1]), 1, {}, 'old-member'); + await vi.waitFor(() => expect(importKey).toHaveBeenCalledOnce()); + provider.attach(newSession as never); + pendingImport.resolve(importedKey); + await Promise.resolve(); + await Promise.resolve(); + + expect(onSetEncryptionKey).not.toHaveBeenCalled(); + }); + + it('keeps the effective key index from regressing when imports complete out of order', async () => { + const testSession = session(); + const firstImport = deferred(); + const secondImport = deferred(); + importKey + .mockImplementationOnce(() => firstImport.promise) + .mockImplementationOnce(() => secondImport.promise); + const provider = new LivekitMatrixKeyProvider(); + provider.setLocalOutboundIdentity('member'); + const onSetEncryptionKey = vi.spyOn( + provider as unknown as { + onSetEncryptionKey: (key: CryptoKey, identity: string, index: number) => void; + }, + 'onSetEncryptionKey' + ); + + provider.attach(testSession as never); + const handler = handlerFor(testSession); + handler(new Uint8Array([1]), 1, {}, 'member'); + handler(new Uint8Array([2]), 2, {}, 'member'); + await vi.waitFor(() => expect(importKey).toHaveBeenCalledTimes(2)); + + secondImport.resolve(importedKey); + await Promise.resolve(); + expect(onSetEncryptionKey).not.toHaveBeenCalled(); + firstImport.resolve(importedKey); + await vi.waitFor(() => expect(onSetEncryptionKey).toHaveBeenCalledTimes(2)); + + expect(onSetEncryptionKey.mock.calls[0]).toEqual([importedKey, 'member', 1]); + expect(onSetEncryptionKey.mock.calls[1]).toEqual([importedKey, 'member', 2]); + expect(provider.getKeyState().keyIndex).toBe(2); + }); + + it('tracks key-index regression independently for each participant', async () => { + const testSession = session(); + const provider = new LivekitMatrixKeyProvider(); + provider.setLocalOutboundIdentity('local-member'); + const onSetEncryptionKey = vi.spyOn( + provider as unknown as { + onSetEncryptionKey: (key: CryptoKey, identity: string, index: number) => void; + }, + 'onSetEncryptionKey' + ); + + provider.attach(testSession as never); + const handler = handlerFor(testSession); + handler(new Uint8Array([1]), 10, {}, 'remote-member'); + handler(new Uint8Array([2]), 1, {}, 'local-member'); + + await vi.waitFor(() => expect(onSetEncryptionKey).toHaveBeenCalledTimes(2)); + + expect(onSetEncryptionKey).toHaveBeenNthCalledWith(1, importedKey, 'remote-member', 10); + expect(onSetEncryptionKey).toHaveBeenNthCalledWith(2, importedKey, 'local-member', 1); + expect(provider.getKeyState()).toMatchObject({ + ready: true, + localOutboundIdentity: 'local-member', + keyIndex: 1, + lastImportFailure: null, + }); + }); + + it('records a safe failure when key import is rejected', async () => { + const testSession = session(); + importKey.mockRejectedValueOnce(new Error('raw key internals')); + const provider = new LivekitMatrixKeyProvider(); + provider.setLocalOutboundIdentity('member'); + + provider.attach(testSession as never); + handlerFor(testSession)(new Uint8Array([1]), 1, {}, 'member'); + + await vi.waitFor(() => expect(provider.getKeyState().lastImportFailure).toBe('import-failed')); + expect(JSON.stringify(provider.getKeyState())).not.toContain('raw key internals'); + + handlerFor(testSession)(new Uint8Array([2]), 2, {}, 'member'); + await vi.waitFor(() => expect(provider.getKeyState().ready).toBe(true)); + expect(provider.getKeyState().lastImportFailure).toBeNull(); + }); + + it('records a safe failure when WebCrypto import support is missing', async () => { + vi.stubGlobal('crypto', { subtle: {} }); + const testSession = session(); + const provider = new LivekitMatrixKeyProvider(); + + provider.attach(testSession as never); + handlerFor(testSession)(new Uint8Array([1]), 1, {}, 'member'); + + await vi.waitFor(() => + expect(provider.getKeyState().lastImportFailure).toBe('webcrypto-unavailable') + ); + }); +}); + +describe('isLivekitE2EESupported', () => { + beforeEach(() => { + vi.stubGlobal('crypto', { subtle: { importKey: vi.fn() } }); + }); + + it('fails closed when the current LiveKit API reports unsupported E2EE', () => { + Object.defineProperty(window, 'RTCRtpScriptTransform', { + configurable: true, + value: undefined, + }); + Object.defineProperty(window, 'RTCRtpSender', { + configurable: true, + value: undefined, + }); + + expect(isLivekitE2EESupported()).toBe(false); + }); + + it('reports support when LiveKit and WebCrypto are available', () => { + Object.defineProperty(window, 'RTCRtpScriptTransform', { + configurable: true, + value: vi.fn<() => void>(), + }); + + expect(isLivekitE2EESupported()).toBe(true); + }); +}); diff --git a/src/app/features/call/livekitMatrixKeyProvider.ts b/src/app/features/call/livekitMatrixKeyProvider.ts new file mode 100644 index 000000000..716d2fcab --- /dev/null +++ b/src/app/features/call/livekitMatrixKeyProvider.ts @@ -0,0 +1,199 @@ +import { BaseKeyProvider, isE2EESupported } from 'livekit-client'; +import { + MatrixRTCSessionEvent, + type CallMembershipIdentityParts, + type MatrixRTCSession, +} from '$types/matrix-sdk'; + +export const isLivekitE2EESupported = (): boolean => { + const subtle = globalThis.crypto?.subtle; + return typeof subtle?.importKey === 'function' && isE2EESupported(); +}; + +export type LivekitMatrixKeyImportFailure = 'webcrypto-unavailable' | 'import-failed'; + +export type LivekitMatrixKeyProviderState = { + ready: boolean; + localOutboundIdentity: string | null; + keyIndex: number | null; + lastImportFailure: LivekitMatrixKeyImportFailure | null; +}; + +export type LivekitMatrixKeyProviderStateListener = ( + state: Readonly +) => void; + +type KeyImportResult = + | { + keyMaterial: CryptoKey; + rtcBackendIdentity: string; + encryptionKeyIndex: number; + } + | { failure: LivekitMatrixKeyImportFailure }; + +export class LivekitMatrixKeyProvider extends BaseKeyProvider { + private rtcSession?: MatrixRTCSession; + private attachmentGeneration = 0; + private nextImportSequence = 0; + private nextUpdateSequence = 0; + private readonly pendingUpdates = new Map(); + private readonly acceptedKeyIndices = new Map(); + private readonly stateListeners = new Set(); + private localOutboundIdentity: string | null = null; + private state: LivekitMatrixKeyProviderState = { + ready: false, + localOutboundIdentity: null, + keyIndex: null, + lastImportFailure: null, + }; + + public constructor() { + super({ + ratchetWindowSize: 10, + keyringSize: 256, + sharedKey: false, + }); + } + + public attach(session: MatrixRTCSession): void { + this.detach(); + this.rtcSession = session; + session.on(MatrixRTCSessionEvent.EncryptionKeyChanged, this.onEncryptionKeyChanged); + session.reemitEncryptionKeys(); + } + + public detach(): void { + this.attachmentGeneration += 1; + this.nextImportSequence = 0; + this.nextUpdateSequence = 0; + this.pendingUpdates.clear(); + this.acceptedKeyIndices.clear(); + this.updateState({ + ready: false, + keyIndex: null, + lastImportFailure: null, + }); + + if (!this.rtcSession) return; + + this.rtcSession.off(MatrixRTCSessionEvent.EncryptionKeyChanged, this.onEncryptionKeyChanged); + this.rtcSession = undefined; + } + + public setLocalOutboundIdentity(identity: string | undefined): void { + this.localOutboundIdentity = identity ?? null; + const keyIndex = identity === undefined ? undefined : this.acceptedKeyIndices.get(identity); + this.updateState({ + localOutboundIdentity: this.localOutboundIdentity, + ready: keyIndex !== undefined, + keyIndex: keyIndex ?? null, + }); + } + + public getKeyState(): Readonly { + return { ...this.state }; + } + + public subscribe(listener: LivekitMatrixKeyProviderStateListener): () => void { + this.stateListeners.add(listener); + listener(this.getKeyState()); + return () => { + this.stateListeners.delete(listener); + }; + } + + private readonly onEncryptionKeyChanged = ( + encryptionKey: Uint8Array, + encryptionKeyIndex: number, + _membershipParts: CallMembershipIdentityParts, + rtcBackendIdentity: string + ): void => { + const generation = this.attachmentGeneration; + const sequence = this.nextImportSequence++; + const subtle = globalThis.crypto?.subtle; + if (!subtle || typeof subtle.importKey !== 'function') { + this.enqueueUpdate(generation, sequence, { failure: 'webcrypto-unavailable' }); + return; + } + + let importPromise: Promise; + try { + importPromise = subtle.importKey('raw', encryptionKey, 'HKDF', false, [ + 'deriveBits', + 'deriveKey', + ]); + } catch { + this.enqueueUpdate(generation, sequence, { failure: 'import-failed' }); + return; + } + + void importPromise.then( + (keyMaterial) => { + this.enqueueUpdate(generation, sequence, { + keyMaterial, + rtcBackendIdentity, + encryptionKeyIndex, + }); + }, + () => { + this.enqueueUpdate(generation, sequence, { failure: 'import-failed' }); + } + ); + }; + + private enqueueUpdate(generation: number, sequence: number, result: KeyImportResult): void { + if (generation !== this.attachmentGeneration) return; + this.pendingUpdates.set(sequence, result); + + while (this.pendingUpdates.has(this.nextUpdateSequence)) { + const update = this.pendingUpdates.get(this.nextUpdateSequence); + this.pendingUpdates.delete(this.nextUpdateSequence); + this.nextUpdateSequence += 1; + if (!update) continue; + + if ('failure' in update) { + this.updateState({ lastImportFailure: update.failure }); + continue; + } + + const lastParticipantKeyIndex = this.acceptedKeyIndices.get(update.rtcBackendIdentity) ?? -1; + if (update.encryptionKeyIndex < lastParticipantKeyIndex) continue; + + try { + this.onSetEncryptionKey( + update.keyMaterial, + update.rtcBackendIdentity, + update.encryptionKeyIndex + ); + } catch { + this.updateState({ lastImportFailure: 'import-failed' }); + continue; + } + + this.acceptedKeyIndices.set( + update.rtcBackendIdentity, + Math.max( + this.acceptedKeyIndices.get(update.rtcBackendIdentity) ?? -1, + update.encryptionKeyIndex + ) + ); + + this.updateState({ lastImportFailure: null }); + if (update.rtcBackendIdentity === this.localOutboundIdentity) { + this.updateState({ ready: true, keyIndex: update.encryptionKeyIndex }); + } + } + } + + private updateState(changes: Partial): void { + this.state = { ...this.state, ...changes }; + const state = this.getKeyState(); + this.stateListeners.forEach((listener) => { + try { + listener(state); + } catch { + // A state observer must not interrupt key updates. + } + }); + } +} diff --git a/src/app/features/call/matrixRtcCallLifecycle.ts b/src/app/features/call/matrixRtcCallLifecycle.ts new file mode 100644 index 000000000..8cb8b5075 --- /dev/null +++ b/src/app/features/call/matrixRtcCallLifecycle.ts @@ -0,0 +1,212 @@ +import type { AutoDiscoveryInfo } from '../../cs-api'; +import { + MatrixRTCSessionEvent, + type CallMembership, + type CallMembershipIdentityParts, + type JoinSessionConfig, + type MatrixClient, + type MatrixRTCSession, + type Room, +} from '$types/matrix-sdk'; +import { getPreferredLivekitTransport, provisionLivekitToken } from './livekitProvisioning'; +import type { LivekitProvisioningResult } from './livekitProvisioning'; +import type { LivekitTransportConfig } from '$types/matrix-sdk'; + +const membershipWaitTimeoutMs = 10_000; + +export type MatrixRTCJoinProvisionOptions = { + mx: MatrixClient; + room: Room; + session: MatrixRTCSession; + discovery?: Pick; + getPreferredTransport?: typeof getPreferredLivekitTransport; + provisionToken?: typeof provisionLivekitToken; + callIntent: JoinSessionConfig['callIntent']; + notificationType?: JoinSessionConfig['notificationType']; + manageMediaKeys?: boolean; + isCancelled?: () => boolean; + onStage?: (stage: 'joining-matrix' | 'provisioning') => void; + onMembershipWait?: (cancel: (() => void) | undefined) => void; + onJoinStarted?: () => void; + onMembershipError?: (error: unknown) => void; +}; + +export type MatrixRTCJoinProvisionResult = { + session: MatrixRTCSession; + transport: LivekitTransportConfig; + identity: CallMembershipIdentityParts; + ownMembership: CallMembership | undefined; + slotId: string; + provisioned: LivekitProvisioningResult; +}; + +type MembershipWait = { + promise: Promise; + cancel: () => void; +}; + +const waitForOwnMembership = ( + session: MatrixRTCSession, + userId: string, + deviceId: string, + onMembershipError?: (error: unknown) => void +): MembershipWait => { + let resolveWait!: () => void; + let rejectWait!: (reason?: unknown) => void; + let settled = false; + let timeout: ReturnType | undefined; + let membershipsListenerInstalled = false; + let membershipErrorListenerInstalled = false; + + const handleMembershipsChanged = ( + _oldMemberships: CallMembership[], + memberships: CallMembership[] + ): void => { + if ( + memberships.some( + (membership) => membership.userId === userId && membership.deviceId === deviceId + ) + ) { + settle(resolveWait); + } + }; + + const handleMembershipManagerError = (error: unknown): void => { + onMembershipError?.(error); + settle(() => rejectWait(new Error('MatrixRTC membership publication failed'))); + }; + + const removeListeners = (): void => { + if (membershipsListenerInstalled) { + try { + session.removeListener(MatrixRTCSessionEvent.MembershipsChanged, handleMembershipsChanged); + } catch {} + membershipsListenerInstalled = false; + } + if (membershipErrorListenerInstalled) { + try { + session.removeListener( + MatrixRTCSessionEvent.MembershipManagerError, + handleMembershipManagerError + ); + } catch {} + membershipErrorListenerInstalled = false; + } + }; + + const settle = (settlePromise: () => void): void => { + if (settled) return; + settled = true; + if (timeout !== undefined) clearTimeout(timeout); + removeListeners(); + settlePromise(); + }; + + const promise = new Promise((resolve, reject) => { + resolveWait = resolve; + rejectWait = reject; + }); + + try { + session.on(MatrixRTCSessionEvent.MembershipsChanged, handleMembershipsChanged); + membershipsListenerInstalled = true; + session.on(MatrixRTCSessionEvent.MembershipManagerError, handleMembershipManagerError); + membershipErrorListenerInstalled = true; + timeout = setTimeout( + () => settle(() => rejectWait(new Error('MatrixRTC membership publication timed out'))), + membershipWaitTimeoutMs + ); + } catch { + settle(() => rejectWait(new Error('MatrixRTC membership listener setup failed'))); + } + + return { + promise, + cancel: () => settle(() => rejectWait(new Error('MatrixRTC membership wait cancelled'))), + }; +}; + +export const joinAndProvisionMatrixRTC = async ({ + mx, + room, + session, + discovery, + getPreferredTransport = getPreferredLivekitTransport, + provisionToken = provisionLivekitToken, + callIntent, + notificationType, + manageMediaKeys = false, + isCancelled, + onStage, + onMembershipWait, + onJoinStarted, + onMembershipError, +}: MatrixRTCJoinProvisionOptions): Promise => { + const deviceId = mx.getDeviceId(); + if (!deviceId) throw new Error('MatrixRTC device unavailable'); + + const transport = await getPreferredTransport(mx, discovery); + if (!transport) throw new Error('No LiveKit transport available'); + + const userId = mx.getSafeUserId(); + const identity = { userId, deviceId, memberId: `${userId}:${deviceId}` }; + if (isCancelled?.()) throw new Error('MatrixRTC setup cancelled'); + const membershipWait = waitForOwnMembership( + session, + identity.userId, + identity.deviceId, + onMembershipError + ); + onMembershipWait?.(membershipWait.cancel); + onStage?.('joining-matrix'); + + try { + const joinConfig: JoinSessionConfig = { + callIntent, + ...(notificationType ? { notificationType } : {}), + ...(manageMediaKeys ? { manageMediaKeys: true } : {}), + }; + onJoinStarted?.(); + session.joinRTCSession(identity, [transport], undefined, joinConfig); + await membershipWait.promise; + } catch (error) { + membershipWait.cancel(); + throw error; + } finally { + onMembershipWait?.(undefined); + } + + if (isCancelled?.()) throw new Error('MatrixRTC setup cancelled'); + const slotId = session.slotId; + if (!slotId) throw new Error('MatrixRTC slot was not assigned'); + const ownMembership = session.memberships?.find( + (membership) => + membership.userId === identity.userId && membership.deviceId === identity.deviceId + ); + + onStage?.('provisioning'); + const provisioned = await provisionToken({ + mx, + roomId: room.roomId, + slotId, + deviceId, + serviceUrl: transport.livekit_service_url, + memberId: identity.memberId, + userId: identity.userId, + }); + if (isCancelled?.()) throw new Error('MatrixRTC setup cancelled'); + + return { session, transport, identity, ownMembership, slotId, provisioned }; +}; + +export const disconnectLivekitThenLeaveMatrixRTC = async ( + disconnect: () => Promise, + session: MatrixRTCSession +): Promise => { + try { + await disconnect(); + } catch {} + try { + await session.leaveRoomSession(5000); + } catch {} +}; diff --git a/src/app/features/call/nativeCallController.test.ts b/src/app/features/call/nativeCallController.test.ts index a61224e07..72b2a72cc 100644 --- a/src/app/features/call/nativeCallController.test.ts +++ b/src/app/features/call/nativeCallController.test.ts @@ -1,10 +1,18 @@ -import { describe, expect, it, vi } from 'vitest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; import { MatrixRTCSessionEvent, type CallMembership } from '$types/matrix-sdk'; import type { MatrixClient, MatrixRTCSession, Room } from '$types/matrix-sdk'; import { createNativeCallController } from './nativeCallController'; import type { NativeCallSession } from '$state/nativeCall'; import type { CallLifecycleError, CallState } from '$plugins/call/callLifecycle'; +const { debugError } = vi.hoisted(() => ({ + debugError: vi.fn<(...args: unknown[]) => void>(), +})); + +vi.mock('$utils/debugLogger', () => ({ + createDebugLogger: () => ({ error: debugError }), +})); + const room = { roomId: '!room:example.org' } as Room; const makeSession = (slotId = 'm.call#real-slot') => @@ -66,6 +74,10 @@ const waitForMembershipErrorListener = async (session: MatrixRTCSession): Promis }; describe('native call controller', () => { + beforeEach(() => { + debugError.mockClear(); + }); + it('joins with one identity, provisions the session slot, then connects', async () => { const session = makeSession(); const setSession = vi.fn<(session: NativeCallSession | undefined) => void>(); @@ -146,7 +158,9 @@ describe('native call controller', () => { }, getPreferredTransport: async () => transport, provisionToken: async () => { - throw new Error('openid-secret jwt-secret'); + throw Object.assign(new Error('openid-secret jwt-secret'), { + cause: { name: 'PluginError', message: 'request-body-secret' }, + }); }, }); @@ -177,11 +191,67 @@ describe('native call controller', () => { expect(setSession).toHaveBeenLastCalledWith( expect.objectContaining({ lifecycle: 'error', - error: 'Native call setup failed during token provisioning.', + error: 'Native call setup failed during authorizing.', }) ); expect(setSession.mock.calls.flat()).not.toContain('openid-secret'); expect(setSession.mock.calls.flat()).not.toContain('jwt-secret'); + expect(JSON.stringify(setSession.mock.calls)).not.toContain('openid-secret'); + expect(JSON.stringify(setSession.mock.calls)).not.toContain('jwt-secret'); + expect(debugError).toHaveBeenCalledWith( + 'call', + 'Native call setup failed', + expect.objectContaining({ + stage: 'authorizing', + errorName: 'Error', + errorMessage: 'redacted', + cause: { errorName: 'UnknownError', errorMessage: 'redacted' }, + }) + ); + expect(JSON.stringify(debugError.mock.calls)).not.toContain('openid-secret'); + expect(JSON.stringify(debugError.mock.calls)).not.toContain('jwt-secret'); + expect(JSON.stringify(debugError.mock.calls)).not.toContain('request-body-secret'); + }); + + it('logs allowlisted transport failures with a stage-safe session error', async () => { + const session = makeSession(); + const setSession = vi.fn<(session: NativeCallSession | undefined) => void>(); + const disconnect = vi + .fn<(request: { connectionId: string }) => Promise>() + .mockResolvedValue({ revision: 1, state: 'idle', connectionId: null }); + const controller = createNativeCallController({ + setSession, + connectionId: () => 'connection-id', + disconnect, + onState: async () => vi.fn<() => void>(), + onError: async () => vi.fn<() => void>(), + getPreferredTransport: async () => undefined, + }); + + await controller.start({ + mx: makeClient(session), + room, + elementCallActive: false, + dm: false, + video: false, + ongoing: false, + }); + + expect(setSession).toHaveBeenLastCalledWith( + expect.objectContaining({ + lifecycle: 'error', + error: 'Native call setup failed during no transport.', + }) + ); + expect(debugError).toHaveBeenCalledWith( + 'call', + 'Native call setup failed', + expect.objectContaining({ + stage: 'no transport', + errorName: 'Error', + errorMessage: 'No LiveKit transport available', + }) + ); }); it('cleans up when the membership manager fails before publication', async () => { @@ -235,10 +305,21 @@ describe('native call controller', () => { expect(setSession).toHaveBeenLastCalledWith( expect.objectContaining({ lifecycle: 'error', - error: 'Native call setup failed during MatrixRTC.', + error: 'Native call setup failed during joining the call.', }) ); expect(setSession.mock.calls.flat()).not.toContain('secret-membership-error'); + expect(JSON.stringify(setSession.mock.calls)).not.toContain('secret-membership-error'); + expect(debugError).toHaveBeenCalledWith( + 'call', + 'Native call setup failed', + expect.objectContaining({ + stage: 'joining the call', + errorName: 'Error', + errorMessage: 'redacted', + }) + ); + expect(JSON.stringify(debugError.mock.calls)).not.toContain('secret-membership-error'); }); it('cleans up when local membership publication times out', async () => { @@ -297,7 +378,7 @@ describe('native call controller', () => { expect(setSession).toHaveBeenLastCalledWith( expect.objectContaining({ lifecycle: 'error', - error: 'Native call setup failed during MatrixRTC.', + error: 'Native call setup failed during joining the call.', }) ); } finally { @@ -343,7 +424,7 @@ describe('native call controller', () => { expect(setSession).toHaveBeenLastCalledWith( expect.objectContaining({ lifecycle: 'error', - error: 'Native call setup failed during MatrixRTC.', + error: 'Native call setup failed during joining the call.', }) ); expect(setSession.mock.calls.flat()).not.toContain('secret-device'); @@ -480,4 +561,276 @@ describe('native call controller', () => { expect.objectContaining({ lifecycle: 'error', error: 'Native call ended.' }) ); }); + + it('redacts secret-bearing lifecycle errors and keeps the allowlisted code', async () => { + const session = makeSession(); + const setSession = vi.fn<(session: NativeCallSession | undefined) => void>(); + let errorHandler: ((error: CallLifecycleError) => void) | undefined; + const disconnect = vi.fn<(request: { connectionId: string }) => Promise>(); + disconnect.mockResolvedValue({ revision: 3, state: 'idle', connectionId: null }); + const controller = createNativeCallController({ + setSession, + connectionId: () => 'connection-id', + disconnect, + onState: async () => vi.fn<() => void>(), + onError: async (handler: (error: CallLifecycleError) => void) => { + errorHandler = handler; + return vi.fn<() => void>(); + }, + getPreferredTransport: async () => transport, + provisionToken: async () => ({ url: 'wss://livekit.example', jwt: 'jwt' }), + connect: async () => ({ revision: 2, state: 'connected', connectionId: 'connection-id' }), + }); + + const startPromise = controller.start({ + mx: makeClient(session), + room, + elementCallActive: false, + dm: false, + video: false, + ongoing: false, + }); + await waitForMembershipListener(session); + emitOwnMembership(session); + await startPromise; + errorHandler?.({ + revision: 3, + code: 'connect_failed', + message: 'wss://sfu-secret.example jwt-secret-token', + connectionId: 'connection-id', + }); + + await vi.waitFor(() => + expect(disconnect).toHaveBeenCalledWith({ connectionId: 'connection-id' }) + ); + expect(setSession).toHaveBeenLastCalledWith( + expect.objectContaining({ lifecycle: 'error', error: 'Native call setup failed.' }) + ); + expect(debugError).toHaveBeenCalledWith( + 'call', + 'Native call connection failed', + expect.objectContaining({ + stage: 'connecting', + errorName: 'UnknownError', + errorMessage: 'redacted', + code: 'connect_failed', + }) + ); + expect(JSON.stringify(debugError.mock.calls)).not.toContain('sfu-secret'); + expect(JSON.stringify(debugError.mock.calls)).not.toContain('jwt-secret-token'); + expect(JSON.stringify(debugError.mock.calls)).not.toContain('connection-id'); + expect(JSON.stringify(setSession.mock.calls)).not.toContain('sfu-secret'); + expect(JSON.stringify(setSession.mock.calls)).not.toContain('jwt-secret-token'); + }); + + it('redacts unknown lifecycle error codes instead of logging them', async () => { + const session = makeSession(); + const setSession = vi.fn<(session: NativeCallSession | undefined) => void>(); + let errorHandler: ((error: CallLifecycleError) => void) | undefined; + const disconnect = vi.fn<(request: { connectionId: string }) => Promise>(); + disconnect.mockResolvedValue({ revision: 3, state: 'idle', connectionId: null }); + const controller = createNativeCallController({ + setSession, + connectionId: () => 'connection-id', + disconnect, + onState: async () => vi.fn<() => void>(), + onError: async (handler: (error: CallLifecycleError) => void) => { + errorHandler = handler; + return vi.fn<() => void>(); + }, + getPreferredTransport: async () => transport, + provisionToken: async () => ({ url: 'wss://livekit.example', jwt: 'jwt' }), + connect: async () => ({ revision: 2, state: 'connected', connectionId: 'connection-id' }), + }); + + const startPromise = controller.start({ + mx: makeClient(session), + room, + elementCallActive: false, + dm: false, + video: false, + ongoing: false, + }); + await waitForMembershipListener(session); + emitOwnMembership(session); + await startPromise; + errorHandler?.({ + revision: 3, + code: 'secret-lifecycle-code', + message: 'payload-secret', + connectionId: 'connection-id', + }); + + await vi.waitFor(() => + expect(disconnect).toHaveBeenCalledWith({ connectionId: 'connection-id' }) + ); + expect(debugError).toHaveBeenCalledWith( + 'call', + 'Native call connection failed', + expect.objectContaining({ + stage: 'connecting', + errorName: 'UnknownError', + errorMessage: 'redacted', + code: 'unknown', + }) + ); + expect(JSON.stringify(debugError.mock.calls)).not.toContain('secret-lifecycle-code'); + expect(JSON.stringify(debugError.mock.calls)).not.toContain('payload-secret'); + }); + + it.each([ + ['audio_failed', 'native audio failed'], + ['camera_failed', 'native camera failed'], + ['screen_share_failed', 'native screen share failed'], + ['video_failed', 'native video failed'], + ['media_unsupported', 'media kind is not supported on this platform'], + ])('keeps the call alive when a media control fails with %s', async (code, message) => { + const session = makeSession(); + const setSession = vi.fn<(session: NativeCallSession | undefined) => void>(); + let errorHandler: ((error: CallLifecycleError) => void) | undefined; + const disconnect = vi.fn<(request: { connectionId: string }) => Promise>(); + disconnect.mockResolvedValue({ revision: 3, state: 'idle', connectionId: null }); + const controller = createNativeCallController({ + setSession, + connectionId: () => 'connection-id', + disconnect, + onState: async () => vi.fn<() => void>(), + onError: async (handler: (error: CallLifecycleError) => void) => { + errorHandler = handler; + return vi.fn<() => void>(); + }, + getPreferredTransport: async () => transport, + provisionToken: async () => ({ url: 'wss://livekit.example', jwt: 'jwt' }), + connect: async () => ({ revision: 2, state: 'connected', connectionId: 'connection-id' }), + }); + + const startPromise = controller.start({ + mx: makeClient(session), + room, + elementCallActive: false, + dm: false, + video: false, + ongoing: false, + }); + await waitForMembershipListener(session); + emitOwnMembership(session); + await startPromise; + errorHandler?.({ revision: 3, code, message, connectionId: 'connection-id' }); + + expect(disconnect).not.toHaveBeenCalled(); + expect(session.leaveRoomSession).not.toHaveBeenCalled(); + expect(setSession).toHaveBeenLastCalledWith( + expect.objectContaining({ lifecycle: 'connected' }) + ); + expect(debugError).toHaveBeenCalledWith( + 'call', + 'Native call connection failed', + expect.objectContaining({ stage: 'media control', code, errorMessage: message }) + ); + }); + + it('redacts raw media failure details without hanging up the call', async () => { + const session = makeSession(); + const setSession = vi.fn<(session: NativeCallSession | undefined) => void>(); + let errorHandler: ((error: CallLifecycleError) => void) | undefined; + const disconnect = vi.fn<(request: { connectionId: string }) => Promise>(); + disconnect.mockResolvedValue({ revision: 3, state: 'idle', connectionId: null }); + const controller = createNativeCallController({ + setSession, + connectionId: () => 'connection-id', + disconnect, + onState: async () => vi.fn<() => void>(), + onError: async (handler: (error: CallLifecycleError) => void) => { + errorHandler = handler; + return vi.fn<() => void>(); + }, + getPreferredTransport: async () => transport, + provisionToken: async () => ({ url: 'wss://livekit.example', jwt: 'jwt' }), + connect: async () => ({ revision: 2, state: 'connected', connectionId: 'connection-id' }), + }); + + const startPromise = controller.start({ + mx: makeClient(session), + room, + elementCallActive: false, + dm: false, + video: false, + ongoing: false, + }); + await waitForMembershipListener(session); + emitOwnMembership(session); + await startPromise; + errorHandler?.({ + revision: 3, + code: 'camera_failed', + message: 'https://sfu-secret.example/track?access_token=jwt-secret-token', + connectionId: 'connection-id', + }); + + expect(disconnect).not.toHaveBeenCalled(); + expect(session.leaveRoomSession).not.toHaveBeenCalled(); + expect(debugError).toHaveBeenCalledWith( + 'call', + 'Native call connection failed', + expect.objectContaining({ + stage: 'media control', + code: 'camera_failed', + errorMessage: 'redacted', + }) + ); + expect(JSON.stringify(debugError.mock.calls)).not.toContain('sfu-secret'); + expect(JSON.stringify(debugError.mock.calls)).not.toContain('jwt-secret-token'); + }); + + it('ignores lifecycle errors for stale or anonymous connections', async () => { + const session = makeSession(); + const setSession = vi.fn<(session: NativeCallSession | undefined) => void>(); + let errorHandler: ((error: CallLifecycleError) => void) | undefined; + const disconnect = vi.fn<(request: { connectionId: string }) => Promise>(); + disconnect.mockResolvedValue({ revision: 3, state: 'idle', connectionId: null }); + const controller = createNativeCallController({ + setSession, + connectionId: () => 'connection-id', + disconnect, + onState: async () => vi.fn<() => void>(), + onError: async (handler: (error: CallLifecycleError) => void) => { + errorHandler = handler; + return vi.fn<() => void>(); + }, + getPreferredTransport: async () => transport, + provisionToken: async () => ({ url: 'wss://livekit.example', jwt: 'jwt' }), + connect: async () => ({ revision: 2, state: 'connected', connectionId: 'connection-id' }), + }); + + const startPromise = controller.start({ + mx: makeClient(session), + room, + elementCallActive: false, + dm: false, + video: false, + ongoing: false, + }); + await waitForMembershipListener(session); + emitOwnMembership(session); + await startPromise; + debugError.mockClear(); + setSession.mockClear(); + + errorHandler?.({ + revision: 3, + code: 'stale_connection', + message: 'stale-secret', + connectionId: 'other-connection', + }); + errorHandler?.({ + revision: 4, + code: 'connect_failed', + message: 'null-secret', + connectionId: null, + }); + + expect(debugError).not.toHaveBeenCalled(); + expect(disconnect).not.toHaveBeenCalled(); + expect(setSession).not.toHaveBeenCalled(); + }); }); diff --git a/src/app/features/call/nativeCallController.ts b/src/app/features/call/nativeCallController.ts index 69c5e4e90..7bb4c9b83 100644 --- a/src/app/features/call/nativeCallController.ts +++ b/src/app/features/call/nativeCallController.ts @@ -1,12 +1,5 @@ import type { AutoDiscoveryInfo } from '../../cs-api'; -import { - MatrixRTCSessionEvent, - type CallMembership, - type JoinSessionConfig, - type MatrixClient, - type MatrixRTCSession, - type Room, -} from '$types/matrix-sdk'; +import { type MatrixClient, type MatrixRTCSession, type Room } from '$types/matrix-sdk'; import { connect, disconnect, @@ -18,6 +11,11 @@ import { } from '$plugins/call/callLifecycle'; import { getPreferredLivekitTransport, provisionLivekitToken } from './livekitProvisioning'; import type { NativeCallLifecycle, NativeCallSession } from '$state/nativeCall'; +import { createDebugLogger } from '$utils/debugLogger'; +import { + joinAndProvisionMatrixRTC, + disconnectLivekitThenLeaveMatrixRTC, +} from './matrixRtcCallLifecycle'; type SetNativeCall = (session: NativeCallSession | undefined) => void; @@ -30,6 +28,7 @@ type NativeCallControllerDependencies = { getPreferredTransport: typeof getPreferredLivekitTransport; provisionToken: typeof provisionLivekitToken; connectionId?: () => string; + onCleanup?: () => void; }; type NativeCallStartOptions = { @@ -54,97 +53,130 @@ type NativeCallRecord = { cleanupPromise?: Promise; }; +const debugLog = createDebugLogger('nativeCallController'); + const errorMessage = 'Native call setup failed.'; const endedMessage = 'Native call ended.'; -type SetupStage = 'MatrixRTC' | 'LiveKit transport' | 'token provisioning' | 'LiveKit connection'; +type SetupStage = + | 'joining the call' + | 'no transport' + | 'authorizing' + | 'connecting' + | 'media control'; -const membershipWaitTimeoutMs = 10_000; +const safeErrorNames = new Set([ + 'AggregateError', + 'AbortError', + 'DataError', + 'DOMException', + 'Error', + 'EvalError', + 'InvalidStateError', + 'NetworkError', + 'NotAllowedError', + 'NotSupportedError', + 'OperationError', + 'QuotaExceededError', + 'RangeError', + 'ReferenceError', + 'SecurityError', + 'SyntaxError', + 'TimeoutError', + 'TypeError', + 'URIError', + 'UnknownError', +]); -type MembershipWait = { - promise: Promise; - cancel: () => void; +const safeErrorMessages = new Set([ + 'MatrixRTC device unavailable', + 'MatrixRTC membership listener setup failed', + 'MatrixRTC membership publication failed', + 'MatrixRTC membership wait cancelled', + 'media kind is not supported on this platform', + 'native audio failed', + 'native camera failed', + 'native screen share failed', + 'native video failed', + 'MatrixRTC slot was not assigned', + 'No LiveKit transport available', +]); + +const safeLifecycleErrorCodes = new Set([ + 'actor_unavailable', + 'audio_failed', + 'busy', + 'camera_failed', + 'close_failed', + 'connect_failed', + 'media_unsupported', + 'screen_share_failed', + 'stale_connection', + 'video_failed', +]); + +const mediaControlErrorCodes = new Set([ + 'audio_failed', + 'camera_failed', + 'media_unsupported', + 'screen_share_failed', + 'video_failed', +]); + +const isMediaControlError = (error: CallLifecycleError): boolean => + typeof error.code === 'string' && mediaControlErrorCodes.has(error.code); + +const isRecord = (value: unknown): value is Record => + typeof value === 'object' && value !== null; + +const getErrorProperty = (error: unknown, property: 'name' | 'message' | 'cause'): unknown => { + if (error instanceof Error) return error[property]; + return isRecord(error) ? error[property] : undefined; }; -const waitForOwnMembership = ( - session: MatrixRTCSession, - userId: string, - deviceId: string -): MembershipWait => { - let resolveWait!: () => void; - let rejectWait!: (reason?: unknown) => void; - let settled = false; - let timeout: ReturnType | undefined; - let membershipsListenerInstalled = false; - let membershipErrorListenerInstalled = false; +const safeErrorName = (error: unknown): string => { + const name = getErrorProperty(error, 'name'); + return typeof name === 'string' && safeErrorNames.has(name) ? name : 'UnknownError'; +}; - const removeListeners = (): void => { - if (membershipsListenerInstalled) { - try { - session.removeListener(MatrixRTCSessionEvent.MembershipsChanged, handleMembershipsChanged); - } catch {} - membershipsListenerInstalled = false; - } - if (membershipErrorListenerInstalled) { - try { - session.removeListener( - MatrixRTCSessionEvent.MembershipManagerError, - handleMembershipManagerError - ); - } catch {} - membershipErrorListenerInstalled = false; - } - }; - - const settle = (settlePromise: () => void): void => { - if (settled) return; - settled = true; - if (timeout !== undefined) clearTimeout(timeout); - removeListeners(); - settlePromise(); - }; - - const handleMembershipsChanged = ( - _oldMemberships: CallMembership[], - memberships: CallMembership[] - ): void => { - if ( - memberships.some( - (membership) => membership.userId === userId && membership.deviceId === deviceId - ) - ) { - settle(resolveWait); - } - }; - - const handleMembershipManagerError = (): void => { - settle(() => rejectWait(new Error('MatrixRTC membership publication failed'))); - }; - - const promise = new Promise((resolve, reject) => { - resolveWait = resolve; - rejectWait = reject; - }); - - try { - session.on(MatrixRTCSessionEvent.MembershipsChanged, handleMembershipsChanged); - membershipsListenerInstalled = true; - session.on(MatrixRTCSessionEvent.MembershipManagerError, handleMembershipManagerError); - membershipErrorListenerInstalled = true; - timeout = setTimeout( - () => settle(() => rejectWait(new Error('MatrixRTC membership publication timed out'))), - membershipWaitTimeoutMs - ); - } catch { - settle(() => rejectWait(new Error('MatrixRTC membership listener setup failed'))); - } +const safeErrorMessage = (error: unknown): string => { + const message = getErrorProperty(error, 'message'); + return typeof message === 'string' && safeErrorMessages.has(message) ? message : 'redacted'; +}; +const setupFailureDiagnostics = (stage: SetupStage, error: unknown): Record => { + const cause = getErrorProperty(error, 'cause'); return { - promise, - cancel: () => settle(() => rejectWait(new Error('MatrixRTC membership wait cancelled'))), + stage, + errorName: safeErrorName(error), + errorMessage: safeErrorMessage(error), + ...(cause !== undefined + ? { + cause: { + errorName: safeErrorName(cause), + errorMessage: safeErrorMessage(cause), + }, + } + : {}), }; }; +const logSetupFailure = (stage: SetupStage, error: unknown): void => { + debugLog.error('call', 'Native call setup failed', setupFailureDiagnostics(stage, error)); +}; + +const safeLifecycleErrorCode = (code: unknown): string => + typeof code === 'string' && safeLifecycleErrorCodes.has(code) ? code : 'unknown'; + +const lifecycleFailureDiagnostics = (error: CallLifecycleError): Record => ({ + ...setupFailureDiagnostics(isMediaControlError(error) ? 'media control' : 'connecting', error), + code: safeLifecycleErrorCode(error.code), +}); + +const logLifecycleFailure = (error: CallLifecycleError): void => { + debugLog.error('call', 'Native call connection failed', lifecycleFailureDiagnostics(error)); +}; + const setupErrorMessage = (stage: SetupStage): string => `Native call setup failed during ${stage}.`; @@ -170,6 +202,7 @@ export const createNativeCallController = ( | 'getPreferredTransport' | 'provisionToken' | 'connectionId' + | 'onCleanup' > > ) => { @@ -182,6 +215,7 @@ export const createNativeCallController = ( getPreferredTransport: dependencies.getPreferredTransport ?? getPreferredLivekitTransport, provisionToken: dependencies.provisionToken ?? provisionLivekitToken, connectionId: dependencies.connectionId ?? (() => crypto.randomUUID()), + onCleanup: dependencies.onCleanup, }; let activeRecord: NativeCallRecord | undefined; let displayedRecord: NativeCallRecord | undefined; @@ -254,12 +288,10 @@ export const createNativeCallController = ( } record.cleanupPromise = (async () => { - try { - await deps.disconnect({ connectionId: record.connectionId }); - } catch {} - try { - await record.session.leaveRoomSession(5000); - } catch {} + await disconnectLivekitThenLeaveMatrixRTC( + () => deps.disconnect({ connectionId: record.connectionId }).then(() => undefined), + record.session + ); await Promise.allSettled([ record.stateUnlistenPromise?.then((unlisten) => unlisten()) ?? Promise.resolve(), record.errorUnlistenPromise?.then((unlisten) => unlisten()) ?? Promise.resolve(), @@ -273,6 +305,7 @@ export const createNativeCallController = ( deps.setSession(undefined); } catch {} } + deps.onCleanup?.(); })(); await record.cleanupPromise; }; @@ -295,6 +328,10 @@ export const createNativeCallController = ( const handleError = (record: NativeCallRecord, error: CallLifecycleError): void => { if (!isCurrent(record)) return; if (!isMatchingConnection(error.connectionId, record)) return; + logLifecycleFailure(error); + // Media toggle failures are recoverable: keep the call alive and only + // record a safe diagnostic. + if (isMediaControlError(error)) return; void cleanup(record, errorMessage, false); }; @@ -307,15 +344,17 @@ export const createNativeCallController = ( video, ongoing, }: NativeCallStartOptions) => { - if (activeRecord || elementCallActive) return; + if (activeRecord || elementCallActive) { + deps.onCleanup?.(); + return; + } let record: NativeCallRecord | undefined; - let stage: SetupStage = 'MatrixRTC'; + let stage: SetupStage = 'joining the call'; try { const deviceId = mx.getDeviceId(); if (!deviceId) { - setSetupError(room.roomId, setupErrorMessage(stage)); - return; + throw new Error('MatrixRTC device unavailable'); } const connectionId = deps.connectionId!(); @@ -335,53 +374,44 @@ export const createNativeCallController = ( record.errorUnlistenPromise = deps.onError((error) => handleError(currentRecord, error)); await Promise.all([record.stateUnlistenPromise, record.errorUnlistenPromise]); - stage = 'LiveKit transport'; - const transport = await deps.getPreferredTransport(mx, discovery); - if (!transport) throw new Error('No LiveKit transport available'); - - const userId = mx.getSafeUserId(); - const identity = { userId, deviceId, memberId: `${userId}:${deviceId}` }; - const joinConfig: JoinSessionConfig = { + stage = 'no transport'; + const joined = await joinAndProvisionMatrixRTC({ + mx, + room, + session, + discovery, + getPreferredTransport: deps.getPreferredTransport, + provisionToken: deps.provisionToken, callIntent: video ? 'video' : 'audio', ...(ongoing ? {} : { notificationType: dm ? 'ring' : 'notification' }), - }; - stage = 'MatrixRTC'; - const membershipWait = waitForOwnMembership(session, identity.userId, identity.deviceId); - record.cancelMembershipWait = membershipWait.cancel; - session.joinRTCSession(identity, [transport], undefined, joinConfig); - await membershipWait.promise; - record.cancelMembershipWait = undefined; - const slotId = session.slotId; - if (!slotId) throw new Error('MatrixRTC slot was not assigned'); - if (!isCurrent(record)) return; - - stage = 'token provisioning'; - const provisioned = await deps.provisionToken({ - mx, - roomId: room.roomId, - slotId, - deviceId, - serviceUrl: transport.livekit_service_url, - memberId: identity.memberId, - userId: identity.userId, + isCancelled: () => !isCurrent(currentRecord), + onStage: (joinStage) => { + stage = joinStage === 'joining-matrix' ? 'joining the call' : 'authorizing'; + }, + onMembershipWait: (cancel) => { + currentRecord.cancelMembershipWait = cancel; + }, + onMembershipError: (error) => logSetupFailure('joining the call', error), }); - if (!isCurrent(record)) return; + if (!isCurrent(currentRecord)) return; - stage = 'LiveKit connection'; + stage = 'connecting'; const state = await deps.connect({ connectionId, - serverUrl: provisioned.url, - participantToken: provisioned.jwt, + serverUrl: joined.provisioned.url, + participantToken: joined.provisioned.jwt, audio: true, video, screenShare: false, }); - handleState(record, state); - } catch { + handleState(currentRecord, state); + } catch (error) { + logSetupFailure(stage, error); if (record) { await cleanup(record, setupErrorMessage(stage), false); } else { setSetupError(room.roomId, setupErrorMessage(stage)); + deps.onCleanup?.(); } } }; diff --git a/src/app/features/call/useCallSignaling.ts b/src/app/features/call/useCallSignaling.ts index 878a6fb74..41bf833cb 100644 --- a/src/app/features/call/useCallSignaling.ts +++ b/src/app/features/call/useCallSignaling.ts @@ -12,6 +12,7 @@ import { type IncomingCall, } from '$state/callEmbed'; import { nativeCallAtom } from '$state/nativeCall'; +import { livekitJsCallAtom, selectActiveCallSession } from '$state/livekitJsCall'; import { settingsAtom } from '$state/settings'; import { parseIncomingRtcNotification, @@ -54,6 +55,7 @@ export function useIncomingCallSignaling() { const store = useStore(); const callEmbed = useAtomValue(callEmbedAtom); const nativeCall = useAtomValue(nativeCallAtom); + const livekitJsCall = useAtomValue(livekitJsCallAtom); const mDirects = useAtomValue(mDirectAtom); const settings = useAtomValue(settingsAtom); const incomingCall = useAtomValue(incomingCallAtom); @@ -92,6 +94,7 @@ export function useIncomingCallSignaling() { type SignalingHandlerRefs = { callEmbed: typeof callEmbed; nativeCall: typeof nativeCall; + livekitJsCall: typeof livekitJsCall; mDirects: typeof mDirects; outgoingRingbackAllowed: boolean; handleIncomingCall: (incoming: IncomingCall) => void; @@ -120,7 +123,7 @@ export function useIncomingCallSignaling() { hasCallBeenActiveRef.current = false; outgoingRingRoomIdRef.current = null; outgoingStartRef.current = null; - }, [callEmbed?.roomId, nativeCall?.connectionId]); + }, [callEmbed?.roomId, nativeCall?.connectionId, livekitJsCall?.roomId]); useEffect(() => { ringtoneManager.syncSources( @@ -150,7 +153,7 @@ export function useIncomingCallSignaling() { const handleOutgoingDecline = useCallback( (decline: OutgoingDeclineEvent) => { - const activeCall = callEmbed ?? nativeCall; + const activeCall = selectActiveCallSession(callEmbed, nativeCall, livekitJsCall); if (!activeCall || activeCall.roomId !== decline.roomId) { return; } @@ -211,7 +214,9 @@ export function useIncomingCallSignaling() { Sentry.metrics.count('sable.call.outgoing.declined', 1); stopOutgoingRing(); - const hangup = callEmbed?.hangup() ?? nativeCall?.hangup() ?? Promise.resolve(); + const hangup = + selectActiveCallSession(callEmbed, nativeCall, livekitJsCall)?.hangup() ?? + Promise.resolve(); void hangup .catch((error) => { debugLog.warn('call', 'Failed to hang up after outgoing decline', { @@ -227,7 +232,7 @@ export function useIncomingCallSignaling() { }, OUTGOING_DECLINE_EMBED_CLEAR_MS); }); }, - [callEmbed, mDirects, mx, nativeCall, setCallEmbed, stopOutgoingRing, store] + [callEmbed, livekitJsCall, mDirects, mx, nativeCall, setCallEmbed, stopOutgoingRing, store] ); const callAudioAllowed = canPlayCallAudio({ @@ -287,6 +292,7 @@ export function useIncomingCallSignaling() { signalingHandlerRefs.current = { callEmbed, nativeCall, + livekitJsCall, mDirects, outgoingRingbackAllowed, handleIncomingCall, @@ -425,11 +431,12 @@ export function useIncomingCallSignaling() { if (!senderId || !eventId) return; if (senderId === myUserId) { - if ( - type === RTC_NOTIFICATION_EVENT_TYPE && - (handlers().callEmbed?.roomId === room.roomId || - handlers().nativeCall?.roomId === room.roomId) - ) { + const activeCall = selectActiveCallSession( + handlers().callEmbed, + handlers().nativeCall, + handlers().livekitJsCall + ); + if (type === RTC_NOTIFICATION_EVENT_TYPE && activeCall?.roomId === room.roomId) { activeOutgoingNotificationIdRef.current = eventId; } return; @@ -444,7 +451,11 @@ export function useIncomingCallSignaling() { // Only inspect declines for the active outgoing call room. Cleartext declines are // cheap; encrypted events are decrypted only when they might be RTC declines. - const activeCall = handlers().callEmbed ?? handlers().nativeCall; + const activeCall = selectActiveCallSession( + handlers().callEmbed, + handlers().nativeCall, + handlers().livekitJsCall + ); if (!activeCall || activeCall.roomId !== room.roomId) { return; } @@ -504,8 +515,11 @@ export function useIncomingCallSignaling() { let outgoingRingTimeoutId: number | undefined; const evaluateOutgoingFallback = () => { - const activeCallRoomId = - handlers().callEmbed?.roomId ?? handlers().nativeCall?.roomId; + const activeCallRoomId = selectActiveCallSession( + handlers().callEmbed, + handlers().nativeCall, + handlers().livekitJsCall + )?.roomId; const stop = () => { handlers().stopOutgoingRing(); diff --git a/src/app/features/call/useCallStartCapabilities.ts b/src/app/features/call/useCallStartCapabilities.ts index bad149ca5..f3c3c7f7f 100644 --- a/src/app/features/call/useCallStartCapabilities.ts +++ b/src/app/features/call/useCallStartCapabilities.ts @@ -2,6 +2,8 @@ import { useCallback, useMemo } from 'react'; import type { Room } from '$types/matrix-sdk'; import { EventType } from '$types/matrix-sdk'; import { useCallEmbed } from '$hooks/useCallEmbed'; +import { useAtomValue } from 'jotai'; +import { livekitJsCallAtom } from '$state/livekitJsCall'; import { useLivekitSupport } from '$hooks/useLivekitSupport'; import { useMatrixClient } from '$hooks/useMatrixClient'; import { useStateEventCallback } from '$hooks/useStateEventCallback'; @@ -15,6 +17,7 @@ import { export const useCallStartCapabilities = (room: Room): CallStartCapabilities => { const mx = useMatrixClient(); const callEmbed = useCallEmbed(); + const livekitJsCall = useAtomValue(livekitJsCallAtom); const livekitSupported = useLivekitSupport(); const rtcSupported = webRTCSupported(); const myUserId = mx.getSafeUserId(); @@ -42,9 +45,17 @@ export const useCallStartCapabilities = (room: Room): CallStartCapabilities => { return evaluateCallStartCapabilities({ room, myUserId, - activeCallRoomId: callEmbed?.roomId, + activeCallRoomId: callEmbed?.roomId ?? livekitJsCall?.roomId, livekitSupported, rtcSupported, }); - }, [room, myUserId, callEmbed?.roomId, livekitSupported, rtcSupported, updateCount]); + }, [ + room, + myUserId, + callEmbed?.roomId, + livekitJsCall?.roomId, + livekitSupported, + rtcSupported, + updateCount, + ]); }; diff --git a/src/app/features/settings/experimental/Experimental.test.tsx b/src/app/features/settings/experimental/Experimental.test.tsx new file mode 100644 index 000000000..b15118900 --- /dev/null +++ b/src/app/features/settings/experimental/Experimental.test.tsx @@ -0,0 +1,164 @@ +import type { ReactNode } from 'react'; +import type * as SettingsModule from '$state/settings'; +import { fireEvent, render, screen } from '@testing-library/react'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { Experimental } from './Experimental'; + +const { + mockPlatformSupported, + mockSetLiveKitJsCallsEnabled, + mockSetLiveKitJsMediaTestEnabled, + mockUseSetting, +} = vi.hoisted(() => ({ + mockPlatformSupported: vi.fn<() => boolean>(), + mockSetLiveKitJsCallsEnabled: vi.fn<(value: boolean) => void>(), + mockSetLiveKitJsMediaTestEnabled: vi.fn<(value: boolean) => void>(), + mockUseSetting: + vi.fn<(_atom: unknown, key: string) => readonly [boolean, (value: boolean) => void]>(), +})); + +vi.mock('$state/settings', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, settingsAtom: {} }; +}); + +vi.mock('$state/hooks/settings', () => ({ + useSetting: mockUseSetting, +})); + +vi.mock('$features/call/nativeCallProbe', () => ({ + isNativeCallProbePlatformSupported: mockPlatformSupported, +})); + +vi.mock('$components/page', () => ({ + PageContent: ({ children }: { children: ReactNode }) =>
{children}
, + SettingsSectionPage: ({ children }: { children: ReactNode }) =>
{children}
, +})); + +vi.mock('$components/info-card', () => ({ + InfoCard: () =>
Experimental warning
, +})); + +vi.mock('$components/setting-tile', () => ({ + SettingToggle: ({ + title, + description, + focusId, + value, + onChange, + }: { + title: string; + description: ReactNode; + focusId: string; + value: boolean; + onChange: (value: boolean) => void; + }) => ( +
+ {title} + {description} + +
+ ), +})); + +vi.mock('./BandwithSavingEmojis', () => ({ BandwidthSavingEmojis: () => null })); +vi.mock('./MSC4268HistoryShare', () => ({ MSC4268HistoryShare: () => null })); +vi.mock('./MSC4274MediaGalleries', () => ({ MSC4274MediaGalleries: () => null })); + +vi.mock('folds', () => ({ + Box: ({ children }: { children: ReactNode }) =>
{children}
, + Scroll: ({ children }: { children: ReactNode }) =>
{children}
, + Text: ({ children }: { children: ReactNode }) => {children}, +})); + +beforeEach(() => { + mockPlatformSupported.mockReset(); + mockPlatformSupported.mockReturnValue(true); + mockSetLiveKitJsCallsEnabled.mockReset(); + mockSetLiveKitJsMediaTestEnabled.mockReset(); + mockUseSetting.mockImplementation((_atom: unknown, key: string) => { + if (key === 'livekitJsCallsEnabled') return [false, mockSetLiveKitJsCallsEnabled]; + if (key === 'livekitJsMediaTestEnabled') return [false, mockSetLiveKitJsMediaTestEnabled]; + return [false, vi.fn<() => void>()]; + }); +}); + +describe('Experimental LiveKit JS calls setting', () => { + it('shows the connection probe toggle on supported Tauri platforms', () => { + render( {}} />); + + expect(screen.getByText('LiveKit JS Calls')).toBeInTheDocument(); + expect(screen.getByText('Try the LiveKit JS connection probe')).toBeInTheDocument(); + expect( + screen.getByText( + 'Runs an experimental LiveKit JS connection probe. It does not publish media. Element Call remains the normal fallback.' + ) + ).toBeInTheDocument(); + }); + + it('hides the connection probe toggle outside supported Tauri platforms', () => { + mockPlatformSupported.mockReturnValue(false); + + render( {}} />); + + expect(screen.queryByText('LiveKit JS Calls')).not.toBeInTheDocument(); + expect(screen.queryByText('Try the LiveKit JS connection probe')).not.toBeInTheDocument(); + }); + + it('persists the opt-in through the settings hook', () => { + render( {}} />); + + fireEvent.click(screen.getByRole('button', { name: 'livekit-js-calls' })); + + expect(mockSetLiveKitJsCallsEnabled).toHaveBeenCalledWith(true); + }); + + it('shows the manual media test only when the connection probe is enabled', () => { + const { rerender } = render( {}} />); + + expect(screen.queryByText('Enable the manual LiveKit JS media test')).not.toBeInTheDocument(); + + mockUseSetting.mockImplementation((_atom: unknown, key: string) => { + if (key === 'livekitJsCallsEnabled') return [true, mockSetLiveKitJsCallsEnabled]; + if (key === 'livekitJsMediaTestEnabled') return [false, mockSetLiveKitJsMediaTestEnabled]; + return [false, vi.fn<() => void>()]; + }); + rerender( {}} />); + + expect(screen.getByText('Enable the manual LiveKit JS media test')).toBeInTheDocument(); + expect( + screen.getByText( + 'Manual local media test only. Encrypted media is required. There is no fallback or automatic call selection. This is not release-ready.' + ) + ).toBeInTheDocument(); + }); + + it('does not show the manual media test on unsupported platforms', () => { + mockPlatformSupported.mockReturnValue(false); + mockUseSetting.mockImplementation((_atom: unknown, key: string) => { + if (key === 'livekitJsCallsEnabled') return [true, mockSetLiveKitJsCallsEnabled]; + if (key === 'livekitJsMediaTestEnabled') return [false, mockSetLiveKitJsMediaTestEnabled]; + return [false, vi.fn<() => void>()]; + }); + + render( {}} />); + + expect(screen.queryByText('Enable the manual LiveKit JS media test')).not.toBeInTheDocument(); + }); + + it('persists the manual media test opt-in through the settings hook', () => { + mockUseSetting.mockImplementation((_atom: unknown, key: string) => { + if (key === 'livekitJsCallsEnabled') return [true, mockSetLiveKitJsCallsEnabled]; + if (key === 'livekitJsMediaTestEnabled') return [false, mockSetLiveKitJsMediaTestEnabled]; + return [false, vi.fn<() => void>()]; + }); + + render( {}} />); + + fireEvent.click(screen.getByRole('button', { name: 'livekit-js-media-test' })); + + expect(mockSetLiveKitJsMediaTestEnabled).toHaveBeenCalledWith(true); + }); +}); diff --git a/src/app/features/settings/experimental/Experimental.tsx b/src/app/features/settings/experimental/Experimental.tsx index 910272fe4..93eb06d01 100644 --- a/src/app/features/settings/experimental/Experimental.tsx +++ b/src/app/features/settings/experimental/Experimental.tsx @@ -52,6 +52,41 @@ function NativeCallsToggle() { ); } +function LiveKitJsCallsToggle() { + const [livekitJsCallsEnabled, setLivekitJsCallsEnabled] = useSetting( + settingsAtom, + 'livekitJsCallsEnabled' + ); + const [livekitJsMediaTestEnabled, setLivekitJsMediaTestEnabled] = useSetting( + settingsAtom, + 'livekitJsMediaTestEnabled' + ); + + if (!isNativeCallProbePlatformSupported()) return null; + + return ( + + LiveKit JS Calls + + {livekitJsCallsEnabled && ( + + )} + + ); +} + type ExperimentalProps = { requestBack?: () => void; requestClose: () => void; @@ -80,6 +115,7 @@ export function Experimental({ requestBack, requestClose }: Readonly + diff --git a/src/app/generated/tauri/commands.ts b/src/app/generated/tauri/commands.ts index 2f1add560..aa3af5f22 100644 --- a/src/app/generated/tauri/commands.ts +++ b/src/app/generated/tauri/commands.ts @@ -43,6 +43,14 @@ export async function exportDiagnostics(params: types.ExportDiagnosticsParams): return invoke('export_diagnostics', params); } +export async function getPlatformCallCapabilities(): Promise { + return invoke('getPlatformCallCapabilities'); +} + +export async function getPlatformCallState(): Promise { + return invoke('getPlatformCallState'); +} + export async function getDesktopRuntimeState(): Promise { return invoke('get_desktop_runtime_state'); } @@ -79,6 +87,10 @@ export async function saveDownload(params: types.SaveDownloadParams): Promise { + return invoke('set_media_enabled', params); +} + export async function setMediaEncryption(params: types.SetMediaEncryptionParams): Promise { return invoke('set_media_encryption', params); } @@ -119,6 +131,10 @@ export async function showSnapOverlay(): Promise { return invoke('show_snap_overlay'); } +export async function startPlatformCallLifecycle(params: types.StartPlatformCallLifecycleParams): Promise { + return invoke('startPlatformCallLifecycle', params); +} + export async function startCallForegroundService(): Promise { return invoke('start_call_foreground_service'); } @@ -127,6 +143,10 @@ export async function startWindowTrackingWithTarget(params: types.StartWindowTra return invoke('start_window_tracking_with_target', params); } +export async function stopPlatformCallLifecycle(params: types.StopPlatformCallLifecycleParams): Promise { + return invoke('stopPlatformCallLifecycle', params); +} + export async function stopCallForegroundService(): Promise { return invoke('stop_call_foreground_service'); } diff --git a/src/app/generated/tauri/types.ts b/src/app/generated/tauri/types.ts index 6fa575bb1..60fae4cdf 100644 --- a/src/app/generated/tauri/types.ts +++ b/src/app/generated/tauri/types.ts @@ -49,6 +49,9 @@ export interface LoopbackFetchResponse { body: number[]; } + +export type MediaKind = "Microphone" | "Camera" | "ScreenShare"; + export interface NativeUploadResponse { status: number; body: string; @@ -59,6 +62,12 @@ export interface ProgressPayload { total: number; } +export interface SetMediaEnabledRequest { + connectionId: string; + kind: MediaKind; + enabled: boolean; +} + export interface ShareBatch { batchId: string; items: ShareItem[]; @@ -71,6 +80,16 @@ export interface ShareItem { mime?: string | null; } +export interface StartPlatformCallLifecycleRequest { + sessionId: string; + microphone: boolean; + playback: boolean; +} + +export interface StopPlatformCallLifecycleRequest { + sessionId: string; +} + export interface WindowTarget { window_class?: string | null; exe_name?: string | null; @@ -131,6 +150,11 @@ export interface SaveDownloadParams { [key: string]: unknown; } +export interface SetMediaEnabledParams { + payload: SetMediaEnabledRequest; + [key: string]: unknown; +} + export interface SetMediaEncryptionParams { url: string; key: string; @@ -179,11 +203,21 @@ export interface ShareInboxReadParams { [key: string]: unknown; } +export interface StartPlatformCallLifecycleParams { + payload: StartPlatformCallLifecycleRequest; + [key: string]: unknown; +} + export interface StartWindowTrackingWithTargetParams { target: WindowTarget; [key: string]: unknown; } +export interface StopPlatformCallLifecycleParams { + payload: StopPlatformCallLifecycleRequest; + [key: string]: unknown; +} + export interface SyncDesktopSettingsParams { settings: DesktopSettings; [key: string]: unknown; diff --git a/src/app/hooks/useCallEmbed.ts b/src/app/hooks/useCallEmbed.ts index 81da3b6dd..316233035 100644 --- a/src/app/hooks/useCallEmbed.ts +++ b/src/app/hooks/useCallEmbed.ts @@ -1,5 +1,13 @@ import type { RefObject } from 'react'; -import { createContext, useCallback, useContext, useEffect, useMemo, useState } from 'react'; +import { + createContext, + useCallback, + useContext, + useEffect, + useMemo, + useRef, + useState, +} from 'react'; import type { MatrixClient, Room } from '$types/matrix-sdk'; import { useSetAtom } from 'jotai'; import * as Sentry from '@sentry/react'; @@ -22,6 +30,11 @@ import { useAutoDiscoveryInfo } from './useAutoDiscoveryInfo'; import { useStore } from 'jotai'; import { settingsAtom } from '$state/settings'; import { useSetting } from '$state/hooks/settings'; +import { livekitJsCallAtom, isLivekitJsCallActive } from '$state/livekitJsCall'; +import { acquireCallOwner, type CallOwnerLease } from '$state/callOwner'; +import { createLivekitJsController } from '$features/call/livekitJsController'; +import { isLivekitJsCallProbeEnabled } from '$features/call/livekitJsCallProbe'; +import { selectCallStartOwner } from '$features/call/callStartSelection'; const debugLog = createDebugLogger('useCallEmbed'); @@ -76,19 +89,89 @@ export const useCallStart = (dm = false) => { const callEmbedRef = useCallEmbedRef(); const store = useStore(); const [nativeCallsEnabled] = useSetting(settingsAtom, 'nativeCallsEnabled'); + const [livekitJsCallsEnabled] = useSetting(settingsAtom, 'livekitJsCallsEnabled'); + const [livekitJsMediaTestEnabled] = useSetting(settingsAtom, 'livekitJsMediaTestEnabled'); const discovery = useAutoDiscoveryInfo(); + const nativeOwnerLeaseRef = useRef(undefined); + const livekitJsRoomIdRef = useRef(undefined); + const livekitJsController = useMemo( + () => createLivekitJsController(undefined, { manualMediaTest: livekitJsMediaTestEnabled }), + [livekitJsMediaTestEnabled] + ); const nativeCallController = useMemo( () => createNativeCallController({ setSession: setNativeCall, + onCleanup: () => { + nativeOwnerLeaseRef.current?.release(); + nativeOwnerLeaseRef.current = undefined; + }, }), [setNativeCall] ); + useEffect(() => { + const unsubscribe = livekitJsController.subscribe((controllerState) => { + const roomId = livekitJsRoomIdRef.current; + if (!roomId) return; + if (controllerState.lifecycle === 'idle') { + store.set(livekitJsCallAtom, undefined); + return; + } + store.set(livekitJsCallAtom, { + roomId, + lifecycle: controllerState.lifecycle, + failure: controllerState.failure, + room: controllerState.lifecycle === 'active' ? controllerState.room : undefined, + media: controllerState.lifecycle === 'active' ? controllerState.media : undefined, + hangup: () => livekitJsController.disconnect(), + }); + }); + return () => { + unsubscribe(); + livekitJsRoomIdRef.current = undefined; + void livekitJsController.disconnect().finally(() => { + store.set(livekitJsCallAtom, undefined); + }); + }; + }, [livekitJsController, store]); + const startCall = useCallback( (room: Room, pref?: CallPreferences) => { - if (isNativeCallProbeEnabled(nativeCallsEnabled)) { - if (store.get(callEmbedAtom) || isNativeCallActive(store.get(nativeCallAtom))) return; + const startOwner = selectCallStartOwner({ + livekitJsProbeEnabled: isLivekitJsCallProbeEnabled(livekitJsCallsEnabled), + nativeProbeEnabled: isNativeCallProbeEnabled(nativeCallsEnabled), + }); + if (startOwner === 'livekit-js') { + if ( + store.get(callEmbedAtom) || + isNativeCallActive(store.get(nativeCallAtom)) || + isLivekitJsCallActive(store.get(livekitJsCallAtom)) + ) + return; + livekitJsRoomIdRef.current = room.roomId; + void livekitJsController + .connect({ + mx, + room, + discovery, + callIntent: pref?.video ? 'video' : 'audio', + dm, + ongoing: mx.matrixRTC.getRoomSession(room).memberships.length > 0, + }) + .catch(() => undefined); + return; + } + if (startOwner === 'native') { + if ( + store.get(callEmbedAtom) || + isNativeCallActive(store.get(nativeCallAtom)) || + isLivekitJsCallActive(store.get(livekitJsCallAtom)) + ) + return; + const ownerLease = acquireCallOwner('native', room.roomId); + if (!ownerLease) return; + nativeOwnerLeaseRef.current = ownerLease; const ongoing = mx.matrixRTC.getRoomSession(room).memberships.length > 0; void nativeCallController .start({ @@ -103,8 +186,11 @@ export const useCallStart = (dm = false) => { .catch(() => undefined); return; } + const ownerLease = acquireCallOwner('element', room.roomId); + if (!ownerLease) return; const container = callEmbedRef.current; if (!container) { + ownerLease.release(); debugLog.error('call', 'Failed to start call — no embed container', { roomId: room.roomId, }); @@ -130,6 +216,7 @@ export const useCallStart = (dm = false) => { ); setCallEmbed(callEmbed); } catch (err) { + ownerLease.release(); debugLog.error('call', 'Call embed creation failed', { roomId: room.roomId, error: err instanceof Error ? err.message : String(err), @@ -152,6 +239,8 @@ export const useCallStart = (dm = false) => { discovery, nativeCallController, nativeCallsEnabled, + livekitJsCallsEnabled, + livekitJsController, ] ); diff --git a/src/app/plugins/call/callLifecycle.test.ts b/src/app/plugins/call/callLifecycle.test.ts index 8a213bd50..78dfdb172 100644 --- a/src/app/plugins/call/callLifecycle.test.ts +++ b/src/app/plugins/call/callLifecycle.test.ts @@ -3,6 +3,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; const pluginConnect = vi.hoisted(() => vi.fn<(request: unknown) => Promise>()); const pluginDisconnect = vi.hoisted(() => vi.fn<(request: unknown) => Promise>()); const pluginGetState = vi.hoisted(() => vi.fn<() => Promise>()); +const invoke = vi.hoisted(() => vi.fn<(...args: unknown[]) => Promise>()); const listen = vi.hoisted(() => vi.fn<(...args: unknown[]) => Promise>()); vi.mock('tauri-plugin-call-lifecycle-api', () => ({ @@ -10,6 +11,7 @@ vi.mock('tauri-plugin-call-lifecycle-api', () => ({ disconnect: pluginDisconnect, getState: pluginGetState, })); +vi.mock('@tauri-apps/api/core', () => ({ invoke })); vi.mock('@tauri-apps/api/event', () => ({ listen })); import { @@ -18,6 +20,7 @@ import { getState, onError, onState, + setMediaEnabled, type CallLifecycleError, type CallState, } from './callLifecycle'; @@ -27,10 +30,28 @@ describe('call lifecycle wrapper', () => { pluginConnect.mockReset(); pluginDisconnect.mockReset(); pluginGetState.mockReset(); + invoke.mockReset(); listen.mockReset(); listen.mockResolvedValue(vi.fn()); }); + it('uses the namespaced media command and native snake-case payload', async () => { + const state = { + revision: 1, + state: 'connected', + connectionId: 'connection-id', + } satisfies CallState; + invoke.mockResolvedValue(state); + + await expect( + setMediaEnabled({ connectionId: 'connection-id', kind: 'screen_share', enabled: true }) + ).resolves.toEqual(state); + + expect(invoke).toHaveBeenCalledWith('plugin:call-lifecycle|set_media_enabled', { + payload: { connectionId: 'connection-id', kind: 'screen_share', enabled: true }, + }); + }); + it('delegates lifecycle commands to the guest API', async () => { const state = { revision: 1, diff --git a/src/app/plugins/call/callLifecycle.ts b/src/app/plugins/call/callLifecycle.ts index dc2d20161..410cdcab7 100644 --- a/src/app/plugins/call/callLifecycle.ts +++ b/src/app/plugins/call/callLifecycle.ts @@ -8,6 +8,7 @@ import { type ConnectionState, type DisconnectRequest, } from 'tauri-plugin-call-lifecycle-api'; +import { invoke } from '@tauri-apps/api/core'; import { listen, type UnlistenFn } from '@tauri-apps/api/event'; const STATE_EVENT = 'plugin:call-lifecycle://state'; @@ -15,10 +16,22 @@ const ERROR_EVENT = 'plugin:call-lifecycle://error'; export type { CallLifecycleError, CallState, ConnectRequest, ConnectionState, DisconnectRequest }; +export type MediaKind = 'microphone' | 'camera' | 'screen_share'; + +export type SetMediaEnabledRequest = { + connectionId: string; + kind: MediaKind; + enabled: boolean; +}; + export const connect = pluginConnect; export const disconnect = pluginDisconnect; export const getState = pluginGetState; +export function setMediaEnabled(request: SetMediaEnabledRequest): Promise { + return invoke('plugin:call-lifecycle|set_media_enabled', { payload: request }); +} + export function onState(handler: (state: CallState) => void): Promise { return listen(STATE_EVENT, (event) => handler(event.payload)); } diff --git a/src/app/plugins/call/platformCallLifecycle.test.ts b/src/app/plugins/call/platformCallLifecycle.test.ts new file mode 100644 index 000000000..b4c743f03 --- /dev/null +++ b/src/app/plugins/call/platformCallLifecycle.test.ts @@ -0,0 +1,99 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const pluginGetCapabilities = vi.hoisted(() => vi.fn<() => Promise>()); +const pluginGetState = vi.hoisted(() => vi.fn<() => Promise>()); +const pluginListenEvent = vi.hoisted(() => + vi.fn<(handler: (event: unknown) => void) => Promise>() +); +const pluginStart = vi.hoisted(() => vi.fn<(request: unknown) => Promise>()); +const pluginStop = vi.hoisted(() => vi.fn<(request: unknown) => Promise>()); + +vi.mock('tauri-plugin-call-lifecycle-api', () => ({ + getPlatformCallCapabilities: pluginGetCapabilities, + getPlatformCallState: pluginGetState, + listenPlatformCallEvent: pluginListenEvent, + startPlatformCallLifecycle: pluginStart, + stopPlatformCallLifecycle: pluginStop, +})); + +import { + getPlatformCapabilities, + getPlatformState, + onPlatformCallEvent, + startPlatformLifecycle, + stopPlatformLifecycle, + type PlatformCallEvent, + type PlatformCallState, +} from './platformCallLifecycle'; + +describe('platform call lifecycle wrapper', () => { + beforeEach(() => { + pluginGetCapabilities.mockReset(); + pluginGetState.mockReset(); + pluginListenEvent.mockReset(); + pluginStart.mockReset(); + pluginStop.mockReset(); + pluginListenEvent.mockResolvedValue(vi.fn()); + }); + + it('delegates capabilities/start/stop/state commands with opaque session payloads', async () => { + const capabilities = { supported: true, microphone: true, playback: true }; + const state = { + revision: 2, + state: 'active', + sessionId: 'opaque-session', + microphone: true, + playback: true, + capabilities, + } satisfies PlatformCallState; + pluginGetCapabilities.mockResolvedValue(capabilities); + pluginStart.mockResolvedValue(state); + pluginStop.mockResolvedValue(state); + pluginGetState.mockResolvedValue(state); + + await expect(getPlatformCapabilities()).resolves.toEqual(capabilities); + await expect( + startPlatformLifecycle({ sessionId: 'opaque-session', microphone: true, playback: true }) + ).resolves.toEqual(state); + await expect(stopPlatformLifecycle({ sessionId: 'opaque-session' })).resolves.toEqual(state); + await expect(getPlatformState()).resolves.toEqual(state); + + expect(pluginStart).toHaveBeenCalledWith({ + sessionId: 'opaque-session', + microphone: true, + playback: true, + }); + expect(pluginStop).toHaveBeenCalledWith({ sessionId: 'opaque-session' }); + }); + + it('forwards bounded platform events and drops malformed payloads', async () => { + const handler = vi.fn<(event: PlatformCallEvent) => void>(); + await onPlatformCallEvent(handler); + const listener = pluginListenEvent.mock.calls[0]?.[0] as (event: unknown) => void; + + const routeEvent: PlatformCallEvent = { + revision: 4, + sessionId: 'opaque-session', + type: 'route_changed', + route: 'speaker', + }; + listener(routeEvent); + listener({ revision: 5, sessionId: 'opaque-session', type: 'media_reset' }); + listener({ + revision: 6, + sessionId: 'opaque-session', + type: 'failed', + code: 'audio_unavailable', + }); + + // Malformed or out-of-contract payloads never reach the handler. + listener({ revision: 7, sessionId: 'opaque-session', type: 'route_changed', route: 'eth0' }); + listener({ revision: 8, sessionId: 'opaque-session', type: 'publish_track' }); + listener({ revision: '9', sessionId: 'opaque-session', type: 'media_reset' }); + listener({ revision: 10, type: 'media_reset' }); + listener({ revision: 11, sessionId: 'opaque-session', type: 'failed', code: 'raw-native' }); + + expect(handler).toHaveBeenCalledTimes(3); + expect(handler).toHaveBeenNthCalledWith(1, routeEvent); + }); +}); diff --git a/src/app/plugins/call/platformCallLifecycle.ts b/src/app/plugins/call/platformCallLifecycle.ts new file mode 100644 index 000000000..3b73c3bab --- /dev/null +++ b/src/app/plugins/call/platformCallLifecycle.ts @@ -0,0 +1,81 @@ +import { + getPlatformCallCapabilities as pluginGetCapabilities, + getPlatformCallState as pluginGetState, + listenPlatformCallEvent as pluginListenEvent, + startPlatformCallLifecycle as pluginStart, + stopPlatformCallLifecycle as pluginStop, + type PlatformCallCapabilities, + type PlatformCallEvent, + type PlatformCallFailureCode, + type PlatformCallRoute, + type PlatformCallState, + type StartPlatformCallLifecycleRequest, + type StopPlatformCallLifecycleRequest, +} from 'tauri-plugin-call-lifecycle-api'; +import type { UnlistenFn } from '@tauri-apps/api/event'; + +export type { + PlatformCallCapabilities, + PlatformCallEvent, + PlatformCallFailureCode, + PlatformCallRoute, + PlatformCallState, + StartPlatformCallLifecycleRequest, + StopPlatformCallLifecycleRequest, +}; + +const EVENT_TYPES = new Set([ + 'focus_changed', + 'route_changed', + 'interrupted', + 'media_reset', + 'failed', +]); + +const ROUTES = new Set(['earpiece', 'speaker', 'wired', 'bluetooth', 'unknown']); + +const FAILURE_CODES = new Set([ + 'unsupported', + 'permission_denied', + 'audio_unavailable', + 'start_failed', + 'stop_failed', + 'stale_session', + 'busy', +]); + +const isBoundedEvent = (payload: unknown): payload is PlatformCallEvent => { + if (!payload || typeof payload !== 'object') return false; + const event = payload as Record; + if (typeof event.revision !== 'number' || typeof event.sessionId !== 'string') return false; + if (typeof event.type !== 'string' || !EVENT_TYPES.has(event.type)) return false; + switch (event.type) { + case 'focus_changed': + return typeof event.focused === 'boolean'; + case 'route_changed': + return typeof event.route === 'string' && ROUTES.has(event.route as PlatformCallRoute); + case 'interrupted': + return event.state === 'began' || event.state === 'ended'; + case 'media_reset': + return true; + case 'failed': + return ( + typeof event.code === 'string' && FAILURE_CODES.has(event.code as PlatformCallFailureCode) + ); + default: + return false; + } +}; + +export const getPlatformCapabilities = pluginGetCapabilities; +export const startPlatformLifecycle = pluginStart; +export const stopPlatformLifecycle = pluginStop; +export const getPlatformState = pluginGetState; + +export function onPlatformCallEvent( + handler: (event: PlatformCallEvent) => void +): Promise { + return pluginListenEvent((payload) => { + if (isBoundedEvent(payload)) handler(payload); + }); +} diff --git a/src/app/state/callEmbed.ts b/src/app/state/callEmbed.ts index 3dcc6c745..6a8112454 100644 --- a/src/app/state/callEmbed.ts +++ b/src/app/state/callEmbed.ts @@ -2,6 +2,7 @@ import { atom } from 'jotai'; import * as Sentry from '@sentry/react'; import type { CallEmbed } from '../plugins/call'; import type { CallEmbedStartError } from '$plugins/call/callEmbedError'; +import { releaseCallOwner } from './callOwner'; const baseCallEmbedAtom = atom(undefined); const baseCallEmbedStartErrorAtom = atom(null); @@ -25,6 +26,7 @@ export const callEmbedAtom = atom resetCallOwnerForTests()); + +describe('call owner guard', () => { + it('allows one owner and rejects competing owners until release', () => { + const native = acquireCallOwner('native', '!room:example.org'); + + expect(native).toBeDefined(); + expect(acquireCallOwner('livekit-js', '!room:example.org')).toBeUndefined(); + expect(getActiveCallOwner()).toMatchObject({ kind: 'native' }); + + native?.release(); + expect(acquireCallOwner('element', '!room:example.org')).toBeDefined(); + }); + + it('makes release idempotent and cannot release a replacement lease', () => { + const first = acquireCallOwner('element', '!room:example.org'); + first?.release(); + const second = acquireCallOwner('livekit-js', '!room:example.org'); + + first?.release(); + expect(getActiveCallOwner()).toMatchObject({ kind: 'livekit-js' }); + second?.release(); + expect(getActiveCallOwner()).toBeUndefined(); + }); +}); diff --git a/src/app/state/callOwner.ts b/src/app/state/callOwner.ts new file mode 100644 index 000000000..a266a1300 --- /dev/null +++ b/src/app/state/callOwner.ts @@ -0,0 +1,40 @@ +export type CallOwnerKind = 'element' | 'native' | 'livekit-js'; + +export type CallOwnerLease = { + kind: CallOwnerKind; + roomId: string; + release: () => void; +}; + +let activeOwner: CallOwnerLease | undefined; + +export const acquireCallOwner = ( + kind: CallOwnerKind, + roomId: string +): CallOwnerLease | undefined => { + if (activeOwner) return undefined; + + let released = false; + const lease: CallOwnerLease = { + kind, + roomId, + release: () => { + if (released || activeOwner !== lease) return; + released = true; + activeOwner = undefined; + }, + }; + activeOwner = lease; + return lease; +}; + +export const getActiveCallOwner = (): Pick | undefined => + activeOwner; + +export const releaseCallOwner = (kind: CallOwnerKind, roomId: string): void => { + if (activeOwner?.kind === kind && activeOwner.roomId === roomId) activeOwner.release(); +}; + +export const resetCallOwnerForTests = (): void => { + activeOwner = undefined; +}; diff --git a/src/app/state/livekitJsCall.test.ts b/src/app/state/livekitJsCall.test.ts new file mode 100644 index 000000000..bb5a075c6 --- /dev/null +++ b/src/app/state/livekitJsCall.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from 'vitest'; +import { selectActiveCallSession } from './livekitJsCall'; +import { acquireCallOwner, getActiveCallOwner, resetCallOwnerForTests } from './callOwner'; + +describe('selectActiveCallSession', () => { + it('selects the JS owner when Element Call and native sessions are absent', () => { + const livekitSession = { + roomId: '!room:example.org', + lifecycle: 'active' as const, + failure: null, + hangup: async () => undefined, + }; + + expect(selectActiveCallSession(undefined, undefined, livekitSession)).toBe(livekitSession); + }); + + it('preserves Element Call and native precedence', () => { + const element = { roomId: '!element:example.org' }; + const native = { + roomId: '!native:example.org', + connectionId: 'native', + lifecycle: 'connected' as const, + hangup: async () => undefined, + }; + const livekit = { + roomId: '!livekit:example.org', + lifecycle: 'active' as const, + failure: null, + hangup: async () => undefined, + }; + + expect(selectActiveCallSession(element, native, livekit)).toBe(element); + expect(selectActiveCallSession(undefined, native, livekit)).toBe(native); + }); + + it('routes past a failed native session to the active JS owner and releases its lease', async () => { + resetCallOwnerForTests(); + const lease = acquireCallOwner('livekit-js', '!room:example.org'); + const livekit = { + roomId: '!room:example.org', + lifecycle: 'active' as const, + failure: null, + hangup: async () => lease?.release(), + }; + const native = { + roomId: '!room:example.org', + connectionId: 'native', + lifecycle: 'error' as const, + hangup: async () => undefined, + }; + + await selectActiveCallSession(undefined, native, livekit)?.hangup(); + + expect(getActiveCallOwner()).toBeUndefined(); + }); + + it('routes past a failed JS session to Element Call and releases its lease', async () => { + resetCallOwnerForTests(); + const lease = acquireCallOwner('element', '!room:example.org'); + const element = { + roomId: '!room:example.org', + hangup: async () => lease?.release(), + }; + const livekit = { + roomId: '!room:example.org', + lifecycle: 'failed' as const, + failure: 'setup-failed' as const, + hangup: async () => undefined, + }; + + await selectActiveCallSession(element, undefined, livekit)?.hangup(); + + expect(getActiveCallOwner()).toBeUndefined(); + }); +}); diff --git a/src/app/state/livekitJsCall.ts b/src/app/state/livekitJsCall.ts new file mode 100644 index 000000000..278c4951f --- /dev/null +++ b/src/app/state/livekitJsCall.ts @@ -0,0 +1,35 @@ +import { atom } from 'jotai'; +import type { + LivekitJsControllerFailure, + LivekitJsControllerLifecycle, + LivekitJsMediaFacade, +} from '$features/call/livekitJsController'; +import type { Room as LivekitRoom } from 'livekit-client'; +import type { NativeCallSession } from './nativeCall'; + +export type LivekitJsCallSession = { + roomId: string; + lifecycle: LivekitJsControllerLifecycle; + failure: LivekitJsControllerFailure | null; + room?: LivekitRoom; + media?: LivekitJsMediaFacade; + hangup: () => Promise; +}; + +export const livekitJsCallAtom = atom(undefined); + +export const isLivekitJsCallActive = (session: LivekitJsCallSession | undefined): boolean => + session?.lifecycle !== undefined && + session.lifecycle !== 'idle' && + session.lifecycle !== 'failed'; + +export const selectActiveCallSession = ( + elementCall: Element | undefined, + nativeCall: NativeCallSession | undefined, + livekitJsCall: LivekitJsCallSession | undefined +): Element | NativeCallSession | LivekitJsCallSession | undefined => { + if (elementCall) return elementCall; + if (nativeCall && nativeCall.lifecycle !== 'error') return nativeCall; + if (isLivekitJsCallActive(livekitJsCall)) return livekitJsCall; + return undefined; +}; diff --git a/src/app/state/settings.defaults.test.ts b/src/app/state/settings.defaults.test.ts index 159d3f7a6..377abb661 100644 --- a/src/app/state/settings.defaults.test.ts +++ b/src/app/state/settings.defaults.test.ts @@ -21,6 +21,24 @@ describe('mergePersistedSettings', () => { ); }); + it('defaults the LiveKit JS connection probe off and persists the opt-in', () => { + expect(defaultSettings.livekitJsCallsEnabled).toBe(false); + + localStorage.setItem('settings', JSON.stringify({ livekitJsCallsEnabled: true })); + expect(mergePersistedSettings(localStorage.getItem('settings'), {}).livekitJsCallsEnabled).toBe( + true + ); + }); + + it('defaults the LiveKit JS media test off and persists the opt-in', () => { + expect(defaultSettings.livekitJsMediaTestEnabled).toBe(false); + + localStorage.setItem('settings', JSON.stringify({ livekitJsMediaTestEnabled: true })); + expect( + mergePersistedSettings(localStorage.getItem('settings'), {}).livekitJsMediaTestEnabled + ).toBe(true); + }); + it('layers deployer defaults over code defaults when localStorage is empty', () => { const merged = mergePersistedSettings(null, { twitterEmoji: false }); expect(merged.twitterEmoji).toBe(false); @@ -97,6 +115,20 @@ describe('sanitizeSettingsDefaults', () => { }); }); + it('accepts the LiveKit JS connection probe setting', () => { + expect(sanitizeSettingsDefaults({ livekitJsCallsEnabled: true })).toEqual({ + livekitJsCallsEnabled: true, + }); + expect(sanitizeSettingsDefaults({ livekitJsCallsEnabled: 'yes' })).toEqual({}); + }); + + it('accepts the LiveKit JS media test setting', () => { + expect(sanitizeSettingsDefaults({ livekitJsMediaTestEnabled: true })).toEqual({ + livekitJsMediaTestEnabled: true, + }); + expect(sanitizeSettingsDefaults({ livekitJsMediaTestEnabled: 'yes' })).toEqual({}); + }); + it('drops unknown keys', () => { expect(sanitizeSettingsDefaults({ notARealSetting: true, hour24Clock: true })).toEqual({ hour24Clock: true, diff --git a/src/app/state/settings.ts b/src/app/state/settings.ts index a007f9e24..de1905923 100644 --- a/src/app/state/settings.ts +++ b/src/app/state/settings.ts @@ -181,6 +181,8 @@ export interface Settings { enableMSC4268CMD: boolean; enableMediaGalleries: boolean; nativeCallsEnabled: boolean; + livekitJsCallsEnabled: boolean; + livekitJsMediaTestEnabled: boolean; settingsSyncEnabled: boolean; // Cosmetics! @@ -339,6 +341,8 @@ export const defaultSettings: Settings = { enableMSC4268CMD: false, enableMediaGalleries: false, nativeCallsEnabled: false, + livekitJsCallsEnabled: false, + livekitJsMediaTestEnabled: false, // Push notifications (SW/Sygnal): default on for mobile, opt-in on desktop. // In-app pill banner: default on for mobile (primary foreground alert), opt-in on desktop.