/* * WiVRn VR streaming * Copyright (C) 2024 galister * Copyright (C) 2025 Sapphire * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ #include "pico_face_tracker.h" #include "spdlog/common.h" #include "spdlog/spdlog.h" #include "wivrn_packets.h" #include "xr/instance.h" #include "xr/pico_eye_types.h" #include "xr/pico_eye_types_reflection.h" #include "xr/session.h" #include "xr/to_string.h" #include #include struct xr::pico_face_tracker::impl { const XrSession s; const PFN_xrStopEyeTrackingPICO xrStopEyeTrackingPICO; const PFN_xrGetFaceTrackingDataPICO xrGetFaceTrackingDataPICO; impl(instance & inst, session & s) : s(s), xrStopEyeTrackingPICO(inst.get_proc("xrStopEyeTrackingPICO")), xrGetFaceTrackingDataPICO(inst.get_proc("xrGetFaceTrackingDataPICO")) { auto xrStartEyeTrackingPICO = inst.get_proc("xrStartEyeTrackingPICO"); auto xrSetTrackingModePICO = inst.get_proc("xrSetTrackingModePICO"); CHECK_XR(xrStartEyeTrackingPICO(s)); CHECK_XR(xrSetTrackingModePICO(s, XR_TRACKING_MODE_FACE_BIT_PICO)); } ~impl() { if (auto res = xrStopEyeTrackingPICO(s, XR_TRACKING_MODE_FACE_BIT_PICO); not XR_SUCCEEDED(res)) spdlog::warn("Failed to deactivate face tracking: {}", xr::to_string(res)); } void get_weights(XrTime time, wivrn::from_headset::tracking::fb_face2 & out_expressions) { out_expressions.is_valid = false; out_expressions.is_eye_following_blendshapes_valid = false; XrFaceTrackingDataPICO face_tracking{.time = 0}; if (auto res = xrGetFaceTrackingDataPICO(s, time, XR_GET_FACE_DATA_DEFAULT_PICO, &face_tracking); !XR_SUCCEEDED(res)) { spdlog::warn("Unable to get face tracking data: xrGetFaceTrackingDataPICO returned {}", xr::to_string(res)); return; } if (face_tracking.time == 0) return; out_expressions.time = face_tracking.time; #define MAP_EXPRESSION(fb, pico) \ out_expressions.weights[fb] = face_tracking.blendShapeWeight[pico]; // Map the blendshapes. out_expressions.weights.fill(0.f); out_expressions.confidences.fill(1.f); MAP_EXPRESSION(XR_FACE_EXPRESSION2_BROW_LOWERER_L_FB, XR_BS_BROWDOWN_L_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_BROW_LOWERER_R_FB, XR_BS_BROWDOWN_R_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_CHEEK_PUFF_L_FB, XR_BS_CHEEKPUFF_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_CHEEK_PUFF_R_FB, XR_BS_CHEEKPUFF_PICO); // XR_FACE_EXPRESSION2_CHEEK_RAISER_L_FB // XR_FACE_EXPRESSION2_CHEEK_RAISER_R_FB // XR_FACE_EXPRESSION2_CHEEK_SUCK_L_FB // XR_FACE_EXPRESSION2_CHEEK_SUCK_R_FB MAP_EXPRESSION(XR_FACE_EXPRESSION2_CHIN_RAISER_B_FB, XR_BS_MOUTHSHRUGLOWER_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_CHIN_RAISER_T_FB, XR_BS_MOUTHSHRUGUPPER_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_DIMPLER_L_FB, XR_BS_MOUTHDIMPLE_L_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_DIMPLER_R_FB, XR_BS_MOUTHDIMPLE_R_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_EYES_CLOSED_L_FB, XR_BS_EYEBLINK_L_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_EYES_CLOSED_R_FB, XR_BS_EYEBLINK_R_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_EYES_LOOK_DOWN_L_FB, XR_BS_EYELOOKDOWN_L_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_EYES_LOOK_DOWN_R_FB, XR_BS_EYELOOKDOWN_R_PICO); // "out" is left for the left eye, right for the right eye MAP_EXPRESSION(XR_FACE_EXPRESSION2_EYES_LOOK_LEFT_L_FB, XR_BS_EYELOOKOUT_L_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_EYES_LOOK_RIGHT_L_FB, XR_BS_EYELOOKIN_L_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_EYES_LOOK_LEFT_R_FB, XR_BS_EYELOOKIN_R_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_EYES_LOOK_RIGHT_R_FB, XR_BS_EYELOOKOUT_R_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_EYES_LOOK_UP_L_FB, XR_BS_EYELOOKUP_L_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_EYES_LOOK_UP_R_FB, XR_BS_EYELOOKUP_R_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_INNER_BROW_RAISER_L_FB, XR_BS_BROWINNERUP_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_INNER_BROW_RAISER_R_FB, XR_BS_BROWINNERUP_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_JAW_DROP_FB, XR_BS_JAWOPEN_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_JAW_SIDEWAYS_LEFT_FB, XR_BS_JAWLEFT_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_JAW_SIDEWAYS_RIGHT_FB, XR_BS_JAWRIGHT_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_JAW_THRUST_FB, XR_BS_JAWFORWARD_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_LID_TIGHTENER_L_FB, XR_BS_EYESQUINT_L_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_LID_TIGHTENER_R_FB, XR_BS_EYESQUINT_R_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_CORNER_DEPRESSOR_L_FB, XR_BS_MOUTHFROWN_L_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_CORNER_DEPRESSOR_R_FB, XR_BS_MOUTHFROWN_R_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_CORNER_PULLER_L_FB, XR_BS_MOUTHSMILE_L_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_CORNER_PULLER_R_FB, XR_BS_MOUTHSMILE_R_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_FUNNELER_LB_FB, XR_BS_MOUTHFUNNEL_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_FUNNELER_LT_FB, XR_BS_MOUTHFUNNEL_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_FUNNELER_RB_FB, XR_BS_MOUTHFUNNEL_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_FUNNELER_RT_FB, XR_BS_MOUTHFUNNEL_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_PRESSOR_L_FB, XR_BS_MOUTHPRESS_L_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_PRESSOR_R_FB, XR_BS_MOUTHPRESS_R_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_PUCKER_L_FB, XR_BS_MOUTHPUCKER_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_PUCKER_R_FB, XR_BS_MOUTHPUCKER_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_STRETCHER_L_FB, XR_BS_MOUTHSTRETCH_L_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_STRETCHER_R_FB, XR_BS_MOUTHSTRETCH_R_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_SUCK_LB_FB, XR_BS_MOUTHROLLLOWER_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_SUCK_LT_FB, XR_BS_MOUTHROLLLOWER_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_SUCK_RT_FB, XR_BS_MOUTHROLLUPPER_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_SUCK_RB_FB, XR_BS_MOUTHROLLUPPER_PICO); // XR_FACE_EXPRESSION2_LIP_TIGHTENER_L_FB // XR_FACE_EXPRESSION2_LIP_TIGHTENER_R_FB MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIPS_TOWARD_FB, XR_BS_MOUTHCLOSE_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_LOWER_LIP_DEPRESSOR_L_FB, XR_BS_MOUTHLOWERDOWN_L_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_LOWER_LIP_DEPRESSOR_R_FB, XR_BS_MOUTHLOWERDOWN_R_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_MOUTH_LEFT_FB, XR_BS_MOUTHLEFT_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_MOUTH_RIGHT_FB, XR_BS_MOUTHRIGHT_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_NOSE_WRINKLER_L_FB, XR_BS_NOSESNEER_L_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_NOSE_WRINKLER_R_FB, XR_BS_NOSESNEER_R_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_OUTER_BROW_RAISER_L_FB, XR_BS_BROWOUTERUP_L_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_OUTER_BROW_RAISER_R_FB, XR_BS_BROWOUTERUP_R_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_UPPER_LID_RAISER_L_FB, XR_BS_EYEWIDE_L_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_UPPER_LID_RAISER_R_FB, XR_BS_EYEWIDE_R_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_UPPER_LIP_RAISER_L_FB, XR_BS_MOUTHUPPERUP_L_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_UPPER_LIP_RAISER_R_FB, XR_BS_MOUTHUPPERUP_R_PICO); MAP_EXPRESSION(XR_FACE_EXPRESSION2_TONGUE_OUT_FB, XR_BS_TONGUEOUT_PICO); // XR_FACE_EXPRESSION2_TONGUE_TIP_INTERDENTAL_FB // XR_FACE_EXPRESSION2_TONGUE_TIP_ALVEOLAR_FB // XR_FACE_EXPRESSION2_TONGUE_FRONT_DORSAL_PALATE_FB // XR_FACE_EXPRESSION2_TONGUE_MID_DORSAL_PALATE_FB // XR_FACE_EXPRESSION2_TONGUE_BACK_DORSAL_VELAR_FB // XR_FACE_EXPRESSION2_TONGUE_RETREAT_FB #undef MAP_EXPRESSION out_expressions.is_valid = true; out_expressions.is_eye_following_blendshapes_valid = true; } }; xr::pico_face_tracker::pico_face_tracker(instance & inst, session & s) { // The API has a global start and stop method // use a shared handle so we can have independent objects // and start/stop accordingly static std::mutex mutex; std::unique_lock lock(mutex); static std::weak_ptr instance; if (auto shared = instance.lock()) { i = std::move(shared); return; } i = std::make_unique(inst, s); instance = i; } void xr::pico_face_tracker::get_weights(XrTime time, packet_type & out_expressions) { return i->get_weights(time, out_expressions); }