mirror of
https://github.com/WiVRn/WiVRn.git
synced 2026-11-20 00:51:32 -08:00
182 lines
8.7 KiB
C++
182 lines
8.7 KiB
C++
/*
|
|
* WiVRn VR streaming
|
|
* Copyright (C) 2024 galister <[email protected]>
|
|
* Copyright (C) 2025 Sapphire <[email protected]>
|
|
*
|
|
* This program is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation, either version 3 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
#include "pico_face_tracker.h"
|
|
|
|
#include "spdlog/common.h"
|
|
#include "spdlog/spdlog.h"
|
|
#include "wivrn_packets.h"
|
|
#include "xr/instance.h"
|
|
#include "xr/pico_eye_types.h"
|
|
#include "xr/pico_eye_types_reflection.h"
|
|
#include "xr/session.h"
|
|
#include "xr/to_string.h"
|
|
|
|
#include <mutex>
|
|
#include <openxr/openxr.h>
|
|
|
|
struct xr::pico_face_tracker::impl
|
|
{
|
|
const XrSession s;
|
|
const PFN_xrStopEyeTrackingPICO xrStopEyeTrackingPICO;
|
|
const PFN_xrGetFaceTrackingDataPICO xrGetFaceTrackingDataPICO;
|
|
|
|
impl(instance & inst, session & s) :
|
|
s(s),
|
|
xrStopEyeTrackingPICO(inst.get_proc<PFN_xrStopEyeTrackingPICO>("xrStopEyeTrackingPICO")),
|
|
xrGetFaceTrackingDataPICO(inst.get_proc<PFN_xrGetFaceTrackingDataPICO>("xrGetFaceTrackingDataPICO"))
|
|
{
|
|
auto xrStartEyeTrackingPICO = inst.get_proc<PFN_xrStartEyeTrackingPICO>("xrStartEyeTrackingPICO");
|
|
auto xrSetTrackingModePICO = inst.get_proc<PFN_xrSetTrackingModePICO>("xrSetTrackingModePICO");
|
|
CHECK_XR(xrStartEyeTrackingPICO(s));
|
|
CHECK_XR(xrSetTrackingModePICO(s, XR_TRACKING_MODE_FACE_BIT_PICO));
|
|
}
|
|
|
|
~impl()
|
|
{
|
|
if (auto res = xrStopEyeTrackingPICO(s, XR_TRACKING_MODE_FACE_BIT_PICO); not XR_SUCCEEDED(res))
|
|
spdlog::warn("Failed to deactivate face tracking: {}", xr::to_string(res));
|
|
}
|
|
|
|
void get_weights(XrTime time, wivrn::from_headset::tracking::fb_face2 & out_expressions)
|
|
{
|
|
out_expressions.is_valid = false;
|
|
out_expressions.is_eye_following_blendshapes_valid = false;
|
|
|
|
XrFaceTrackingDataPICO face_tracking{.time = 0};
|
|
|
|
if (auto res = xrGetFaceTrackingDataPICO(s, time, XR_GET_FACE_DATA_DEFAULT_PICO, &face_tracking); !XR_SUCCEEDED(res))
|
|
{
|
|
spdlog::warn("Unable to get face tracking data: xrGetFaceTrackingDataPICO returned {}", xr::to_string(res));
|
|
return;
|
|
}
|
|
|
|
if (face_tracking.time == 0)
|
|
return;
|
|
out_expressions.time = face_tracking.time;
|
|
|
|
#define MAP_EXPRESSION(fb, pico) \
|
|
out_expressions.weights[fb] = face_tracking.blendShapeWeight[pico];
|
|
|
|
// Map the blendshapes.
|
|
out_expressions.weights.fill(0.f);
|
|
out_expressions.confidences.fill(1.f);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_BROW_LOWERER_L_FB, XR_BS_BROWDOWN_L_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_BROW_LOWERER_R_FB, XR_BS_BROWDOWN_R_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_CHEEK_PUFF_L_FB, XR_BS_CHEEKPUFF_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_CHEEK_PUFF_R_FB, XR_BS_CHEEKPUFF_PICO);
|
|
// XR_FACE_EXPRESSION2_CHEEK_RAISER_L_FB
|
|
// XR_FACE_EXPRESSION2_CHEEK_RAISER_R_FB
|
|
// XR_FACE_EXPRESSION2_CHEEK_SUCK_L_FB
|
|
// XR_FACE_EXPRESSION2_CHEEK_SUCK_R_FB
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_CHIN_RAISER_B_FB, XR_BS_MOUTHSHRUGLOWER_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_CHIN_RAISER_T_FB, XR_BS_MOUTHSHRUGUPPER_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_DIMPLER_L_FB, XR_BS_MOUTHDIMPLE_L_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_DIMPLER_R_FB, XR_BS_MOUTHDIMPLE_R_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_EYES_CLOSED_L_FB, XR_BS_EYEBLINK_L_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_EYES_CLOSED_R_FB, XR_BS_EYEBLINK_R_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_EYES_LOOK_DOWN_L_FB, XR_BS_EYELOOKDOWN_L_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_EYES_LOOK_DOWN_R_FB, XR_BS_EYELOOKDOWN_R_PICO);
|
|
|
|
// "out" is left for the left eye, right for the right eye
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_EYES_LOOK_LEFT_L_FB, XR_BS_EYELOOKOUT_L_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_EYES_LOOK_RIGHT_L_FB, XR_BS_EYELOOKIN_L_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_EYES_LOOK_LEFT_R_FB, XR_BS_EYELOOKIN_R_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_EYES_LOOK_RIGHT_R_FB, XR_BS_EYELOOKOUT_R_PICO);
|
|
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_EYES_LOOK_UP_L_FB, XR_BS_EYELOOKUP_L_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_EYES_LOOK_UP_R_FB, XR_BS_EYELOOKUP_R_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_INNER_BROW_RAISER_L_FB, XR_BS_BROWINNERUP_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_INNER_BROW_RAISER_R_FB, XR_BS_BROWINNERUP_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_JAW_DROP_FB, XR_BS_JAWOPEN_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_JAW_SIDEWAYS_LEFT_FB, XR_BS_JAWLEFT_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_JAW_SIDEWAYS_RIGHT_FB, XR_BS_JAWRIGHT_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_JAW_THRUST_FB, XR_BS_JAWFORWARD_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_LID_TIGHTENER_L_FB, XR_BS_EYESQUINT_L_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_LID_TIGHTENER_R_FB, XR_BS_EYESQUINT_R_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_CORNER_DEPRESSOR_L_FB, XR_BS_MOUTHFROWN_L_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_CORNER_DEPRESSOR_R_FB, XR_BS_MOUTHFROWN_R_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_CORNER_PULLER_L_FB, XR_BS_MOUTHSMILE_L_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_CORNER_PULLER_R_FB, XR_BS_MOUTHSMILE_R_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_FUNNELER_LB_FB, XR_BS_MOUTHFUNNEL_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_FUNNELER_LT_FB, XR_BS_MOUTHFUNNEL_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_FUNNELER_RB_FB, XR_BS_MOUTHFUNNEL_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_FUNNELER_RT_FB, XR_BS_MOUTHFUNNEL_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_PRESSOR_L_FB, XR_BS_MOUTHPRESS_L_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_PRESSOR_R_FB, XR_BS_MOUTHPRESS_R_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_PUCKER_L_FB, XR_BS_MOUTHPUCKER_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_PUCKER_R_FB, XR_BS_MOUTHPUCKER_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_STRETCHER_L_FB, XR_BS_MOUTHSTRETCH_L_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_STRETCHER_R_FB, XR_BS_MOUTHSTRETCH_R_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_SUCK_LB_FB, XR_BS_MOUTHROLLLOWER_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_SUCK_LT_FB, XR_BS_MOUTHROLLLOWER_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_SUCK_RT_FB, XR_BS_MOUTHROLLUPPER_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIP_SUCK_RB_FB, XR_BS_MOUTHROLLUPPER_PICO);
|
|
// XR_FACE_EXPRESSION2_LIP_TIGHTENER_L_FB
|
|
// XR_FACE_EXPRESSION2_LIP_TIGHTENER_R_FB
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_LIPS_TOWARD_FB, XR_BS_MOUTHCLOSE_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_LOWER_LIP_DEPRESSOR_L_FB, XR_BS_MOUTHLOWERDOWN_L_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_LOWER_LIP_DEPRESSOR_R_FB, XR_BS_MOUTHLOWERDOWN_R_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_MOUTH_LEFT_FB, XR_BS_MOUTHLEFT_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_MOUTH_RIGHT_FB, XR_BS_MOUTHRIGHT_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_NOSE_WRINKLER_L_FB, XR_BS_NOSESNEER_L_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_NOSE_WRINKLER_R_FB, XR_BS_NOSESNEER_R_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_OUTER_BROW_RAISER_L_FB, XR_BS_BROWOUTERUP_L_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_OUTER_BROW_RAISER_R_FB, XR_BS_BROWOUTERUP_R_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_UPPER_LID_RAISER_L_FB, XR_BS_EYEWIDE_L_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_UPPER_LID_RAISER_R_FB, XR_BS_EYEWIDE_R_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_UPPER_LIP_RAISER_L_FB, XR_BS_MOUTHUPPERUP_L_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_UPPER_LIP_RAISER_R_FB, XR_BS_MOUTHUPPERUP_R_PICO);
|
|
MAP_EXPRESSION(XR_FACE_EXPRESSION2_TONGUE_OUT_FB, XR_BS_TONGUEOUT_PICO);
|
|
// XR_FACE_EXPRESSION2_TONGUE_TIP_INTERDENTAL_FB
|
|
// XR_FACE_EXPRESSION2_TONGUE_TIP_ALVEOLAR_FB
|
|
// XR_FACE_EXPRESSION2_TONGUE_FRONT_DORSAL_PALATE_FB
|
|
// XR_FACE_EXPRESSION2_TONGUE_MID_DORSAL_PALATE_FB
|
|
// XR_FACE_EXPRESSION2_TONGUE_BACK_DORSAL_VELAR_FB
|
|
// XR_FACE_EXPRESSION2_TONGUE_RETREAT_FB
|
|
|
|
#undef MAP_EXPRESSION
|
|
|
|
out_expressions.is_valid = true;
|
|
out_expressions.is_eye_following_blendshapes_valid = true;
|
|
}
|
|
};
|
|
|
|
xr::pico_face_tracker::pico_face_tracker(instance & inst, session & s)
|
|
{
|
|
// The API has a global start and stop method
|
|
// use a shared handle so we can have independent objects
|
|
// and start/stop accordingly
|
|
static std::mutex mutex;
|
|
std::unique_lock lock(mutex);
|
|
static std::weak_ptr<impl> instance;
|
|
if (auto shared = instance.lock())
|
|
{
|
|
i = std::move(shared);
|
|
return;
|
|
}
|
|
i = std::make_unique<impl>(inst, s);
|
|
instance = i;
|
|
}
|
|
|
|
void xr::pico_face_tracker::get_weights(XrTime time, packet_type & out_expressions)
|
|
{
|
|
return i->get_weights(time, out_expressions);
|
|
}
|