From 261bb2c600c3aff84f1cadec2d9339d374e75fa7 Mon Sep 17 00:00:00 2001 From: galister <22305755+galister@users.noreply.github.com> Date: Sat, 16 Nov 2024 16:33:47 +0900 Subject: [PATCH] pass through HTC_facial_tracking --- AndroidManifest.xml | 2 + client/application.cpp | 20 +++ client/application.h | 26 ++++ client/configuration.cpp | 16 ++- client/scenes/lobby_gui.cpp | 11 ++ client/scenes/stream.cpp | 10 +- client/scenes/stream_tracking.cpp | 30 ++++- client/xr/htc_face_tracker.cpp | 71 +++++++++++ client/xr/htc_face_tracker.h | 43 +++++++ client/xr/session.cpp | 17 +++ client/xr/session.h | 2 + client/xr/system.cpp | 18 +++ client/xr/system.h | 1 + common/wivrn_packets.h | 18 ++- server/CMakeLists.txt | 1 + server/driver/wivrn_htc_face_tracker.cpp | 154 +++++++++++++++++++++++ server/driver/wivrn_htc_face_tracker.h | 114 +++++++++++++++++ server/driver/wivrn_session.cpp | 22 +++- server/driver/wivrn_session.h | 2 + server/main.cpp | 2 +- 20 files changed, 567 insertions(+), 13 deletions(-) create mode 100644 client/xr/htc_face_tracker.cpp create mode 100644 client/xr/htc_face_tracker.h create mode 100644 server/driver/wivrn_htc_face_tracker.cpp create mode 100644 server/driver/wivrn_htc_face_tracker.h diff --git a/AndroidManifest.xml b/AndroidManifest.xml index 159db79c..9914169e 100644 --- a/AndroidManifest.xml +++ b/AndroidManifest.xml @@ -55,6 +55,8 @@ + + #endif @@ -127,6 +128,11 @@ class application : public singleton bool fb_face_tracking2_supported = false; xr::fb_face_tracker2 fb_face_tracker2; + bool htc_face_tracking_eye_supported = false; + xr::htc_face_tracker htc_face_tracker_eye; + bool htc_face_tracking_lip_supported = false; + xr::htc_face_tracker htc_face_tracker_lip; + bool eye_gaze_supported = false; bool session_running = false; @@ -418,6 +424,16 @@ public: return instance().fb_face_tracking2_supported; } + static bool get_htc_face_tracking_eye_supported() + { + return instance().htc_face_tracking_eye_supported; + } + + static bool get_htc_face_tracking_lip_supported() + { + return instance().htc_face_tracking_lip_supported; + } + static bool get_eye_gaze_supported() { return instance().eye_gaze_supported; @@ -438,6 +454,16 @@ public: return instance().fb_face_tracker2; } + static xr::htc_face_tracker & get_htc_face_tracker_eye() + { + return instance().htc_face_tracker_eye; + } + + static xr::htc_face_tracker & get_htc_face_tracker_lip() + { + return instance().htc_face_tracker_lip; + } + static const std::vector & get_xr_extensions() { return instance().xr_extensions; diff --git a/client/configuration.cpp b/client/configuration.cpp index ffb55492..2fc7bd71 100644 --- a/client/configuration.cpp +++ b/client/configuration.cpp @@ -104,9 +104,19 @@ bool configuration::check_feature(feature f) const return false; break; case feature::face_tracking: - if (not application::get_fb_face_tracking2_supported()) - return false; - break; + switch (guess_model()) + { + case model::htc_vive_focus_3: + [[fallthrough]]; + case model::htc_vive_xr_elite: + if (not(application::get_htc_face_tracking_eye_supported() or application::get_htc_face_tracking_lip_supported())) + return false; + break; + default: + if (not application::get_fb_face_tracking2_supported()) + return false; + break; + } } } #ifdef __ANDROID__ diff --git a/client/scenes/lobby_gui.cpp b/client/scenes/lobby_gui.cpp index 7c4665b3..7725dd73 100644 --- a/client/scenes/lobby_gui.cpp +++ b/client/scenes/lobby_gui.cpp @@ -1206,6 +1206,17 @@ void scenes::lobby::draw_features_status(XrTime predicted_display_time) }); } + if (application::get_htc_face_tracking_eye_supported() or application::get_htc_face_tracking_lip_supported()) + { + items.push_back({ + .f = feature::face_tracking, + .tooltip_enabled = _("Face tracking is enabled"), + .tooltip_disabled = _("Face tracking is disabled"), + .icon_enabled = ICON_FA_FACE_KISS_WINK_HEART, + .icon_disabled = ICON_FA_FACE_MEH_BLANK, + }); + } + // Get statuses for (auto & i: items) { diff --git a/client/scenes/stream.cpp b/client/scenes/stream.cpp index 2b772f22..2934af9b 100644 --- a/client/scenes/stream.cpp +++ b/client/scenes/stream.cpp @@ -155,7 +155,15 @@ std::shared_ptr scenes::stream::create(std::unique_ptrsystem.passthrough_supported() != xr::system::passthrough_type::no_passthrough; info.system_name = std::string(self->system.properties().systemName); diff --git a/client/scenes/stream_tracking.cpp b/client/scenes/stream_tracking.cpp index 66c18ece..8fc0d8a3 100644 --- a/client/scenes/stream_tracking.cpp +++ b/client/scenes/stream_tracking.cpp @@ -19,6 +19,7 @@ #include "application.h" #include "stream.h" +#include "wivrn_packets.h" #include #include #include @@ -210,7 +211,14 @@ void scenes::stream::tracking() std::vector packets; const bool hand_tracking = config.check_feature(feature::hand_tracking); - const bool face_tracking = config.check_feature(feature::face_tracking); + from_headset::face_type face_tracking = from_headset::face_type::none; + if (config.check_feature(feature::face_tracking)) + { + if (application::get_fb_face_tracking2_supported()) + face_tracking = from_headset::face_type::fb2; + else if (application::get_htc_face_tracking_eye_supported() or application::get_htc_face_tracking_lip_supported()) + face_tracking = from_headset::face_type::htc; + } on_interaction_profile_changed({}); @@ -291,9 +299,25 @@ void scenes::stream::tracking() locate_hands(application::get_right_hand(), world_space, t0 + Δt)); } - if (face_tracking and control.enabled[size_t(tid::face)]) + if (control.enabled[size_t(tid::face)]) { - application::get_fb_face_tracker2().get_weights(t0 + Δt, packet.face.emplace()); + switch (face_tracking) + { + case wivrn::from_headset::face_type::none: + break; + case wivrn::from_headset::face_type::fb2: + application::get_fb_face_tracker2().get_weights(t0 + Δt, packet.face.emplace()); + break; + case wivrn::from_headset::face_type::htc: + auto face_htc = packet.face_htc.emplace(); + face_htc.eye_active = false; + face_htc.lip_active = false; + if (application::get_htc_face_tracking_eye_supported()) + application::get_htc_face_tracker_eye().get_weights(t0 + Δt, face_htc); + if (application::get_htc_face_tracking_lip_supported()) + application::get_htc_face_tracker_lip().get_weights(t0 + Δt, face_htc); + break; + } } } catch (const std::system_error & e) diff --git a/client/xr/htc_face_tracker.cpp b/client/xr/htc_face_tracker.cpp new file mode 100644 index 00000000..b0ad1ede --- /dev/null +++ b/client/xr/htc_face_tracker.cpp @@ -0,0 +1,71 @@ +/* + * WiVRn VR streaming + * Copyright (C) 2024 galister + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "htc_face_tracker.h" +#include "wivrn_packets.h" +#include "xr/instance.h" +#include + +static PFN_xrDestroyFacialTrackerHTC xrDestroyFacialTrackerHTC{}; + +XrResult xr::destroy_htc_face_tracker(XrFacialTrackerHTC id) +{ + return xrDestroyFacialTrackerHTC(id); +} + +xr::htc_face_tracker::htc_face_tracker(instance & inst, XrFacialTrackerHTC h, XrFacialTrackingTypeHTC facialTrackingTypeHTC) +{ + id = h; + xrGetFacialExpressionsHTC = inst.get_proc("xrGetFacialExpressionsHTC"); + xrDestroyFacialTrackerHTC = inst.get_proc("xrDestroyFacialTrackerHTC"); + trackerType = facialTrackingTypeHTC; +} + +void xr::htc_face_tracker::get_weights(XrTime time, wivrn::from_headset::tracking::htc_face & out_expressions) +{ + if (!id || !xrGetFacialExpressionsHTC) + return; + + XrFacialExpressionsHTC expressions{ + .type = XR_TYPE_FACIAL_EXPRESSIONS_HTC, + .next = nullptr, + .sampleTime = time, + }; + + bool * is_active; + + switch (trackerType) + { + case XR_FACIAL_TRACKING_TYPE_EYE_DEFAULT_HTC: + expressions.expressionCount = out_expressions.eye.size(); + expressions.expressionWeightings = out_expressions.eye.data(); + is_active = &out_expressions.eye_active; + break; + case XR_FACIAL_TRACKING_TYPE_LIP_DEFAULT_HTC: + expressions.expressionCount = out_expressions.lip.size(); + expressions.expressionWeightings = out_expressions.lip.data(); + is_active = &out_expressions.lip_active; + break; + default: + throw std::system_error(1, std::generic_category(), "htc_face_tracker with unknown trackerType"); + } + + CHECK_XR(xrGetFacialExpressionsHTC(id, &expressions)); + + *is_active = expressions.isActive; +} diff --git a/client/xr/htc_face_tracker.h b/client/xr/htc_face_tracker.h new file mode 100644 index 00000000..f8381b54 --- /dev/null +++ b/client/xr/htc_face_tracker.h @@ -0,0 +1,43 @@ +/* + * WiVRn VR streaming + * Copyright (C) 2024 galister + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once + +#include "utils/handle.h" +#include "wivrn_packets.h" +#include + +namespace xr +{ +class instance; + +XrResult destroy_htc_face_tracker(XrFacialTrackerHTC); + +class htc_face_tracker : public utils::handle +{ + PFN_xrGetFacialExpressionsHTC xrGetFacialExpressionsHTC{}; + PFN_xrDestroyFacialTrackerHTC xrDestroyFacialTrackerHTC{}; + XrFacialTrackingTypeHTC trackerType{}; + +public: + htc_face_tracker() = default; + htc_face_tracker(instance & inst, XrFacialTrackerHTC h, XrFacialTrackingTypeHTC facialTrackingTypeHTC); + + void get_weights(XrTime time, struct wivrn::from_headset::tracking::htc_face & out_expressions); +}; +} // namespace xr diff --git a/client/xr/session.cpp b/client/xr/session.cpp index 428cc488..a97f1734 100644 --- a/client/xr/session.cpp +++ b/client/xr/session.cpp @@ -21,6 +21,7 @@ #include "application.h" #include "details/enumerate.h" +#include "openxr/openxr.h" #include "utils/contains.h" #include "xr/instance.h" #include "xr/system.h" @@ -119,6 +120,22 @@ xr::fb_face_tracker2 xr::session::create_fb_face_tracker2() return {*inst, ft}; } +xr::htc_face_tracker xr::session::create_htc_face_tracker(XrFacialTrackingTypeHTC facialTrackingType) +{ + XrFacialTrackerCreateInfoHTC create_info{ + .type = XR_TYPE_FACIAL_TRACKER_CREATE_INFO_HTC, + .next = nullptr, + .facialTrackingType = facialTrackingType, + }; + + XrFacialTrackerHTC ft; + + auto xrCreateFacialTrackerHTC = inst->get_proc("xrCreateFacialTrackerHTC"); + assert(xrCreateFacialTrackerHTC); + + CHECK_XR(xrCreateFacialTrackerHTC(id, &create_info, &ft)); + return {*inst, ft, facialTrackingType}; +} std::vector xr::session::get_swapchain_formats() const { std::vector formats; diff --git a/client/xr/session.h b/client/xr/session.h index 205884ea..d4a4bcd4 100644 --- a/client/xr/session.h +++ b/client/xr/session.h @@ -28,6 +28,7 @@ #include "fb_face_tracker2.h" #include "hand_tracker.h" +#include "htc_face_tracker.h" #include "passthrough.h" #include "space.h" @@ -50,6 +51,7 @@ public: space create_action_space(XrAction action, const XrPosef & pose = {{0, 0, 0, 1}, {0, 0, 0}}); hand_tracker create_hand_tracker(XrHandEXT hand, XrHandJointSetEXT hand_joint_set = XR_HAND_JOINT_SET_DEFAULT_EXT); fb_face_tracker2 create_fb_face_tracker2(); + htc_face_tracker create_htc_face_tracker(XrFacialTrackingTypeHTC); std::vector get_swapchain_formats() const; diff --git a/client/xr/system.cpp b/client/xr/system.cpp index 00217a4c..33769ba7 100644 --- a/client/xr/system.cpp +++ b/client/xr/system.cpp @@ -123,6 +123,24 @@ XrSystemFaceTrackingProperties2FB xr::system::fb_face_tracking2_properties() con return face_tracking_prop; } +XrSystemFacialTrackingPropertiesHTC xr::system::htc_face_tracking_properties() const +{ + if (!id) + throw std::invalid_argument("this"); + + XrSystemFacialTrackingPropertiesHTC face_tracking_prop{ + .type = XR_TYPE_SYSTEM_FACIAL_TRACKING_PROPERTIES_HTC, + }; + + XrSystemProperties prop{ + .type = XR_TYPE_SYSTEM_PROPERTIES, + .next = &face_tracking_prop, + }; + CHECK_XR(xrGetSystemProperties(*inst, id, &prop)); + + return face_tracking_prop; +} + xr::system::passthrough_type xr::system::passthrough_supported() const { if (utils::contains(environment_blend_modes(XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO), XR_ENVIRONMENT_BLEND_MODE_ALPHA_BLEND)) diff --git a/client/xr/system.h b/client/xr/system.h index e41c4c36..0a1b76fc 100644 --- a/client/xr/system.h +++ b/client/xr/system.h @@ -63,6 +63,7 @@ public: XrSystemHandTrackingPropertiesEXT hand_tracking_properties() const; XrSystemEyeGazeInteractionPropertiesEXT eye_gaze_interaction_properties() const; XrSystemFaceTrackingProperties2FB fb_face_tracking2_properties() const; + XrSystemFacialTrackingPropertiesHTC htc_face_tracking_properties() const; passthrough_type passthrough_supported() const; XrGraphicsRequirementsVulkan2KHR graphics_requirements() const; diff --git a/common/wivrn_packets.h b/common/wivrn_packets.h index c52a9b90..be69c3b8 100644 --- a/common/wivrn_packets.h +++ b/common/wivrn_packets.h @@ -130,6 +130,13 @@ struct visibility_mask_changed uint8_t view_index; }; +enum face_type : uint8_t +{ + none, + fb2, + htc, +}; + struct headset_info_packet { uint32_t recommended_eye_width; @@ -146,9 +153,9 @@ struct headset_info_packet std::array fov; bool hand_tracking; bool eye_gaze; - bool face_tracking2_fb; bool palm_pose; bool passthrough; + face_type face_tracking; std::vector supported_codecs; // from preferred to least preferred std::string system_name; }; @@ -208,6 +215,15 @@ struct tracking bool is_eye_following_blendshapes_valid; }; std::optional face; + + struct htc_face + { + std::array eye; + std::array lip; + bool eye_active; + bool lip_active; + }; + std::optional face_htc; }; struct trackings diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index c7e92172..d4497f12 100644 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -84,6 +84,7 @@ add_executable(wivrn-server driver/wivrn_controller.cpp driver/wivrn_eye_tracker.cpp driver/wivrn_fb_face2_tracker.cpp + driver/wivrn_htc_face_tracker.cpp driver/wivrn_foveation.cpp driver/pose_list.cpp driver/view_list.cpp diff --git a/server/driver/wivrn_htc_face_tracker.cpp b/server/driver/wivrn_htc_face_tracker.cpp new file mode 100644 index 00000000..9fd92c32 --- /dev/null +++ b/server/driver/wivrn_htc_face_tracker.cpp @@ -0,0 +1,154 @@ +/* + * WiVRn VR streaming + * Copyright (C) 2024 Guillaume Meunier + * Copyright (C) 2024 Patrick Nicolas + * Copyright (C) 2024 galister + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "wivrn_htc_face_tracker.h" + +#include "wivrn_packets.h" +#include "wivrn_session.h" + +#include "util/u_logging.h" +#include "xrt/xrt_defines.h" +#include "xrt/xrt_device.h" +#include "xrt/xrt_results.h" + +#include +#include +#include +#include + +namespace wivrn +{ + +static void wivrn_htc_face_tracker_destroy(xrt_device * xdev); + +static xrt_result_t wivrn_htc_face_tracker_update_inputs(xrt_device * xdev) +{ + static_cast(xdev)->update_inputs(); + return XRT_SUCCESS; +} + +static xrt_result_t wivrn_htc_face_tracker_get_face_tracking(struct xrt_device * xdev, + enum xrt_input_name facial_expression_type, + int64_t at_timestamp_ns, + struct xrt_facial_expression_set * out_value); + +wivrn_htc_face_tracker::wivrn_htc_face_tracker(xrt_device * hmd, + wivrn::wivrn_session & cnx) : + xrt_device{}, cnx(cnx) +{ + xrt_device * base = this; + base->tracking_origin = hmd->tracking_origin; + base->get_face_tracking = wivrn_htc_face_tracker_get_face_tracking; + base->update_inputs = wivrn_htc_face_tracker_update_inputs; + base->destroy = wivrn_htc_face_tracker_destroy; + name = XRT_DEVICE_HTC_FACE_TRACKING; + device_type = XRT_DEVICE_TYPE_FACE_TRACKER; + face_tracking_supported = true; + + // Print name. + strcpy(str, "WiVRn HTC Face Tracker"); + strcpy(serial, "WiVRn HTC Face Tracker"); + + // Setup input. + inputs_array[0].name = XRT_INPUT_HTC_EYE_FACE_TRACKING; + inputs_array[0].active = true; + + inputs_array[1].name = XRT_INPUT_HTC_LIP_FACE_TRACKING; + inputs_array[1].active = true; + + inputs = inputs_array.data(); + input_count = inputs_array.size(); +} + +void wivrn_htc_face_tracker::update_inputs() +{ + // Empty +} + +void wivrn_htc_face_tracker::update_tracking(const from_headset::tracking & tracking, const clock_offset & offset) +{ + if (not(tracking.face_htc)) + return; + const auto & face = *tracking.face_htc; + + wivrn_htc_face_data data{ + .eye = face.eye, + .lip = face.lip, + .eye_active = face.eye_active, + .lip_active = face.lip_active}; + + if (not face_list.update_tracking(tracking.production_timestamp, tracking.timestamp, data, offset)) + cnx.set_enabled(to_headset::tracking_control::id::face, false); +} + +xrt_result_t wivrn_htc_face_tracker::get_face_tracking(enum xrt_input_name facial_expression_type, int64_t at_timestamp_ns, struct xrt_facial_expression_set * inout_value) +{ + if (facial_expression_type == XRT_INPUT_HTC_EYE_FACE_TRACKING) + { + cnx.set_enabled(to_headset::tracking_control::id::face, true); + auto [_, data] = face_list.get_at(at_timestamp_ns); + + inout_value->base_expression_set_htc.is_active = data.eye_active; + inout_value->base_expression_set_htc.sample_time_ns = at_timestamp_ns; + + if (not data.eye_active) + return XRT_SUCCESS; + + memcpy(&inout_value->eye_expression_set_htc.expression_weights, data.eye.data(), sizeof(float) * data.eye.size()); + + return XRT_SUCCESS; + } + else if (facial_expression_type == XRT_INPUT_HTC_LIP_FACE_TRACKING) + { + cnx.set_enabled(to_headset::tracking_control::id::face, true); + auto [_, data] = face_list.get_at(at_timestamp_ns); + + inout_value->base_expression_set_htc.is_active = data.lip_active; + inout_value->base_expression_set_htc.sample_time_ns = at_timestamp_ns; + + if (not data.lip_active) + return XRT_SUCCESS; + + memcpy(&inout_value->lip_expression_set_htc.expression_weights, data.lip.data(), sizeof(float) * data.lip.size()); + + return XRT_SUCCESS; + } + + return XRT_ERROR_NOT_IMPLEMENTED; +} + +/* + * + * Functions + * + */ + +static void wivrn_htc_face_tracker_destroy(xrt_device * xdev) +{ +} + +static xrt_result_t wivrn_htc_face_tracker_get_face_tracking(struct xrt_device * xdev, + enum xrt_input_name facial_expression_type, + int64_t at_timestamp_ns, + struct xrt_facial_expression_set * inout_value) +{ + return static_cast(xdev)->get_face_tracking(facial_expression_type, at_timestamp_ns, inout_value); +} +} // namespace wivrn diff --git a/server/driver/wivrn_htc_face_tracker.h b/server/driver/wivrn_htc_face_tracker.h new file mode 100644 index 00000000..84c6de23 --- /dev/null +++ b/server/driver/wivrn_htc_face_tracker.h @@ -0,0 +1,114 @@ +/* + * WiVRn VR streaming + * Copyright (C) 2024 Guillaume Meunier + * Copyright (C) 2024 Patrick Nicolas + * Copyright (C) 2024 galister + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#pragma once + +#include "wivrn_packets.h" +#include "xrt/xrt_defines.h" +#include "xrt/xrt_device.h" + +#include "history.h" + +#include +#include +#include + +namespace wivrn +{ + +struct clock_offset; +class wivrn_session; + +struct wivrn_htc_face_data +{ + std::array eye; + std::array lip; + bool eye_active; + bool lip_active; +}; + +class htc_face_list : public history +{ +public: + static wivrn_htc_face_data interpolate(const wivrn_htc_face_data & a, const wivrn_htc_face_data & b, float t) + { + wivrn_htc_face_data result = b; + + if (not a.eye_active) + { + result.eye = b.eye; + result.eye_active = b.eye_active; + } + else if (not b.eye_active) + { + result.eye = a.eye; + result.eye_active = a.eye_active; + } + else + { + result.eye_active = true; + for (size_t i = 0; i < result.eye.size(); i++) + { + result.eye[i] = std::clamp(std::lerp(a.eye[i], b.eye[i], t), 0.0f, 1.0f); + } + } + + if (not a.lip_active) + { + result.lip = b.lip; + result.lip_active = b.lip_active; + } + else if (not b.lip_active) + { + result.lip = a.lip; + result.lip_active = a.lip_active; + } + else + { + result.lip_active = true; + for (size_t i = 0; i < result.lip.size(); i++) + { + result.lip[i] = std::clamp(std::lerp(a.lip[i], b.lip[i], t), 0.0f, 1.0f); + } + } + return result; + } + + bool update_tracking(const XrTime & production_timestamp, const XrTime & timestamp, const wivrn_htc_face_data & data, const clock_offset & offset) + { + return this->add_sample(production_timestamp, timestamp, data, offset); + } +}; + +class wivrn_htc_face_tracker : public xrt_device +{ + htc_face_list face_list; + std::array inputs_array; + + wivrn::wivrn_session & cnx; + +public: + wivrn_htc_face_tracker(xrt_device * hmd, wivrn::wivrn_session & cnx); + + void update_inputs(); + void update_tracking(const from_headset::tracking &, const clock_offset &); + xrt_result_t get_face_tracking(enum xrt_input_name facial_expression_type, int64_t at_timestamp_ns, struct xrt_facial_expression_set * out_value); +}; +} // namespace wivrn diff --git a/server/driver/wivrn_session.cpp b/server/driver/wivrn_session.cpp index 5af128e5..0c8ba398 100644 --- a/server/driver/wivrn_session.cpp +++ b/server/driver/wivrn_session.cpp @@ -34,8 +34,10 @@ #include "wivrn_eye_tracker.h" #include "wivrn_fb_face2_tracker.h" #include "wivrn_foveation.h" +#include "wivrn_htc_face_tracker.h" #include "wivrn_ipc.h" +#include "wivrn_packets.h" #include "xrt/xrt_session.h" #include #include @@ -197,11 +199,20 @@ wivrn::wivrn_session::wivrn_session(std::unique_ptr connection static_roles.eyes = eye_tracker.get(); xdevs[xdev_count++] = eye_tracker.get(); } - if (get_info().face_tracking2_fb || is_forced_extension("FB_face_tracking2")) + switch (get_info().face_tracking) { - fb_face2_tracker = std::make_unique(&hmd, *this); - static_roles.face = fb_face2_tracker.get(); - xdevs[xdev_count++] = fb_face2_tracker.get(); + case wivrn::from_headset::face_type::fb2: + fb_face2_tracker = std::make_unique(&hmd, *this); + static_roles.face = fb_face2_tracker.get(); + xdevs[xdev_count++] = fb_face2_tracker.get(); + break; + case wivrn::from_headset::face_type::htc: + htc_face_tracker = std::make_unique(&hmd, *this); + static_roles.face = htc_face_tracker.get(); + xdevs[xdev_count++] = htc_face_tracker.get(); + break; + default: + break; } #if WIVRN_FEATURE_SOLARXR @@ -322,8 +333,11 @@ void wivrn_session::operator()(const from_headset::tracking & tracking) eye_tracker->update_tracking(tracking, offset); if (foveation) foveation->update_tracking(tracking, offset); + if (fb_face2_tracker) fb_face2_tracker->update_tracking(tracking, offset); + else if (htc_face_tracker) + htc_face_tracker->update_tracking(tracking, offset); } void wivrn_session::operator()(from_headset::derived_pose && derived) diff --git a/server/driver/wivrn_session.h b/server/driver/wivrn_session.h index 50f1280b..43b3e0d9 100644 --- a/server/driver/wivrn_session.h +++ b/server/driver/wivrn_session.h @@ -42,6 +42,7 @@ namespace wivrn { class wivrn_eye_tracker; class wivrn_fb_face2_tracker; +class wivrn_htc_face_tracker; class wivrn_foveation; class wivrn_foveation_renderer; struct audio_device; @@ -105,6 +106,7 @@ class wivrn_session : public xrt_system_devices wivrn_controller right_hand; std::unique_ptr eye_tracker; std::unique_ptr fb_face2_tracker; + std::unique_ptr htc_face_tracker; std::unique_ptr foveation; wivrn_comp_target * comp_target; diff --git a/server/main.cpp b/server/main.cpp index 4c3c2406..be682556 100644 --- a/server/main.cpp +++ b/server/main.cpp @@ -782,7 +782,7 @@ void on_headset_info_packet(const wivrn::from_headset::headset_info_packet & inf wivrn_server_set_hand_tracking(dbus_server, info.hand_tracking); wivrn_server_set_eye_gaze(dbus_server, info.eye_gaze); - wivrn_server_set_face_tracking(dbus_server, info.face_tracking2_fb); + wivrn_server_set_face_tracking(dbus_server, info.face_tracking != wivrn::from_headset::face_type::none); std::vector codecs; static const std::map codec_names = {