From 261bb2c600c3aff84f1cadec2d9339d374e75fa7 Mon Sep 17 00:00:00 2001
From: galister <22305755+galister@users.noreply.github.com>
Date: Sat, 16 Nov 2024 16:33:47 +0900
Subject: [PATCH] pass through HTC_facial_tracking
---
AndroidManifest.xml | 2 +
client/application.cpp | 20 +++
client/application.h | 26 ++++
client/configuration.cpp | 16 ++-
client/scenes/lobby_gui.cpp | 11 ++
client/scenes/stream.cpp | 10 +-
client/scenes/stream_tracking.cpp | 30 ++++-
client/xr/htc_face_tracker.cpp | 71 +++++++++++
client/xr/htc_face_tracker.h | 43 +++++++
client/xr/session.cpp | 17 +++
client/xr/session.h | 2 +
client/xr/system.cpp | 18 +++
client/xr/system.h | 1 +
common/wivrn_packets.h | 18 ++-
server/CMakeLists.txt | 1 +
server/driver/wivrn_htc_face_tracker.cpp | 154 +++++++++++++++++++++++
server/driver/wivrn_htc_face_tracker.h | 114 +++++++++++++++++
server/driver/wivrn_session.cpp | 22 +++-
server/driver/wivrn_session.h | 2 +
server/main.cpp | 2 +-
20 files changed, 567 insertions(+), 13 deletions(-)
create mode 100644 client/xr/htc_face_tracker.cpp
create mode 100644 client/xr/htc_face_tracker.h
create mode 100644 server/driver/wivrn_htc_face_tracker.cpp
create mode 100644 server/driver/wivrn_htc_face_tracker.h
diff --git a/AndroidManifest.xml b/AndroidManifest.xml
index 159db79c..9914169e 100644
--- a/AndroidManifest.xml
+++ b/AndroidManifest.xml
@@ -55,6 +55,8 @@
+
+
#endif
@@ -127,6 +128,11 @@ class application : public singleton
bool fb_face_tracking2_supported = false;
xr::fb_face_tracker2 fb_face_tracker2;
+ bool htc_face_tracking_eye_supported = false;
+ xr::htc_face_tracker htc_face_tracker_eye;
+ bool htc_face_tracking_lip_supported = false;
+ xr::htc_face_tracker htc_face_tracker_lip;
+
bool eye_gaze_supported = false;
bool session_running = false;
@@ -418,6 +424,16 @@ public:
return instance().fb_face_tracking2_supported;
}
+ static bool get_htc_face_tracking_eye_supported()
+ {
+ return instance().htc_face_tracking_eye_supported;
+ }
+
+ static bool get_htc_face_tracking_lip_supported()
+ {
+ return instance().htc_face_tracking_lip_supported;
+ }
+
static bool get_eye_gaze_supported()
{
return instance().eye_gaze_supported;
@@ -438,6 +454,16 @@ public:
return instance().fb_face_tracker2;
}
+ static xr::htc_face_tracker & get_htc_face_tracker_eye()
+ {
+ return instance().htc_face_tracker_eye;
+ }
+
+ static xr::htc_face_tracker & get_htc_face_tracker_lip()
+ {
+ return instance().htc_face_tracker_lip;
+ }
+
static const std::vector & get_xr_extensions()
{
return instance().xr_extensions;
diff --git a/client/configuration.cpp b/client/configuration.cpp
index ffb55492..2fc7bd71 100644
--- a/client/configuration.cpp
+++ b/client/configuration.cpp
@@ -104,9 +104,19 @@ bool configuration::check_feature(feature f) const
return false;
break;
case feature::face_tracking:
- if (not application::get_fb_face_tracking2_supported())
- return false;
- break;
+ switch (guess_model())
+ {
+ case model::htc_vive_focus_3:
+ [[fallthrough]];
+ case model::htc_vive_xr_elite:
+ if (not(application::get_htc_face_tracking_eye_supported() or application::get_htc_face_tracking_lip_supported()))
+ return false;
+ break;
+ default:
+ if (not application::get_fb_face_tracking2_supported())
+ return false;
+ break;
+ }
}
}
#ifdef __ANDROID__
diff --git a/client/scenes/lobby_gui.cpp b/client/scenes/lobby_gui.cpp
index 7c4665b3..7725dd73 100644
--- a/client/scenes/lobby_gui.cpp
+++ b/client/scenes/lobby_gui.cpp
@@ -1206,6 +1206,17 @@ void scenes::lobby::draw_features_status(XrTime predicted_display_time)
});
}
+ if (application::get_htc_face_tracking_eye_supported() or application::get_htc_face_tracking_lip_supported())
+ {
+ items.push_back({
+ .f = feature::face_tracking,
+ .tooltip_enabled = _("Face tracking is enabled"),
+ .tooltip_disabled = _("Face tracking is disabled"),
+ .icon_enabled = ICON_FA_FACE_KISS_WINK_HEART,
+ .icon_disabled = ICON_FA_FACE_MEH_BLANK,
+ });
+ }
+
// Get statuses
for (auto & i: items)
{
diff --git a/client/scenes/stream.cpp b/client/scenes/stream.cpp
index 2b772f22..2934af9b 100644
--- a/client/scenes/stream.cpp
+++ b/client/scenes/stream.cpp
@@ -155,7 +155,15 @@ std::shared_ptr scenes::stream::create(std::unique_ptrsystem.passthrough_supported() != xr::system::passthrough_type::no_passthrough;
info.system_name = std::string(self->system.properties().systemName);
diff --git a/client/scenes/stream_tracking.cpp b/client/scenes/stream_tracking.cpp
index 66c18ece..8fc0d8a3 100644
--- a/client/scenes/stream_tracking.cpp
+++ b/client/scenes/stream_tracking.cpp
@@ -19,6 +19,7 @@
#include "application.h"
#include "stream.h"
+#include "wivrn_packets.h"
#include
#include
#include
@@ -210,7 +211,14 @@ void scenes::stream::tracking()
std::vector packets;
const bool hand_tracking = config.check_feature(feature::hand_tracking);
- const bool face_tracking = config.check_feature(feature::face_tracking);
+ from_headset::face_type face_tracking = from_headset::face_type::none;
+ if (config.check_feature(feature::face_tracking))
+ {
+ if (application::get_fb_face_tracking2_supported())
+ face_tracking = from_headset::face_type::fb2;
+ else if (application::get_htc_face_tracking_eye_supported() or application::get_htc_face_tracking_lip_supported())
+ face_tracking = from_headset::face_type::htc;
+ }
on_interaction_profile_changed({});
@@ -291,9 +299,25 @@ void scenes::stream::tracking()
locate_hands(application::get_right_hand(), world_space, t0 + Δt));
}
- if (face_tracking and control.enabled[size_t(tid::face)])
+ if (control.enabled[size_t(tid::face)])
{
- application::get_fb_face_tracker2().get_weights(t0 + Δt, packet.face.emplace());
+ switch (face_tracking)
+ {
+ case wivrn::from_headset::face_type::none:
+ break;
+ case wivrn::from_headset::face_type::fb2:
+ application::get_fb_face_tracker2().get_weights(t0 + Δt, packet.face.emplace());
+ break;
+ case wivrn::from_headset::face_type::htc:
+ auto face_htc = packet.face_htc.emplace();
+ face_htc.eye_active = false;
+ face_htc.lip_active = false;
+ if (application::get_htc_face_tracking_eye_supported())
+ application::get_htc_face_tracker_eye().get_weights(t0 + Δt, face_htc);
+ if (application::get_htc_face_tracking_lip_supported())
+ application::get_htc_face_tracker_lip().get_weights(t0 + Δt, face_htc);
+ break;
+ }
}
}
catch (const std::system_error & e)
diff --git a/client/xr/htc_face_tracker.cpp b/client/xr/htc_face_tracker.cpp
new file mode 100644
index 00000000..b0ad1ede
--- /dev/null
+++ b/client/xr/htc_face_tracker.cpp
@@ -0,0 +1,71 @@
+/*
+ * WiVRn VR streaming
+ * Copyright (C) 2024 galister
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see .
+ */
+
+#include "htc_face_tracker.h"
+#include "wivrn_packets.h"
+#include "xr/instance.h"
+#include
+
+static PFN_xrDestroyFacialTrackerHTC xrDestroyFacialTrackerHTC{};
+
+XrResult xr::destroy_htc_face_tracker(XrFacialTrackerHTC id)
+{
+ return xrDestroyFacialTrackerHTC(id);
+}
+
+xr::htc_face_tracker::htc_face_tracker(instance & inst, XrFacialTrackerHTC h, XrFacialTrackingTypeHTC facialTrackingTypeHTC)
+{
+ id = h;
+ xrGetFacialExpressionsHTC = inst.get_proc("xrGetFacialExpressionsHTC");
+ xrDestroyFacialTrackerHTC = inst.get_proc("xrDestroyFacialTrackerHTC");
+ trackerType = facialTrackingTypeHTC;
+}
+
+void xr::htc_face_tracker::get_weights(XrTime time, wivrn::from_headset::tracking::htc_face & out_expressions)
+{
+ if (!id || !xrGetFacialExpressionsHTC)
+ return;
+
+ XrFacialExpressionsHTC expressions{
+ .type = XR_TYPE_FACIAL_EXPRESSIONS_HTC,
+ .next = nullptr,
+ .sampleTime = time,
+ };
+
+ bool * is_active;
+
+ switch (trackerType)
+ {
+ case XR_FACIAL_TRACKING_TYPE_EYE_DEFAULT_HTC:
+ expressions.expressionCount = out_expressions.eye.size();
+ expressions.expressionWeightings = out_expressions.eye.data();
+ is_active = &out_expressions.eye_active;
+ break;
+ case XR_FACIAL_TRACKING_TYPE_LIP_DEFAULT_HTC:
+ expressions.expressionCount = out_expressions.lip.size();
+ expressions.expressionWeightings = out_expressions.lip.data();
+ is_active = &out_expressions.lip_active;
+ break;
+ default:
+ throw std::system_error(1, std::generic_category(), "htc_face_tracker with unknown trackerType");
+ }
+
+ CHECK_XR(xrGetFacialExpressionsHTC(id, &expressions));
+
+ *is_active = expressions.isActive;
+}
diff --git a/client/xr/htc_face_tracker.h b/client/xr/htc_face_tracker.h
new file mode 100644
index 00000000..f8381b54
--- /dev/null
+++ b/client/xr/htc_face_tracker.h
@@ -0,0 +1,43 @@
+/*
+ * WiVRn VR streaming
+ * Copyright (C) 2024 galister
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see .
+ */
+
+#pragma once
+
+#include "utils/handle.h"
+#include "wivrn_packets.h"
+#include
+
+namespace xr
+{
+class instance;
+
+XrResult destroy_htc_face_tracker(XrFacialTrackerHTC);
+
+class htc_face_tracker : public utils::handle
+{
+ PFN_xrGetFacialExpressionsHTC xrGetFacialExpressionsHTC{};
+ PFN_xrDestroyFacialTrackerHTC xrDestroyFacialTrackerHTC{};
+ XrFacialTrackingTypeHTC trackerType{};
+
+public:
+ htc_face_tracker() = default;
+ htc_face_tracker(instance & inst, XrFacialTrackerHTC h, XrFacialTrackingTypeHTC facialTrackingTypeHTC);
+
+ void get_weights(XrTime time, struct wivrn::from_headset::tracking::htc_face & out_expressions);
+};
+} // namespace xr
diff --git a/client/xr/session.cpp b/client/xr/session.cpp
index 428cc488..a97f1734 100644
--- a/client/xr/session.cpp
+++ b/client/xr/session.cpp
@@ -21,6 +21,7 @@
#include "application.h"
#include "details/enumerate.h"
+#include "openxr/openxr.h"
#include "utils/contains.h"
#include "xr/instance.h"
#include "xr/system.h"
@@ -119,6 +120,22 @@ xr::fb_face_tracker2 xr::session::create_fb_face_tracker2()
return {*inst, ft};
}
+xr::htc_face_tracker xr::session::create_htc_face_tracker(XrFacialTrackingTypeHTC facialTrackingType)
+{
+ XrFacialTrackerCreateInfoHTC create_info{
+ .type = XR_TYPE_FACIAL_TRACKER_CREATE_INFO_HTC,
+ .next = nullptr,
+ .facialTrackingType = facialTrackingType,
+ };
+
+ XrFacialTrackerHTC ft;
+
+ auto xrCreateFacialTrackerHTC = inst->get_proc("xrCreateFacialTrackerHTC");
+ assert(xrCreateFacialTrackerHTC);
+
+ CHECK_XR(xrCreateFacialTrackerHTC(id, &create_info, &ft));
+ return {*inst, ft, facialTrackingType};
+}
std::vector xr::session::get_swapchain_formats() const
{
std::vector formats;
diff --git a/client/xr/session.h b/client/xr/session.h
index 205884ea..d4a4bcd4 100644
--- a/client/xr/session.h
+++ b/client/xr/session.h
@@ -28,6 +28,7 @@
#include "fb_face_tracker2.h"
#include "hand_tracker.h"
+#include "htc_face_tracker.h"
#include "passthrough.h"
#include "space.h"
@@ -50,6 +51,7 @@ public:
space create_action_space(XrAction action, const XrPosef & pose = {{0, 0, 0, 1}, {0, 0, 0}});
hand_tracker create_hand_tracker(XrHandEXT hand, XrHandJointSetEXT hand_joint_set = XR_HAND_JOINT_SET_DEFAULT_EXT);
fb_face_tracker2 create_fb_face_tracker2();
+ htc_face_tracker create_htc_face_tracker(XrFacialTrackingTypeHTC);
std::vector get_swapchain_formats() const;
diff --git a/client/xr/system.cpp b/client/xr/system.cpp
index 00217a4c..33769ba7 100644
--- a/client/xr/system.cpp
+++ b/client/xr/system.cpp
@@ -123,6 +123,24 @@ XrSystemFaceTrackingProperties2FB xr::system::fb_face_tracking2_properties() con
return face_tracking_prop;
}
+XrSystemFacialTrackingPropertiesHTC xr::system::htc_face_tracking_properties() const
+{
+ if (!id)
+ throw std::invalid_argument("this");
+
+ XrSystemFacialTrackingPropertiesHTC face_tracking_prop{
+ .type = XR_TYPE_SYSTEM_FACIAL_TRACKING_PROPERTIES_HTC,
+ };
+
+ XrSystemProperties prop{
+ .type = XR_TYPE_SYSTEM_PROPERTIES,
+ .next = &face_tracking_prop,
+ };
+ CHECK_XR(xrGetSystemProperties(*inst, id, &prop));
+
+ return face_tracking_prop;
+}
+
xr::system::passthrough_type xr::system::passthrough_supported() const
{
if (utils::contains(environment_blend_modes(XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO), XR_ENVIRONMENT_BLEND_MODE_ALPHA_BLEND))
diff --git a/client/xr/system.h b/client/xr/system.h
index e41c4c36..0a1b76fc 100644
--- a/client/xr/system.h
+++ b/client/xr/system.h
@@ -63,6 +63,7 @@ public:
XrSystemHandTrackingPropertiesEXT hand_tracking_properties() const;
XrSystemEyeGazeInteractionPropertiesEXT eye_gaze_interaction_properties() const;
XrSystemFaceTrackingProperties2FB fb_face_tracking2_properties() const;
+ XrSystemFacialTrackingPropertiesHTC htc_face_tracking_properties() const;
passthrough_type passthrough_supported() const;
XrGraphicsRequirementsVulkan2KHR graphics_requirements() const;
diff --git a/common/wivrn_packets.h b/common/wivrn_packets.h
index c52a9b90..be69c3b8 100644
--- a/common/wivrn_packets.h
+++ b/common/wivrn_packets.h
@@ -130,6 +130,13 @@ struct visibility_mask_changed
uint8_t view_index;
};
+enum face_type : uint8_t
+{
+ none,
+ fb2,
+ htc,
+};
+
struct headset_info_packet
{
uint32_t recommended_eye_width;
@@ -146,9 +153,9 @@ struct headset_info_packet
std::array fov;
bool hand_tracking;
bool eye_gaze;
- bool face_tracking2_fb;
bool palm_pose;
bool passthrough;
+ face_type face_tracking;
std::vector supported_codecs; // from preferred to least preferred
std::string system_name;
};
@@ -208,6 +215,15 @@ struct tracking
bool is_eye_following_blendshapes_valid;
};
std::optional face;
+
+ struct htc_face
+ {
+ std::array eye;
+ std::array lip;
+ bool eye_active;
+ bool lip_active;
+ };
+ std::optional face_htc;
};
struct trackings
diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt
index c7e92172..d4497f12 100644
--- a/server/CMakeLists.txt
+++ b/server/CMakeLists.txt
@@ -84,6 +84,7 @@ add_executable(wivrn-server
driver/wivrn_controller.cpp
driver/wivrn_eye_tracker.cpp
driver/wivrn_fb_face2_tracker.cpp
+ driver/wivrn_htc_face_tracker.cpp
driver/wivrn_foveation.cpp
driver/pose_list.cpp
driver/view_list.cpp
diff --git a/server/driver/wivrn_htc_face_tracker.cpp b/server/driver/wivrn_htc_face_tracker.cpp
new file mode 100644
index 00000000..9fd92c32
--- /dev/null
+++ b/server/driver/wivrn_htc_face_tracker.cpp
@@ -0,0 +1,154 @@
+/*
+ * WiVRn VR streaming
+ * Copyright (C) 2024 Guillaume Meunier
+ * Copyright (C) 2024 Patrick Nicolas
+ * Copyright (C) 2024 galister
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see .
+ */
+
+#include "wivrn_htc_face_tracker.h"
+
+#include "wivrn_packets.h"
+#include "wivrn_session.h"
+
+#include "util/u_logging.h"
+#include "xrt/xrt_defines.h"
+#include "xrt/xrt_device.h"
+#include "xrt/xrt_results.h"
+
+#include
+#include
+#include
+#include
+
+namespace wivrn
+{
+
+static void wivrn_htc_face_tracker_destroy(xrt_device * xdev);
+
+static xrt_result_t wivrn_htc_face_tracker_update_inputs(xrt_device * xdev)
+{
+ static_cast(xdev)->update_inputs();
+ return XRT_SUCCESS;
+}
+
+static xrt_result_t wivrn_htc_face_tracker_get_face_tracking(struct xrt_device * xdev,
+ enum xrt_input_name facial_expression_type,
+ int64_t at_timestamp_ns,
+ struct xrt_facial_expression_set * out_value);
+
+wivrn_htc_face_tracker::wivrn_htc_face_tracker(xrt_device * hmd,
+ wivrn::wivrn_session & cnx) :
+ xrt_device{}, cnx(cnx)
+{
+ xrt_device * base = this;
+ base->tracking_origin = hmd->tracking_origin;
+ base->get_face_tracking = wivrn_htc_face_tracker_get_face_tracking;
+ base->update_inputs = wivrn_htc_face_tracker_update_inputs;
+ base->destroy = wivrn_htc_face_tracker_destroy;
+ name = XRT_DEVICE_HTC_FACE_TRACKING;
+ device_type = XRT_DEVICE_TYPE_FACE_TRACKER;
+ face_tracking_supported = true;
+
+ // Print name.
+ strcpy(str, "WiVRn HTC Face Tracker");
+ strcpy(serial, "WiVRn HTC Face Tracker");
+
+ // Setup input.
+ inputs_array[0].name = XRT_INPUT_HTC_EYE_FACE_TRACKING;
+ inputs_array[0].active = true;
+
+ inputs_array[1].name = XRT_INPUT_HTC_LIP_FACE_TRACKING;
+ inputs_array[1].active = true;
+
+ inputs = inputs_array.data();
+ input_count = inputs_array.size();
+}
+
+void wivrn_htc_face_tracker::update_inputs()
+{
+ // Empty
+}
+
+void wivrn_htc_face_tracker::update_tracking(const from_headset::tracking & tracking, const clock_offset & offset)
+{
+ if (not(tracking.face_htc))
+ return;
+ const auto & face = *tracking.face_htc;
+
+ wivrn_htc_face_data data{
+ .eye = face.eye,
+ .lip = face.lip,
+ .eye_active = face.eye_active,
+ .lip_active = face.lip_active};
+
+ if (not face_list.update_tracking(tracking.production_timestamp, tracking.timestamp, data, offset))
+ cnx.set_enabled(to_headset::tracking_control::id::face, false);
+}
+
+xrt_result_t wivrn_htc_face_tracker::get_face_tracking(enum xrt_input_name facial_expression_type, int64_t at_timestamp_ns, struct xrt_facial_expression_set * inout_value)
+{
+ if (facial_expression_type == XRT_INPUT_HTC_EYE_FACE_TRACKING)
+ {
+ cnx.set_enabled(to_headset::tracking_control::id::face, true);
+ auto [_, data] = face_list.get_at(at_timestamp_ns);
+
+ inout_value->base_expression_set_htc.is_active = data.eye_active;
+ inout_value->base_expression_set_htc.sample_time_ns = at_timestamp_ns;
+
+ if (not data.eye_active)
+ return XRT_SUCCESS;
+
+ memcpy(&inout_value->eye_expression_set_htc.expression_weights, data.eye.data(), sizeof(float) * data.eye.size());
+
+ return XRT_SUCCESS;
+ }
+ else if (facial_expression_type == XRT_INPUT_HTC_LIP_FACE_TRACKING)
+ {
+ cnx.set_enabled(to_headset::tracking_control::id::face, true);
+ auto [_, data] = face_list.get_at(at_timestamp_ns);
+
+ inout_value->base_expression_set_htc.is_active = data.lip_active;
+ inout_value->base_expression_set_htc.sample_time_ns = at_timestamp_ns;
+
+ if (not data.lip_active)
+ return XRT_SUCCESS;
+
+ memcpy(&inout_value->lip_expression_set_htc.expression_weights, data.lip.data(), sizeof(float) * data.lip.size());
+
+ return XRT_SUCCESS;
+ }
+
+ return XRT_ERROR_NOT_IMPLEMENTED;
+}
+
+/*
+ *
+ * Functions
+ *
+ */
+
+static void wivrn_htc_face_tracker_destroy(xrt_device * xdev)
+{
+}
+
+static xrt_result_t wivrn_htc_face_tracker_get_face_tracking(struct xrt_device * xdev,
+ enum xrt_input_name facial_expression_type,
+ int64_t at_timestamp_ns,
+ struct xrt_facial_expression_set * inout_value)
+{
+ return static_cast(xdev)->get_face_tracking(facial_expression_type, at_timestamp_ns, inout_value);
+}
+} // namespace wivrn
diff --git a/server/driver/wivrn_htc_face_tracker.h b/server/driver/wivrn_htc_face_tracker.h
new file mode 100644
index 00000000..84c6de23
--- /dev/null
+++ b/server/driver/wivrn_htc_face_tracker.h
@@ -0,0 +1,114 @@
+/*
+ * WiVRn VR streaming
+ * Copyright (C) 2024 Guillaume Meunier
+ * Copyright (C) 2024 Patrick Nicolas
+ * Copyright (C) 2024 galister
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see .
+ */
+
+#pragma once
+
+#include "wivrn_packets.h"
+#include "xrt/xrt_defines.h"
+#include "xrt/xrt_device.h"
+
+#include "history.h"
+
+#include
+#include
+#include
+
+namespace wivrn
+{
+
+struct clock_offset;
+class wivrn_session;
+
+struct wivrn_htc_face_data
+{
+ std::array eye;
+ std::array lip;
+ bool eye_active;
+ bool lip_active;
+};
+
+class htc_face_list : public history
+{
+public:
+ static wivrn_htc_face_data interpolate(const wivrn_htc_face_data & a, const wivrn_htc_face_data & b, float t)
+ {
+ wivrn_htc_face_data result = b;
+
+ if (not a.eye_active)
+ {
+ result.eye = b.eye;
+ result.eye_active = b.eye_active;
+ }
+ else if (not b.eye_active)
+ {
+ result.eye = a.eye;
+ result.eye_active = a.eye_active;
+ }
+ else
+ {
+ result.eye_active = true;
+ for (size_t i = 0; i < result.eye.size(); i++)
+ {
+ result.eye[i] = std::clamp(std::lerp(a.eye[i], b.eye[i], t), 0.0f, 1.0f);
+ }
+ }
+
+ if (not a.lip_active)
+ {
+ result.lip = b.lip;
+ result.lip_active = b.lip_active;
+ }
+ else if (not b.lip_active)
+ {
+ result.lip = a.lip;
+ result.lip_active = a.lip_active;
+ }
+ else
+ {
+ result.lip_active = true;
+ for (size_t i = 0; i < result.lip.size(); i++)
+ {
+ result.lip[i] = std::clamp(std::lerp(a.lip[i], b.lip[i], t), 0.0f, 1.0f);
+ }
+ }
+ return result;
+ }
+
+ bool update_tracking(const XrTime & production_timestamp, const XrTime & timestamp, const wivrn_htc_face_data & data, const clock_offset & offset)
+ {
+ return this->add_sample(production_timestamp, timestamp, data, offset);
+ }
+};
+
+class wivrn_htc_face_tracker : public xrt_device
+{
+ htc_face_list face_list;
+ std::array inputs_array;
+
+ wivrn::wivrn_session & cnx;
+
+public:
+ wivrn_htc_face_tracker(xrt_device * hmd, wivrn::wivrn_session & cnx);
+
+ void update_inputs();
+ void update_tracking(const from_headset::tracking &, const clock_offset &);
+ xrt_result_t get_face_tracking(enum xrt_input_name facial_expression_type, int64_t at_timestamp_ns, struct xrt_facial_expression_set * out_value);
+};
+} // namespace wivrn
diff --git a/server/driver/wivrn_session.cpp b/server/driver/wivrn_session.cpp
index 5af128e5..0c8ba398 100644
--- a/server/driver/wivrn_session.cpp
+++ b/server/driver/wivrn_session.cpp
@@ -34,8 +34,10 @@
#include "wivrn_eye_tracker.h"
#include "wivrn_fb_face2_tracker.h"
#include "wivrn_foveation.h"
+#include "wivrn_htc_face_tracker.h"
#include "wivrn_ipc.h"
+#include "wivrn_packets.h"
#include "xrt/xrt_session.h"
#include
#include
@@ -197,11 +199,20 @@ wivrn::wivrn_session::wivrn_session(std::unique_ptr connection
static_roles.eyes = eye_tracker.get();
xdevs[xdev_count++] = eye_tracker.get();
}
- if (get_info().face_tracking2_fb || is_forced_extension("FB_face_tracking2"))
+ switch (get_info().face_tracking)
{
- fb_face2_tracker = std::make_unique(&hmd, *this);
- static_roles.face = fb_face2_tracker.get();
- xdevs[xdev_count++] = fb_face2_tracker.get();
+ case wivrn::from_headset::face_type::fb2:
+ fb_face2_tracker = std::make_unique(&hmd, *this);
+ static_roles.face = fb_face2_tracker.get();
+ xdevs[xdev_count++] = fb_face2_tracker.get();
+ break;
+ case wivrn::from_headset::face_type::htc:
+ htc_face_tracker = std::make_unique(&hmd, *this);
+ static_roles.face = htc_face_tracker.get();
+ xdevs[xdev_count++] = htc_face_tracker.get();
+ break;
+ default:
+ break;
}
#if WIVRN_FEATURE_SOLARXR
@@ -322,8 +333,11 @@ void wivrn_session::operator()(const from_headset::tracking & tracking)
eye_tracker->update_tracking(tracking, offset);
if (foveation)
foveation->update_tracking(tracking, offset);
+
if (fb_face2_tracker)
fb_face2_tracker->update_tracking(tracking, offset);
+ else if (htc_face_tracker)
+ htc_face_tracker->update_tracking(tracking, offset);
}
void wivrn_session::operator()(from_headset::derived_pose && derived)
diff --git a/server/driver/wivrn_session.h b/server/driver/wivrn_session.h
index 50f1280b..43b3e0d9 100644
--- a/server/driver/wivrn_session.h
+++ b/server/driver/wivrn_session.h
@@ -42,6 +42,7 @@ namespace wivrn
{
class wivrn_eye_tracker;
class wivrn_fb_face2_tracker;
+class wivrn_htc_face_tracker;
class wivrn_foveation;
class wivrn_foveation_renderer;
struct audio_device;
@@ -105,6 +106,7 @@ class wivrn_session : public xrt_system_devices
wivrn_controller right_hand;
std::unique_ptr eye_tracker;
std::unique_ptr fb_face2_tracker;
+ std::unique_ptr htc_face_tracker;
std::unique_ptr foveation;
wivrn_comp_target * comp_target;
diff --git a/server/main.cpp b/server/main.cpp
index 4c3c2406..be682556 100644
--- a/server/main.cpp
+++ b/server/main.cpp
@@ -782,7 +782,7 @@ void on_headset_info_packet(const wivrn::from_headset::headset_info_packet & inf
wivrn_server_set_hand_tracking(dbus_server, info.hand_tracking);
wivrn_server_set_eye_gaze(dbus_server, info.eye_gaze);
- wivrn_server_set_face_tracking(dbus_server, info.face_tracking2_fb);
+ wivrn_server_set_face_tracking(dbus_server, info.face_tracking != wivrn::from_headset::face_type::none);
std::vector codecs;
static const std::map codec_names = {