mirror of
https://github.com/WiVRn/WiVRn.git
synced 2026-11-21 09:31:33 -08:00
pass through HTC_facial_tracking
This commit is contained in:
1 parent
22186f0043
commit
261bb2c600
20 files changed
+567
-13
No files matched your search
@@ -55,6 +55,8 @@
|
||||
|
||||
<!-- See https://hub.vive.com/storage/docs/en-us/ConfigureAppCapabilities.html -->
|
||||
<uses-feature android:name="wave.feature.handtracking" android:required="false" />
|
||||
<uses-feature android:name="wave.feature.eyetracking" android:required="false" />
|
||||
<uses-feature android:name="wave.feature.lipexpression" android:required="false" />
|
||||
|
||||
<application
|
||||
android:allowBackup="false"
|
||||
|
||||
@@ -777,6 +777,7 @@ void application::initialize()
|
||||
opt_extensions.push_back(XR_EXT_EYE_GAZE_INTERACTION_EXTENSION_NAME);
|
||||
opt_extensions.push_back(XR_FB_PASSTHROUGH_EXTENSION_NAME);
|
||||
opt_extensions.push_back(XR_HTC_PASSTHROUGH_EXTENSION_NAME);
|
||||
opt_extensions.push_back(XR_HTC_FACIAL_TRACKING_EXTENSION_NAME);
|
||||
opt_extensions.push_back(XR_FB_FACE_TRACKING2_EXTENSION_NAME);
|
||||
opt_extensions.push_back(XR_EXT_PALM_POSE_EXTENSION_NAME);
|
||||
opt_extensions.push_back(XR_KHR_COMPOSITION_LAYER_DEPTH_EXTENSION_NAME);
|
||||
@@ -849,6 +850,15 @@ void application::initialize()
|
||||
fb_face_tracking2_supported = fb_face2_properties.supportsVisualFaceTracking;
|
||||
}
|
||||
|
||||
if (utils::contains(xr_extensions, XR_HTC_FACIAL_TRACKING_EXTENSION_NAME))
|
||||
{
|
||||
XrSystemFacialTrackingPropertiesHTC htc_face_properties = xr_system_id.htc_face_tracking_properties();
|
||||
spdlog::info(" HTC eye tracking support: {}", (bool)htc_face_properties.supportEyeFacialTracking);
|
||||
spdlog::info(" HTC lip tracking support: {}", (bool)htc_face_properties.supportLipFacialTracking);
|
||||
htc_face_tracking_eye_supported = htc_face_properties.supportEyeFacialTracking;
|
||||
htc_face_tracking_lip_supported = htc_face_properties.supportLipFacialTracking;
|
||||
}
|
||||
|
||||
switch (xr_system_id.passthrough_supported())
|
||||
{
|
||||
case xr::system::passthrough_type::no_passthrough:
|
||||
@@ -892,6 +902,16 @@ void application::initialize()
|
||||
fb_face_tracker2 = xr_session.create_fb_face_tracker2();
|
||||
}
|
||||
|
||||
if (htc_face_tracking_eye_supported)
|
||||
{
|
||||
htc_face_tracker_eye = xr_session.create_htc_face_tracker(XR_FACIAL_TRACKING_TYPE_EYE_DEFAULT_HTC);
|
||||
}
|
||||
|
||||
if (htc_face_tracking_lip_supported)
|
||||
{
|
||||
htc_face_tracker_lip = xr_session.create_htc_face_tracker(XR_FACIAL_TRACKING_TYPE_LIP_DEFAULT_HTC);
|
||||
}
|
||||
|
||||
vk::CommandPoolCreateInfo cmdpool_create_info;
|
||||
cmdpool_create_info.queueFamilyIndex = vk_queue_family_index;
|
||||
cmdpool_create_info.flags = vk::CommandPoolCreateFlagBits::eResetCommandBuffer;
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
#include "wifi_lock.h"
|
||||
#include "xr/fb_face_tracker2.h"
|
||||
#include "xr/hand_tracker.h"
|
||||
#include "xr/htc_face_tracker.h"
|
||||
#ifdef __ANDROID__
|
||||
#include <android_native_app_glue.h>
|
||||
#endif
|
||||
@@ -127,6 +128,11 @@ class application : public singleton<application>
|
||||
bool fb_face_tracking2_supported = false;
|
||||
xr::fb_face_tracker2 fb_face_tracker2;
|
||||
|
||||
bool htc_face_tracking_eye_supported = false;
|
||||
xr::htc_face_tracker htc_face_tracker_eye;
|
||||
bool htc_face_tracking_lip_supported = false;
|
||||
xr::htc_face_tracker htc_face_tracker_lip;
|
||||
|
||||
bool eye_gaze_supported = false;
|
||||
|
||||
bool session_running = false;
|
||||
@@ -418,6 +424,16 @@ public:
|
||||
return instance().fb_face_tracking2_supported;
|
||||
}
|
||||
|
||||
static bool get_htc_face_tracking_eye_supported()
|
||||
{
|
||||
return instance().htc_face_tracking_eye_supported;
|
||||
}
|
||||
|
||||
static bool get_htc_face_tracking_lip_supported()
|
||||
{
|
||||
return instance().htc_face_tracking_lip_supported;
|
||||
}
|
||||
|
||||
static bool get_eye_gaze_supported()
|
||||
{
|
||||
return instance().eye_gaze_supported;
|
||||
@@ -438,6 +454,16 @@ public:
|
||||
return instance().fb_face_tracker2;
|
||||
}
|
||||
|
||||
static xr::htc_face_tracker & get_htc_face_tracker_eye()
|
||||
{
|
||||
return instance().htc_face_tracker_eye;
|
||||
}
|
||||
|
||||
static xr::htc_face_tracker & get_htc_face_tracker_lip()
|
||||
{
|
||||
return instance().htc_face_tracker_lip;
|
||||
}
|
||||
|
||||
static const std::vector<std::string> & get_xr_extensions()
|
||||
{
|
||||
return instance().xr_extensions;
|
||||
|
||||
@@ -104,9 +104,19 @@ bool configuration::check_feature(feature f) const
|
||||
return false;
|
||||
break;
|
||||
case feature::face_tracking:
|
||||
if (not application::get_fb_face_tracking2_supported())
|
||||
return false;
|
||||
break;
|
||||
switch (guess_model())
|
||||
{
|
||||
case model::htc_vive_focus_3:
|
||||
[[fallthrough]];
|
||||
case model::htc_vive_xr_elite:
|
||||
if (not(application::get_htc_face_tracking_eye_supported() or application::get_htc_face_tracking_lip_supported()))
|
||||
return false;
|
||||
break;
|
||||
default:
|
||||
if (not application::get_fb_face_tracking2_supported())
|
||||
return false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
#ifdef __ANDROID__
|
||||
|
||||
@@ -1206,6 +1206,17 @@ void scenes::lobby::draw_features_status(XrTime predicted_display_time)
|
||||
});
|
||||
}
|
||||
|
||||
if (application::get_htc_face_tracking_eye_supported() or application::get_htc_face_tracking_lip_supported())
|
||||
{
|
||||
items.push_back({
|
||||
.f = feature::face_tracking,
|
||||
.tooltip_enabled = _("Face tracking is enabled"),
|
||||
.tooltip_disabled = _("Face tracking is disabled"),
|
||||
.icon_enabled = ICON_FA_FACE_KISS_WINK_HEART,
|
||||
.icon_disabled = ICON_FA_FACE_MEH_BLANK,
|
||||
});
|
||||
}
|
||||
|
||||
// Get statuses
|
||||
for (auto & i: items)
|
||||
{
|
||||
|
||||
@@ -155,7 +155,15 @@ std::shared_ptr<scenes::stream> scenes::stream::create(std::unique_ptr<wivrn_ses
|
||||
|
||||
info.hand_tracking = config.check_feature(feature::hand_tracking);
|
||||
info.eye_gaze = config.check_feature(feature::eye_gaze);
|
||||
info.face_tracking2_fb = config.check_feature(feature::face_tracking);
|
||||
|
||||
if (config.check_feature(feature::face_tracking))
|
||||
{
|
||||
if (application::get_fb_face_tracking2_supported())
|
||||
info.face_tracking = from_headset::face_type::fb2;
|
||||
else if (application::get_htc_face_tracking_eye_supported() or application::get_htc_face_tracking_lip_supported())
|
||||
info.face_tracking = from_headset::face_type::htc;
|
||||
}
|
||||
|
||||
info.palm_pose = application::space(xr::spaces::palm_left) or application::space(xr::spaces::palm_right);
|
||||
info.passthrough = self->system.passthrough_supported() != xr::system::passthrough_type::no_passthrough;
|
||||
info.system_name = std::string(self->system.properties().systemName);
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
|
||||
#include "application.h"
|
||||
#include "stream.h"
|
||||
#include "wivrn_packets.h"
|
||||
#include <ranges>
|
||||
#include <spdlog/spdlog.h>
|
||||
#include <thread>
|
||||
@@ -210,7 +211,14 @@ void scenes::stream::tracking()
|
||||
std::vector<serialization_packet> packets;
|
||||
|
||||
const bool hand_tracking = config.check_feature(feature::hand_tracking);
|
||||
const bool face_tracking = config.check_feature(feature::face_tracking);
|
||||
from_headset::face_type face_tracking = from_headset::face_type::none;
|
||||
if (config.check_feature(feature::face_tracking))
|
||||
{
|
||||
if (application::get_fb_face_tracking2_supported())
|
||||
face_tracking = from_headset::face_type::fb2;
|
||||
else if (application::get_htc_face_tracking_eye_supported() or application::get_htc_face_tracking_lip_supported())
|
||||
face_tracking = from_headset::face_type::htc;
|
||||
}
|
||||
|
||||
on_interaction_profile_changed({});
|
||||
|
||||
@@ -291,9 +299,25 @@ void scenes::stream::tracking()
|
||||
locate_hands(application::get_right_hand(), world_space, t0 + Δt));
|
||||
}
|
||||
|
||||
if (face_tracking and control.enabled[size_t(tid::face)])
|
||||
if (control.enabled[size_t(tid::face)])
|
||||
{
|
||||
application::get_fb_face_tracker2().get_weights(t0 + Δt, packet.face.emplace());
|
||||
switch (face_tracking)
|
||||
{
|
||||
case wivrn::from_headset::face_type::none:
|
||||
break;
|
||||
case wivrn::from_headset::face_type::fb2:
|
||||
application::get_fb_face_tracker2().get_weights(t0 + Δt, packet.face.emplace());
|
||||
break;
|
||||
case wivrn::from_headset::face_type::htc:
|
||||
auto face_htc = packet.face_htc.emplace();
|
||||
face_htc.eye_active = false;
|
||||
face_htc.lip_active = false;
|
||||
if (application::get_htc_face_tracking_eye_supported())
|
||||
application::get_htc_face_tracker_eye().get_weights(t0 + Δt, face_htc);
|
||||
if (application::get_htc_face_tracking_lip_supported())
|
||||
application::get_htc_face_tracker_lip().get_weights(t0 + Δt, face_htc);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (const std::system_error & e)
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
/*
|
||||
* WiVRn VR streaming
|
||||
* Copyright (C) 2024 galister <[email protected]>
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "htc_face_tracker.h"
|
||||
#include "wivrn_packets.h"
|
||||
#include "xr/instance.h"
|
||||
#include <openxr/openxr.h>
|
||||
|
||||
static PFN_xrDestroyFacialTrackerHTC xrDestroyFacialTrackerHTC{};
|
||||
|
||||
XrResult xr::destroy_htc_face_tracker(XrFacialTrackerHTC id)
|
||||
{
|
||||
return xrDestroyFacialTrackerHTC(id);
|
||||
}
|
||||
|
||||
xr::htc_face_tracker::htc_face_tracker(instance & inst, XrFacialTrackerHTC h, XrFacialTrackingTypeHTC facialTrackingTypeHTC)
|
||||
{
|
||||
id = h;
|
||||
xrGetFacialExpressionsHTC = inst.get_proc<PFN_xrGetFacialExpressionsHTC>("xrGetFacialExpressionsHTC");
|
||||
xrDestroyFacialTrackerHTC = inst.get_proc<PFN_xrDestroyFacialTrackerHTC>("xrDestroyFacialTrackerHTC");
|
||||
trackerType = facialTrackingTypeHTC;
|
||||
}
|
||||
|
||||
void xr::htc_face_tracker::get_weights(XrTime time, wivrn::from_headset::tracking::htc_face & out_expressions)
|
||||
{
|
||||
if (!id || !xrGetFacialExpressionsHTC)
|
||||
return;
|
||||
|
||||
XrFacialExpressionsHTC expressions{
|
||||
.type = XR_TYPE_FACIAL_EXPRESSIONS_HTC,
|
||||
.next = nullptr,
|
||||
.sampleTime = time,
|
||||
};
|
||||
|
||||
bool * is_active;
|
||||
|
||||
switch (trackerType)
|
||||
{
|
||||
case XR_FACIAL_TRACKING_TYPE_EYE_DEFAULT_HTC:
|
||||
expressions.expressionCount = out_expressions.eye.size();
|
||||
expressions.expressionWeightings = out_expressions.eye.data();
|
||||
is_active = &out_expressions.eye_active;
|
||||
break;
|
||||
case XR_FACIAL_TRACKING_TYPE_LIP_DEFAULT_HTC:
|
||||
expressions.expressionCount = out_expressions.lip.size();
|
||||
expressions.expressionWeightings = out_expressions.lip.data();
|
||||
is_active = &out_expressions.lip_active;
|
||||
break;
|
||||
default:
|
||||
throw std::system_error(1, std::generic_category(), "htc_face_tracker with unknown trackerType");
|
||||
}
|
||||
|
||||
CHECK_XR(xrGetFacialExpressionsHTC(id, &expressions));
|
||||
|
||||
*is_active = expressions.isActive;
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
* WiVRn VR streaming
|
||||
* Copyright (C) 2024 galister <[email protected]>
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "utils/handle.h"
|
||||
#include "wivrn_packets.h"
|
||||
#include <openxr/openxr.h>
|
||||
|
||||
namespace xr
|
||||
{
|
||||
class instance;
|
||||
|
||||
XrResult destroy_htc_face_tracker(XrFacialTrackerHTC);
|
||||
|
||||
class htc_face_tracker : public utils::handle<XrFacialTrackerHTC, destroy_htc_face_tracker>
|
||||
{
|
||||
PFN_xrGetFacialExpressionsHTC xrGetFacialExpressionsHTC{};
|
||||
PFN_xrDestroyFacialTrackerHTC xrDestroyFacialTrackerHTC{};
|
||||
XrFacialTrackingTypeHTC trackerType{};
|
||||
|
||||
public:
|
||||
htc_face_tracker() = default;
|
||||
htc_face_tracker(instance & inst, XrFacialTrackerHTC h, XrFacialTrackingTypeHTC facialTrackingTypeHTC);
|
||||
|
||||
void get_weights(XrTime time, struct wivrn::from_headset::tracking::htc_face & out_expressions);
|
||||
};
|
||||
} // namespace xr
|
||||
@@ -21,6 +21,7 @@
|
||||
|
||||
#include "application.h"
|
||||
#include "details/enumerate.h"
|
||||
#include "openxr/openxr.h"
|
||||
#include "utils/contains.h"
|
||||
#include "xr/instance.h"
|
||||
#include "xr/system.h"
|
||||
@@ -119,6 +120,22 @@ xr::fb_face_tracker2 xr::session::create_fb_face_tracker2()
|
||||
return {*inst, ft};
|
||||
}
|
||||
|
||||
xr::htc_face_tracker xr::session::create_htc_face_tracker(XrFacialTrackingTypeHTC facialTrackingType)
|
||||
{
|
||||
XrFacialTrackerCreateInfoHTC create_info{
|
||||
.type = XR_TYPE_FACIAL_TRACKER_CREATE_INFO_HTC,
|
||||
.next = nullptr,
|
||||
.facialTrackingType = facialTrackingType,
|
||||
};
|
||||
|
||||
XrFacialTrackerHTC ft;
|
||||
|
||||
auto xrCreateFacialTrackerHTC = inst->get_proc<PFN_xrCreateFacialTrackerHTC>("xrCreateFacialTrackerHTC");
|
||||
assert(xrCreateFacialTrackerHTC);
|
||||
|
||||
CHECK_XR(xrCreateFacialTrackerHTC(id, &create_info, &ft));
|
||||
return {*inst, ft, facialTrackingType};
|
||||
}
|
||||
std::vector<vk::Format> xr::session::get_swapchain_formats() const
|
||||
{
|
||||
std::vector<vk::Format> formats;
|
||||
|
||||
@@ -28,6 +28,7 @@
|
||||
|
||||
#include "fb_face_tracker2.h"
|
||||
#include "hand_tracker.h"
|
||||
#include "htc_face_tracker.h"
|
||||
#include "passthrough.h"
|
||||
#include "space.h"
|
||||
|
||||
@@ -50,6 +51,7 @@ public:
|
||||
space create_action_space(XrAction action, const XrPosef & pose = {{0, 0, 0, 1}, {0, 0, 0}});
|
||||
hand_tracker create_hand_tracker(XrHandEXT hand, XrHandJointSetEXT hand_joint_set = XR_HAND_JOINT_SET_DEFAULT_EXT);
|
||||
fb_face_tracker2 create_fb_face_tracker2();
|
||||
htc_face_tracker create_htc_face_tracker(XrFacialTrackingTypeHTC);
|
||||
|
||||
std::vector<vk::Format> get_swapchain_formats() const;
|
||||
|
||||
|
||||
@@ -123,6 +123,24 @@ XrSystemFaceTrackingProperties2FB xr::system::fb_face_tracking2_properties() con
|
||||
return face_tracking_prop;
|
||||
}
|
||||
|
||||
XrSystemFacialTrackingPropertiesHTC xr::system::htc_face_tracking_properties() const
|
||||
{
|
||||
if (!id)
|
||||
throw std::invalid_argument("this");
|
||||
|
||||
XrSystemFacialTrackingPropertiesHTC face_tracking_prop{
|
||||
.type = XR_TYPE_SYSTEM_FACIAL_TRACKING_PROPERTIES_HTC,
|
||||
};
|
||||
|
||||
XrSystemProperties prop{
|
||||
.type = XR_TYPE_SYSTEM_PROPERTIES,
|
||||
.next = &face_tracking_prop,
|
||||
};
|
||||
CHECK_XR(xrGetSystemProperties(*inst, id, &prop));
|
||||
|
||||
return face_tracking_prop;
|
||||
}
|
||||
|
||||
xr::system::passthrough_type xr::system::passthrough_supported() const
|
||||
{
|
||||
if (utils::contains(environment_blend_modes(XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO), XR_ENVIRONMENT_BLEND_MODE_ALPHA_BLEND))
|
||||
|
||||
@@ -63,6 +63,7 @@ public:
|
||||
XrSystemHandTrackingPropertiesEXT hand_tracking_properties() const;
|
||||
XrSystemEyeGazeInteractionPropertiesEXT eye_gaze_interaction_properties() const;
|
||||
XrSystemFaceTrackingProperties2FB fb_face_tracking2_properties() const;
|
||||
XrSystemFacialTrackingPropertiesHTC htc_face_tracking_properties() const;
|
||||
passthrough_type passthrough_supported() const;
|
||||
|
||||
XrGraphicsRequirementsVulkan2KHR graphics_requirements() const;
|
||||
|
||||
+17
-1
@@ -130,6 +130,13 @@ struct visibility_mask_changed
|
||||
uint8_t view_index;
|
||||
};
|
||||
|
||||
enum face_type : uint8_t
|
||||
{
|
||||
none,
|
||||
fb2,
|
||||
htc,
|
||||
};
|
||||
|
||||
struct headset_info_packet
|
||||
{
|
||||
uint32_t recommended_eye_width;
|
||||
@@ -146,9 +153,9 @@ struct headset_info_packet
|
||||
std::array<XrFovf, 2> fov;
|
||||
bool hand_tracking;
|
||||
bool eye_gaze;
|
||||
bool face_tracking2_fb;
|
||||
bool palm_pose;
|
||||
bool passthrough;
|
||||
face_type face_tracking;
|
||||
std::vector<video_codec> supported_codecs; // from preferred to least preferred
|
||||
std::string system_name;
|
||||
};
|
||||
@@ -208,6 +215,15 @@ struct tracking
|
||||
bool is_eye_following_blendshapes_valid;
|
||||
};
|
||||
std::optional<fb_face2> face;
|
||||
|
||||
struct htc_face
|
||||
{
|
||||
std::array<float, XR_FACIAL_EXPRESSION_EYE_COUNT_HTC> eye;
|
||||
std::array<float, XR_FACIAL_EXPRESSION_LIP_COUNT_HTC> lip;
|
||||
bool eye_active;
|
||||
bool lip_active;
|
||||
};
|
||||
std::optional<htc_face> face_htc;
|
||||
};
|
||||
|
||||
struct trackings
|
||||
|
||||
@@ -84,6 +84,7 @@ add_executable(wivrn-server
|
||||
driver/wivrn_controller.cpp
|
||||
driver/wivrn_eye_tracker.cpp
|
||||
driver/wivrn_fb_face2_tracker.cpp
|
||||
driver/wivrn_htc_face_tracker.cpp
|
||||
driver/wivrn_foveation.cpp
|
||||
driver/pose_list.cpp
|
||||
driver/view_list.cpp
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
/*
|
||||
* WiVRn VR streaming
|
||||
* Copyright (C) 2024 Guillaume Meunier <[email protected]>
|
||||
* Copyright (C) 2024 Patrick Nicolas <[email protected]>
|
||||
* Copyright (C) 2024 galister <[email protected]>
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "wivrn_htc_face_tracker.h"
|
||||
|
||||
#include "wivrn_packets.h"
|
||||
#include "wivrn_session.h"
|
||||
|
||||
#include "util/u_logging.h"
|
||||
#include "xrt/xrt_defines.h"
|
||||
#include "xrt/xrt_device.h"
|
||||
#include "xrt/xrt_results.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
#include <openxr/openxr.h>
|
||||
|
||||
namespace wivrn
|
||||
{
|
||||
|
||||
static void wivrn_htc_face_tracker_destroy(xrt_device * xdev);
|
||||
|
||||
static xrt_result_t wivrn_htc_face_tracker_update_inputs(xrt_device * xdev)
|
||||
{
|
||||
static_cast<wivrn_htc_face_tracker *>(xdev)->update_inputs();
|
||||
return XRT_SUCCESS;
|
||||
}
|
||||
|
||||
static xrt_result_t wivrn_htc_face_tracker_get_face_tracking(struct xrt_device * xdev,
|
||||
enum xrt_input_name facial_expression_type,
|
||||
int64_t at_timestamp_ns,
|
||||
struct xrt_facial_expression_set * out_value);
|
||||
|
||||
wivrn_htc_face_tracker::wivrn_htc_face_tracker(xrt_device * hmd,
|
||||
wivrn::wivrn_session & cnx) :
|
||||
xrt_device{}, cnx(cnx)
|
||||
{
|
||||
xrt_device * base = this;
|
||||
base->tracking_origin = hmd->tracking_origin;
|
||||
base->get_face_tracking = wivrn_htc_face_tracker_get_face_tracking;
|
||||
base->update_inputs = wivrn_htc_face_tracker_update_inputs;
|
||||
base->destroy = wivrn_htc_face_tracker_destroy;
|
||||
name = XRT_DEVICE_HTC_FACE_TRACKING;
|
||||
device_type = XRT_DEVICE_TYPE_FACE_TRACKER;
|
||||
face_tracking_supported = true;
|
||||
|
||||
// Print name.
|
||||
strcpy(str, "WiVRn HTC Face Tracker");
|
||||
strcpy(serial, "WiVRn HTC Face Tracker");
|
||||
|
||||
// Setup input.
|
||||
inputs_array[0].name = XRT_INPUT_HTC_EYE_FACE_TRACKING;
|
||||
inputs_array[0].active = true;
|
||||
|
||||
inputs_array[1].name = XRT_INPUT_HTC_LIP_FACE_TRACKING;
|
||||
inputs_array[1].active = true;
|
||||
|
||||
inputs = inputs_array.data();
|
||||
input_count = inputs_array.size();
|
||||
}
|
||||
|
||||
void wivrn_htc_face_tracker::update_inputs()
|
||||
{
|
||||
// Empty
|
||||
}
|
||||
|
||||
void wivrn_htc_face_tracker::update_tracking(const from_headset::tracking & tracking, const clock_offset & offset)
|
||||
{
|
||||
if (not(tracking.face_htc))
|
||||
return;
|
||||
const auto & face = *tracking.face_htc;
|
||||
|
||||
wivrn_htc_face_data data{
|
||||
.eye = face.eye,
|
||||
.lip = face.lip,
|
||||
.eye_active = face.eye_active,
|
||||
.lip_active = face.lip_active};
|
||||
|
||||
if (not face_list.update_tracking(tracking.production_timestamp, tracking.timestamp, data, offset))
|
||||
cnx.set_enabled(to_headset::tracking_control::id::face, false);
|
||||
}
|
||||
|
||||
xrt_result_t wivrn_htc_face_tracker::get_face_tracking(enum xrt_input_name facial_expression_type, int64_t at_timestamp_ns, struct xrt_facial_expression_set * inout_value)
|
||||
{
|
||||
if (facial_expression_type == XRT_INPUT_HTC_EYE_FACE_TRACKING)
|
||||
{
|
||||
cnx.set_enabled(to_headset::tracking_control::id::face, true);
|
||||
auto [_, data] = face_list.get_at(at_timestamp_ns);
|
||||
|
||||
inout_value->base_expression_set_htc.is_active = data.eye_active;
|
||||
inout_value->base_expression_set_htc.sample_time_ns = at_timestamp_ns;
|
||||
|
||||
if (not data.eye_active)
|
||||
return XRT_SUCCESS;
|
||||
|
||||
memcpy(&inout_value->eye_expression_set_htc.expression_weights, data.eye.data(), sizeof(float) * data.eye.size());
|
||||
|
||||
return XRT_SUCCESS;
|
||||
}
|
||||
else if (facial_expression_type == XRT_INPUT_HTC_LIP_FACE_TRACKING)
|
||||
{
|
||||
cnx.set_enabled(to_headset::tracking_control::id::face, true);
|
||||
auto [_, data] = face_list.get_at(at_timestamp_ns);
|
||||
|
||||
inout_value->base_expression_set_htc.is_active = data.lip_active;
|
||||
inout_value->base_expression_set_htc.sample_time_ns = at_timestamp_ns;
|
||||
|
||||
if (not data.lip_active)
|
||||
return XRT_SUCCESS;
|
||||
|
||||
memcpy(&inout_value->lip_expression_set_htc.expression_weights, data.lip.data(), sizeof(float) * data.lip.size());
|
||||
|
||||
return XRT_SUCCESS;
|
||||
}
|
||||
|
||||
return XRT_ERROR_NOT_IMPLEMENTED;
|
||||
}
|
||||
|
||||
/*
|
||||
*
|
||||
* Functions
|
||||
*
|
||||
*/
|
||||
|
||||
static void wivrn_htc_face_tracker_destroy(xrt_device * xdev)
|
||||
{
|
||||
}
|
||||
|
||||
static xrt_result_t wivrn_htc_face_tracker_get_face_tracking(struct xrt_device * xdev,
|
||||
enum xrt_input_name facial_expression_type,
|
||||
int64_t at_timestamp_ns,
|
||||
struct xrt_facial_expression_set * inout_value)
|
||||
{
|
||||
return static_cast<wivrn_htc_face_tracker *>(xdev)->get_face_tracking(facial_expression_type, at_timestamp_ns, inout_value);
|
||||
}
|
||||
} // namespace wivrn
|
||||
@@ -0,0 +1,114 @@
|
||||
/*
|
||||
* WiVRn VR streaming
|
||||
* Copyright (C) 2024 Guillaume Meunier <[email protected]>
|
||||
* Copyright (C) 2024 Patrick Nicolas <[email protected]>
|
||||
* Copyright (C) 2024 galister <[email protected]>
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "wivrn_packets.h"
|
||||
#include "xrt/xrt_defines.h"
|
||||
#include "xrt/xrt_device.h"
|
||||
|
||||
#include "history.h"
|
||||
|
||||
#include <array>
|
||||
#include <cmath>
|
||||
#include <openxr/openxr.h>
|
||||
|
||||
namespace wivrn
|
||||
{
|
||||
|
||||
struct clock_offset;
|
||||
class wivrn_session;
|
||||
|
||||
struct wivrn_htc_face_data
|
||||
{
|
||||
std::array<float, XRT_FACIAL_EXPRESSION_EYE_COUNT_HTC> eye;
|
||||
std::array<float, XRT_FACIAL_EXPRESSION_LIP_COUNT_HTC> lip;
|
||||
bool eye_active;
|
||||
bool lip_active;
|
||||
};
|
||||
|
||||
class htc_face_list : public history<htc_face_list, wivrn_htc_face_data>
|
||||
{
|
||||
public:
|
||||
static wivrn_htc_face_data interpolate(const wivrn_htc_face_data & a, const wivrn_htc_face_data & b, float t)
|
||||
{
|
||||
wivrn_htc_face_data result = b;
|
||||
|
||||
if (not a.eye_active)
|
||||
{
|
||||
result.eye = b.eye;
|
||||
result.eye_active = b.eye_active;
|
||||
}
|
||||
else if (not b.eye_active)
|
||||
{
|
||||
result.eye = a.eye;
|
||||
result.eye_active = a.eye_active;
|
||||
}
|
||||
else
|
||||
{
|
||||
result.eye_active = true;
|
||||
for (size_t i = 0; i < result.eye.size(); i++)
|
||||
{
|
||||
result.eye[i] = std::clamp(std::lerp(a.eye[i], b.eye[i], t), 0.0f, 1.0f);
|
||||
}
|
||||
}
|
||||
|
||||
if (not a.lip_active)
|
||||
{
|
||||
result.lip = b.lip;
|
||||
result.lip_active = b.lip_active;
|
||||
}
|
||||
else if (not b.lip_active)
|
||||
{
|
||||
result.lip = a.lip;
|
||||
result.lip_active = a.lip_active;
|
||||
}
|
||||
else
|
||||
{
|
||||
result.lip_active = true;
|
||||
for (size_t i = 0; i < result.lip.size(); i++)
|
||||
{
|
||||
result.lip[i] = std::clamp(std::lerp(a.lip[i], b.lip[i], t), 0.0f, 1.0f);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
bool update_tracking(const XrTime & production_timestamp, const XrTime & timestamp, const wivrn_htc_face_data & data, const clock_offset & offset)
|
||||
{
|
||||
return this->add_sample(production_timestamp, timestamp, data, offset);
|
||||
}
|
||||
};
|
||||
|
||||
class wivrn_htc_face_tracker : public xrt_device
|
||||
{
|
||||
htc_face_list face_list;
|
||||
std::array<xrt_input, 2> inputs_array;
|
||||
|
||||
wivrn::wivrn_session & cnx;
|
||||
|
||||
public:
|
||||
wivrn_htc_face_tracker(xrt_device * hmd, wivrn::wivrn_session & cnx);
|
||||
|
||||
void update_inputs();
|
||||
void update_tracking(const from_headset::tracking &, const clock_offset &);
|
||||
xrt_result_t get_face_tracking(enum xrt_input_name facial_expression_type, int64_t at_timestamp_ns, struct xrt_facial_expression_set * out_value);
|
||||
};
|
||||
} // namespace wivrn
|
||||
@@ -34,8 +34,10 @@
|
||||
#include "wivrn_eye_tracker.h"
|
||||
#include "wivrn_fb_face2_tracker.h"
|
||||
#include "wivrn_foveation.h"
|
||||
#include "wivrn_htc_face_tracker.h"
|
||||
#include "wivrn_ipc.h"
|
||||
|
||||
#include "wivrn_packets.h"
|
||||
#include "xrt/xrt_session.h"
|
||||
#include <cmath>
|
||||
#include <magic_enum.hpp>
|
||||
@@ -197,11 +199,20 @@ wivrn::wivrn_session::wivrn_session(std::unique_ptr<wivrn_connection> connection
|
||||
static_roles.eyes = eye_tracker.get();
|
||||
xdevs[xdev_count++] = eye_tracker.get();
|
||||
}
|
||||
if (get_info().face_tracking2_fb || is_forced_extension("FB_face_tracking2"))
|
||||
switch (get_info().face_tracking)
|
||||
{
|
||||
fb_face2_tracker = std::make_unique<wivrn_fb_face2_tracker>(&hmd, *this);
|
||||
static_roles.face = fb_face2_tracker.get();
|
||||
xdevs[xdev_count++] = fb_face2_tracker.get();
|
||||
case wivrn::from_headset::face_type::fb2:
|
||||
fb_face2_tracker = std::make_unique<wivrn_fb_face2_tracker>(&hmd, *this);
|
||||
static_roles.face = fb_face2_tracker.get();
|
||||
xdevs[xdev_count++] = fb_face2_tracker.get();
|
||||
break;
|
||||
case wivrn::from_headset::face_type::htc:
|
||||
htc_face_tracker = std::make_unique<wivrn_htc_face_tracker>(&hmd, *this);
|
||||
static_roles.face = htc_face_tracker.get();
|
||||
xdevs[xdev_count++] = htc_face_tracker.get();
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
#if WIVRN_FEATURE_SOLARXR
|
||||
@@ -322,8 +333,11 @@ void wivrn_session::operator()(const from_headset::tracking & tracking)
|
||||
eye_tracker->update_tracking(tracking, offset);
|
||||
if (foveation)
|
||||
foveation->update_tracking(tracking, offset);
|
||||
|
||||
if (fb_face2_tracker)
|
||||
fb_face2_tracker->update_tracking(tracking, offset);
|
||||
else if (htc_face_tracker)
|
||||
htc_face_tracker->update_tracking(tracking, offset);
|
||||
}
|
||||
|
||||
void wivrn_session::operator()(from_headset::derived_pose && derived)
|
||||
|
||||
@@ -42,6 +42,7 @@ namespace wivrn
|
||||
{
|
||||
class wivrn_eye_tracker;
|
||||
class wivrn_fb_face2_tracker;
|
||||
class wivrn_htc_face_tracker;
|
||||
class wivrn_foveation;
|
||||
class wivrn_foveation_renderer;
|
||||
struct audio_device;
|
||||
@@ -105,6 +106,7 @@ class wivrn_session : public xrt_system_devices
|
||||
wivrn_controller right_hand;
|
||||
std::unique_ptr<wivrn_eye_tracker> eye_tracker;
|
||||
std::unique_ptr<wivrn_fb_face2_tracker> fb_face2_tracker;
|
||||
std::unique_ptr<wivrn_htc_face_tracker> htc_face_tracker;
|
||||
std::unique_ptr<wivrn_foveation> foveation;
|
||||
wivrn_comp_target * comp_target;
|
||||
|
||||
|
||||
+1
-1
@@ -782,7 +782,7 @@ void on_headset_info_packet(const wivrn::from_headset::headset_info_packet & inf
|
||||
|
||||
wivrn_server_set_hand_tracking(dbus_server, info.hand_tracking);
|
||||
wivrn_server_set_eye_gaze(dbus_server, info.eye_gaze);
|
||||
wivrn_server_set_face_tracking(dbus_server, info.face_tracking2_fb);
|
||||
wivrn_server_set_face_tracking(dbus_server, info.face_tracking != wivrn::from_headset::face_type::none);
|
||||
|
||||
std::vector<const char *> codecs;
|
||||
static const std::map<video_codec, const char *> codec_names = {
|
||||
|
||||
Reference in new issue
Block a user