From f0ca7ff8daafef87e9977b5073e7d840f4ec030b Mon Sep 17 00:00:00 2001
From: galister <22305755+galister@users.noreply.github.com>
Date: Thu, 1 Aug 2024 08:18:57 +0900
Subject: [PATCH] passthru EXT_eye_gaze_interaction
---
AndroidManifest.xml | 5 +++++
client/application.cpp | 16 ++++++++++++++++
client/application.h | 12 ++++++++++++
client/scenes/stream.cpp | 1 +
client/scenes/stream_tracking.cpp | 3 +++
client/xr/system.cpp | 18 ++++++++++++++++++
client/xr/system.h | 1 +
common/wivrn_packets.h | 2 ++
server/driver/wivrn_hmd.cpp | 22 +++++++++++++++-------
server/driver/wivrn_hmd.h | 1 +
server/driver/wivrn_session.cpp | 4 ++++
11 files changed, 78 insertions(+), 7 deletions(-)
diff --git a/AndroidManifest.xml b/AndroidManifest.xml
index b6b7cce8..b1ecee87 100644
--- a/AndroidManifest.xml
+++ b/AndroidManifest.xml
@@ -31,8 +31,11 @@
+
+
+
@@ -56,6 +59,7 @@
+
@@ -65,6 +69,7 @@
+
diff --git a/client/application.cpp b/client/application.cpp
index 3ea9fb79..a095c8d6 100644
--- a/client/application.cpp
+++ b/client/application.cpp
@@ -250,6 +250,12 @@ static std::vector interaction_profiles{
"/user/hand/right/input/thumbstick/touch",
"/user/hand/right/input/thumbrest/touch",
}},
+ interaction_profile{
+ "/interaction_profiles/ext/eye_gaze_interaction",
+ {"XR_EXT_eye_gaze_interaction"},
+ {
+ "/user/eyes_ext/input/gaze_ext/pose",
+ }},
};
static const std::pair action_suffixes[] =
@@ -648,6 +654,8 @@ void application::initialize_actions()
right_grip_space = xr_session.create_action_space(a);
else if (name == "/user/hand/right/input/aim/pose")
right_aim_space = xr_session.create_action_space(a);
+ else if (name == "/user/eyes_ext/input/gaze_ext/pose")
+ eye_gaze_space = xr_session.create_action_space(a);
}
// Build an action set for each scene
@@ -727,6 +735,7 @@ void application::initialize()
std::vector opt_extensions;
opt_extensions.push_back(XR_FB_DISPLAY_REFRESH_RATE_EXTENSION_NAME);
opt_extensions.push_back(XR_EXT_HAND_TRACKING_EXTENSION_NAME);
+ opt_extensions.push_back(XR_EXT_EYE_GAZE_INTERACTION_EXTENSION_NAME);
opt_extensions.push_back(XR_FB_PASSTHROUGH_EXTENSION_NAME);
opt_extensions.push_back(XR_HTC_PASSTHROUGH_EXTENSION_NAME);
@@ -781,6 +790,13 @@ void application::initialize()
hand_tracking_supported = hand_tracking_properties.supportsHandTracking;
}
+ if (utils::contains(xr_extensions, XR_EXT_EYE_GAZE_INTERACTION_EXTENSION_NAME))
+ {
+ XrSystemEyeGazeInteractionPropertiesEXT eye_gaze_properties = xr_system_id.eye_gaze_interaction_properties();
+ spdlog::info(" Eye gaze support: {}", (bool)eye_gaze_properties.supportsEyeGazeInteraction);
+ eye_gaze_supported = eye_gaze_properties.supportsEyeGazeInteraction;
+ }
+
switch (xr_system_id.passthrough_supported())
{
case xr::system::passthrough_type::no_passthrough:
diff --git a/client/application.h b/client/application.h
index 4763654c..8e78e779 100644
--- a/client/application.h
+++ b/client/application.h
@@ -121,11 +121,14 @@ class application : public singleton
xr::space left_aim_space;
xr::space right_grip_space;
xr::space right_aim_space;
+ xr::space eye_gaze_space;
bool hand_tracking_supported = false;
xr::hand_tracker left_hand;
xr::hand_tracker right_hand;
+ bool eye_gaze_supported = false;
+
bool session_running = false;
bool session_focused = false;
bool session_visible = false;
@@ -304,6 +307,10 @@ public:
{
return instance().right_aim_space;
};
+ static XrSpace eye_gaze()
+ {
+ return instance().eye_gaze_space;
+ };
static void ignore_debug_reports_for(void * object)
{
@@ -404,6 +411,11 @@ public:
return instance().hand_tracking_supported;
}
+ static bool get_eye_gaze_supported()
+ {
+ return instance().eye_gaze_supported;
+ }
+
static xr::hand_tracker & get_left_hand()
{
return instance().left_hand;
diff --git a/client/scenes/stream.cpp b/client/scenes/stream.cpp
index 860a34e4..7cb7ac54 100644
--- a/client/scenes/stream.cpp
+++ b/client/scenes/stream.cpp
@@ -120,6 +120,7 @@ std::shared_ptr scenes::stream::create(std::unique_ptr microphone;
std::array fov;
bool hand_tracking;
+ bool eye_gaze;
};
struct handshake
diff --git a/server/driver/wivrn_hmd.cpp b/server/driver/wivrn_hmd.cpp
index e0b4720c..c3835520 100644
--- a/server/driver/wivrn_hmd.cpp
+++ b/server/driver/wivrn_hmd.cpp
@@ -180,7 +180,7 @@ bool wivrn_hmd::wivrn_hmd_compute_distortion(xrt_device * xdev, uint32_t view_in
wivrn_hmd::wivrn_hmd(std::shared_ptr cnx,
const from_headset::headset_info_packet & info) :
- xrt_device{}, cnx(cnx)
+ xrt_device{}, gaze(device_id::EYE_GAZE), cnx(cnx)
{
xrt_device * base = this;
@@ -196,6 +196,7 @@ wivrn_hmd::wivrn_hmd(std::shared_ptr cnx,
orientation_tracking_supported = true;
// hand_tracking_supported = true;
position_tracking_supported = true;
+ eye_gaze_supported = info.eye_gaze;
// Print name.
strcpy(str, "WiVRn HMD");
@@ -245,20 +246,27 @@ void wivrn_hmd::update_inputs()
xrt_space_relation wivrn_hmd::get_tracked_pose(xrt_input_name name, uint64_t at_timestamp_ns)
{
- if (name != XRT_INPUT_GENERIC_HEAD_POSE)
+ if (name == XRT_INPUT_GENERIC_HEAD_POSE)
{
- U_LOG_E("Unknown input name");
- return {};
+ auto [extrapolation_time, res] = views.get_at(at_timestamp_ns);
+ cnx->add_predict_offset(extrapolation_time);
+ return res.relation;
}
- auto [extrapolation_time, res] = views.get_at(at_timestamp_ns);
- cnx->add_predict_offset(extrapolation_time);
- return res.relation;
+ if (name == XRT_INPUT_GENERIC_EYE_GAZE_POSE)
+ {
+ auto [_, relation] = gaze.get_at(at_timestamp_ns);
+ return relation;
+ }
+
+ U_LOG_E("Unknown input name");
+ return {};
}
void wivrn_hmd::update_tracking(const from_headset::tracking & tracking, const clock_offset & offset)
{
views.update_tracking(tracking, offset);
+ gaze.update_tracking(tracking, offset);
}
void wivrn_hmd::get_view_poses(const xrt_vec3 * default_eye_relation,
diff --git a/server/driver/wivrn_hmd.h b/server/driver/wivrn_hmd.h
index aabeecfb..065762aa 100644
--- a/server/driver/wivrn_hmd.h
+++ b/server/driver/wivrn_hmd.h
@@ -47,6 +47,7 @@ class wivrn_hmd : public xrt_device
};
view_list views;
+ pose_list gaze;
std::array foveation_parameters{};
std::shared_ptr cnx;
diff --git a/server/driver/wivrn_session.cpp b/server/driver/wivrn_session.cpp
index 4ca771b5..1a5389d3 100644
--- a/server/driver/wivrn_session.cpp
+++ b/server/driver/wivrn_session.cpp
@@ -142,6 +142,10 @@ xrt_result_t xrt::drivers::wivrn::wivrn_session::create_session(xrt::drivers::wi
int n = 0;
if (self->hmd)
usysds->base.base.static_roles.head = devices->xdevs[n++] = self->hmd.get();
+
+ if (self->hmd->eye_gaze_supported)
+ usysds->base.base.static_roles.eyes = self->hmd.get();
+
if (self->left_hand)
{
devices->xdevs[n++] = self->left_hand.get();