passthru EXT_eye_gaze_interaction

This commit is contained in:
galister authored and xytovl committed 2024-08-01 12:08:05 +02:00
1 parent 4cfdc4fa24
commit f0ca7ff8da
11 files changed
+78 -7

No files matched your search

+5
View File
@@ -31,8 +31,11 @@
<!-- Allow starting the application even without controllers -->
<uses-feature android:name="oculus.software.handtracking" android:required="false" />
<uses-feature android:name="oculus.software.eye_tracking" android:required="false" />
<uses-permission android:name="com.magicleap.permission.HAND_TRACKING" />
<uses-permission android:name="com.oculus.permission.HAND_TRACKING" />
<uses-permission android:name="com.oculus.permission.EYE_TRACKING" />
<uses-permission android:name="com.picovr.permission.EYE_TRACKING" />
<!-- Required for XR_FB_render_model -->
<uses-feature android:name="com.oculus.feature.RENDER_MODEL" android:required="false" />
@@ -56,6 +59,7 @@
<meta-data android:name="pvr.app.type" android:value="vr" />
<meta-data android:name="pvr.sdk.version" android:value="OpenXR" />
<meta-data android:name="picovr.software.eye_tracking" android:value="true" />
<meta-data android:name="com.oculus.intent.category.VR" android:value="vr_only"/>
<meta-data android:name="com.oculus.supportedDevices" android:value="all"/>
<meta-data android:name="com.oculus.vr.focusaware" android:value="true"/>
@@ -65,6 +69,7 @@
<meta-data android:name="com.htc.vr.content.NumDoFController" android:value="6DoF"/>
<meta-data android:name="handtracking" android:value="1" />
<meta-data android:name="eyetracking_calibration" android:value="true" />
<!-- The activity is the built-in NativeActivity framework class. -->
<!-- launchMode is set to singleTask because there should never be multiple copies of the app running. -->
+16
View File
@@ -250,6 +250,12 @@ static std::vector<interaction_profile> interaction_profiles{
"/user/hand/right/input/thumbstick/touch",
"/user/hand/right/input/thumbrest/touch",
}},
interaction_profile{
"/interaction_profiles/ext/eye_gaze_interaction",
{"XR_EXT_eye_gaze_interaction"},
{
"/user/eyes_ext/input/gaze_ext/pose",
}},
};
static const std::pair<std::string_view, XrActionType> action_suffixes[] =
@@ -648,6 +654,8 @@ void application::initialize_actions()
right_grip_space = xr_session.create_action_space(a);
else if (name == "/user/hand/right/input/aim/pose")
right_aim_space = xr_session.create_action_space(a);
else if (name == "/user/eyes_ext/input/gaze_ext/pose")
eye_gaze_space = xr_session.create_action_space(a);
}
// Build an action set for each scene
@@ -727,6 +735,7 @@ void application::initialize()
std::vector<std::string> opt_extensions;
opt_extensions.push_back(XR_FB_DISPLAY_REFRESH_RATE_EXTENSION_NAME);
opt_extensions.push_back(XR_EXT_HAND_TRACKING_EXTENSION_NAME);
opt_extensions.push_back(XR_EXT_EYE_GAZE_INTERACTION_EXTENSION_NAME);
opt_extensions.push_back(XR_FB_PASSTHROUGH_EXTENSION_NAME);
opt_extensions.push_back(XR_HTC_PASSTHROUGH_EXTENSION_NAME);
@@ -781,6 +790,13 @@ void application::initialize()
hand_tracking_supported = hand_tracking_properties.supportsHandTracking;
}
if (utils::contains(xr_extensions, XR_EXT_EYE_GAZE_INTERACTION_EXTENSION_NAME))
{
XrSystemEyeGazeInteractionPropertiesEXT eye_gaze_properties = xr_system_id.eye_gaze_interaction_properties();
spdlog::info(" Eye gaze support: {}", (bool)eye_gaze_properties.supportsEyeGazeInteraction);
eye_gaze_supported = eye_gaze_properties.supportsEyeGazeInteraction;
}
switch (xr_system_id.passthrough_supported())
{
case xr::system::passthrough_type::no_passthrough:
+12
View File
@@ -121,11 +121,14 @@ class application : public singleton<application>
xr::space left_aim_space;
xr::space right_grip_space;
xr::space right_aim_space;
xr::space eye_gaze_space;
bool hand_tracking_supported = false;
xr::hand_tracker left_hand;
xr::hand_tracker right_hand;
bool eye_gaze_supported = false;
bool session_running = false;
bool session_focused = false;
bool session_visible = false;
@@ -304,6 +307,10 @@ public:
{
return instance().right_aim_space;
};
static XrSpace eye_gaze()
{
return instance().eye_gaze_space;
};
static void ignore_debug_reports_for(void * object)
{
@@ -404,6 +411,11 @@ public:
return instance().hand_tracking_supported;
}
static bool get_eye_gaze_supported()
{
return instance().eye_gaze_supported;
}
static xr::hand_tracker & get_left_hand()
{
return instance().left_hand;
+1
View File
@@ -120,6 +120,7 @@ std::shared_ptr<scenes::stream> scenes::stream::create(std::unique_ptr<wivrn_ses
spdlog::warn("Unable to detect refresh rates");
info.hand_tracking = application::get_hand_tracking_supported();
info.eye_gaze = application::get_eye_gaze_supported();
audio::get_audio_description(info);
if (not application::get_config().microphone)
+3
View File
@@ -146,6 +146,9 @@ void scenes::stream::tracking()
{device_id::RIGHT_AIM, application::right_aim()},
{device_id::RIGHT_GRIP, application::right_grip()}};
if (application::get_eye_gaze_supported())
spaces.push_back({device_id::EYE_GAZE, application::eye_gaze()});
XrSpace view_space = application::view();
XrDuration tracking_period = 1'000'000; // Send tracking data every 1ms
const XrDuration dt = 100'000; // Wake up 0.1ms before measuring the position
+18
View File
@@ -87,6 +87,24 @@ XrSystemHandTrackingPropertiesEXT xr::system::hand_tracking_properties() const
return hand_tracking_prop;
}
XrSystemEyeGazeInteractionPropertiesEXT xr::system::eye_gaze_interaction_properties() const
{
if (!id)
throw std::invalid_argument("this");
XrSystemEyeGazeInteractionPropertiesEXT eye_gaze_prop{
.type = XR_TYPE_SYSTEM_EYE_GAZE_INTERACTION_PROPERTIES_EXT,
};
XrSystemProperties prop{
.type = XR_TYPE_SYSTEM_PROPERTIES,
.next = &eye_gaze_prop,
};
CHECK_XR(xrGetSystemProperties(*inst, id, &prop));
return eye_gaze_prop;
}
xr::system::passthrough_type xr::system::passthrough_supported() const
{
if (utils::contains(environment_blend_modes(XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO), XR_ENVIRONMENT_BLEND_MODE_ALPHA_BLEND))
+1
View File
@@ -61,6 +61,7 @@ public:
XrSystemProperties properties() const;
XrSystemHandTrackingPropertiesEXT hand_tracking_properties() const;
XrSystemEyeGazeInteractionPropertiesEXT eye_gaze_interaction_properties() const;
passthrough_type passthrough_supported() const;
XrGraphicsRequirementsVulkan2KHR graphics_requirements() const;
+2
View File
@@ -73,6 +73,7 @@ enum class device_id : uint8_t
RIGHT_THUMBSTICK_CLICK, // /user/hand/right/input/thumbstick/click
RIGHT_THUMBSTICK_TOUCH, // /user/hand/right/input/thumbstick/touch
RIGHT_THUMBREST_TOUCH, // /user/hand/right/input/thumbrest/touch
EYE_GAZE, // /user/eyes_ext/input/gaze_ext/pose
};
enum video_codec
@@ -107,6 +108,7 @@ struct headset_info_packet
std::optional<audio_description> microphone;
std::array<XrFovf, 2> fov;
bool hand_tracking;
bool eye_gaze;
};
struct handshake
+15 -7
View File
@@ -180,7 +180,7 @@ bool wivrn_hmd::wivrn_hmd_compute_distortion(xrt_device * xdev, uint32_t view_in
wivrn_hmd::wivrn_hmd(std::shared_ptr<xrt::drivers::wivrn::wivrn_session> cnx,
const from_headset::headset_info_packet & info) :
xrt_device{}, cnx(cnx)
xrt_device{}, gaze(device_id::EYE_GAZE), cnx(cnx)
{
xrt_device * base = this;
@@ -196,6 +196,7 @@ wivrn_hmd::wivrn_hmd(std::shared_ptr<xrt::drivers::wivrn::wivrn_session> cnx,
orientation_tracking_supported = true;
// hand_tracking_supported = true;
position_tracking_supported = true;
eye_gaze_supported = info.eye_gaze;
// Print name.
strcpy(str, "WiVRn HMD");
@@ -245,20 +246,27 @@ void wivrn_hmd::update_inputs()
xrt_space_relation wivrn_hmd::get_tracked_pose(xrt_input_name name, uint64_t at_timestamp_ns)
{
if (name != XRT_INPUT_GENERIC_HEAD_POSE)
if (name == XRT_INPUT_GENERIC_HEAD_POSE)
{
U_LOG_E("Unknown input name");
return {};
auto [extrapolation_time, res] = views.get_at(at_timestamp_ns);
cnx->add_predict_offset(extrapolation_time);
return res.relation;
}
auto [extrapolation_time, res] = views.get_at(at_timestamp_ns);
cnx->add_predict_offset(extrapolation_time);
return res.relation;
if (name == XRT_INPUT_GENERIC_EYE_GAZE_POSE)
{
auto [_, relation] = gaze.get_at(at_timestamp_ns);
return relation;
}
U_LOG_E("Unknown input name");
return {};
}
void wivrn_hmd::update_tracking(const from_headset::tracking & tracking, const clock_offset & offset)
{
views.update_tracking(tracking, offset);
gaze.update_tracking(tracking, offset);
}
void wivrn_hmd::get_view_poses(const xrt_vec3 * default_eye_relation,
+1
View File
@@ -47,6 +47,7 @@ class wivrn_hmd : public xrt_device
};
view_list views;
pose_list gaze;
std::array<to_headset::video_stream_description::foveation_parameter, 2> foveation_parameters{};
std::shared_ptr<xrt::drivers::wivrn::wivrn_session> cnx;
+4
View File
@@ -142,6 +142,10 @@ xrt_result_t xrt::drivers::wivrn::wivrn_session::create_session(xrt::drivers::wi
int n = 0;
if (self->hmd)
usysds->base.base.static_roles.head = devices->xdevs[n++] = self->hmd.get();
if (self->hmd->eye_gaze_supported)
usysds->base.base.static_roles.eyes = self->hmd.get();
if (self->left_hand)
{
devices->xdevs[n++] = self->left_hand.get();