mirror of
https://github.com/WiVRn/WiVRn.git
synced 2026-11-21 17:41:54 -08:00
passthru EXT_eye_gaze_interaction
This commit is contained in:
1 parent
4cfdc4fa24
commit
f0ca7ff8da
11 files changed
+78
-7
No files matched your search
@@ -31,8 +31,11 @@
|
||||
|
||||
<!-- Allow starting the application even without controllers -->
|
||||
<uses-feature android:name="oculus.software.handtracking" android:required="false" />
|
||||
<uses-feature android:name="oculus.software.eye_tracking" android:required="false" />
|
||||
<uses-permission android:name="com.magicleap.permission.HAND_TRACKING" />
|
||||
<uses-permission android:name="com.oculus.permission.HAND_TRACKING" />
|
||||
<uses-permission android:name="com.oculus.permission.EYE_TRACKING" />
|
||||
<uses-permission android:name="com.picovr.permission.EYE_TRACKING" />
|
||||
|
||||
<!-- Required for XR_FB_render_model -->
|
||||
<uses-feature android:name="com.oculus.feature.RENDER_MODEL" android:required="false" />
|
||||
@@ -56,6 +59,7 @@
|
||||
|
||||
<meta-data android:name="pvr.app.type" android:value="vr" />
|
||||
<meta-data android:name="pvr.sdk.version" android:value="OpenXR" />
|
||||
<meta-data android:name="picovr.software.eye_tracking" android:value="true" />
|
||||
<meta-data android:name="com.oculus.intent.category.VR" android:value="vr_only"/>
|
||||
<meta-data android:name="com.oculus.supportedDevices" android:value="all"/>
|
||||
<meta-data android:name="com.oculus.vr.focusaware" android:value="true"/>
|
||||
@@ -65,6 +69,7 @@
|
||||
<meta-data android:name="com.htc.vr.content.NumDoFController" android:value="6DoF"/>
|
||||
|
||||
<meta-data android:name="handtracking" android:value="1" />
|
||||
<meta-data android:name="eyetracking_calibration" android:value="true" />
|
||||
|
||||
<!-- The activity is the built-in NativeActivity framework class. -->
|
||||
<!-- launchMode is set to singleTask because there should never be multiple copies of the app running. -->
|
||||
|
||||
@@ -250,6 +250,12 @@ static std::vector<interaction_profile> interaction_profiles{
|
||||
"/user/hand/right/input/thumbstick/touch",
|
||||
"/user/hand/right/input/thumbrest/touch",
|
||||
}},
|
||||
interaction_profile{
|
||||
"/interaction_profiles/ext/eye_gaze_interaction",
|
||||
{"XR_EXT_eye_gaze_interaction"},
|
||||
{
|
||||
"/user/eyes_ext/input/gaze_ext/pose",
|
||||
}},
|
||||
};
|
||||
|
||||
static const std::pair<std::string_view, XrActionType> action_suffixes[] =
|
||||
@@ -648,6 +654,8 @@ void application::initialize_actions()
|
||||
right_grip_space = xr_session.create_action_space(a);
|
||||
else if (name == "/user/hand/right/input/aim/pose")
|
||||
right_aim_space = xr_session.create_action_space(a);
|
||||
else if (name == "/user/eyes_ext/input/gaze_ext/pose")
|
||||
eye_gaze_space = xr_session.create_action_space(a);
|
||||
}
|
||||
|
||||
// Build an action set for each scene
|
||||
@@ -727,6 +735,7 @@ void application::initialize()
|
||||
std::vector<std::string> opt_extensions;
|
||||
opt_extensions.push_back(XR_FB_DISPLAY_REFRESH_RATE_EXTENSION_NAME);
|
||||
opt_extensions.push_back(XR_EXT_HAND_TRACKING_EXTENSION_NAME);
|
||||
opt_extensions.push_back(XR_EXT_EYE_GAZE_INTERACTION_EXTENSION_NAME);
|
||||
opt_extensions.push_back(XR_FB_PASSTHROUGH_EXTENSION_NAME);
|
||||
opt_extensions.push_back(XR_HTC_PASSTHROUGH_EXTENSION_NAME);
|
||||
|
||||
@@ -781,6 +790,13 @@ void application::initialize()
|
||||
hand_tracking_supported = hand_tracking_properties.supportsHandTracking;
|
||||
}
|
||||
|
||||
if (utils::contains(xr_extensions, XR_EXT_EYE_GAZE_INTERACTION_EXTENSION_NAME))
|
||||
{
|
||||
XrSystemEyeGazeInteractionPropertiesEXT eye_gaze_properties = xr_system_id.eye_gaze_interaction_properties();
|
||||
spdlog::info(" Eye gaze support: {}", (bool)eye_gaze_properties.supportsEyeGazeInteraction);
|
||||
eye_gaze_supported = eye_gaze_properties.supportsEyeGazeInteraction;
|
||||
}
|
||||
|
||||
switch (xr_system_id.passthrough_supported())
|
||||
{
|
||||
case xr::system::passthrough_type::no_passthrough:
|
||||
|
||||
@@ -121,11 +121,14 @@ class application : public singleton<application>
|
||||
xr::space left_aim_space;
|
||||
xr::space right_grip_space;
|
||||
xr::space right_aim_space;
|
||||
xr::space eye_gaze_space;
|
||||
|
||||
bool hand_tracking_supported = false;
|
||||
xr::hand_tracker left_hand;
|
||||
xr::hand_tracker right_hand;
|
||||
|
||||
bool eye_gaze_supported = false;
|
||||
|
||||
bool session_running = false;
|
||||
bool session_focused = false;
|
||||
bool session_visible = false;
|
||||
@@ -304,6 +307,10 @@ public:
|
||||
{
|
||||
return instance().right_aim_space;
|
||||
};
|
||||
static XrSpace eye_gaze()
|
||||
{
|
||||
return instance().eye_gaze_space;
|
||||
};
|
||||
|
||||
static void ignore_debug_reports_for(void * object)
|
||||
{
|
||||
@@ -404,6 +411,11 @@ public:
|
||||
return instance().hand_tracking_supported;
|
||||
}
|
||||
|
||||
static bool get_eye_gaze_supported()
|
||||
{
|
||||
return instance().eye_gaze_supported;
|
||||
}
|
||||
|
||||
static xr::hand_tracker & get_left_hand()
|
||||
{
|
||||
return instance().left_hand;
|
||||
|
||||
@@ -120,6 +120,7 @@ std::shared_ptr<scenes::stream> scenes::stream::create(std::unique_ptr<wivrn_ses
|
||||
spdlog::warn("Unable to detect refresh rates");
|
||||
|
||||
info.hand_tracking = application::get_hand_tracking_supported();
|
||||
info.eye_gaze = application::get_eye_gaze_supported();
|
||||
|
||||
audio::get_audio_description(info);
|
||||
if (not application::get_config().microphone)
|
||||
|
||||
@@ -146,6 +146,9 @@ void scenes::stream::tracking()
|
||||
{device_id::RIGHT_AIM, application::right_aim()},
|
||||
{device_id::RIGHT_GRIP, application::right_grip()}};
|
||||
|
||||
if (application::get_eye_gaze_supported())
|
||||
spaces.push_back({device_id::EYE_GAZE, application::eye_gaze()});
|
||||
|
||||
XrSpace view_space = application::view();
|
||||
XrDuration tracking_period = 1'000'000; // Send tracking data every 1ms
|
||||
const XrDuration dt = 100'000; // Wake up 0.1ms before measuring the position
|
||||
|
||||
@@ -87,6 +87,24 @@ XrSystemHandTrackingPropertiesEXT xr::system::hand_tracking_properties() const
|
||||
return hand_tracking_prop;
|
||||
}
|
||||
|
||||
XrSystemEyeGazeInteractionPropertiesEXT xr::system::eye_gaze_interaction_properties() const
|
||||
{
|
||||
if (!id)
|
||||
throw std::invalid_argument("this");
|
||||
|
||||
XrSystemEyeGazeInteractionPropertiesEXT eye_gaze_prop{
|
||||
.type = XR_TYPE_SYSTEM_EYE_GAZE_INTERACTION_PROPERTIES_EXT,
|
||||
};
|
||||
|
||||
XrSystemProperties prop{
|
||||
.type = XR_TYPE_SYSTEM_PROPERTIES,
|
||||
.next = &eye_gaze_prop,
|
||||
};
|
||||
CHECK_XR(xrGetSystemProperties(*inst, id, &prop));
|
||||
|
||||
return eye_gaze_prop;
|
||||
}
|
||||
|
||||
xr::system::passthrough_type xr::system::passthrough_supported() const
|
||||
{
|
||||
if (utils::contains(environment_blend_modes(XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO), XR_ENVIRONMENT_BLEND_MODE_ALPHA_BLEND))
|
||||
|
||||
@@ -61,6 +61,7 @@ public:
|
||||
|
||||
XrSystemProperties properties() const;
|
||||
XrSystemHandTrackingPropertiesEXT hand_tracking_properties() const;
|
||||
XrSystemEyeGazeInteractionPropertiesEXT eye_gaze_interaction_properties() const;
|
||||
passthrough_type passthrough_supported() const;
|
||||
|
||||
XrGraphicsRequirementsVulkan2KHR graphics_requirements() const;
|
||||
|
||||
@@ -73,6 +73,7 @@ enum class device_id : uint8_t
|
||||
RIGHT_THUMBSTICK_CLICK, // /user/hand/right/input/thumbstick/click
|
||||
RIGHT_THUMBSTICK_TOUCH, // /user/hand/right/input/thumbstick/touch
|
||||
RIGHT_THUMBREST_TOUCH, // /user/hand/right/input/thumbrest/touch
|
||||
EYE_GAZE, // /user/eyes_ext/input/gaze_ext/pose
|
||||
};
|
||||
|
||||
enum video_codec
|
||||
@@ -107,6 +108,7 @@ struct headset_info_packet
|
||||
std::optional<audio_description> microphone;
|
||||
std::array<XrFovf, 2> fov;
|
||||
bool hand_tracking;
|
||||
bool eye_gaze;
|
||||
};
|
||||
|
||||
struct handshake
|
||||
|
||||
@@ -180,7 +180,7 @@ bool wivrn_hmd::wivrn_hmd_compute_distortion(xrt_device * xdev, uint32_t view_in
|
||||
|
||||
wivrn_hmd::wivrn_hmd(std::shared_ptr<xrt::drivers::wivrn::wivrn_session> cnx,
|
||||
const from_headset::headset_info_packet & info) :
|
||||
xrt_device{}, cnx(cnx)
|
||||
xrt_device{}, gaze(device_id::EYE_GAZE), cnx(cnx)
|
||||
{
|
||||
xrt_device * base = this;
|
||||
|
||||
@@ -196,6 +196,7 @@ wivrn_hmd::wivrn_hmd(std::shared_ptr<xrt::drivers::wivrn::wivrn_session> cnx,
|
||||
orientation_tracking_supported = true;
|
||||
// hand_tracking_supported = true;
|
||||
position_tracking_supported = true;
|
||||
eye_gaze_supported = info.eye_gaze;
|
||||
|
||||
// Print name.
|
||||
strcpy(str, "WiVRn HMD");
|
||||
@@ -245,20 +246,27 @@ void wivrn_hmd::update_inputs()
|
||||
|
||||
xrt_space_relation wivrn_hmd::get_tracked_pose(xrt_input_name name, uint64_t at_timestamp_ns)
|
||||
{
|
||||
if (name != XRT_INPUT_GENERIC_HEAD_POSE)
|
||||
if (name == XRT_INPUT_GENERIC_HEAD_POSE)
|
||||
{
|
||||
U_LOG_E("Unknown input name");
|
||||
return {};
|
||||
auto [extrapolation_time, res] = views.get_at(at_timestamp_ns);
|
||||
cnx->add_predict_offset(extrapolation_time);
|
||||
return res.relation;
|
||||
}
|
||||
|
||||
auto [extrapolation_time, res] = views.get_at(at_timestamp_ns);
|
||||
cnx->add_predict_offset(extrapolation_time);
|
||||
return res.relation;
|
||||
if (name == XRT_INPUT_GENERIC_EYE_GAZE_POSE)
|
||||
{
|
||||
auto [_, relation] = gaze.get_at(at_timestamp_ns);
|
||||
return relation;
|
||||
}
|
||||
|
||||
U_LOG_E("Unknown input name");
|
||||
return {};
|
||||
}
|
||||
|
||||
void wivrn_hmd::update_tracking(const from_headset::tracking & tracking, const clock_offset & offset)
|
||||
{
|
||||
views.update_tracking(tracking, offset);
|
||||
gaze.update_tracking(tracking, offset);
|
||||
}
|
||||
|
||||
void wivrn_hmd::get_view_poses(const xrt_vec3 * default_eye_relation,
|
||||
|
||||
@@ -47,6 +47,7 @@ class wivrn_hmd : public xrt_device
|
||||
};
|
||||
|
||||
view_list views;
|
||||
pose_list gaze;
|
||||
std::array<to_headset::video_stream_description::foveation_parameter, 2> foveation_parameters{};
|
||||
|
||||
std::shared_ptr<xrt::drivers::wivrn::wivrn_session> cnx;
|
||||
|
||||
@@ -142,6 +142,10 @@ xrt_result_t xrt::drivers::wivrn::wivrn_session::create_session(xrt::drivers::wi
|
||||
int n = 0;
|
||||
if (self->hmd)
|
||||
usysds->base.base.static_roles.head = devices->xdevs[n++] = self->hmd.get();
|
||||
|
||||
if (self->hmd->eye_gaze_supported)
|
||||
usysds->base.base.static_roles.eyes = self->hmd.get();
|
||||
|
||||
if (self->left_hand)
|
||||
{
|
||||
devices->xdevs[n++] = self->left_hand.get();
|
||||
|
||||
Reference in new issue
Block a user