Files
WiVRn/client/xr/hand_tracker.cpp

261 lines
8.4 KiB
C++

/*
* WiVRn VR streaming
* Copyright (C) 2024 Guillaume Meunier <[email protected]>
* Copyright (C) 2024 Patrick Nicolas <[email protected]>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
#include "hand_tracker.h"
#include "xr/check.h"
#include "xr/instance.h"
#include "xr/session.h"
#include "xr/to_string.h"
#include <algorithm>
#include <cassert>
#include <spdlog/spdlog.h>
xr::hand_tracker::hand_tracker(instance & inst, session & session, const XrHandTrackerCreateInfoEXT & info) :
handle(inst.get_proc<PFN_xrDestroyHandTrackerEXT>("xrDestroyHandTrackerEXT")),
hand_{info.hand}
{
auto xrCreateHandTrackerEXT = inst.get_proc<PFN_xrCreateHandTrackerEXT>("xrCreateHandTrackerEXT");
assert(xrCreateHandTrackerEXT);
xrLocateHandJointsEXT = inst.get_proc<PFN_xrLocateHandJointsEXT>("xrLocateHandJointsEXT");
hand_tracking_data_source_supported = inst.has_extension(XR_EXT_HAND_TRACKING_DATA_SOURCE_EXTENSION_NAME);
if (inst.has_extension(XR_FB_HAND_TRACKING_MESH_EXTENSION_NAME))
xrGetHandMeshFB = inst.get_proc<PFN_xrGetHandMeshFB>("xrGetHandMeshFB");
if (hand_tracking_data_source_supported)
{
// request both sources, callers of locate() decide how to use the results
std::array<XrHandTrackingDataSourceEXT, 2> data_sources = {
XR_HAND_TRACKING_DATA_SOURCE_UNOBSTRUCTED_EXT,
XR_HAND_TRACKING_DATA_SOURCE_CONTROLLER_EXT,
};
XrHandTrackingDataSourceInfoEXT data_source_info{
.type = XR_TYPE_HAND_TRACKING_DATA_SOURCE_INFO_EXT,
.next = info.next,
.requestedDataSourceCount = static_cast<uint32_t>(data_sources.size()),
.requestedDataSources = data_sources.data(),
};
auto create_info = info;
create_info.next = &data_source_info;
CHECK_XR(xrCreateHandTrackerEXT(session, &create_info, &id));
}
else
{
CHECK_XR(xrCreateHandTrackerEXT(session, &info, &id));
}
}
xr::hand_tracker::located_hand xr::hand_tracker::locate(XrSpace space, XrTime time)
{
if (!id || !xrLocateHandJointsEXT)
return {};
XrHandJointsLocateInfoEXT info{
.type = XR_TYPE_HAND_JOINTS_LOCATE_INFO_EXT,
.next = nullptr,
.baseSpace = space,
.time = time,
};
std::array<XrHandJointLocationEXT, XR_HAND_JOINT_COUNT_EXT> joints_pos;
std::array<XrHandJointVelocityEXT, XR_HAND_JOINT_COUNT_EXT> joints_vel;
#ifndef NDEBUG
// Silence the OpenXR validation layer by setting valid flags
for (auto & i: joints_pos)
i.locationFlags = XR_SPACE_LOCATION_POSITION_VALID_BIT;
for (auto & i: joints_vel)
i.velocityFlags = XR_SPACE_VELOCITY_LINEAR_VALID_BIT;
#endif
XrHandJointVelocitiesEXT velocities{
.type = XR_TYPE_HAND_JOINT_VELOCITIES_EXT,
.next = nullptr,
.jointCount = joints_vel.size(),
.jointVelocities = joints_vel.data(),
};
XrHandTrackingDataSourceStateEXT data_source_state{
.type = XR_TYPE_HAND_TRACKING_DATA_SOURCE_STATE_EXT,
.next = &velocities,
};
XrHandJointLocationsEXT locations{
.type = XR_TYPE_HAND_JOINT_LOCATIONS_EXT,
.next = &velocities,
.jointCount = joints_pos.size(),
.jointLocations = joints_pos.data(),
};
if (hand_tracking_data_source_supported)
locations.next = &data_source_state;
CHECK_XR(xrLocateHandJointsEXT(id, &info, &locations));
located_hand res{};
if (locations.isActive)
{
res.joints.emplace();
auto & joints = *res.joints;
for (int i = 0; i < XR_HAND_JOINT_COUNT_EXT; i++)
{
if (joints_pos[i].locationFlags == 0)
{
// bail if any of the joint is invalid
res.joints.reset();
break;
}
joints[i] = {joints_pos[i], joints_vel[i]};
}
}
if (hand_tracking_data_source_supported)
{
res.data_source = data_source_state.dataSource;
if (not data_source_state.isActive)
// returned joints are not valid
res.joints.reset();
}
return res;
}
const xr::hand_tracker::mesh_data * xr::hand_tracker::mesh()
{
if (hand_mesh_fb_fetched)
return cached_hand_mesh_fb ? &*cached_hand_mesh_fb : nullptr;
hand_mesh_fb_fetched = true;
if (!id || !xrGetHandMeshFB)
return nullptr;
XrHandTrackingMeshFB mesh{
.type = XR_TYPE_HAND_TRACKING_MESH_FB,
};
XrResult result = xrGetHandMeshFB(id, &mesh);
if (XR_FAILED(result))
{
spdlog::warn("xrGetHandMeshFB count query failed: {}", xr::to_string(result));
return nullptr;
}
if (mesh.jointCountOutput != XR_HAND_JOINT_COUNT_EXT)
{
spdlog::warn("Unsupported hand mesh joint count {}", mesh.jointCountOutput);
return nullptr;
}
if (mesh.jointCountOutput > 32)
{
spdlog::warn("Hand mesh joint count {} exceeds renderer limit", mesh.jointCountOutput);
return nullptr;
}
if (mesh.vertexCountOutput == 0 || mesh.indexCountOutput == 0)
{
spdlog::warn("Hand mesh data is empty");
return nullptr;
}
mesh_data data;
data.joint_bind_poses.resize(mesh.jointCountOutput);
data.joint_radii.resize(mesh.jointCountOutput);
data.joint_parents.resize(mesh.jointCountOutput);
data.vertex_positions.resize(mesh.vertexCountOutput);
data.vertex_normals.resize(mesh.vertexCountOutput);
data.vertex_uvs.resize(mesh.vertexCountOutput);
data.vertex_blend_indices.resize(mesh.vertexCountOutput);
data.vertex_blend_weights.resize(mesh.vertexCountOutput);
data.indices.resize(mesh.indexCountOutput);
mesh.jointCapacityInput = data.joint_bind_poses.size();
mesh.jointBindPoses = data.joint_bind_poses.data();
mesh.jointRadii = data.joint_radii.data();
mesh.jointParents = data.joint_parents.data();
mesh.vertexCapacityInput = data.vertex_positions.size();
mesh.vertexPositions = data.vertex_positions.data();
mesh.vertexNormals = data.vertex_normals.data();
mesh.vertexUVs = data.vertex_uvs.data();
mesh.vertexBlendIndices = data.vertex_blend_indices.data();
mesh.vertexBlendWeights = data.vertex_blend_weights.data();
mesh.indexCapacityInput = data.indices.size();
mesh.indices = data.indices.data();
result = xrGetHandMeshFB(id, &mesh);
if (XR_FAILED(result))
{
spdlog::warn("xrGetHandMeshFB data query failed: {}", xr::to_string(result));
return nullptr;
}
auto count_changed = [](const char * what, uint32_t output, size_t expected) {
if (output == expected)
return false;
spdlog::warn("Hand mesh {} count changed from {} to {}", what, expected, output);
return true;
};
if (count_changed("joint", mesh.jointCountOutput, data.joint_bind_poses.size()) ||
count_changed("vertex", mesh.vertexCountOutput, data.vertex_positions.size()) ||
count_changed("index", mesh.indexCountOutput, data.indices.size()))
{
return nullptr;
}
if (mesh.indexCountOutput % 3 != 0)
{
spdlog::warn("Hand mesh index count {} is not a triangle list", mesh.indexCountOutput);
return nullptr;
}
const int joint_count = static_cast<int>(mesh.jointCountOutput);
if (!std::ranges::all_of(data.vertex_blend_indices, [joint_count](const XrVector4sFB & blend_indices) {
return 0 <= blend_indices.x && blend_indices.x < joint_count &&
0 <= blend_indices.y && blend_indices.y < joint_count &&
0 <= blend_indices.z && blend_indices.z < joint_count &&
0 <= blend_indices.w && blend_indices.w < joint_count;
}))
{
spdlog::warn("Hand mesh has out-of-range blend indices");
return nullptr;
}
const int vertex_count = static_cast<int>(mesh.vertexCountOutput);
if (!std::ranges::all_of(data.indices, [vertex_count](int16_t index) {
return 0 <= index && index < vertex_count;
}))
{
spdlog::warn("Hand mesh has out-of-range indices");
return nullptr;
}
cached_hand_mesh_fb = std::move(data);
return &*cached_hand_mesh_fb;
}
bool xr::hand_tracker::check_flags(const joint_array & joints, XrSpaceLocationFlags position, XrSpaceVelocityFlags velocity)
{
return std::ranges::all_of(joints, [position, velocity](const auto & joint) {
return (joint.first.locationFlags & position) == position and (joint.second.velocityFlags & velocity) == velocity;
});
}