Files
WiVRn/client/scenes/hand_model.cpp

374 lines
14 KiB
C++

/*
* WiVRn VR streaming
* Copyright (C) 2024 Guillaume Meunier <[email protected]>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
#include "hand_model.h"
#include "openxr/openxr.h"
#include "render/gpu_buffer.h"
#include "render/scene_components.h"
#include "vk/shader.h"
#include <algorithm>
#include <cctype>
#include <cstddef>
#include <cstring>
#include <entt/entt.hpp>
#include <glm/gtc/matrix_transform.hpp>
#include <glm/gtx/quaternion.hpp>
#include <magic_enum.hpp>
#include <spdlog/spdlog.h>
#include <stdexcept>
#include <string>
#include <string_view>
namespace components
{
struct hand_joint
{
XrHandEXT hand;
XrHandJointEXT joint;
};
struct hand_mesh_fb_root
{
XrHandEXT hand;
};
} // namespace components
static glm::vec3 to_vec3(const XrVector3f & value)
{
return {value.x, value.y, value.z};
}
static glm::quat to_quat(const XrQuaternionf & value)
{
return glm::quat(value.w, value.x, value.y, value.z);
}
static glm::mat4 to_matrix(const XrPosef & pose)
{
return glm::translate(glm::mat4(1), to_vec3(pose.position)) * glm::mat4_cast(to_quat(pose.orientation));
}
template <typename T>
static void write_vertex_attribute(std::vector<std::byte> & buffer, size_t stride, size_t offset, size_t index, const T & value)
{
std::memcpy(buffer.data() + index * stride + offset, &value, sizeof(value));
}
static std::string normalize_hand_mesh_fb_semantic(std::string semantic)
{
std::transform(semantic.begin(), semantic.end(), semantic.begin(), [](unsigned char c) {
return static_cast<char>(std::toupper(c));
});
if (semantic.starts_with("IN_"))
semantic = semantic.substr(3);
return semantic;
}
static renderer::vertex_layout create_hand_mesh_fb_layout(scene & scene)
{
renderer::vertex_layout layout;
auto shader = load_shader(scene.device, "lit_skinned.vert");
for (const auto & input: shader->inputs)
{
std::string semantic = normalize_hand_mesh_fb_semantic(input.name);
const uint32_t binding = semantic == "POSITION" || semantic == "JOINTS" || semantic == "WEIGHTS" ? 0 : 1;
layout.add_vertex_attribute(semantic, input.format, binding, input.location, input.array_size);
}
return layout;
}
static const vk::VertexInputBindingDescription & find_hand_mesh_fb_binding(const renderer::vertex_layout & layout, uint32_t binding)
{
for (const auto & description: layout.bindings)
if (description.binding == binding)
return description;
throw std::runtime_error("Missing hand mesh FB vertex binding");
}
static const vk::VertexInputAttributeDescription & find_hand_mesh_fb_attribute(const renderer::vertex_layout & layout, std::string_view semantic)
{
for (size_t i = 0; i < layout.attribute_names.size(); ++i)
if (layout.attribute_names[i] == semantic)
return layout.attributes.at(i);
throw std::runtime_error("Missing hand mesh FB vertex attribute");
}
static std::shared_ptr<renderer::material> create_hand_mesh_fb_material(scene & scene, XrHandEXT hand)
{
return scene.create_material([hand](renderer::material & material) {
material.staging.base_color_factor = {0.5f, 0.5f, 0.5f, 1.0f};
material.staging.metallic_factor = 0.0f;
material.staging.roughness_factor = 0.5527864f;
material.name = hand == XR_HAND_LEFT_EXT ? "left-hand-mesh-material" : "right-hand-mesh-material";
material.double_sided = false;
material.blend_enable = false;
});
}
static std::shared_ptr<renderer::mesh> create_hand_mesh_fb_mesh(scene & scene, const xr::hand_tracker::mesh_data & mesh)
{
std::vector<uint16_t> indices;
indices.reserve(mesh.indices.size());
for (int16_t index: mesh.indices)
indices.push_back(static_cast<uint16_t>(index));
auto layout = create_hand_mesh_fb_layout(scene);
const auto & binding_0_layout = find_hand_mesh_fb_binding(layout, 0);
const auto & binding_1_layout = find_hand_mesh_fb_binding(layout, 1);
const size_t position_offset = find_hand_mesh_fb_attribute(layout, "POSITION").offset;
const size_t joints_offset = find_hand_mesh_fb_attribute(layout, "JOINTS").offset;
const size_t weights_offset = find_hand_mesh_fb_attribute(layout, "WEIGHTS").offset;
const size_t normal_offset = find_hand_mesh_fb_attribute(layout, "NORMAL").offset;
const size_t tangent_offset = find_hand_mesh_fb_attribute(layout, "TANGENT").offset;
const size_t texcoord_0_offset = find_hand_mesh_fb_attribute(layout, "TEXCOORD_0").offset;
const size_t texcoord_1_offset = find_hand_mesh_fb_attribute(layout, "TEXCOORD_1").offset;
const size_t color_offset = find_hand_mesh_fb_attribute(layout, "COLOR").offset;
std::vector<std::byte> binding_0(mesh.vertex_positions.size() * binding_0_layout.stride);
std::vector<std::byte> binding_1(mesh.vertex_positions.size() * binding_1_layout.stride);
glm::vec3 obb_min;
glm::vec3 obb_max;
for (size_t i = 0; i < mesh.vertex_positions.size(); ++i)
{
glm::vec3 position = to_vec3(mesh.vertex_positions.at(i));
glm::vec3 normal = to_vec3(mesh.vertex_normals.at(i));
glm::vec4 tangent{};
glm::vec2 texcoord_0{mesh.vertex_uvs.at(i).x, mesh.vertex_uvs.at(i).y};
glm::vec2 texcoord_1{};
glm::vec4 color{1, 1, 1, 1};
glm::vec4 joints{
static_cast<float>(mesh.vertex_blend_indices.at(i).x),
static_cast<float>(mesh.vertex_blend_indices.at(i).y),
static_cast<float>(mesh.vertex_blend_indices.at(i).z),
static_cast<float>(mesh.vertex_blend_indices.at(i).w)};
glm::vec4 weights{
mesh.vertex_blend_weights.at(i).x,
mesh.vertex_blend_weights.at(i).y,
mesh.vertex_blend_weights.at(i).z,
mesh.vertex_blend_weights.at(i).w};
if (i == 0)
obb_min = obb_max = position;
else
{
obb_min.x = std::min(obb_min.x, position.x);
obb_min.y = std::min(obb_min.y, position.y);
obb_min.z = std::min(obb_min.z, position.z);
obb_max.x = std::max(obb_max.x, position.x);
obb_max.y = std::max(obb_max.y, position.y);
obb_max.z = std::max(obb_max.z, position.z);
}
write_vertex_attribute(binding_0, binding_0_layout.stride, position_offset, i, position);
write_vertex_attribute(binding_0, binding_0_layout.stride, joints_offset, i, joints);
write_vertex_attribute(binding_0, binding_0_layout.stride, weights_offset, i, weights);
write_vertex_attribute(binding_1, binding_1_layout.stride, normal_offset, i, normal);
write_vertex_attribute(binding_1, binding_1_layout.stride, tangent_offset, i, tangent);
write_vertex_attribute(binding_1, binding_1_layout.stride, texcoord_0_offset, i, texcoord_0);
write_vertex_attribute(binding_1, binding_1_layout.stride, texcoord_1_offset, i, texcoord_1);
write_vertex_attribute(binding_1, binding_1_layout.stride, color_offset, i, color);
}
fastgltf::Asset asset;
gpu_buffer staging_buffer(scene.physical_device, asset);
auto hand_mesh_fb_mesh = std::make_shared<renderer::mesh>();
auto & primitive = hand_mesh_fb_mesh->primitives.emplace_back();
primitive.indexed = true;
primitive.index_count = indices.size();
primitive.vertex_count = mesh.vertex_positions.size();
primitive.index_type = vk::IndexType::eUint16;
primitive.index_offset = staging_buffer.add_indices(indices);
primitive.vertex_offset.push_back(staging_buffer.add_vertices(binding_0));
primitive.vertex_offset.push_back(staging_buffer.add_vertices(binding_1));
primitive.layout = std::move(layout);
primitive.obb_min = obb_min;
primitive.obb_max = obb_max;
primitive.vertex_shader = "lit_skinned.vert";
primitive.cull_mode = vk::CullModeFlagBits::eBack;
primitive.front_face = vk::FrontFace::eCounterClockwise;
primitive.topology = vk::PrimitiveTopology::eTriangleList;
hand_mesh_fb_mesh->buffer = std::make_shared<buffer_allocation>(staging_buffer.copy_to_gpu(scene.device));
return hand_mesh_fb_mesh;
}
static const std::array<std::pair<XrHandJointEXT, std::string_view>, XR_HAND_JOINT_COUNT_EXT - 1 /* No palm in gltf */> joints{{
// clang-format off
{XR_HAND_JOINT_WRIST_EXT, "wrist"},
{XR_HAND_JOINT_THUMB_METACARPAL_EXT, "thumb-metacarpal"},
{XR_HAND_JOINT_THUMB_PROXIMAL_EXT, "thumb-phalanx-proximal"},
{XR_HAND_JOINT_THUMB_DISTAL_EXT, "thumb-phalanx-distal"},
{XR_HAND_JOINT_THUMB_TIP_EXT, "thumb-tip"},
{XR_HAND_JOINT_INDEX_METACARPAL_EXT, "index-finger-metacarpal"},
{XR_HAND_JOINT_INDEX_PROXIMAL_EXT, "index-finger-phalanx-proximal"},
{XR_HAND_JOINT_INDEX_INTERMEDIATE_EXT, "index-finger-phalanx-intermediate"},
{XR_HAND_JOINT_INDEX_DISTAL_EXT, "index-finger-phalanx-distal"},
{XR_HAND_JOINT_INDEX_TIP_EXT, "index-finger-tip"},
{XR_HAND_JOINT_MIDDLE_METACARPAL_EXT, "middle-finger-metacarpal"},
{XR_HAND_JOINT_MIDDLE_PROXIMAL_EXT, "middle-finger-phalanx-proximal"},
{XR_HAND_JOINT_MIDDLE_INTERMEDIATE_EXT, "middle-finger-phalanx-intermediate"},
{XR_HAND_JOINT_MIDDLE_DISTAL_EXT, "middle-finger-phalanx-distal"},
{XR_HAND_JOINT_MIDDLE_TIP_EXT, "middle-finger-tip"},
{XR_HAND_JOINT_RING_METACARPAL_EXT, "ring-finger-metacarpal"},
{XR_HAND_JOINT_RING_PROXIMAL_EXT, "ring-finger-phalanx-proximal"},
{XR_HAND_JOINT_RING_INTERMEDIATE_EXT, "ring-finger-phalanx-intermediate"},
{XR_HAND_JOINT_RING_DISTAL_EXT, "ring-finger-phalanx-distal"},
{XR_HAND_JOINT_RING_TIP_EXT, "ring-finger-tip"},
{XR_HAND_JOINT_LITTLE_METACARPAL_EXT, "pinky-finger-metacarpal"},
{XR_HAND_JOINT_LITTLE_PROXIMAL_EXT, "pinky-finger-phalanx-proximal"},
{XR_HAND_JOINT_LITTLE_INTERMEDIATE_EXT, "pinky-finger-phalanx-intermediate"},
{XR_HAND_JOINT_LITTLE_DISTAL_EXT, "pinky-finger-phalanx-distal"},
{XR_HAND_JOINT_LITTLE_TIP_EXT, "pinky-finger-tip"},
// clang-format on
}};
void hand_model::add_hand(scene & scene,
XrHandEXT hand,
const std::filesystem::path & gltf_path,
uint32_t layer_mask)
{
auto && [entity, node] = scene.add_gltf(gltf_path, layer_mask);
node.name = gltf_path.stem();
for (auto [joint, name]: joints)
{
entt::entity joint_entity = find_node_by_name(scene.world, name, entity);
scene.world.emplace<components::hand_joint>(joint_entity, hand, joint);
}
// Fake joint to hide the hand when it should not be visible
scene.world.emplace<components::hand_joint>(entity, hand, (XrHandJointEXT)-1);
}
void hand_model::add_hand(scene & scene,
XrHandEXT hand,
const xr::hand_tracker::mesh_data & mesh,
uint32_t layer_mask)
{
auto material = create_hand_mesh_fb_material(scene, hand);
entt::entity entity = scene.world.create();
auto & node = scene.world.emplace<components::node>(entity);
node.name = hand == XR_HAND_LEFT_EXT ? "left-hand-mesh" : "right-hand-mesh";
node.mesh = create_hand_mesh_fb_mesh(scene, mesh);
for (auto & primitive: node.mesh->primitives)
primitive.material_ = material;
node.visible = false;
node.layer_mask = layer_mask;
node.joints.reserve(mesh.joint_bind_poses.size());
scene.world.emplace<components::hand_mesh_fb_root>(entity, hand);
for (size_t i = 0; i < mesh.joint_bind_poses.size(); ++i)
{
entt::entity joint_entity = scene.world.create();
auto & joint_node = scene.world.emplace<components::node>(joint_entity);
joint_node.parent = entity;
joint_node.name = std::string(magic_enum::enum_name(static_cast<XrHandJointEXT>(i)));
node.joints.emplace_back(joint_entity, glm::inverse(to_matrix(mesh.joint_bind_poses[i])));
}
}
void hand_model::apply(entt::registry & scene,
const xr::hand_tracker::joint_array * left_hand,
const xr::hand_tracker::joint_array * right_hand)
{
for (auto && [entity, hj, node]: scene.view<components::hand_joint, components::node>().each())
{
auto f = [&](const xr::hand_tracker::joint_array * joints) {
if (joints)
{
node.visible = true;
if (hj.joint < 0 or hj.joint > XR_HAND_JOINT_COUNT_EXT)
return;
const auto & pose = (*joints)[hj.joint].first.pose;
node.position = {
pose.position.x,
pose.position.y,
pose.position.z};
node.orientation = {
pose.orientation.w,
pose.orientation.x,
pose.orientation.y,
pose.orientation.z};
}
else
{
node.visible = false;
}
};
switch (hj.hand)
{
case XR_HAND_LEFT_EXT:
f(left_hand);
break;
case XR_HAND_RIGHT_EXT:
f(right_hand);
break;
default:
break;
}
}
for (auto && [entity, hand_root, node]: scene.view<components::hand_mesh_fb_root, components::node>().each())
{
const auto & joints = hand_root.hand == XR_HAND_LEFT_EXT ? left_hand : right_hand;
if (!joints)
{
node.visible = false;
continue;
}
node.visible = true;
const auto & wrist_pose = (*joints)[XR_HAND_JOINT_WRIST_EXT].first.pose;
glm::vec3 wrist_position = to_vec3(wrist_pose.position);
glm::quat wrist_orientation = to_quat(wrist_pose.orientation);
glm::quat wrist_orientation_inv = glm::conjugate(wrist_orientation);
node.position = wrist_position;
node.orientation = wrist_orientation;
for (size_t i = 0; i < node.joints.size(); ++i)
{
auto & joint_node = scene.get<components::node>(node.joints[i].first);
if (i == XR_HAND_JOINT_WRIST_EXT)
{
joint_node.position = {};
joint_node.orientation = {1, 0, 0, 0};
continue;
}
const auto & pose = (*joints)[i].first.pose;
glm::vec3 position = to_vec3(pose.position);
glm::quat orientation = to_quat(pose.orientation);
joint_node.position = wrist_orientation_inv * (position - wrist_position);
joint_node.orientation = glm::normalize(wrist_orientation_inv * orientation);
}
}
}