Files
WiVRn/client/scene.cpp
EasonZxp 6a72c537c2 Add FB hand mesh support with GLB fallback (#934)
Use the XR_FB_HAND_TRACKING_MESH extension to render hand models when supported,
and fall back to the bundled GLB hand models otherwise. This fixes incorrect hand
model rendering on PFDM MR device, Have tested with PFDM MR V4.5.0 version.
2026-05-26 09:40:37 +02:00

612 lines
18 KiB
C++

/*
* WiVRn VR streaming
* Copyright (C) 2022 Guillaume Meunier <[email protected]>
* Copyright (C) 2022 Patrick Nicolas <[email protected]>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
#include "scene.h"
#include "application.h"
#include "render/scene_components.h"
#include "utils/contains.h"
#include "utils/i18n.h"
#include "utils/overloaded.h"
#include "utils/ranges.h"
#include <cstring>
#include <entt/core/fwd.hpp>
#include <magic_enum.hpp>
#include <spdlog/spdlog.h>
#include <vulkan/vulkan_raii.hpp>
#include <openxr/openxr.h>
// TODO remove constants
#include "constants.h"
std::vector<scene::meta *> scene::scene_registry;
scene::~scene() {}
scene::scene(key, const meta & current_meta, std::span<const vk::Format> supported_color_formats, std::span<const vk::Format> supported_depth_formats, scene * parent_scene) :
instance(application::instance().xr_instance),
system(application::instance().xr_system_id),
session(application::instance().xr_session),
viewconfig(application::instance().app_info.viewconfig),
vk_instance(application::instance().vk_instance),
device(application::instance().vk_device),
physical_device(application::instance().vk_physical_device),
queue(application::instance().vk_queue),
queue_family_index(application::instance().vk_queue_family_index),
commandpool(application::instance().vk_cmdpool),
current_meta(current_meta)
{
swapchain_format = vk::Format::eUndefined;
spdlog::info("Supported swapchain formats:");
for (auto format: session.get_swapchain_formats())
{
spdlog::info(" {}", vk::to_string(format));
}
for (auto format: session.get_swapchain_formats())
{
if (utils::contains(supported_color_formats, format))
{
swapchain_format = format;
break;
}
}
if (swapchain_format == vk::Format::eUndefined)
throw std::runtime_error(_("No supported swapchain format"));
auto views = system.view_configuration_views(viewconfig);
depth_format = scene_renderer::find_usable_image_format(
physical_device,
supported_depth_formats,
{
views[0].recommendedImageRectWidth,
views[0].recommendedImageRectHeight,
1,
},
vk::ImageUsageFlagBits::eDepthStencilAttachment);
composition_layer_depth_test_supported =
instance.has_extension(XR_KHR_COMPOSITION_LAYER_DEPTH_EXTENSION_NAME) and
instance.has_extension(XR_FB_COMPOSITION_LAYER_DEPTH_TEST_EXTENSION_NAME);
composition_layer_color_scale_bias_supported = instance.has_extension(XR_KHR_COMPOSITION_LAYER_COLOR_SCALE_BIAS_EXTENSION_NAME);
if (parent_scene)
{
renderer = parent_scene->renderer;
gltf_cache = parent_scene->gltf_cache;
image_cache = parent_scene->image_cache;
}
else
{
renderer = std::make_shared<scene_renderer>(device, physical_device, queue, queue_family_index);
gltf_cache = std::make_shared<gltf_cache_type>(device, physical_device, queue, queue_family_index, renderer->get_default_material(), application::get_cache_path() / "textures");
image_cache = std::make_shared<image_cache_type>(device, physical_device, queue, queue_family_index);
}
}
void scene::set_focused(bool status)
{
if (status != focused)
{
focused = status;
if (focused)
on_focused();
else
on_unfocused();
}
}
glm::mat4 scene::projection_matrix(XrFovf fov, float zn)
{
float r = tan(fov.angleRight);
float l = tan(fov.angleLeft);
float t = tan(fov.angleUp);
float b = tan(fov.angleDown);
// reversed Z projection, infinite far plane
// clang-format off
return glm::mat4{
{ 2/(r-l), 0, 0, 0 },
{ 0, 2/(b-t), 0, 0 },
{ (l+r)/(r-l), (t+b)/(b-t), 0, -1 },
{ 0, 0, zn, 0 }
};
// clang-format on
}
glm::mat4 scene::view_matrix(XrPosef pose)
{
XrQuaternionf q = pose.orientation;
XrVector3f pos = pose.position;
glm::mat4 inv_view_matrix = glm::mat4_cast(glm::quat(q.w, q.x, q.y, q.z));
inv_view_matrix = glm::translate(glm::mat4(1), glm::vec3(pos.x, pos.y, pos.z)) * inv_view_matrix;
return glm::inverse(inv_view_matrix);
}
void scene::render_world(
XrCompositionLayerFlags flags,
XrSpace space,
std::span<XrView> views,
uint32_t width,
uint32_t height,
bool keep_depth_buffer,
uint32_t layer_mask,
XrColor4f clear_color,
bool render_debug_draws)
{
assert(views.size() <= layer::max_views);
beman::inplace_vector::inplace_vector<scene_renderer::frame_info, layer::max_views> frames;
beman::inplace_vector::inplace_vector<XrCompositionLayerProjectionView, layer::max_views> composition_layer_color;
beman::inplace_vector::inplace_vector<XrCompositionLayerDepthInfoKHR, layer::max_views> composition_layer_depth;
auto [color_swapchain, color_image] = [&]() -> std::tuple<XrSwapchain, vk::Image> {
xr::swapchain & color_swapchain = get_swapchain(swapchain_format, width, height, 1, views.size());
int color_image_index = color_swapchain.acquire();
vk::Image color_image = color_swapchain.image(color_image_index);
color_swapchain.wait();
return {color_swapchain, color_image};
}();
auto [depth_swapchain, depth_image] = [&]() -> std::pair<XrSwapchain, vk::Image> {
if (keep_depth_buffer)
{
xr::swapchain & depth_swapchain = get_swapchain(depth_format, width, height, 1, views.size());
int depth_image_index = depth_swapchain.acquire();
vk::Image depth_image = depth_swapchain.image(depth_image_index);
depth_swapchain.wait();
return {depth_swapchain, depth_image};
}
else
return {};
}();
for (auto && [index, view]: utils::enumerate(views))
{
frames.push_back({
.projection = projection_matrix(view.fov, constants::lobby::near_plane),
.view = view_matrix(view.pose),
});
composition_layer_color.push_back({
.type = XR_TYPE_COMPOSITION_LAYER_PROJECTION_VIEW,
.pose = view.pose,
.fov = view.fov,
.subImage = {
.swapchain = color_swapchain,
.imageRect = {
.offset = {0, 0},
.extent = {(int)width, (int)height},
},
.imageArrayIndex = (uint32_t)index,
},
});
if (keep_depth_buffer)
{
composition_layer_depth.push_back({
.type = XR_TYPE_COMPOSITION_LAYER_DEPTH_INFO_KHR,
.subImage = {
.swapchain = depth_swapchain,
.imageRect = {
.offset = {0, 0},
.extent = {(int)width, (int)height},
},
.imageArrayIndex = (uint32_t)index,
},
.minDepth = 0,
.maxDepth = 1,
.nearZ = std::bit_cast<float>(0x7f800000), // infinity
.farZ = constants::lobby::near_plane,
});
static_assert(std::numeric_limits<float>::is_iec559);
}
}
// TODO: compute transforms only once
renderer->render(
world,
{clear_color.r, clear_color.g, clear_color.b, clear_color.a},
layer_mask,
vk::Extent2D{width, height},
swapchain_format,
depth_format,
color_image,
depth_image,
frames,
render_debug_draws);
add_projection_layer(
XR_COMPOSITION_LAYER_BLEND_TEXTURE_SOURCE_ALPHA_BIT,
space,
composition_layer_color,
composition_layer_depth);
}
xr::swapchain & scene::get_swapchain(vk::Format format, int32_t width, int32_t height, int sample_count, uint32_t array_size)
{
// Look for an exact match
for (swapchain_entry & entry: swapchains)
{
if (not entry.used and
entry.format == format and
entry.width == width and
entry.height == height and
entry.sample_count == sample_count and
entry.array_size == array_size)
{
entry.used = true;
return entry.swapchain;
}
}
spdlog::info("Creating new swapchain: {}, {}x{}, {} sample(s), {} level(s)", magic_enum::enum_name(format), width, height, sample_count, array_size);
swapchain_entry new_swapchain{
.format = format,
.width = width,
.height = height,
.sample_count = sample_count,
.array_size = array_size,
.used = true,
.swapchain = xr::swapchain(
instance,
session,
device,
format,
width,
height,
sample_count,
array_size)};
spdlog::info("Created swapchain");
return swapchains.emplace_back(std::move(new_swapchain)).swapchain;
}
void scene::clear_swapchains()
{
swapchains.clear();
}
void scene::render_start(bool passthrough, XrTime predicted_display_time_)
{
blend_mode = XR_ENVIRONMENT_BLEND_MODE_OPAQUE;
predicted_display_time = predicted_display_time_;
layers.clear();
openxr_layers.clear();
for (swapchain_entry & swapchain: swapchains)
swapchain.used = false;
if (renderer)
renderer->start_frame();
if (passthrough)
{
std::visit(
utils::overloaded{
[&](std::monostate &) {
assert(false);
},
[&](xr::passthrough_alpha_blend & p) {
blend_mode = XR_ENVIRONMENT_BLEND_MODE_ALPHA_BLEND;
},
[&](auto & p) {
layers.push_back(layer{
.composition_layer = p.layer(),
});
}},
session.get_passthrough());
}
}
void scene::add_projection_layer(
XrCompositionLayerFlags flags,
XrSpace space,
std::span<XrCompositionLayerProjectionView> color_views,
std::span<XrCompositionLayerDepthInfoKHR> depth_views)
{
layers.push_back(layer{
.composition_layer = XrCompositionLayerProjection{
.type = XR_TYPE_COMPOSITION_LAYER_PROJECTION,
.next = nullptr,
.layerFlags = flags,
.space = space,
.viewCount = (uint32_t)color_views.size(),
.views = nullptr,
},
.color_views{color_views.begin(), color_views.end()},
.depth_views{depth_views.begin(), depth_views.end()},
});
}
void scene::add_quad_layer(
XrCompositionLayerFlags flags,
XrSpace space,
XrEyeVisibility eyeVisibility,
XrSwapchainSubImage subImage,
XrPosef pose,
XrExtent2Df size)
{
layers.push_back(layer{
.composition_layer = XrCompositionLayerQuad{
.type = XR_TYPE_COMPOSITION_LAYER_QUAD,
.next = nullptr,
.layerFlags = flags,
.space = space,
.eyeVisibility = eyeVisibility,
.subImage = subImage,
.pose = pose,
.size = size,
},
});
}
void scene::set_color_scale_bias(XrColor4f scale, XrColor4f bias)
{
assert(not layers.empty());
layers.back().color_scale_bias = XrCompositionLayerColorScaleBiasKHR{
.type = XR_TYPE_COMPOSITION_LAYER_COLOR_SCALE_BIAS_KHR,
.colorScale = scale,
.colorBias = bias,
};
}
void scene::set_depth_test(bool write, XrCompareOpFB op)
{
assert(not layers.empty());
layers.back().depth_test = XrCompositionLayerDepthTestFB{
.type = XR_TYPE_COMPOSITION_LAYER_DEPTH_TEST_FB,
.depthMask = write,
.compareOp = op,
};
}
void scene::set_layer_settings(XrCompositionLayerSettingsFlagsFB flags)
{
assert(not layers.empty());
layers.back().settings = XrCompositionLayerSettingsFB{
.type = XR_TYPE_COMPOSITION_LAYER_SETTINGS_FB,
.layerFlags = flags,
};
}
void scene::render_end()
{
// Fixup all pointers
for (auto & i: layers)
{
XrCompositionLayerBaseHeader * base = std::visit(
utils::overloaded{
[&](XrCompositionLayerProjection & layer) {
layer.views = i.color_views.data();
for (auto [color, depth]: std::views::zip(i.color_views, i.depth_views))
color.next = &depth;
return reinterpret_cast<XrCompositionLayerBaseHeader *>(&layer);
},
[&](XrCompositionLayerQuad & layer) {
return reinterpret_cast<XrCompositionLayerBaseHeader *>(&layer);
},
[&](XrCompositionLayerBaseHeader * layer) {
return layer;
},
},
i.composition_layer);
if (i.color_scale_bias)
{
i.color_scale_bias->next = base->next;
base->next = &*i.color_scale_bias;
}
if (i.depth_test)
{
i.depth_test->next = base->next;
base->next = &*i.depth_test;
}
if (i.settings)
{
i.settings->type = XR_TYPE_COMPOSITION_LAYER_SETTINGS_FB;
i.settings->next = base->next;
base->next = &*i.settings;
}
openxr_layers.push_back(base);
}
if (renderer)
renderer->end_frame();
// Release all swapchains after the renderer has submitted the command buffers
for (auto & swapchain: swapchains)
{
if (swapchain.used)
swapchain.swapchain.release();
}
session.end_frame(predicted_display_time, openxr_layers, blend_mode);
}
template <typename T>
void copy_components(entt::registry & scene, const entt::registry & prefab, const std::unordered_map<entt::entity, entt::entity> & entity_map)
{
for (const auto & [entity, component]: prefab.view<T>().each())
{
scene.emplace<T>(entity_map.at(entity), component);
}
}
std::pair<entt::entity, components::node &> scene::add_gltf(std::shared_ptr<entt::registry> gltf, uint32_t layer_mask)
{
auto root = world.create();
auto & node = world.emplace<components::node>(root);
node.layer_mask = layer_mask;
auto prefab_entities = gltf->view<entt::entity>();
std::vector<entt::entity> scene_entities{prefab_entities.size()};
world.create(scene_entities.begin(), scene_entities.end());
std::unordered_map<entt::entity, entt::entity> entity_map; // key: prefab entity, value: scene entity
entity_map.emplace(entt::null, root);
for (auto [prefab_entity, scene_entity]: std::ranges::zip_view(prefab_entities, scene_entities))
entity_map.emplace(prefab_entity, scene_entity);
copy_components<components::node>(world, *gltf, entity_map);
copy_components<components::animation>(world, *gltf, entity_map);
// update links
for (auto [prefab_entity, scene_entity]: entity_map)
{
if (prefab_entity == entt::null)
continue;
auto * node = world.try_get<components::node>(scene_entity);
auto * anim = world.try_get<components::animation>(scene_entity);
if (node)
{
node->parent = entity_map.at(node->parent);
for (auto & joint: node->joints)
{
joint.first = entity_map.at(joint.first);
}
}
if (anim)
{
for (auto & track: anim->tracks)
{
std::visit(utils::overloaded{[&](auto & t) { t.target = entity_map.at(t.target); }}, track);
}
}
}
return {root, node};
}
std::shared_ptr<entt::registry> scene::load_gltf(const std::filesystem::path & path, std::function<void(float)> progress_cb)
{
return gltf_cache->load(path, path, progress_cb);
}
void scene::unload_gltf(const std::filesystem::path & path)
{
gltf_cache->remove(path);
}
std::pair<entt::entity, components::node &> scene::add_gltf(const std::filesystem::path & path, uint32_t layer_mask)
{
return add_gltf(load_gltf(path), layer_mask);
}
std::shared_ptr<renderer::material> scene::create_material(std::function<void(renderer::material &)> init) const
{
auto material = std::make_shared<renderer::material>(*renderer->get_default_material());
if (init)
init(*material);
material->buffer = std::make_shared<buffer_allocation>(
device,
vk::BufferCreateInfo{
.size = sizeof(material->staging),
.usage = vk::BufferUsageFlagBits::eUniformBuffer},
VmaAllocationCreateInfo{
.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT,
.usage = VMA_MEMORY_USAGE_AUTO});
material->offset = 0;
std::memcpy(material->buffer->map(), &material->staging, sizeof(material->staging));
material->buffer->unmap();
return material;
}
void scene::remove(entt::entity entity)
{
std::vector to_be_removed{entity};
while (not to_be_removed.empty())
{
entt::entity parent = to_be_removed.back();
to_be_removed.pop_back();
world.destroy(parent);
for (auto [e, node]: world.view<components::node>().each())
{
if (node.parent == parent)
to_be_removed.push_back(e);
}
for (auto [e, anim]: world.view<components::animation>().each())
{
if (std::ranges::any_of(anim.tracks,
[&](const auto & track) { return std::visit(
[&](const components::animation_track_base & track) {
return track.target == parent;
},
track); }))
{
to_be_removed.push_back(e);
}
}
}
}
void scene::on_unfocused() {}
void scene::on_focused() {}
void scene::on_xr_event(const xr::event &) {}
bool scene::on_input_key_down(uint8_t key_code)
{
return false;
}
bool scene::on_input_key_up(uint8_t key_code)
{
return false;
}
bool scene::on_input_mouse_move(float x, float y)
{
return false;
}
bool scene::on_input_button_down(uint8_t button)
{
return false;
}
bool scene::on_input_button_up(uint8_t button)
{
return false;
}
bool scene::on_input_scroll(float h, float v)
{
return false;
}