diff --git a/CMakeLists.txt b/CMakeLists.txt index f3a5f225..91584c70 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,6 +40,8 @@ endif() find_package(PkgConfig REQUIRED) find_package(Vulkan REQUIRED) +find_package(Git) + if (WIVRN_USE_SYSTEM_BOOST STREQUAL "AUTO") find_package(Boost 1.75.0) @@ -130,10 +132,11 @@ FetchContent_Declare(monado GIT_TAG 79bf8eb8fa168f65f4e5505e0525ee74aa88783e ) -# Use the docking branch of imgui to use multi-viewport FetchContent_Declare(imgui - GIT_REPOSITORY https://github.com/ocornut/imgui.git - GIT_TAG ce0d0ac8298ce164b5d862577e8b087d92f6e90e + URL https://github.com/ocornut/imgui/archive/refs/tags/v1.90.1.tar.gz +# Use the docking branch of imgui to use multi-viewport + # GIT_REPOSITORY https://github.com/ocornut/imgui.git + # GIT_TAG ce0d0ac8298ce164b5d862577e8b087d92f6e90e ) add_subdirectory(external) diff --git a/README.md b/README.md index 73edb52e..b9ea7f8a 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,7 @@ Identifier (number) of the encoder group. Encoders with the same identifier are "codec": "h265" } ] +} ``` Creates a single encoder, using vaapi hardware encoding, h265 video codec (HEVC) and 50Mb/s bitrate. @@ -246,5 +247,6 @@ WiVRn uses the following software: - [spdlog](https://github.com/gabime/spdlog) - [VulkanMemoryAllocator](https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator) - [x264](https://www.videolan.org/developers/x264.html) optional, for software encoding - -The controller models are from the [WebXR input profiles](https://www.npmjs.com/package/@webxr-input-profiles/motion-controllers) +- [ambientCG](https://ambientcg.com/) +- [Roboto](https://fonts.google.com/specimen/Roboto) +- [WebXR input profiles](https://www.npmjs.com/package/@webxr-input-profiles/motion-controllers) diff --git a/assets/Roboto-Regular.ttf b/assets/Roboto-Regular.ttf new file mode 100644 index 00000000..ddf4bfac Binary files /dev/null and b/assets/Roboto-Regular.ttf differ diff --git a/assets/ground.gltf b/assets/ground.gltf new file mode 100644 index 00000000..3b17d1cd --- /dev/null +++ b/assets/ground.gltf @@ -0,0 +1,153 @@ +{ + "accessors": [ + { + "bufferView": 0, + "byteOffset": 0, + "componentType": 5123, + "count": 6, + "type": "SCALAR", + "max": [ + 5 + ], + "min": [ + 0 + ] + }, + { + "bufferView": 1, + "byteOffset": 0, + "componentType": 5126, + "count": 6, + "type": "VEC3", + "max": [ + 100, + 0, + 100 + ], + "min": [ + -100, + 0, + -100 + ], + "name": "POSITION" + }, + { + "bufferView": 1, + "byteOffset": 12, + "componentType": 5126, + "count": 6, + "type": "VEC2", + "max": [ + 100, + 1 + ], + "min": [ + 0, + -99 + ], + "name": "TEXCOORD_0" + }, + { + "bufferView": 1, + "byteOffset": 20, + "componentType": 5126, + "count": 6, + "type": "VEC3", + "max": [ + 0, + 1.0000044107437134, + 0 + ], + "min": [ + 0, + 1.0000044107437134, + 0 + ], + "name": "NORMAL" + } + ], + "asset": { + "generator": "Aspose.3D 23.12.0", + "version": "2.0" + }, + "buffers": [ + { + "uri": "data:application/octet-stream;base64,AAABAAIAAwAEAAUAAAAAAAAAyMIAAAAAAADIwgAAAAAAAIA/AAAAACUAgD8AAAAAAADIwgAAAAAAAMhCAADIQgAAgD8AAAAAJQCAPwAAAAAAAMhCAAAAAAAAyMIAAAAAAADGwgAAAAAlAIA/AAAAAAAAyEIAAAAAAADIQgAAyEIAAMbCAAAAACUAgD8AAAAAAADIQgAAAAAAAMjCAAAAAAAAxsIAAAAAJQCAPwAAAAAAAMjCAAAAAAAAyEIAAMhCAACAPwAAAAAlAIA/AAAAAAAAAAA=", + "byteLength": 212 + } + ], + "images" : [ + { + "uri": "WoodFloor051_1K-JPG_Color.jpg" + } + ], + "bufferViews": [ + { + "buffer": 0, + "byteOffset": 0, + "byteLength": 12, + "target": 34963 + }, + { + "buffer": 0, + "byteOffset": 16, + "byteLength": 192, + "byteStride": 32, + "target": 34962 + } + ], + "meshes": [ + { + "primitives": [ + { + "attributes": { + "POSITION": 1, + "TEXCOORD_0": 2, + "NORMAL": 3 + }, + "indices": 0, + "mode": 4, + "material": 0 + } + ], + "name": "" + } + ], + "samplers" : [ + { + "wrapS": 10497, + "wrapT": 10497, + "minFilter": 9987 + } + ], + "textures" : [ + { + "sampler" : 0, + "source" : 0 + } + ], + "materials" : [ + { + "name": "ground", + "pbrMetallicRoughness": { + "baseColorTexture": { + "index": 0 + } + } + } + ], + "nodes": [ + { + "mesh": 0, + "name": "ground" + } + ], + "scene": 0, + "scenes": [ + { + "nodes": [ + 0 + ] + } + ] +} diff --git a/assets/imgui.gltf b/assets/imgui.gltf index 1255aa32..07db31d5 100644 --- a/assets/imgui.gltf +++ b/assets/imgui.gltf @@ -114,7 +114,11 @@ } ], "samplers" : [ - {} + { + "wrapS": 33071, + "wrapT": 33071, + "minFilter": 9987 + } ], "textures" : [ { @@ -124,11 +128,11 @@ ], "materials" : [ { - "name" : "imgui", - "pbrMetallicRoughness" : { - "baseColorTexture" : { - "index" : 0 - } + "name": "imgui", + "pbrMetallicRoughness": { + "baseColorTexture": { + "index": 0 + } } } ], @@ -136,7 +140,8 @@ { "mesh": 0, "name": "imgui", - "translation": [0, 1, -1.5] + "translation": [0, 1.5, -1.5], + "scale": [1.5, 1, 1] } ], "scene": 0, diff --git a/client/CMakeLists.txt b/client/CMakeLists.txt index 98185521..9cdb85a6 100644 --- a/client/CMakeLists.txt +++ b/client/CMakeLists.txt @@ -15,10 +15,12 @@ add_library(imgui OBJECT ) target_include_directories(imgui PUBLIC ${imgui_SOURCE_DIR}) +# target_compile_definitions(imgui PUBLIC "IMGUI_USER_CONFIG=\"render/imgui_config.h\"") target_compile_definitions(imgui PUBLIC IMGUI_USE_WCHAR32) target_compile_definitions(imgui PUBLIC IMGUI_ENABLE_FREETYPE) target_compile_definitions(imgui PUBLIC IMGUI_DISABLE_OBSOLETE_FUNCTIONS) # target_compile_definitions(imgui PUBLIC IMGUI_ENABLE_FREETYPE_LUNASVG) +target_compile_definitions(imgui PUBLIC IMGUI_DEFINE_MATH_OPERATORS) target_link_libraries(imgui PUBLIC FreetypeHarfbuzz) add_library(stb INTERFACE) diff --git a/client/application.cpp b/client/application.cpp index 0e45f731..89684423 100644 --- a/client/application.cpp +++ b/client/application.cpp @@ -26,11 +26,13 @@ #include "spdlog/spdlog.h" #include "utils/check.h" #include "utils/files.h" +#include "utils/named_thread.h" #include "xr/actionset.h" #include "xr/xr.h" #include #include "utils/contains.h" #include +#include #include #include #include @@ -491,7 +493,9 @@ void application::initialize_vulkan() .pQueuePriorities = &queuePriority }; - vk::PhysicalDeviceFeatures device_features; + vk::PhysicalDeviceFeatures device_features{ + // .samplerAnisotropy = true, + }; vk::StructureChain device_create_info{ vk::DeviceCreateInfo{ @@ -753,32 +757,7 @@ void application::initialize() view_space = xr_session.create_reference_space(XR_REFERENCE_SPACE_TYPE_VIEW); world_space = xr_session.create_reference_space(XR_REFERENCE_SPACE_TYPE_STAGE); - - swapchain_format = vk::Format::eUndefined; - for (auto i: xr_session.get_swapchain_formats()) - { - if (std::find(supported_formats.begin(), supported_formats.end(), i) != supported_formats.end()) - { - swapchain_format = i; - break; - } - } - - if (swapchain_format == vk::Format::eUndefined) - throw std::runtime_error("No supported swapchain format"); - - spdlog::info("Using format {}", vk::to_string(swapchain_format)); - - auto views = xr_system_id.view_configuration_views(app_info.viewconfig); - - xr_swapchains.reserve(views.size()); - for (auto view: views) - { - view = override_view(view, guess_model()); - xr_swapchains.emplace_back(xr_session, vk_device, swapchain_format, view.recommendedImageRectWidth, view.recommendedImageRectHeight); - - spdlog::info("Created swapchain {}: {}x{}", xr_swapchains.size(), xr_swapchains.back().width(), xr_swapchains.back().height()); - } + // world_space = xr_session.create_reference_space(XR_REFERENCE_SPACE_TYPE_LOCAL); vk::CommandPoolCreateInfo cmdpool_create_info; cmdpool_create_info.queueFamilyIndex = vk_queue_family_index; @@ -987,11 +966,11 @@ void application::loop() } else { - poll_actions(); auto scene = current_scene(); if (scene) { + poll_actions(); if (auto tmp = last_scene.lock(); scene != tmp) { if (tmp) @@ -1006,6 +985,7 @@ void application::loop() } else { + spdlog::info("Last scene popped: exiting"); exit_requested = true; } } @@ -1014,7 +994,7 @@ void application::loop() #ifdef __ANDROID__ void application::run() { - std::thread application_thread{[&]() { + auto application_thread = utils::named_thread("application_thread", [&]() { setup_jni(); while (!is_exit_requested()) @@ -1034,8 +1014,7 @@ void application::run() exit_requested = true; } } - }}; - pthread_setname_np(application_thread.native_handle(), "application_thread"); + }); // Read all pending events. while (!exit_requested) @@ -1043,7 +1022,8 @@ void application::run() int events; struct android_poll_source * source; - while (ALooper_pollAll(-1, nullptr, &events, (void **)&source) >= 0) + // TODO signal with a file descriptor instead of a 100ms timeout + while (ALooper_pollAll(100, nullptr, &events, (void **)&source) >= 0) { // Process this event. if (source != nullptr) diff --git a/client/application.h b/client/application.h index 94b1ff43..f7e02913 100644 --- a/client/application.h +++ b/client/application.h @@ -38,6 +38,7 @@ #include "singleton.h" #include #include +#include class scene; @@ -67,9 +68,6 @@ class application : public singleton static inline const char engine_name[] = "No engine"; static inline const int engine_version = VK_MAKE_VERSION(1, 0, 0); - static inline const std::array supported_formats = {vk::Format::eR8G8B8A8Srgb, - vk::Format::eB8G8R8A8Srgb}; - void initialize_vulkan(); void log_views(); @@ -90,7 +88,6 @@ class application : public singleton uint32_t vk_queue_family_index; vk::raii::Queue vk_queue = nullptr; vk::raii::CommandPool vk_cmdpool = nullptr; - vk::Format swapchain_format; vk::raii::PipelineCache pipeline_cache = nullptr; // OpenXR stuff @@ -99,7 +96,6 @@ class application : public singleton xr::instance xr_instance; xr::system xr_system_id; xr::session xr_session; - std::vector xr_swapchains; XrSessionState session_state = XR_SESSION_STATE_UNKNOWN; xr::space world_space; diff --git a/client/audio/android/audio.cpp b/client/audio/android/audio.cpp index 828d189e..19871ba7 100644 --- a/client/audio/android/audio.cpp +++ b/client/audio/android/audio.cpp @@ -157,7 +157,7 @@ wivrn::android::audio::audio(const xrt::drivers::wivrn::to_headset::audio_stream if (result != AAUDIO_OK) spdlog::error("Cannot create output stream: {}", AAudio_convertResultToText(result)); else - output_thread = std::thread(&audio::output, this, stream, *desc.speaker); + output_thread = utils::named_thread("audio_output_thread", &audio::output, this, stream, *desc.speaker); } if (desc.microphone) @@ -173,7 +173,7 @@ wivrn::android::audio::audio(const xrt::drivers::wivrn::to_headset::audio_stream if (result != AAUDIO_OK) spdlog::error("Cannot create input stream: {}", AAudio_convertResultToText(result)); else - input_thread = std::thread(&audio::input, this, stream, *desc.microphone); + input_thread = utils::named_thread("audio_input_thread", &audio::input, this, stream, *desc.microphone); } AAudioStreamBuilder_delete(builder); diff --git a/client/audio/android/audio.h b/client/audio/android/audio.h index c7d30035..ce1ed15e 100644 --- a/client/audio/android/audio.h +++ b/client/audio/android/audio.h @@ -19,6 +19,7 @@ #pragma once +#include "utils/named_thread.h" #include "utils/sync_queue.h" #include "wivrn_packets.h" #include diff --git a/client/decoder/android/android_decoder.cpp b/client/decoder/android/android_decoder.cpp index bccefaa1..cf84db1b 100644 --- a/client/decoder/android/android_decoder.cpp +++ b/client/decoder/android/android_decoder.cpp @@ -56,6 +56,8 @@ struct wivrn::android::decoder::pipeline_context pipeline_context(vk::raii::Device& device, const AHardwareBuffer_Desc& buffer_desc, vk::AndroidHardwareBufferFormatPropertiesANDROID & ahb_format, vk::raii::RenderPass& renderpass, const to_headset::video_stream_description::item& description) : device(device), ahb_format(ahb_format) { + spdlog::info("descriptor_pool_mutex.native_handle() = {}", (void*)descriptor_pool_mutex.native_handle()); + assert(ahb_format.externalFormat != 0); spdlog::info("AndroidHardwareBufferProperties"); spdlog::info(" Vulkan format: {}", vk::to_string(ahb_format.format)); @@ -435,6 +437,8 @@ decoder::decoder( shard_accumulator * accumulator) : description(description), fps(fps), device(device), weak_scene(weak_scene), accumulator(accumulator) { + spdlog::info("hbm_mutex.native_handle() = {}", (void*)hbm_mutex.native_handle()); + AImageReader * ir; check(AImageReader_newWithUsage(description.width, description.height, AIMAGE_FORMAT_PRIVATE, AHARDWAREBUFFER_USAGE_CPU_READ_NEVER | AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE, 5 /* maxImages */, &ir), "AImageReader_newWithUsage"); @@ -447,7 +451,7 @@ decoder::decoder( application::get_vulkan_proc( "vkGetAndroidHardwareBufferPropertiesANDROID"); - output_releaser = std::thread([this]() { + output_releaser = utils::named_thread("decoder-" + std::to_string(stream_index), [this]() { while (true) { try @@ -469,14 +473,17 @@ decoder::decoder( } } }); - pthread_setname_np(output_releaser.native_handle(), ("decoder-" + std::to_string(stream_index)).c_str()); } decoder::~decoder() { input_buffers.close(); output_buffers.close(); - output_releaser.join(); + + if (output_releaser.joinable()) + output_releaser.join(); + + spdlog::info("decoder::~decoder"); } void decoder::set_blit_targets(std::vector targets, vk::Format format) @@ -652,7 +659,7 @@ std::shared_ptr decoder::map_hardware_buffer(AI AHardwareBuffer * hardware_buffer; check(AImage_getHardwareBuffer(image, &hardware_buffer), "AImage_getHardwareBuffer"); - std::lock_guard lock(hbm_mutex); + std::unique_lock lock(hbm_mutex); AHardwareBuffer_Desc buffer_desc{}; AHardwareBuffer_describe(hardware_buffer, &buffer_desc); diff --git a/client/decoder/android/android_decoder.h b/client/decoder/android/android_decoder.h index e25557a6..208cfd89 100644 --- a/client/decoder/android/android_decoder.h +++ b/client/decoder/android/android_decoder.h @@ -19,6 +19,7 @@ #pragma once +#include "utils/named_thread.h" #include "utils/sync_queue.h" #include "wivrn_packets.h" #include @@ -93,6 +94,7 @@ private: vk::raii::Device& device; + std::mutex hbm_mutex; AImageReader_ptr image_reader; AMediaCodec_ptr media_codec; @@ -119,7 +121,6 @@ private: std::vector blit_targets; std::shared_ptr pipeline; - std::mutex hbm_mutex; std::unordered_map> hardware_buffer_map; vk::raii::RenderPass renderpass = nullptr; diff --git a/client/decoder/ffmpeg/ffmpeg_decoder.cpp b/client/decoder/ffmpeg/ffmpeg_decoder.cpp index bd6c7b38..ec740506 100644 --- a/client/decoder/ffmpeg/ffmpeg_decoder.cpp +++ b/client/decoder/ffmpeg/ffmpeg_decoder.cpp @@ -62,7 +62,7 @@ static AVCodecID codec_id(xrt::drivers::wivrn::video_codec codec) decoder::blit_handle::~blit_handle() { - std::lock_guard lock(self->mutex); + std::unique_lock lock(self->mutex); if (image_index < 0) return; diff --git a/client/main.cpp b/client/main.cpp index 4225635c..a90b9ce8 100644 --- a/client/main.cpp +++ b/client/main.cpp @@ -73,7 +73,7 @@ void real_main() .ai_socktype = SOCK_STREAM, }; struct addrinfo * addresses; - if (int err = getaddrinfo(server_address.c_str(), NULL, &hint, &addresses)) + if (int err = getaddrinfo(server_address.c_str(), nullptr, &hint, &addresses)) { throw std::runtime_error("Unable to resolve address for " + server_address + ":" + gai_strerror(err)); } diff --git a/client/render/gpu_buffer.h b/client/render/gpu_buffer.h index 4a817911..3a754cf9 100644 --- a/client/render/gpu_buffer.h +++ b/client/render/gpu_buffer.h @@ -23,6 +23,7 @@ #include #include #include "vk/allocation.h" +#include "utils/alignment.h" class gpu_buffer { @@ -33,7 +34,7 @@ class gpu_buffer size_t add(size_t alignment, const void * data_to_add, size_t size) { - size_t offset = ((bytes.size() + alignment - 1) / alignment) * alignment; + size_t offset = utils::align_up(alignment, bytes.size()); bytes.resize(offset + size); @@ -121,7 +122,9 @@ public: .usage = usage}, VmaAllocationCreateInfo{ .flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT, - .usage = VMA_MEMORY_USAGE_AUTO}}; + .usage = VMA_MEMORY_USAGE_AUTO}, + "gpu_buffer::copy_to_gpu" + }; memcpy(gpu_buffer.map(), bytes.data(), bytes.size()); gpu_buffer.unmap(); diff --git a/client/render/image_loader.cpp b/client/render/image_loader.cpp index 957720f9..24725e52 100644 --- a/client/render/image_loader.cpp +++ b/client/render/image_loader.cpp @@ -156,7 +156,7 @@ void image_loader::do_load(vk::raii::Device & device, vk::raii::CommandBuffer & VmaAllocationCreateInfo{ .flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT, .usage = VMA_MEMORY_USAGE_AUTO, - }}; + }, "image_loader::do_load (staging)"}; memcpy(staging_buffer.map(), pixels, byte_size); staging_buffer.unmap(); @@ -176,7 +176,7 @@ void image_loader::do_load(vk::raii::Device & device, vk::raii::CommandBuffer & VmaAllocationCreateInfo{ .flags = 0, .usage = VMA_MEMORY_USAGE_AUTO, - }}; + }, "image_loader::do_load"}; // Transition all mipmap levels layout to eTransferDstOptimal cb.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eTransfer, vk::DependencyFlags{}, {}, {}, vk::ImageMemoryBarrier{ diff --git a/client/render/imgui_impl.cpp b/client/render/imgui_impl.cpp index 6358c809..b09f29d7 100644 --- a/client/render/imgui_impl.cpp +++ b/client/render/imgui_impl.cpp @@ -20,6 +20,7 @@ #include "imgui_impl.h" #include "application.h" +#include "asset.h" #include "openxr/openxr.h" #include "utils/ranges.h" #include "vulkan/vulkan_enums.hpp" @@ -43,7 +44,7 @@ * struct ImGui_ImplVulkanH_Window */ -static vk::raii::RenderPass create_renderpass(vk::raii::Device& device, vk::Format format, vk::ImageLayout final_layout, bool clear) +static vk::raii::RenderPass create_renderpass(vk::raii::Device& device, vk::Format format, bool clear) { vk::AttachmentDescription attachment{ .format = format, @@ -53,7 +54,7 @@ static vk::raii::RenderPass create_renderpass(vk::raii::Device& device, vk::Form .stencilLoadOp = vk::AttachmentLoadOp::eDontCare, .stencilStoreOp = vk::AttachmentStoreOp::eDontCare, .initialLayout = vk::ImageLayout::eUndefined, - .finalLayout = final_layout, + .finalLayout = vk::ImageLayout::eTransferSrcOptimal, }; vk::AttachmentReference color_attachment{ @@ -138,7 +139,7 @@ imgui_context::imgui_viewport::imgui_viewport(vk::raii::Device& device, vk::raii device(device), size(size) { - uint32_t num_mipmaps = 1; + num_mipmaps = std::floor(std::log2(std::max(size.width, size.height))) + 1; for(auto& frame:frames) { @@ -155,7 +156,7 @@ imgui_context::imgui_viewport::imgui_viewport(vk::raii::Device& device, vk::raii .arrayLayers = 1, .samples = vk::SampleCountFlagBits::e1, .tiling = vk::ImageTiling::eOptimal, - .usage = vk::ImageUsageFlagBits::eColorAttachment | vk::ImageUsageFlagBits::eSampled, + .usage = vk::ImageUsageFlagBits::eColorAttachment | vk::ImageUsageFlagBits::eSampled | vk::ImageUsageFlagBits::eTransferSrc | vk::ImageUsageFlagBits::eTransferDst, .initialLayout = vk::ImageLayout::eUndefined, }, VmaAllocationCreateInfo{ @@ -163,7 +164,21 @@ imgui_context::imgui_viewport::imgui_viewport(vk::raii::Device& device, vk::raii } ); - frame.image_view = vk::raii::ImageView(device, vk::ImageViewCreateInfo{ + // Only 1 mipmap level for the framebuffer view + frame.image_view_framebuffer = vk::raii::ImageView(device, vk::ImageViewCreateInfo{ + .image = (vk::Image)frame.image, + .viewType = vk::ImageViewType::e2D, + .format = format, + .subresourceRange = { + .aspectMask = vk::ImageAspectFlagBits::eColor, + .baseMipLevel = 0, + .levelCount = 1, + .baseArrayLayer = 0, + .layerCount = 1, + } + }); + + frame.image_view_texture = vk::raii::ImageView(device, vk::ImageViewCreateInfo{ .image = (vk::Image)frame.image, .viewType = vk::ImageViewType::e2D, .format = format, @@ -179,7 +194,7 @@ imgui_context::imgui_viewport::imgui_viewport(vk::raii::Device& device, vk::raii frame.framebuffer = vk::raii::Framebuffer(device, vk::FramebufferCreateInfo{ .renderPass = renderpass, .attachmentCount = 1, - .pAttachments = &*frame.image_view, + .pAttachments = &*frame.image_view_framebuffer, .width = size.width, .height = size.height, .layers = 1, @@ -194,7 +209,7 @@ imgui_context::imgui_viewport::imgui_viewport(vk::raii::Device& device, vk::raii } } -imgui_context::imgui_context(vk::raii::Device& device, uint32_t queue_family_index, vk::raii::Queue& queue, XrSpace world, std::span controllers_) : +imgui_context::imgui_context(vk::raii::Device& device, uint32_t queue_family_index, vk::raii::Queue& queue, XrSpace world, std::span controllers_, float resolution, glm::vec2 scale) : device(device), queue_family_index(queue_family_index), queue(queue), @@ -205,7 +220,7 @@ imgui_context::imgui_context(vk::raii::Device& device, uint32_t queue_family_ind .poolSizeCount = pool_sizes.size(), .pPoolSizes = pool_sizes.data(), }), - renderpass(create_renderpass(device, vk::Format::eR8G8B8A8Unorm, vk::ImageLayout::eShaderReadOnlyOptimal, true)), + renderpass(create_renderpass(device, vk::Format::eR8G8B8A8Unorm, true)), command_pool(device, vk::CommandPoolCreateInfo{ .flags = vk::CommandPoolCreateFlagBits::eResetCommandBuffer | vk::CommandPoolCreateFlagBits::eTransient, .queueFamilyIndex = queue_family_index, @@ -218,15 +233,15 @@ imgui_context::imgui_context(vk::raii::Device& device, uint32_t queue_family_ind for(const auto& i:controllers_) controllers.emplace_back(i, controller_state{}); - viewport = std::make_shared(device, command_pool, *renderpass, vk::Extent2D{1000, 1000}, vk::Format::eR8G8B8A8Unorm); + vk::Extent2D extent{ + (uint32_t)(resolution * scale.x), + (uint32_t)(resolution * scale.y) + }; - ini_filename = application::get_config_path() / "imgui.ini"; - io.IniFilename = ini_filename.c_str(); + viewport = std::make_shared(device, command_pool, *renderpass, extent, vk::Format::eR8G8B8A8Unorm); + viewport->scale = scale; - // Setup Dear ImGui style - ImGui::StyleColorsDark(); - //ImGui::StyleColorsClassic(); - //ImGui::StyleColorsLight(); + io.IniFilename = nullptr; ImGui_ImplVulkan_InitInfo init_info = { .Instance = *application::get_vulkan_instance(), @@ -246,51 +261,50 @@ imgui_context::imgui_context(vk::raii::Device& device, uint32_t queue_family_ind ImGui_ImplVulkan_Init(&init_info, *renderpass); - // Load Fonts - // - If no fonts are loaded, dear imgui will use the default font. You can also load multiple fonts and use ImGui::PushFont()/PopFont() to select them. - // - AddFontFromFileTTF() will return the ImFont* so you can store it if you need to select the font among multiple. - // - If the file cannot be loaded, the function will return a nullptr. Please handle those errors in your application (e.g. use an assertion, or display an error and quit). - // - The fonts will be rasterized at a given size (w/ oversampling) and stored into a texture when calling ImFontAtlas::Build()/GetTexDataAsXXXX(), which ImGui_ImplXXXX_NewFrame below will call. - // - Use '#define IMGUI_ENABLE_FREETYPE' in your imconfig file to use Freetype for higher quality font rendering. - // - Read 'docs/FONTS.md' for more instructions and details. - // - Remember that in C/C++ if you want to include a backslash \ in a string literal you need to write a double backslash \\ ! - io.Fonts->AddFontDefault(); - //io.Fonts->AddFontFromFileTTF("c:\\Windows\\Fonts\\segoeui.ttf", 18.0f); - //io.Fonts->AddFontFromFileTTF("../../misc/fonts/DroidSans.ttf", 16.0f); - //io.Fonts->AddFontFromFileTTF("../../misc/fonts/Roboto-Medium.ttf", 16.0f); - //io.Fonts->AddFontFromFileTTF("../../misc/fonts/Cousine-Regular.ttf", 15.0f); - //ImFont* font = io.Fonts->AddFontFromFileTTF("c:\\Windows\\Fonts\\ArialUni.ttf", 18.0f, nullptr, io.Fonts->GetGlyphRangesJapanese()); - //IM_ASSERT(font != nullptr); + ImFontConfig config; + asset roboto("Roboto-Regular.ttf"); + config.FontDataOwnedByAtlas = false; + io.Fonts->AddFontFromMemoryTTF(const_cast(roboto.data()), roboto.size(), 45, &config); + large_font = io.Fonts->AddFontFromMemoryTTF(const_cast(roboto.data()), roboto.size(), 90, &config); + + + // Setup Dear ImGui style + ImGui::StyleColorsDark(); + //ImGui::StyleColorsClassic(); + //ImGui::StyleColorsLight(); + + ImGuiStyle& style = ImGui::GetStyle(); + + style.ScaleAllSizes(2.5f); + + style.WindowPadding = { 50, 50 }; + style.WindowBorderSize = 10; // Not scaled by ScaleAllSizes + style.WindowRounding = 25; + style.ItemSpacing = {50, 50}; + + style.FrameRounding = 10; + style.FramePadding = ImVec2(15, 10); + + + // TODO: scroll to drag https://github.com/ocornut/imgui/issues/3379 } - -static float read_action_bool_or_float(imgui_context::controller::typed_action action) +void imgui_context::set_position(glm::vec3 position, glm::quat orientation) { - assert(action.second == XR_ACTION_TYPE_BOOLEAN_INPUT || action.second == XR_ACTION_TYPE_FLOAT_INPUT); - - float value = 0; - switch(action.second) - { - case XR_ACTION_TYPE_BOOLEAN_INPUT: - if (auto act = application::read_action_bool(action.first); act) - value = act->second; - break; - - case XR_ACTION_TYPE_FLOAT_INPUT: - if (auto act = application::read_action_float(action.first); act) - value = act->second; - break; - - default: - __builtin_unreachable(); - } - - return value; + auto& vp = *viewport; + vp.position = position; + vp.orientation = orientation; } void imgui_context::new_frame(XrTime display_time) { + if (last_display_time) + io.DeltaTime = std::min((display_time - last_display_time) * 1e-9f, 0.1f); + last_display_time = display_time; + + float scroll_scale = io.DeltaTime * 3; + size_t new_focused_controller = focused_controller; std::vector new_states; @@ -312,37 +326,39 @@ void imgui_context::new_frame(XrTime display_time) new_state.active = false; } - if (ctrl.squeeze.first) + if (ctrl.squeeze) { - new_state.squeeze_value = read_action_bool_or_float(ctrl.squeeze); + auto squeeze = application::read_action_float(ctrl.squeeze).value_or(std::pair{0,0}); + new_state.squeeze_value = squeeze.second; + // TODO tunable if (new_state.squeeze_value < 0.5) new_state.squeeze_clicked = false; else if (new_state.squeeze_value > 0.8) new_state.squeeze_clicked = true; } - if (ctrl.trigger.first) + if (ctrl.trigger) { - new_state.trigger_value = read_action_bool_or_float(ctrl.trigger); + auto trigger = application::read_action_float(ctrl.trigger).value_or(std::pair{0,0}); + new_state.trigger_value = trigger.second; + // TODO tunable if (new_state.trigger_value < 0.5) new_state.trigger_clicked = false; else if (new_state.trigger_value > 0.8) new_state.trigger_clicked = true; } - if (ctrl.scroll.first) + if (ctrl.scroll) { - assert(ctrl.scroll.second == XR_ACTION_TYPE_VECTOR2F_INPUT); - - if (auto act = application::read_action_vec2(ctrl.scroll.first); act) - new_state.scroll_value = { act->second.x, act->second.y }; + if (auto act = application::read_action_vec2(ctrl.scroll); act) + new_state.scroll_value = { -act->second.x * scroll_scale, act->second.y * scroll_scale }; else new_state.scroll_value = {0, 0}; } - if (new_state.squeeze_clicked || new_state.trigger_clicked || glm::length(new_state.scroll_value) > 0.1f) + if (new_state.squeeze_clicked || new_state.trigger_clicked || glm::length(new_state.scroll_value) > 0.01f) { new_focused_controller = index; } @@ -356,6 +372,7 @@ void imgui_context::new_frame(XrTime display_time) { // Focused controller changed: end the current click io.AddMouseButtonEvent(0, false); + button_pressed = false; } std::optional position; @@ -367,19 +384,29 @@ void imgui_context::new_frame(XrTime display_time) trigger = new_states[new_focused_controller].trigger_value; auto scroll = new_states[new_focused_controller].scroll_value; + bool last_trigger = controllers[new_focused_controller].second.trigger_clicked; + if (position) { io.AddMousePosEvent(position->x, position->y); - bool last_trigger = controllers[new_focused_controller].second.trigger_clicked; - bool next_trigger = new_states[new_focused_controller].trigger_clicked; + button_pressed = new_states[new_focused_controller].trigger_clicked; - if (focused_change || last_trigger != next_trigger) - io.AddMouseButtonEvent(0, next_trigger); + if (focused_change || last_trigger != button_pressed) + { + io.AddMouseButtonEvent(0, button_pressed); + } - if (glm::length(scroll) > 0.1f) + if (glm::length(scroll) > 0.01f) io.AddMouseWheelEvent(scroll.x, scroll.y); } + else + { + if (last_trigger && !button_pressed) + { + io.AddMouseButtonEvent(0, button_pressed); + } + } } else { @@ -400,21 +427,19 @@ void imgui_context::new_frame(XrTime display_time) io.DisplayFramebufferScale = ImVec2(1, 1); // See ImGui_ImplSDL2_ProcessEvent - if (last_display_time) - io.DeltaTime = std::min((display_time - last_display_time) * 1e-9f, 0.1f); - - last_display_time = display_time; - - // io.AddMouseWheelEvent(wheel_x, wheel_y); - ImGui::NewFrame(); ImDrawList* draw_list = ImGui::GetForegroundDrawList(); if (position) - draw_list->AddCircle(*position, 30*(1-0.7*trigger), ImGui::GetColorU32(ImVec4(1, 0, 0, 0.8)), 20, 10); + { + ImU32 color_pressed = ImGui::GetColorU32(ImVec4(0, 0.2, 1, 0.8)); + ImU32 color_unpressed = ImGui::GetColorU32(ImVec4(1, 1, 1, 0.8)); + draw_list->AddCircleFilled(*position, 10, button_pressed ? color_pressed : color_unpressed); + draw_list->AddCircle(*position, 12, ImGui::GetColorU32(ImVec4(0,0,0,0.8)), 0, 4); + } } std::shared_ptr imgui_context::render() @@ -453,6 +478,111 @@ std::shared_ptr imgui_context::render() cb.endRenderPass(); // TODO: create mipmaps + // Create mipmaps + int width = vp.size.width; + int height = vp.size.height; + vk::ImageLayout prev_layout = vk::ImageLayout::eTransferSrcOptimal; + for(uint32_t level = 1; level < vp.num_mipmaps; level++) + { + int next_width = width > 1 ? width / 2 : 1; + int next_height = height > 1 ? height / 2 : 1; + + // Transition source image layout to eTransferSrcOptimal and destination image layout to eTransferDstOptimal + cb.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eFragmentShader, vk::DependencyFlags{}, {}, {}, + { + vk::ImageMemoryBarrier{ + .srcAccessMask = vk::AccessFlagBits::eTransferWrite, + .dstAccessMask = vk::AccessFlagBits::eShaderRead, + .oldLayout = prev_layout, + .newLayout = vk::ImageLayout::eTransferSrcOptimal, + .image = f.image, + .subresourceRange = { + .aspectMask = vk::ImageAspectFlagBits::eColor, + .baseMipLevel = level - 1, + .levelCount = 1, + .baseArrayLayer = 0, + .layerCount = 1, + }, + }, + vk::ImageMemoryBarrier{ + .srcAccessMask = vk::AccessFlagBits::eTransferWrite, // TODO + .dstAccessMask = vk::AccessFlagBits::eShaderRead, // TODO + .oldLayout = vk::ImageLayout::eUndefined, + .newLayout = vk::ImageLayout::eTransferDstOptimal, + .image = f.image, + .subresourceRange = { + .aspectMask = vk::ImageAspectFlagBits::eColor, + .baseMipLevel = level, + .levelCount = 1, + .baseArrayLayer = 0, + .layerCount = 1, + }, + }, + + }); + prev_layout = vk::ImageLayout::eTransferDstOptimal; + + // Blit level n-1 to level n + cb.blitImage(f.image, vk::ImageLayout::eTransferSrcOptimal, f.image, vk::ImageLayout::eTransferDstOptimal, + vk::ImageBlit{ + .srcSubresource = { + .aspectMask = vk::ImageAspectFlagBits::eColor, + .mipLevel = level - 1, + .baseArrayLayer = 0, + .layerCount = 1}, + .srcOffsets = std::array{ + vk::Offset3D{0, 0, 0}, + vk::Offset3D{width, height, 1}, + }, + .dstSubresource = { + .aspectMask = vk::ImageAspectFlagBits::eColor, + .mipLevel = level, + .baseArrayLayer = 0, + .layerCount = 1 + }, + .dstOffsets = std::array{ + vk::Offset3D{0, 0, 0}, + vk::Offset3D{next_width, next_height, 1}, + }, + }, + vk::Filter::eLinear); + + + // Transition source image layout to eShaderReadOnlyOptimal + cb.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eFragmentShader, vk::DependencyFlags{}, {}, {}, vk::ImageMemoryBarrier{ + .srcAccessMask = vk::AccessFlagBits::eTransferRead, + .dstAccessMask = vk::AccessFlagBits::eShaderRead, + .oldLayout = vk::ImageLayout::eTransferSrcOptimal, + .newLayout = vk::ImageLayout::eShaderReadOnlyOptimal, + .image = f.image, + .subresourceRange = { + .aspectMask = vk::ImageAspectFlagBits::eColor, + .baseMipLevel = level - 1, + .levelCount = 1, + .baseArrayLayer = 0, + .layerCount = 1, + }, + }); + + width = next_width; + height = next_height; + } + + // Transition the last level to eShaderReadOnlyOptimal + cb.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eFragmentShader, vk::DependencyFlags{}, {}, {}, vk::ImageMemoryBarrier{ + .srcAccessMask = vk::AccessFlagBits::eTransferWrite, + .dstAccessMask = vk::AccessFlagBits::eShaderRead, + .oldLayout = prev_layout, + .newLayout = vk::ImageLayout::eShaderReadOnlyOptimal, + .image = f.image, + .subresourceRange = { + .aspectMask = vk::ImageAspectFlagBits::eColor, + .baseMipLevel = vp.num_mipmaps - 1, + .levelCount = 1, + .baseArrayLayer = 0, + .layerCount = 1, + }, + }); cb.end(); @@ -462,7 +592,7 @@ std::shared_ptr imgui_context::render() }, *f.fence); // Use the aliasing constructor - return std::shared_ptr{viewport, &vp.frames[vp.frameindex].image_view}; + return std::shared_ptr{viewport, &vp.frames[vp.frameindex].image_view_texture}; } imgui_context::~imgui_context() @@ -481,11 +611,4 @@ imgui_context::~imgui_context() ImGui_ImplVulkan_Shutdown(); ImGui::DestroyContext(context); - // ImGui_ImplSDL2_Shutdown(); - // ImGui::DestroyContext(); - - // ImGui_ImplVulkanH_DestroyWindow(g_Instance, g_Device, &g_MainWindowData, g_Allocator); - - // SDL_DestroyWindow(window); - // SDL_Quit(); } diff --git a/client/render/imgui_impl.h b/client/render/imgui_impl.h index 26ca2880..6111e24d 100644 --- a/client/render/imgui_impl.h +++ b/client/render/imgui_impl.h @@ -18,6 +18,7 @@ #pragma once +#include "render/scene_data.h" #include #include #include @@ -31,7 +32,8 @@ public: struct imgui_frame { image_allocation image; - vk::raii::ImageView image_view = nullptr; + vk::raii::ImageView image_view_framebuffer = nullptr; + vk::raii::ImageView image_view_texture = nullptr; vk::raii::Framebuffer framebuffer = nullptr; vk::raii::CommandBuffer command_buffer = nullptr; vk::raii::Fence fence = nullptr; @@ -40,6 +42,8 @@ public: struct imgui_viewport { static constexpr int frames_in_flight = 2; + uint32_t num_mipmaps; + std::array frames; int frameindex = 0; @@ -58,11 +62,10 @@ public: struct controller { - using typed_action = std::pair; XrSpace aim; - typed_action trigger; - typed_action squeeze; - typed_action scroll; + XrAction trigger; // XR_ACTION_TYPE_FLOAT_INPUT + XrAction squeeze; // XR_ACTION_TYPE_FLOAT_INPUT + XrAction scroll; // XR_ACTION_TYPE_VECTOR2F_INPUT // TODO: thresholds? }; @@ -106,18 +109,27 @@ private: ImGuiContext * context; ImGuiIO& io; - std::string ini_filename; + // std::string ini_filename; std::vector> controllers; XrSpace world; size_t focused_controller = (size_t)-1; XrTime last_display_time = 0; + bool button_pressed = false; + public: imgui_context(vk::raii::Device& device, uint32_t queue_family_index, - vk::raii::Queue& queue, XrSpace world, std::span controllers); + vk::raii::Queue& queue, XrSpace world, std::span controllers, float resolution, glm::vec2 scale); ~imgui_context(); + void set_position(glm::vec3 position, glm::quat orientation); void new_frame(XrTime display_time); std::shared_ptr render(); + + ImFont * large_font; + size_t get_focused_controller() const + { + return focused_controller; + } }; diff --git a/client/render/scene_data.cpp b/client/render/scene_data.cpp index 78b28e3a..5ac94788 100644 --- a/client/render/scene_data.cpp +++ b/client/render/scene_data.cpp @@ -610,6 +610,8 @@ public: material_ref.buffer.reset(); material_ref.ds.reset(); + material_ref.double_sided = gltf_material.doubleSided; + scene_data::material::gpu_data & material_data = material_ref.staging; material_data.base_color_factor = convert(gltf_material.pbrData.baseColorFactor); @@ -718,7 +720,7 @@ public: primitive_ref.vertex_count = vertices.size(); primitive_ref.cull_mode = vk::CullModeFlagBits::eBack; // TBC - primitive_ref.front_face = vk::FrontFace::eClockwise; // TBC + primitive_ref.front_face = vk::FrontFace::eCounterClockwise; // TBC primitive_ref.topology = convert(gltf_primitive.type); if (gltf_primitive.materialIndex) diff --git a/client/render/scene_data.h b/client/render/scene_data.h index 2ea15530..2296225f 100644 --- a/client/render/scene_data.h +++ b/client/render/scene_data.h @@ -117,6 +117,8 @@ struct scene_data bool ds_dirty; std::string name; + std::string shader_name = "lit"; + bool blend_enable = false; }; struct vertex diff --git a/client/render/scene_renderer.cpp b/client/render/scene_renderer.cpp index fdf7a044..19c6d91e 100644 --- a/client/render/scene_renderer.cpp +++ b/client/render/scene_renderer.cpp @@ -173,7 +173,7 @@ scene_renderer::scene_renderer(vk::raii::Device & device, vk::raii::PhysicalDevi .uniform_buffer_dynamic = frames_in_flight, }, frames_in_flight), - ds_pool_material(device, {.combined_image_sampler = 500, .uniform_buffer = 100}, 100) + ds_pool_material(device, {.combined_image_sampler = 500, .uniform_buffer = 100}, 100) // TODO tunable { // Create the default material default_material = create_default_material(cb_pool); @@ -253,7 +253,7 @@ vk::raii::RenderPass scene_renderer::create_renderpass() std::array attachments = { vk::AttachmentDescription{ .format = output_format, - .samples = vk::SampleCountFlagBits::e1, + .samples = vk::SampleCountFlagBits::e4, .loadOp = vk::AttachmentLoadOp::eClear, .storeOp = vk::AttachmentStoreOp::eStore, .initialLayout = vk::ImageLayout::eUndefined, @@ -261,7 +261,7 @@ vk::raii::RenderPass scene_renderer::create_renderpass() }, vk::AttachmentDescription{ .format = depth_format, - .samples = vk::SampleCountFlagBits::e1, + .samples = vk::SampleCountFlagBits::e4, .loadOp = vk::AttachmentLoadOp::eClear, .storeOp = vk::AttachmentStoreOp::eDontCare, .stencilLoadOp = vk::AttachmentLoadOp::eClear, @@ -269,6 +269,17 @@ vk::raii::RenderPass scene_renderer::create_renderpass() .initialLayout = vk::ImageLayout::eUndefined, .finalLayout = vk::ImageLayout::eDepthStencilAttachmentOptimal, + }, + vk::AttachmentDescription{ + .format = output_format, + .samples = vk::SampleCountFlagBits::e1, + .loadOp = vk::AttachmentLoadOp::eDontCare, + .storeOp = vk::AttachmentStoreOp::eStore, + .stencilLoadOp = vk::AttachmentLoadOp::eClear, + .stencilStoreOp = vk::AttachmentStoreOp::eDontCare, + .initialLayout = vk::ImageLayout::eUndefined, + .finalLayout = vk::ImageLayout::eColorAttachmentOptimal, + }}; info.setAttachments(attachments); @@ -282,11 +293,18 @@ vk::raii::RenderPass scene_renderer::create_renderpass() .layout = vk::ImageLayout::eDepthStencilAttachmentOptimal, }; + vk::AttachmentReference resolve_attachment{ + .attachment = 2, + .layout = vk::ImageLayout::eColorAttachmentOptimal, + }; + vk::SubpassDescription subpasses{ .pipelineBindPoint = vk::PipelineBindPoint::eGraphics, + .colorAttachmentCount = 1, + .pColorAttachments = &color_attachment, + .pResolveAttachments = &resolve_attachment, .pDepthStencilAttachment = &depth_attachment, }; - subpasses.setColorAttachments(color_attachment); info.setSubpasses(subpasses); std::array dependencies{ @@ -343,13 +361,14 @@ scene_renderer::output_image scene_renderer::create_output_image_data(vk::Image }, .mipLevels = 1, .arrayLayers = 1, - .samples = vk::SampleCountFlagBits::e1, + .samples = vk::SampleCountFlagBits::e4, .tiling = vk::ImageTiling::eOptimal, - .usage = vk::ImageUsageFlagBits::eDepthStencilAttachment + .usage = vk::ImageUsageFlagBits::eDepthStencilAttachment | vk::ImageUsageFlagBits::eTransientAttachment }, VmaAllocationCreateInfo{ - .flags = VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT, .usage = VMA_MEMORY_USAGE_AUTO, + .flags = VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT, + .usage = VMA_MEMORY_USAGE_AUTO, .requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, // TODO: check }}; @@ -361,6 +380,35 @@ scene_renderer::output_image scene_renderer::create_output_image_data(vk::Image .subresourceRange = {.aspectMask = vk::ImageAspectFlagBits::eDepth, .baseMipLevel = 0, .levelCount = 1, .baseArrayLayer = 0, .layerCount = 1}, }); + out.multisample_image = image_allocation{vk::ImageCreateInfo{ + .imageType = vk::ImageType::e2D, + .format = output_format, + .extent = { + .width = output_size.width, + .height = output_size.height, + .depth = 1, + }, + .mipLevels = 1, + .arrayLayers = 1, + .samples = vk::SampleCountFlagBits::e4, + .tiling = vk::ImageTiling::eOptimal, + .usage = vk::ImageUsageFlagBits::eColorAttachment | vk::ImageUsageFlagBits::eTransientAttachment + + }, + VmaAllocationCreateInfo{ + .flags = VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT, + .usage = VMA_MEMORY_USAGE_AUTO, + .requiredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, // TODO: check + .preferredFlags = VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT, + }}; + out.multisample_view = vk::raii::ImageView(device, vk::ImageViewCreateInfo{ + .image = out.multisample_image, + .viewType = vk::ImageViewType::e2D, + .format = output_format, + .components{}, + .subresourceRange = {.aspectMask = vk::ImageAspectFlagBits::eColor, .baseMipLevel = 0, .levelCount = 1, .baseArrayLayer = 0, .layerCount = 1}, + }); + vk::FramebufferCreateInfo fb_info{ .renderPass = *renderpass, .width = output_size.width, @@ -369,8 +417,10 @@ scene_renderer::output_image scene_renderer::create_output_image_data(vk::Image }; std::array attachments{ + vk::ImageView{*out.multisample_view}, + vk::ImageView{*out.depth_view}, vk::ImageView{*out.image_view}, - vk::ImageView{*out.depth_view}}; + }; fb_info.setAttachments(attachments); out.framebuffer = vk::raii::Framebuffer(device, fb_info); @@ -433,7 +483,7 @@ vk::raii::Pipeline scene_renderer::create_pipeline(const pipeline_info & info) .lineWidth = 1.0, }}, .MultisampleState = {vk::PipelineMultisampleStateCreateInfo{ - .rasterizationSamples = vk::SampleCountFlagBits::e1, + .rasterizationSamples = vk::SampleCountFlagBits::e4, }}, .DepthStencilState = {vk::PipelineDepthStencilStateCreateInfo{ .depthTestEnable = true, @@ -445,6 +495,13 @@ vk::raii::Pipeline scene_renderer::create_pipeline(const pipeline_info & info) }}, .ColorBlendState = {vk::PipelineColorBlendStateCreateInfo{}}, .ColorBlendAttachments = {vk::PipelineColorBlendAttachmentState{ + .blendEnable = info.blend_enable, + .srcColorBlendFactor = vk::BlendFactor::eSrcAlpha, + .dstColorBlendFactor = vk::BlendFactor::eOneMinusSrcAlpha, + .colorBlendOp = vk::BlendOp::eAdd, + .srcAlphaBlendFactor = vk::BlendFactor::eSrcAlpha, + .dstAlphaBlendFactor = vk::BlendFactor::eOneMinusSrcAlpha, + .alphaBlendOp = vk::BlendOp::eAdd, .colorWriteMask = vk::ColorComponentFlagBits::eR | vk::ColorComponentFlagBits::eG | vk::ColorComponentFlagBits::eB}}, .DynamicState = {}, .layout = *pipeline_layout, @@ -465,6 +522,8 @@ vk::Sampler scene_renderer::get_sampler(const sampler_info & info) .mipmapMode = info.min_filter_mipmap, .addressModeU = info.wrapS, .addressModeV = info.wrapT, + // .anisotropyEnable = true, + // .maxAnisotropy = 4, }); samplers.emplace(info, out); @@ -480,7 +539,6 @@ scene_renderer::per_frame_resources & scene_renderer::start_frame() frame_resources[current_frame].resources.clear(); frame_resources[current_frame].cb.begin({.flags = vk::CommandBufferUsageFlagBits::eOneTimeSubmit}); - // frame_resources[current_frame]. return frame_resources[current_frame]; } @@ -534,6 +592,23 @@ void scene_renderer::update_material_descriptor_set(scene_data::material& materi device.updateDescriptorSets(write_ds, {}); } +static void print_scene_hierarchy(const scene_data& scene, std::span model_matrices, size_t root = scene_data::scene_object::root_id, int level = 0) +{ + for(const auto&& [index, node]: utils::enumerate(scene.scene_objects)) + { + if (node.parent_id != root) + continue; + + // glm::mat4 M = model_matrices[index]; + // glm::vec4 pos = glm::column(M, 3); + // spdlog::info("{:{}} {} pos={}, rot={}, pos to root={}", "", level * 2, node.name, node.translation, node.rotation, glm::vec3(pos)); + + spdlog::info("{:{}} {}", "", level * 2, node.name); + + print_scene_hierarchy(scene, model_matrices, index, level + 1); + } +} + void scene_renderer::render(scene_data & scene, std::span frames) { size_t ubo_alignment = physical_device_properties.limits.minUniformBufferOffsetAlignment; @@ -582,7 +657,9 @@ void scene_renderer::render(scene_data & scene, std::span frames) .usage = vk::BufferUsageFlagBits::eUniformBuffer}, VmaAllocationCreateInfo{ .flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT, - .usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}}; + .usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE}, + "scene_renderer::render (UBO staging)" + }; } // if (!resources.uniform_buffer || resources.uniform_buffer.info().size < ubo_size) @@ -610,6 +687,7 @@ void scene_renderer::render(scene_data & scene, std::span frames) std::vector reverse_side(scene.scene_objects.size()); std::vector visible(scene.scene_objects.size()); + for(const auto& [index, object]: utils::enumerate(scene.scene_objects)) { glm::mat4 transform_to_parent = glm::translate(glm::mat4(1), object.translation) * (glm::mat4)object.rotation * glm::scale(glm::mat4(1), object.scale); @@ -634,6 +712,8 @@ void scene_renderer::render(scene_data & scene, std::span frames) } } + // print_scene_hierarchy(scene, transform_to_root); + for (const auto && [frame_index, frame]: utils::enumerate(frames)) { scene_renderer::output_image & output = get_output_image_data(frame.destination); @@ -641,9 +721,16 @@ void scene_renderer::render(scene_data & scene, std::span frames) frame_gpu_data& frame_ubo = *reinterpret_cast(ubo + *current_ubo_frame_offset); - frame_ubo.ambient_color = glm::vec4(0.5,0.5,0.5,0); // TODO - frame_ubo.light_color = glm::vec4(0.5,0.5,0.5,0); // TODO - frame_ubo.light_position = glm::vec4(0,1,0,0); // TODO + // frame_ubo.ambient_color = glm::vec4(0.5,0.5,0.5,0); // TODO + // frame_ubo.light_color = glm::vec4(0.5,0.5,0.5,0); // TODO + + // frame_ubo.ambient_color = glm::vec4(0.2,0.2,0.2,0); // TODO + // frame_ubo.light_color = glm::vec4(0.8,0.8,0.8,0); // TODO + + frame_ubo.ambient_color = glm::vec4(0,0,0,0); // TODO + frame_ubo.light_color = glm::vec4(0.8,0.8,0.8,0); // TODO + + frame_ubo.light_position = glm::vec4(1,1,1,0); // TODO frame_ubo.proj = frame.projection; frame_ubo.view = frame.view; @@ -712,10 +799,11 @@ void scene_renderer::render(scene_data & scene, std::span frames) // Get the pipeline pipeline_info info{ - .shader_name = "lit", + .shader_name = material->shader_name, .cull_mode = primitive.cull_mode, .front_face = primitive.front_face, - .topology = primitive.topology + .topology = primitive.topology, + .blend_enable = material->blend_enable, }; if (material->double_sided) diff --git a/client/render/scene_renderer.h b/client/render/scene_renderer.h index b21c48bb..973be0f5 100644 --- a/client/render/scene_renderer.h +++ b/client/render/scene_renderer.h @@ -42,6 +42,7 @@ struct pipeline_info vk::CullModeFlags cull_mode = vk::CullModeFlagBits::eNone; vk::FrontFace front_face = vk::FrontFace::eClockwise; vk::PrimitiveTopology topology = vk::PrimitiveTopology::eTriangleList; + bool blend_enable = false; bool operator==(const pipeline_info & other) const noexcept = default; }; @@ -75,6 +76,9 @@ public: image_allocation depth_buffer; vk::raii::ImageView depth_view = nullptr; + + image_allocation multisample_image; + vk::raii::ImageView multisample_view = nullptr; }; // Initialization functions @@ -164,7 +168,7 @@ public: void update_material_descriptor_set(scene_data::material& material); public: - scene_renderer(vk::raii::Device & device, vk::raii::PhysicalDevice physical_device, vk::raii::Queue & queue, vk::raii::CommandPool & cb_pool, vk::Extent2D output_size, vk::Format output_format, std::span depth_formats, int frames_in_flight = 3); + scene_renderer(vk::raii::Device & device, vk::raii::PhysicalDevice physical_device, vk::raii::Queue & queue, vk::raii::CommandPool & cb_pool, vk::Extent2D output_size, vk::Format output_format, std::span depth_formats, int frames_in_flight = 2); ~scene_renderer(); diff --git a/client/scene.cpp b/client/scene.cpp index 7ee40b1b..1c11d8e5 100644 --- a/client/scene.cpp +++ b/client/scene.cpp @@ -29,10 +29,10 @@ scene::~scene() {} scene::scene(key, const meta& current_meta) : instance(application::instance().xr_instance), + system(application::instance().xr_system_id), session(application::instance().xr_session), world_space(application::instance().world_space), viewconfig(application::instance().app_info.viewconfig), - swapchains(application::instance().xr_swapchains), vk_instance(application::instance().vk_instance), device(application::instance().vk_device), @@ -46,3 +46,6 @@ scene::scene(key, const meta& current_meta) : void scene::on_unfocused() {} void scene::on_focused() {} +void scene::on_interaction_profile_changed() {} +void scene::on_reference_space_changed(XrReferenceSpaceType) {} +void scene::on_session_state_changed(XrSessionState) {} diff --git a/client/scene.h b/client/scene.h index 8b4d8fd9..8e1eb4ab 100644 --- a/client/scene.h +++ b/client/scene.h @@ -70,10 +70,10 @@ protected: static std::vector scene_registry; xr::instance & instance; + xr::system & system; xr::session & session; xr::space & world_space; XrViewConfigurationType viewconfig; - std::vector & swapchains; vk::raii::Instance& vk_instance; vk::raii::Device& device; @@ -102,6 +102,9 @@ public: virtual void render() = 0; virtual void on_unfocused(); virtual void on_focused(); + virtual void on_interaction_profile_changed(); + virtual void on_reference_space_changed(XrReferenceSpaceType space); + virtual void on_session_state_changed(XrSessionState state); }; template diff --git a/client/scenes/input_profile.h b/client/scenes/input_profile.h index d44f980d..385379c5 100644 --- a/client/scenes/input_profile.h +++ b/client/scenes/input_profile.h @@ -19,6 +19,7 @@ #pragma once #include +#include #include #include #include diff --git a/client/scenes/lobby.cpp b/client/scenes/lobby.cpp index 536b792b..419aca84 100644 --- a/client/scenes/lobby.cpp +++ b/client/scenes/lobby.cpp @@ -20,21 +20,36 @@ #include "lobby.h" #include "../common/version.h" #include "application.h" +#include "glm/geometric.hpp" +#include "imgui.h" #include "input_profile.h" #include "openxr/openxr.h" #include "render/scene_data.h" -#include "render/text_rasterizer.h" #include "stream.h" #include "hardware.h" +#include +#include "wivrn_discover.h" +#include "wivrn_packets.h" #include #include +#include +#include #include #include +#include #include #include #include #include +#include +#include + +#include +#include +#include +#include + static const std::string discover_service = "_wivrn._tcp.local."; @@ -69,11 +84,186 @@ static std::string choose_webxr_profile() case model::unknown: return "generic-trigger-squeeze"; } + + __builtin_unreachable(); } -scenes::lobby::lobby() : - status_string_rasterizer(device, physical_device, commandpool, queue) +static std::string json_string(const std::string& in) { + std::string out; + out.reserve(in.size() + 2); + + out += '"'; + + for(char c: in) + { + switch(c) + { + case '\b': + out += "\\b"; + break; + + case '\f': + out += "\\f"; + break; + + case '\n': + out += "\\n"; + break; + + case '\r': + out += "\\r"; + break; + + case '\t': + out += "\\t"; + break; + + case '"': + out += "\\\""; + break; + + case '\\': + out += "\\\\"; + break; + + default: + out += c; + break; + } + } + + out += '"'; + + return out; +} + +static const std::array supported_formats = +{ + vk::Format::eR8G8B8A8Srgb, + vk::Format::eB8G8R8A8Srgb +}; + +void scenes::lobby::move_gui(glm::vec3 position, glm::quat orientation, XrTime predicted_display_time) +{ + const float gui_target_distance = 1.5; + + glm::vec3 gui_direction = glm::column(glm::mat3_cast(imgui_node->rotation), 2); + float gui_yaw = atan2(gui_direction.x, gui_direction.z); + + glm::vec3 head_direction = -glm::column(glm::mat3_cast(orientation), 2); + float head_yaw = atan2(head_direction.x, head_direction.z) + M_PI; + + head_direction.y = 0; + head_direction = glm::normalize(head_direction); + position.y = 1.5f; + + glm::vec3 gui_target_position = position + gui_target_distance * head_direction; + + glm::vec3 gui_position_error = gui_target_position - imgui_node->translation; + float gui_yaw_error = remainderf(head_yaw - gui_yaw, 2 * M_PI); + + if (move_gui_first_time) + { + move_gui_first_time = false; + gui_yaw += gui_yaw_error; + imgui_node->translation += gui_position_error; + imgui_node->rotation = glm::quat{ cos(gui_yaw/2), 0, sin(gui_yaw/2), 0 }; + } + else + { + // gui_yaw += gui_yaw_error * std::min(1.0f, dt / tau); + // imgui_node->translation += gui_position_error * std::min(1.0f, dt / tau); + } + +} + +scenes::lobby::lobby() +{ + try + { + simdjson::dom::parser parser; + simdjson::dom::element root = parser.load(application::get_config_path() / "client.json"); + for(simdjson::dom::object i: simdjson::dom::array(root["servers"])) + { + server_data data{ + .autoconnect = i["autoconnect"].get_bool(), + .manual = i["manual"].get_bool(), + .visible = false, + .service = { + .name = (std::string)i["pretty_name"], + .hostname = (std::string)i["hostname"], + .port = (int)i["port"].get_int64(), + .txt = {{"cookie", (std::string)i["cookie"]}} + + } + }; + servers.emplace(data.service.txt["cookie"], data); + } + } + catch(std::exception& e) + { + spdlog::warn("Cannot read configuration: {}", e.what()); + servers.clear(); + } + + spdlog::info("{} known server(s):", servers.size()); + for(auto& i: servers) + { + spdlog::info(" {}", i.second.service.name); + } + + std::array imgui_inputs{ + imgui_context::controller{ + .aim = get_action_space("left_aim"), + .trigger = get_action("left_trigger").first, + .squeeze = get_action("left_squeeze").first, + .scroll = get_action("left_scroll").first, + }, + imgui_context::controller{ + .aim = get_action_space("right_aim"), + .trigger = get_action("right_trigger").first, + .squeeze = get_action("right_squeeze").first, + .scroll = get_action("right_scroll").first, + }, + }; + + haptic_output[0] = get_action("left_haptic").first; + haptic_output[1] = get_action("right_haptic").first; + + // TODO Move everything after this to on_focused + + swapchain_format = vk::Format::eUndefined; + spdlog::info("Supported swapchain formats:"); + + for (auto format: session.get_swapchain_formats()) + { + spdlog::info(" {}", vk::to_string(format)); + } + for (auto format: session.get_swapchain_formats()) + { + if (std::find(supported_formats.begin(), supported_formats.end(), format) != supported_formats.end()) + { + swapchain_format = format; + break; + } + } + + if (swapchain_format == vk::Format::eUndefined) + throw std::runtime_error("No supported swapchain format"); + + spdlog::info("Using format {}", vk::to_string(swapchain_format)); + + auto views = system.view_configuration_views(viewconfig); + + swapchains.reserve(views.size()); + for (auto view: views) + { + swapchains.emplace_back(session, device, swapchain_format, view.recommendedImageRectWidth, view.recommendedImageRectHeight); + + spdlog::info("Created lobby swapchain {}: {}x{}", swapchains.size(), swapchains.back().width(), swapchains.back().height()); + } + uint32_t width = swapchains[0].width(); uint32_t height = swapchains[0].height(); vk::Extent2D output_size{width, height}; @@ -88,114 +278,135 @@ scenes::lobby::lobby() : scene_loader loader(device, physical_device, queue, application::queue_family_index(), renderer->get_default_material()); - teapot.import(loader("lobby.gltf")); + teapot.import(loader("ground.gltf")); + input = input_profile("controllers/" + choose_webxr_profile() + "/profile.json", loader, teapot); + spdlog::info("Loaded input profile {}", input->id); + + // Put the imgui node last so that alpha blending works correctly teapot.import(loader("imgui.gltf")); imgui_material = teapot.find_material("imgui"); assert(imgui_material); + imgui_material->shader_name = "unlit"; + imgui_material->blend_enable = true; - input = input_profile("controllers/" + choose_webxr_profile() + "/profile.json", loader, teapot); + imgui_node = teapot.find_node("imgui"); + assert(imgui_node); - spdlog::info("Loaded input profile {}", input->id); - - std::array imgui_inputs{ - imgui_context::controller{ - .aim = get_action_space("left_aim"), - .trigger = get_action("left_trigger"), - .squeeze = get_action("left_squeeze"), - .scroll = get_action("left_scroll"), - }, - imgui_context::controller{ - .aim = get_action_space("right_aim"), - .trigger = get_action("right_trigger"), - .squeeze = get_action("right_squeeze"), - .scroll = get_action("right_scroll"), - }, - }; - - imgui_ctx.emplace(device, queue_family_index, queue, world_space, imgui_inputs); + imgui_ctx.emplace(device, queue_family_index, queue, world_space, imgui_inputs, 1000, imgui_node->scale); + imgui_ctx->set_position(imgui_node->translation, imgui_node->rotation); } -void scenes::lobby::rasterize_status_string() +void scenes::lobby::save_config() { - status_string_rasterized_text = status_string_rasterizer.render(status_string); + std::stringstream ss; - vk::ImageViewCreateInfo iv_info{ - .image = status_string_rasterized_text.image, - .viewType = vk::ImageViewType::e2D, - .format = status_string_rasterized_text.format, - .components = { - .r = vk::ComponentSwizzle::eIdentity, - .g = vk::ComponentSwizzle::eIdentity, - .b = vk::ComponentSwizzle::eIdentity, - .a = vk::ComponentSwizzle::eIdentity, - }, - .subresourceRange = { - .aspectMask = vk::ImageAspectFlagBits::eColor, - .baseMipLevel = 0, - .levelCount = 1, - .baseArrayLayer = 0, - .layerCount = 1, - }, - }; - - status_string_image_view = vk::raii::ImageView(device, iv_info); - - vk::DescriptorImageInfo image_info{ - .sampler = *status_string_sampler, - .imageView = *status_string_image_view, - .imageLayout = vk::ImageLayout::eShaderReadOnlyOptimal, - }; - - vk::WriteDescriptorSet descriptor_write{ - .dstSet = status_string_image_descriptor_set, - .dstBinding = 0, - .dstArrayElement = 0, - .descriptorCount = 1, - .descriptorType = vk::DescriptorType::eCombinedImageSampler, - .pImageInfo = &image_info, - }; - - device.updateDescriptorSets(descriptor_write, {}); - - last_status_string = status_string; -} - -std::unique_ptr connect_to_session(const std::vector & services) -{ - char protocol_string[17]; - sprintf(protocol_string, "%016lx", xrt::drivers::wivrn::protocol_version); - - for (const wivrn_discover::service & service: services) + for (auto& [cookie, server_data]: servers) { + ss << "{"; + ss << "\"autoconnect\":" << std::boolalpha << server_data.autoconnect << ","; + ss << "\"manual\":" << std::boolalpha << server_data.manual << ","; + ss << "\"pretty_name\":" << json_string(server_data.service.name) << ","; + ss << "\"hostname\":" << json_string(server_data.service.hostname) << ","; + ss << "\"port\":" << server_data.service.port << ","; + ss << "\"cookie\":" << json_string(cookie); + ss << "},"; + } + + std::string servers_str = ss.str(); + if (servers_str != "") + servers_str.pop_back(); // Remove last comma + + std::ofstream json(application::get_config_path() / "client.json"); + + json << "{\"servers\":[" << servers_str << "]}"; +} + +static std::string ip_address_to_string(const in_addr& addr) +{ + char buf[100]; + inet_ntop(AF_INET, &addr, buf, sizeof(buf)); + return buf; +} + +static std::string ip_address_to_string(const in6_addr& addr) +{ + char buf[100]; + inet_ntop(AF_INET6, &addr, buf, sizeof(buf)); + return buf; +} + +std::unique_ptr connect_to_session(wivrn_discover::service service, bool manual_connection) +{ + // TODO: make it asynchronous + + if (!manual_connection) + { + char protocol_string[17]; + sprintf(protocol_string, "%016lx", xrt::drivers::wivrn::protocol_version); + auto protocol = service.txt.find("protocol"); if (protocol == service.txt.end()) - continue; + throw std::runtime_error("Incompatible WiVRn server"); if (protocol->second != protocol_string) - continue; + throw std::runtime_error("Incompatible WiVRn server"); + } - int port = service.port; - for (const auto & address: service.addresses) + // Only the automatically discovered servers already have their IP addresses available + if (manual_connection) + { + addrinfo hint { - try + .ai_flags = AI_ADDRCONFIG, + .ai_family = AF_UNSPEC, + .ai_socktype = SOCK_STREAM, + }; + addrinfo * addresses; + if (int err = getaddrinfo(service.hostname.c_str(), nullptr, &hint, &addresses)) + { + spdlog::error("Cannot get address for {}: {}", service.hostname, gai_strerror(err)); + return {}; + } + + for(auto i = addresses; i; i = i->ai_next) + { + switch (i->ai_family) { - return std::visit([port](auto & address) { - return std::make_unique(address, port); - }, - address); - } - catch (std::exception & e) - { - spdlog::warn("Cannot connect to {}: {}", service.hostname, e.what()); + case AF_INET: + service.addresses.push_back(((sockaddr_in*)i->ai_addr)->sin_addr); + break; + case AF_INET6: + service.addresses.push_back(((sockaddr_in6*)i->ai_addr)->sin6_addr); + break; } } + + freeaddrinfo(addresses); + } + + for (const auto & address: service.addresses) + { + try + { + return std::visit([port = service.port](auto & address) { + return std::make_unique(address, port); + }, + address); + } + catch (std::exception & e) + { + std::string address_string = std::visit([](auto & address) { + return ip_address_to_string(address); + }, address); + spdlog::warn("Cannot connect to {} ({}): {}", service.hostname, address_string, e.what()); + } } - return {}; + throw std::runtime_error("No usable address"); } -static glm::mat4 projection_matrix(XrFovf fov, float zn = 0.1) +static glm::mat4 projection_matrix(XrFovf fov, float zn = 0.02) { float r = tan(fov.angleRight); float l = tan(fov.angleLeft); @@ -224,48 +435,394 @@ static glm::mat4 view_matrix(XrPosef pose) return glm::inverse(inv_view_matrix); } +static void CenterText(const std::string& text) +{ + float win_width = ImGui::GetWindowSize().x; + float text_width = ImGui::CalcTextSize(text.c_str()).x; + ImGui::SetCursorPosX((win_width - text_width) / 2); + + ImGui::Text("%s", text.c_str()); +} + +static void CenterNextWindow(ImVec2 size) +{ + ImGui::SetNextWindowPos((ImGui::GetMainViewport()->Size - size) * 0.5f); + ImGui::SetNextWindowSize(size); +} + +void scenes::lobby::update_server_list() +{ + std::vector discovered_services = discover->get_services(); + + // TODO: only if discovered_services changed + for(auto& [cookie, data]: servers) + { + data.visible = false; + } + + for(auto& service: discovered_services) + { + std::string cookie; + if (service.txt.find("cookie") == service.txt.end()) + cookie = service.hostname; + else + cookie = service.txt.at("cookie"); + + if (servers.find(cookie) == servers.end()) + { + // Newly discovered server: add it to the list + servers.emplace(cookie, server_data{ + .autoconnect = false, + .manual = false, + .visible = true, + .service = service + }); + } + else + { + servers.at(cookie).visible = true; + } + } +} + +void scenes::lobby::connect(server_data& data, bool manual) +{ + try + { + if (auto session = connect_to_session(data.service, manual)) + { + next_scene = stream::create(std::move(session)); + server_name = data.service.name; + } + } + catch (const std::exception & e) + { + // TODO: dialog box + spdlog::error("Failed to create stream session: {}", e.what()); + } +} + +void scenes::lobby::gui_connecting() +{ + CenterNextWindow({1000, 500}); + ImGui::Begin("Connection", nullptr, + ImGuiWindowFlags_NoTitleBar | + ImGuiWindowFlags_NoResize | + ImGuiWindowFlags_NoMove); + + ImGui::PushFont(imgui_ctx->large_font); + CenterText("Waiting for video stream"); + ImGui::PopFont(); + + CenterText(fmt::format("Connection to {}", server_name)); + + ImGui::End(); +} + +void scenes::lobby::gui_server_list() +{ + // Build an index of the cookies sorted by server name + std::multimap sorted_cookies; + for(auto&& [cookie, data]: servers) + { + sorted_cookies.emplace(data.service.name, cookie); + + if (data.visible && data.autoconnect && !next_scene) + connect(data, false); + } + + const ImVec2 button_size(220, 80); + const float list_item_height = 100; + auto& style = ImGui::GetStyle(); + + ImGui::SetNextWindowPos({0, 0}); + ImGui::SetNextWindowSize(ImGui::GetMainViewport()->Size); + ImGui::Begin("WiVRn", nullptr, + ImGuiWindowFlags_NoTitleBar | + ImGuiWindowFlags_NoResize | + ImGuiWindowFlags_NoMove); + + ImGui::PushFont(imgui_ctx->large_font); + CenterText(std::string("WiVRn ") + xrt::drivers::wivrn::git_version); + ImGui::PopFont(); + + std::string cookie_to_remove; + + float list_box_height = ImGui::GetWindowContentRegionMax().y - button_size.y - style.WindowPadding.y - ImGui::GetCursorPosY(); + + if (ImGui::BeginListBox("##detected servers", ImVec2(-FLT_MIN, list_box_height))) + { + auto pos = ImGui::GetCursorPos(); + + ImGui::PushStyleColor(ImGuiCol_Header, 0); + ImGui::PushStyleColor(ImGuiCol_HeaderHovered, 0); + ImGui::PushStyleColor(ImGuiCol_HeaderActive, 0); + for(const auto& [name, cookie]: sorted_cookies) + { + server_data& data = servers.at(cookie); + bool is_selected = (cookie == selected_item); + + ImGui::SetCursorPos(pos); + + ImGui::SetNextItemAllowOverlap(); + + // TODO custom widget + if (ImGui::Selectable(("##" + cookie).c_str(), is_selected, ImGuiSelectableFlags_None, ImVec2(0, list_item_height))) + selected_item = cookie; + + if (ImGui::IsItemHovered(ImGuiHoveredFlags_AllowWhenOverlappedByItem)) + { + std::string tooltip = fmt::format("Autoconnect: {}\nManual: {}\nVisible: {}\nCookie: {}", data.autoconnect, data.manual, data.visible, cookie); + + ImGui::SetTooltip("%s", tooltip.c_str()); + } + + ImGui::SetCursorPos(ImVec2(pos.x, pos.y)); + ImGui::Text("%s", name.c_str()); + + ImGui::SetCursorPos(ImVec2(pos.x, pos.y + 50)); + ImGui::PushStyleColor(ImGuiCol_Text, IM_COL32(0,255,0,255)); + + if (ImGui::Checkbox("Autoconnect", &data.autoconnect)) + save_config(); + + if (ImGui::IsItemHovered()) + hovered_item = "autoconnect " + cookie; + + ImGui::PopStyleColor(); + + // TODO + // if (ImGui::IsItemHovered()) + // { + // ImGui::SetTooltip("Tooltip"); + // } + + ImVec2 button_position(ImGui::GetWindowContentRegionMax().x + style.FramePadding.x, pos.y + (list_item_height - button_size.y) / 2); + + button_position.x -= button_size.x + style.WindowPadding.x; + ImGui::SetCursorPos(button_position); + + ImGui::BeginDisabled(!(data.visible || data.manual)); + ImGui::PushStyleColor(ImGuiCol_Button, ImVec4(0.2f, 0.8f, 0.3f, 0.40f)); + ImGui::PushStyleColor(ImGuiCol_ButtonHovered, ImVec4(0.2f, 0.8f, 0.3f, 1.00f)); + ImGui::PushStyleColor(ImGuiCol_ButtonActive, ImVec4(0.1f, 1.0f, 0.2f, 1.00f)); + + if (ImGui::Button("Connect", button_size)) + connect(data, true); + + if (ImGui::IsItemHovered()) + hovered_item = "connect " + cookie; + + ImGui::PopStyleColor(3); + ImGui::EndDisabled(); + + button_position.x -= button_size.x + style.WindowPadding.x; + // if (data.manual) // TODO save manually added servers + { + ImGui::SetCursorPos(button_position); + ImGui::PushStyleColor(ImGuiCol_Button, ImVec4(0.8f, 0.2f, 0.2f, 0.40f)); + ImGui::PushStyleColor(ImGuiCol_ButtonHovered, ImVec4(0.8f, 0.2f, 0.2f, 1.00f)); + ImGui::PushStyleColor(ImGuiCol_ButtonActive, ImVec4(1.0f, 0.1f, 0.1f, 1.00f)); + + if (ImGui::Button("Remove", button_size)) + cookie_to_remove = cookie; + + if (ImGui::IsItemHovered()) + hovered_item = "remove " + cookie; + + ImGui::PopStyleColor(3); + } + + pos.y += 120; + } + ImGui::PopStyleColor(3); + + if (cookie_to_remove != "") + { + servers.erase(cookie_to_remove); + save_config(); + } + + ImGui::EndListBox(); + } + + auto top_left = ImGui::GetWindowContentRegionMin(); + auto bottom_right = ImGui::GetWindowContentRegionMax(); + +#ifndef NDEBUG + ImGui::SetCursorPos(ImVec2(top_left.x, bottom_right.y - button_size.y)); + if (ImGui::Button("Add server", button_size)) + { + show_add_server_window = true; + strcpy(add_server_window_prettyname, ""); + strcpy(add_server_window_hostname, ""); + add_server_window_port = xrt::drivers::wivrn::default_port; + } + + + if (ImGui::IsItemHovered()) + { + hovered_item = "add"; + ImGui::SetTooltip("TODO"); + } +#endif + + ImGui::SetCursorPos(ImVec2(bottom_right.x - button_size.x, bottom_right.y - button_size.y)); + if (ImGui::Button("Exit", button_size)) + application::pop_scene(); + + if (ImGui::IsItemHovered()) + hovered_item = "exit"; + + ImGui::End(); +} + +void scenes::lobby::gui_add_server() +{ + const ImVec2 button_size(220, 80); + + CenterNextWindow({1200, 900}); + ImGui::Begin("Add server", nullptr, + ImGuiWindowFlags_NoTitleBar | + ImGuiWindowFlags_NoResize | + ImGuiWindowFlags_NoMove); + + ImGui::PushFont(imgui_ctx->large_font); + CenterText("Add server"); + ImGui::PopFont(); + + // TODO column widths + ImGui::BeginTable("table", 2); + + ImGui::TableNextRow(); + ImGui::TableNextColumn(); + ImGui::Text("Displayed name"); + + static char buf[100]; + ImGui::TableNextColumn(); + ImGui::InputText("##Name", buf, sizeof(buf)); + if (ImGui::IsItemHovered()) + hovered_item = "name"; + + ImGui::TableNextRow(); + ImGui::TableNextColumn(); + ImGui::Text("Host name"); + + static char buf2[100]; + ImGui::TableNextColumn(); + ImGui::InputText("##Hostname", buf2, sizeof(buf2)); + if (ImGui::IsItemHovered()) + hovered_item = "hostname"; + + ImGui::TableNextRow(); + ImGui::TableNextColumn(); + ImGui::Text("Port"); + + static int port; + ImGui::TableNextColumn(); + ImGui::InputInt("##Port", &port); + if (ImGui::IsItemHovered()) + hovered_item = "port"; + + ImGui::EndTable(); + + // TODO virtual keyboard + // See https://github.com/qt/qtvirtualkeyboard/tree/dev/src/layouts/fallback + // https://github.com/qt/qtvirtualkeyboard/blob/dev/src/layouts/fr_FR/main.qml + // https://doc.qt.io/qt-6/qtvirtualkeyboard-layouts.html + + gui_keyboard(ImVec2(1000, 280)); + + + auto top_left = ImGui::GetWindowContentRegionMin(); + auto bottom_right = ImGui::GetWindowContentRegionMax(); + + // ImGui::SetCursorPos(ImVec2(top_left.x + 250, bottom_right.y - button_size.y)); + // ImGui::Text("ActiveId=%x", ImGui::GetCurrentContext()->ActiveId); + + ImGui::SetCursorPos(ImVec2(top_left.x, bottom_right.y - button_size.y)); + if (ImGui::Button("Cancel", button_size)) + show_add_server_window = false; + if (ImGui::IsItemHovered()) + hovered_item = "cancel"; + + ImGui::SetCursorPos(ImVec2(bottom_right.x - button_size.x, bottom_right.y - button_size.y)); + if (ImGui::Button("Save", button_size)) + { + show_add_server_window = false; + + server_data data + { + .manual = true, + .service = { + .name = add_server_window_prettyname, + .hostname = add_server_window_hostname, + .port = add_server_window_port, + }, + }; + + servers.emplace("manual-" + data.service.name, data); + save_config(); + + } + if (ImGui::IsItemHovered()) + hovered_item = "save"; + + ImGui::End(); +} + +void scenes::lobby::draw_gui(XrTime predicted_display_time) +{ + std::optional> head_position = application::locate_controller(application::view(), world_space, predicted_display_time); + + if (head_position) + { + move_gui(head_position->first, head_position->second, predicted_display_time); + } + + imgui_ctx->set_position(imgui_node->translation, imgui_node->rotation); + imgui_ctx->new_frame(predicted_display_time); + // ImGui::ShowDemoWindow(); + + auto last_hovered = hovered_item; + hovered_item = ""; + + if (next_scene) + gui_connecting(); + else if (show_add_server_window) + gui_add_server(); + else + gui_server_list(); + + + if (hovered_item != last_hovered && hovered_item != "") + { + size_t controller = imgui_ctx->get_focused_controller(); + if (controller < haptic_output.size()) + application::haptic_start(haptic_output[controller], XR_NULL_PATH, 10'000'000, 1000, 1); + } + + // Render the GUI to the imgui material + imgui_material->base_color_texture->image_view = imgui_ctx->render(); + // imgui_material->emissive_texture->image_view = imgui_ctx->render(); + imgui_material->ds_dirty = true; +} + void scenes::lobby::render() { if (next_scene && !next_scene->alive()) next_scene.reset(); - if (!next_scene) - { - auto services = discover->get_services(); - - if (auto session = connect_to_session(services)) - { - try - { - next_scene = stream::create(std::move(session)); - } - catch (const std::exception & e) - { - spdlog::error("Failed to create stream session: {}", e.what()); - } - } - } - if (next_scene) { if (next_scene->ready()) { application::push_scene(next_scene); - discover.reset(); next_scene.reset(); } - else - { - status_string = "Waiting for video stream"; - } - } - else - { - status_string = "Waiting for connection"; } - // if (status_string != last_status_string) - // rasterize_status_string(); + update_server_list(); XrFrameState framestate = session.wait_frame(); @@ -283,13 +840,7 @@ void scenes::lobby::render() input->apply(world_space, framestate.predictedDisplayTime); - - imgui_ctx->new_frame(framestate.predictedDisplayTime); - ImGui::ShowDemoWindow(); - - // Render the GUI to the imgui material - imgui_material->base_color_texture->image_view = imgui_ctx->render(); - imgui_material->ds_dirty = true; + draw_gui(framestate.predictedDisplayTime); std::vector layer_view; std::vector frames; @@ -341,6 +892,7 @@ void scenes::lobby::render() void scenes::lobby::on_focused() { discover.emplace(discover_service); + move_gui_first_time = true; } void scenes::lobby::on_unfocused() @@ -357,10 +909,12 @@ scene::meta& scenes::lobby::get_meta_scene() {"left_trigger", XR_ACTION_TYPE_FLOAT_INPUT}, {"left_squeeze", XR_ACTION_TYPE_FLOAT_INPUT}, {"left_scroll", XR_ACTION_TYPE_VECTOR2F_INPUT}, + {"left_haptic", XR_ACTION_TYPE_VIBRATION_OUTPUT}, {"right_aim", XR_ACTION_TYPE_POSE_INPUT}, {"right_trigger", XR_ACTION_TYPE_FLOAT_INPUT}, {"right_squeeze", XR_ACTION_TYPE_FLOAT_INPUT}, {"right_scroll", XR_ACTION_TYPE_VECTOR2F_INPUT}, + {"right_haptic", XR_ACTION_TYPE_VIBRATION_OUTPUT}, }, .bindings = { suggested_binding{ @@ -370,10 +924,42 @@ scene::meta& scenes::lobby::get_meta_scene() {"left_trigger", "/user/hand/left/input/trigger/value"}, {"left_squeeze", "/user/hand/left/input/squeeze/value"}, {"left_scroll", "/user/hand/left/input/thumbstick"}, + {"left_haptic", "/user/hand/left/output/haptic"}, {"right_aim", "/user/hand/right/input/aim/pose"}, {"right_trigger", "/user/hand/right/input/trigger/value"}, {"right_squeeze", "/user/hand/right/input/squeeze/value"}, {"right_scroll", "/user/hand/right/input/thumbstick"}, + {"right_haptic", "/user/hand/right/output/haptic"}, + } + }, + suggested_binding{ + "/interaction_profiles/bytedance/pico_neo3_controller", + { + {"left_aim", "/user/hand/left/input/aim/pose"}, + {"left_trigger", "/user/hand/left/input/trigger/value"}, + {"left_squeeze", "/user/hand/left/input/squeeze/value"}, + {"left_scroll", "/user/hand/left/input/thumbstick"}, + {"left_haptic", "/user/hand/left/output/haptic"}, + {"right_aim", "/user/hand/right/input/aim/pose"}, + {"right_trigger", "/user/hand/right/input/trigger/value"}, + {"right_squeeze", "/user/hand/right/input/squeeze/value"}, + {"right_scroll", "/user/hand/right/input/thumbstick"}, + {"right_haptic", "/user/hand/right/output/haptic"}, + } + }, + suggested_binding{ + "/interaction_profiles/bytedance/pico4_controller", + { + {"left_aim", "/user/hand/left/input/aim/pose"}, + {"left_trigger", "/user/hand/left/input/trigger/value"}, + {"left_squeeze", "/user/hand/left/input/squeeze/value"}, + {"left_scroll", "/user/hand/left/input/thumbstick"}, + {"left_haptic", "/user/hand/left/output/haptic"}, + {"right_aim", "/user/hand/right/input/aim/pose"}, + {"right_trigger", "/user/hand/right/input/trigger/value"}, + {"right_squeeze", "/user/hand/right/input/squeeze/value"}, + {"right_scroll", "/user/hand/right/input/thumbstick"}, + {"right_haptic", "/user/hand/right/output/haptic"}, } }, suggested_binding{ diff --git a/client/scenes/lobby.h b/client/scenes/lobby.h index 1cea78f8..53b3a061 100644 --- a/client/scenes/lobby.h +++ b/client/scenes/lobby.h @@ -25,9 +25,10 @@ #include "wivrn_discover.h" #include +#include +#include #include "render/scene_renderer.h" -#include "render/text_rasterizer.h" #include "render/imgui_impl.h" #include "input_profile.h" @@ -37,20 +38,25 @@ class stream; class lobby : public scene_impl { - std::string status_string; - std::string last_status_string; - vk::raii::Sampler status_string_sampler = nullptr; - text status_string_rasterized_text; - vk::raii::ImageView status_string_image_view = nullptr; - vk::raii::DescriptorPool status_string_descriptor_pool = nullptr; - vk::raii::DescriptorSetLayout status_string_image_descriptor_set_layout = nullptr; - vk::DescriptorSet status_string_image_descriptor_set; - text_rasterizer status_string_rasterizer; + struct server_data + { + bool autoconnect; + bool manual; + bool visible; + + wivrn_discover::service service; + }; std::optional discover; + std::map servers; + bool show_add_server_window = false; + char add_server_window_prettyname[200]; + char add_server_window_hostname[200]; + int add_server_window_port; std::shared_ptr next_scene; + std::string server_name; std::optional renderer; scene_data teapot; // Must be after the renderer so that the descriptor sets are freed before their pools @@ -58,8 +64,31 @@ class lobby : public scene_impl std::optional imgui_ctx; std::shared_ptr imgui_material; + scene_object_handle imgui_node; + std::array haptic_output; - void rasterize_status_string(); + std::string selected_item; + std::string hovered_item; + + std::vector swapchains; + vk::Format swapchain_format; + + void save_config(); + + void update_server_list(); + + void draw_gui(XrTime predicted_display_time); + + bool move_gui_first_time = true; + + void move_gui(glm::vec3 position, glm::quat orientation, XrTime predicted_display_time); + + void gui_connecting(); + void gui_server_list(); + void gui_add_server(); + void gui_keyboard(ImVec2 size); + + void connect(server_data& data, bool manual); public: virtual ~lobby(); diff --git a/client/scenes/lobby_keyboard.cpp b/client/scenes/lobby_keyboard.cpp new file mode 100644 index 00000000..bf66dd23 --- /dev/null +++ b/client/scenes/lobby_keyboard.cpp @@ -0,0 +1,383 @@ +/* + * WiVRn VR streaming + * Copyright (C) 2022-2024 Guillaume Meunier + * Copyright (C) 2022-2024 Patrick Nicolas + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "imgui.h" +#include "imgui_internal.h" +#include "lobby.h" + +#include "utils/ranges.h" +#include +#include // u_toupper + +struct key +{ + float width; + const std::u32string_view characters; + ImGuiKey key = ImGuiKey_None; + bool visible = true; +}; + +using keyboard_layout = std::vector>; + +keyboard_layout azerty = { + { + { 1, U"aàâæ" }, + { 1, U"z" }, + { 1, U"eéèêë" }, + { 1, U"r" }, + { 1, U"t" }, + { 1, U"yÿ" }, + { 1, U"uùûü" }, + { 1, U"iîï" }, + { 1, U"oôœ" }, + { 1, U"p" }, + { 2, U"", ImGuiKey_Backslash }, + }, + { + { 1, U"q" }, + { 1, U"s" }, + { 1, U"d" }, + { 1, U"f" }, + { 1, U"g" }, + { 1, U"h" }, + { 1, U"j" }, + { 1, U"k" }, + { 1, U"l" }, + { 1, U"m" }, + }, + { + { 2, U"", ImGuiKey_LeftShift }, + { 1, U"w" }, + { 1, U"x" }, + { 1, U"cç" }, + { 1, U"v" }, + { 1, U"b" }, + { 1, U"n" }, + { 1, U",?" }, + { 1, U";." }, + { 1, U":/" }, + { 1, U"!§" }, + } +}; + +keyboard_layout symbols = { + { + { 1, U"1" }, + { 1, U"2" }, + { 1, U"3" }, + { 1, U"4" }, + { 1, U"5" }, + { 1, U"6" }, + { 1, U"7" }, + { 1, U"8" }, + { 1, U"9" }, + { 1, U"0" }, + }, + { + { 1, U"@" }, + { 1, U"#" }, + { 1, U"%" }, + { 1, U"&" }, + { 1, U"*" }, + { 1, U"_" }, + { 1, U"-" }, + { 1, U"+" }, + { 1, U"(" }, + { 1, U")" }, + }, + { + { 1, U"\"" }, + { 1, U"<" }, + { 1, U">" }, + { 1, U"'" }, + { 1, U"*" }, + { 1, U":" }, + { 1, U"/" }, + { 1, U"!" }, + { 1, U"?" }, + { 1, U"" }, + } +}; + +static std::string to_utf8(char32_t c) +{ + std::string s; + + if (c < 0x80) + { + s += (char)c; + } + else if (c < 0x800) + { + s += (char)(0xc0 | ((c >> 6) & 0x1f)); + s += (char)(0x80 | (c & 0x3f)); + } + else if (c < 0x10000) + { + s += (char)(0xe0 | ((c >> 12) & 0x0f)); + s += (char)(0x80 | ((c >> 6) & 0x3f)); + s += (char)(0x80 | (c & 0x3f)); + } + else + { + s += (char)(0xf0 | ((c >> 18) & 0x07)); + s += (char)(0x80 | ((c >> 12) & 0x3f)); + s += (char)(0x80 | ((c >> 6) & 0x3f)); + s += (char)(0x80 | (c & 0x3f)); + } + + return s; +} + +static bool button_behavior(const ImRect& bb, ImGuiID id, bool* out_hovered, bool* out_held, ImGuiButtonFlags flags) +{ + ImGuiContext& g = *GImGui; + ImGuiWindow* window = ImGui::GetCurrentWindow(); + + // Default behavior inherited from item flags + // Note that _both_ ButtonFlags and ItemFlags are valid sources, so copy one into the item_flags and only check that. + ImGuiItemFlags item_flags = (g.LastItemData.ID == id ? g.LastItemData.InFlags : g.CurrentItemFlags); + if (flags & ImGuiButtonFlags_AllowOverlap) + item_flags |= ImGuiItemFlags_AllowOverlap; + if (flags & ImGuiButtonFlags_Repeat) + item_flags |= ImGuiItemFlags_ButtonRepeat; + + bool pressed = false; + bool hovered = ImGui::ItemHoverable(bb, id, item_flags); + + // Mouse handling + const ImGuiID test_owner_id = (flags & ImGuiButtonFlags_NoTestKeyOwner) ? ImGuiKeyOwner_Any : id; + if (hovered) + { + // Poll mouse buttons + // - 'mouse_button_clicked' is generally carried into ActiveIdMouseButton when setting ActiveId. + // - Technically we only need some values in one code path, but since this is gated by hovered test this is fine. + int mouse_button_clicked = -1; + int mouse_button_released = -1; + for (int button = 0; button < 3; button++) + if (flags & (ImGuiButtonFlags_MouseButtonLeft << button)) // Handle ImGuiButtonFlags_MouseButtonRight and ImGuiButtonFlags_MouseButtonMiddle here. + { + if (ImGui::IsMouseClicked(button, test_owner_id) && mouse_button_clicked == -1) { mouse_button_clicked = button; } + if (ImGui::IsMouseReleased(button, test_owner_id) && mouse_button_released == -1) { mouse_button_released = button; } + } + + // Process initial action + if (!(flags & ImGuiButtonFlags_NoKeyModifiers) || (!g.IO.KeyCtrl && !g.IO.KeyShift && !g.IO.KeyAlt)) + { + if (mouse_button_clicked != -1 && g.ActiveId != id) + { + if (!(flags & ImGuiButtonFlags_NoSetKeyOwner)) + ImGui::SetKeyOwner(ImGui::MouseButtonToKey(mouse_button_clicked), id); + if (flags & (ImGuiButtonFlags_PressedOnClickRelease | ImGuiButtonFlags_PressedOnClickReleaseAnywhere)) + { + // ImGui::SetActiveID(id, window); + g.ActiveIdMouseButton = mouse_button_clicked; + if (!(flags & ImGuiButtonFlags_NoNavFocus)) + ImGui::SetFocusID(id, window); + ImGui::FocusWindow(window); + } + if ((flags & ImGuiButtonFlags_PressedOnClick) || ((flags & ImGuiButtonFlags_PressedOnDoubleClick) && g.IO.MouseClickedCount[mouse_button_clicked] == 2)) + { + pressed = true; + // if (flags & ImGuiButtonFlags_NoHoldingActiveId) + // ImGui::ClearActiveID(); + // else + // ImGui::SetActiveID(id, window); // Hold on ID + if (!(flags & ImGuiButtonFlags_NoNavFocus)) + ImGui::SetFocusID(id, window); + g.ActiveIdMouseButton = mouse_button_clicked; + ImGui::FocusWindow(window); + } + } + if (flags & ImGuiButtonFlags_PressedOnRelease) + { + if (mouse_button_released != -1) + { + const bool has_repeated_at_least_once = (item_flags & ImGuiItemFlags_ButtonRepeat) && g.IO.MouseDownDurationPrev[mouse_button_released] >= g.IO.KeyRepeatDelay; // Repeat mode trumps on release behavior + if (!has_repeated_at_least_once) + pressed = true; + if (!(flags & ImGuiButtonFlags_NoNavFocus)) + ImGui::SetFocusID(id, window); + // ImGui::ClearActiveID(); + } + } + + // 'Repeat' mode acts when held regardless of _PressedOn flags (see table above). + // Relies on repeat logic of IsMouseClicked() but we may as well do it ourselves if we end up exposing finer RepeatDelay/RepeatRate settings. + if (g.ActiveId == id && (item_flags & ImGuiItemFlags_ButtonRepeat)) + if (g.IO.MouseDownDuration[g.ActiveIdMouseButton] > 0.0f && ImGui::IsMouseClicked(g.ActiveIdMouseButton, test_owner_id, ImGuiInputFlags_Repeat)) + pressed = true; + } + + if (pressed) + g.NavDisableHighlight = true; + } + + // Process while held + bool held = false; + if (g.ActiveId == id) + { + if (g.ActiveIdSource == ImGuiInputSource_Mouse) + { + if (g.ActiveIdIsJustActivated) + g.ActiveIdClickOffset = g.IO.MousePos - bb.Min; + + const int mouse_button = g.ActiveIdMouseButton; + if (mouse_button == -1) + { + // Fallback for the rare situation were g.ActiveId was set programmatically or from another widget (e.g. #6304). + // ImGui::ClearActiveID(); + } + else if (ImGui::IsMouseDown(mouse_button, test_owner_id)) + { + held = true; + } + else + { + bool release_in = hovered && (flags & ImGuiButtonFlags_PressedOnClickRelease) != 0; + bool release_anywhere = (flags & ImGuiButtonFlags_PressedOnClickReleaseAnywhere) != 0; + if ((release_in || release_anywhere) && !g.DragDropActive) + { + // Report as pressed when releasing the mouse (this is the most common path) + bool is_double_click_release = (flags & ImGuiButtonFlags_PressedOnDoubleClick) && g.IO.MouseReleased[mouse_button] && g.IO.MouseClickedLastCount[mouse_button] == 2; + bool is_repeating_already = (item_flags & ImGuiItemFlags_ButtonRepeat) && g.IO.MouseDownDurationPrev[mouse_button] >= g.IO.KeyRepeatDelay; // Repeat mode trumps + bool is_button_avail_or_owned = ImGui::TestKeyOwner(ImGui::MouseButtonToKey(mouse_button), test_owner_id); + if (!is_double_click_release && !is_repeating_already && is_button_avail_or_owned) + pressed = true; + } + // ImGui::ClearActiveID(); + } + if (!(flags & ImGuiButtonFlags_NoNavFocus)) + g.NavDisableHighlight = true; + } + else if (g.ActiveIdSource == ImGuiInputSource_Keyboard || g.ActiveIdSource == ImGuiInputSource_Gamepad) + { + // When activated using Nav, we hold on the ActiveID until activation button is released + // if (g.NavActivateDownId != id) + // ImGui::ClearActiveID(); + } + if (pressed) + g.ActiveIdHasBeenPressedBefore = true; + } + + if (out_hovered) *out_hovered = hovered; + if (out_held) *out_held = held; + + return pressed; +} + +static void draw_single_key(const key& k, int key_id, ImVec2 size_arg) +{ + std::string label = k.characters == U"" ? "" : to_utf8(k.characters[0]); + label += "##virtual_keyboard_key_" + std::to_string(key_id); + + // ImGui::Button(label.c_str(), size); + + int flags = (int)ImGuiButtonFlags_PressedOnClickRelease | (int)ImGuiButtonFlags_MouseButtonLeft | (int)ImGuiButtonFlags_NoNavFocus; + + ImGuiWindow* window = ImGui::GetCurrentWindow(); + if (window->SkipItems) + return /*false*/; + + ImGuiContext& g = *GImGui; + const ImGuiStyle& style = g.Style; + const ImGuiID id = window->GetID(label.c_str()); + const ImVec2 label_size = ImGui::CalcTextSize(label.c_str(), NULL, true); + + ImVec2 pos = window->DC.CursorPos; + if ((flags & ImGuiButtonFlags_AlignTextBaseLine) && style.FramePadding.y < window->DC.CurrLineTextBaseOffset) // Try to vertically align buttons that are smaller/have no padding so that text baseline matches (bit hacky, since it shouldn't be a flag) + pos.y += window->DC.CurrLineTextBaseOffset - style.FramePadding.y; + ImVec2 size = ImGui::CalcItemSize(size_arg, label_size.x + style.FramePadding.x * 2.0f, label_size.y + style.FramePadding.y * 2.0f); + + const ImRect bb(pos, pos + size); + ImGui::ItemSize(size, style.FramePadding.y); + if (!ImGui::ItemAdd(bb, id)) + return /*false*/; + + bool hovered, held; + bool pressed = button_behavior(bb, id, &hovered, &held, flags); + + // Render + const ImU32 col = ImGui::GetColorU32((held && hovered) ? ImGuiCol_ButtonActive : hovered ? ImGuiCol_ButtonHovered : ImGuiCol_Button); + ImGui::RenderNavHighlight(bb, id); + ImGui::RenderFrame(bb.Min, bb.Max, col, true, style.FrameRounding); + + // if (g.LogEnabled) + // LogSetNextTextDecoration("[", "]"); + ImGui::RenderTextClipped(bb.Min + style.FramePadding, bb.Max - style.FramePadding, label.c_str(), NULL, &label_size, style.ButtonTextAlign, &bb); + + // Automatically close popups + //if (pressed && !(flags & ImGuiButtonFlags_DontClosePopups) && (window->Flags & ImGuiWindowFlags_Popup)) + // CloseCurrentPopup(); + + // IMGUI_TEST_ENGINE_ITEM_INFO(id, label, g.LastItemData.StatusFlags); + // return pressed; + + + + + + + +} + +void scenes::lobby::gui_keyboard(ImVec2 size) +{ + const keyboard_layout& layout = azerty; + + int nb_rows = layout.size(); + + ImVec2 position = ImGui::GetCursorPos(); + ImGuiStyle& style = ImGui::GetStyle(); + + std::vector keys_width; + + // Size of a key including margin, scaled by the key width later + ImVec2 key_size{FLT_MAX, (size.y + style.FramePadding.y) / nb_rows}; + for(const auto& row: layout) + { + float total_width = 0; + for(auto& key: row) + total_width += key.width; + keys_width.push_back(total_width); + + float key_width = (size.x + style.FramePadding.x) / total_width; + key_size.x = std::min(key_size.x, key_width); + } + + int id = 0; + for(const auto& [row, width]: utils::zip(layout, keys_width)) + { + ImVec2 key_position = position; + + for(const auto& key: row) + { + if (key.visible) + { + ImGui::SetCursorPos(key_position); + + draw_single_key(key, id++, ImVec2{key_size.x * key.width - style.FramePadding.x, key_size.y - style.FramePadding.y}); + } + + key_position.x += key_size.x * key.width; + } + + position.y += key_size.y; + } +} diff --git a/client/scenes/stream.cpp b/client/scenes/stream.cpp index fad58ecb..aff26503 100644 --- a/client/scenes/stream.cpp +++ b/client/scenes/stream.cpp @@ -36,6 +36,7 @@ #include #include #include "audio/audio.h" +#include "hardware.h" using namespace xrt::drivers::wivrn; @@ -71,13 +72,18 @@ static const std::unordered_map device_ids = { std::shared_ptr scenes::stream::create(std::unique_ptr network_session) { std::shared_ptr self{new stream}; + spdlog::info("decoder_mutex.native_handle() = {}", (void*)self->decoder_mutex.native_handle()); + self->network_session = std::move(network_session); from_headset::headset_info_packet info{}; - info.recommended_eye_width = self->swapchains[0].width(); - info.recommended_eye_height = self->swapchains[0].height(); + auto view = self->system.view_configuration_views(self->viewconfig)[0]; + view = override_view(view, guess_model()); + + info.recommended_eye_width = view.recommendedImageRectWidth; + info.recommended_eye_height = view.recommendedImageRectHeight; auto [flags, views] = self->session.locate_views( XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO, @@ -102,11 +108,9 @@ std::shared_ptr scenes::stream::create(std::unique_ptrnetwork_session->send_control(info); - self->network_thread = std::thread(&stream::process_packets, self.get()); - pthread_setname_np(self->network_thread.native_handle(), "network_thread"); + self->network_thread = utils::named_thread("network_thread", &stream::process_packets, self.get()); - self->video_thread = std::thread(&stream::video, self.get()); - pthread_setname_np(self->video_thread.native_handle(), "video_thread"); + self->video_thread = utils::named_thread("video_thread", &stream::video, self.get()); self->command_buffer = std::move(self->device.allocateCommandBuffers({ .commandPool = *self->commandpool, @@ -140,15 +144,19 @@ scenes::stream::~stream() cleanup(); exit(); - video_thread.join(); - if (tracking_thread) + if (video_thread.joinable()) + video_thread.join(); + + if (tracking_thread && tracking_thread->joinable()) tracking_thread->join(); - network_thread.join(); + + if (network_thread.joinable()) + network_thread.join(); } void scenes::stream::push_blit_handle(shard_accumulator * decoder, std::shared_ptr handle) { - std::lock_guard lock(decoder_mutex); + std::unique_lock lock(decoder_mutex); if (!application::is_visible()) return; @@ -294,7 +302,7 @@ void scenes::stream::render() std::array pose = {views[0].pose, views[1].pose}; std::array fov = {views[0].fov, views[1].fov}; { - std::lock_guard lock(decoder_mutex); + std::unique_lock lock(decoder_mutex); // Search for the most recent frame available on all decoders. // If no such frame exists, use the most latest frame for each decoder auto common_frame = accumulator_images::common_frame(decoders); @@ -420,8 +428,15 @@ void scenes::stream::cleanup() // Assumes decoder_mutex is locked ready_ = false; decoders.clear(); + swapchains.clear(); } +static const std::array supported_formats = +{ + vk::Format::eR8G8B8A8Srgb, + vk::Format::eB8G8R8A8Srgb +}; + void scenes::stream::setup(const to_headset::video_stream_description & description) { std::unique_lock lock(decoder_mutex); @@ -434,16 +449,50 @@ void scenes::stream::setup(const to_headset::video_stream_description & descript return; } - // Create outputs for the decoders - const uint32_t width = description.width / view_count; - const uint32_t height = description.height; + // Create swapchains + const uint32_t video_width = description.width / view_count; + const uint32_t video_height = description.height; + const uint32_t swapchain_width = video_width / description.foveation[0].x.scale; + const uint32_t swapchain_height = video_height / description.foveation[0].y.scale; - vk::Extent3D decoder_out_size{width, height, 1}; + swapchain_format = vk::Format::eUndefined; + spdlog::info("Supported swapchain formats:"); + + for (auto format: session.get_swapchain_formats()) + { + spdlog::info(" {}", vk::to_string(format)); + } + for (auto format: session.get_swapchain_formats()) + { + if (std::find(supported_formats.begin(), supported_formats.end(), format) != supported_formats.end()) + { + swapchain_format = format; + break; + } + } + + if (swapchain_format == vk::Format::eUndefined) + throw std::runtime_error("No supported swapchain format"); + + spdlog::info("Using format {}", vk::to_string(swapchain_format)); + + auto views = system.view_configuration_views(viewconfig); + + swapchains.reserve(views.size()); + for ([[maybe_unused]] auto view: views) + { + swapchains.emplace_back(session, device, swapchain_format, swapchain_width, swapchain_height); + + spdlog::info("Created stream swapchain {}: {}x{}", swapchains.size(), swapchain_width, swapchain_height); + } + + // Create outputs for the decoders + vk::Extent3D decoder_out_size{video_width, video_height, 1}; for (size_t i = 0; i < view_count; i++) { decoder_output[i].format = vk::Format::eA8B8G8R8SrgbPack32; - decoder_output[i].size.width = width; - decoder_output[i].size.height = height; + decoder_output[i].size.width = video_width; + decoder_output[i].size.height = video_height; vk::ImageCreateInfo image_info{ .flags = vk::ImageCreateFlags{}, @@ -489,9 +538,9 @@ void scenes::stream::setup(const to_headset::video_stream_description & descript { blit_targets[i].image = vk::Image{decoder_output[i].image}; blit_targets[i].image_view = *decoder_output[i].image_view; - blit_targets[i].extent.width = width; - blit_targets[i].extent.height = height; - blit_targets[i].offset.x = width * i; + blit_targets[i].extent.width = video_width; + blit_targets[i].extent.height = video_height; + blit_targets[i].offset.x = video_width * i; blit_targets[i].offset.y = 0; } diff --git a/client/scenes/stream.h b/client/scenes/stream.h index 3e6662fc..1e4b8ce4 100644 --- a/client/scenes/stream.h +++ b/client/scenes/stream.h @@ -19,6 +19,7 @@ #pragma once +#include "utils/named_thread.h" #include "decoder/shard_accumulator.h" #include "scene.h" #include "stream_reprojection.h" @@ -83,6 +84,9 @@ class stream : public scene_impl, public std::enable_shared_from_this swapchains; + vk::Format swapchain_format; + std::optional