mirror of
https://github.com/WiVRn/WiVRn.git
synced 2026-11-20 00:51:32 -08:00
487 lines
24 KiB
C++
487 lines
24 KiB
C++
/*
|
|
* WiVRn VR streaming
|
|
* Copyright (C) 2022 Guillaume Meunier <[email protected]>
|
|
* Copyright (C) 2022 Patrick Nicolas <[email protected]>
|
|
*
|
|
* This program is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation, either version 3 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
#include "image_loader.h"
|
|
|
|
#include <cstdint>
|
|
#include <memory>
|
|
#include <span>
|
|
#include <spdlog/spdlog.h>
|
|
#include <stdexcept>
|
|
#include <variant>
|
|
#include <vulkan/vulkan_raii.hpp>
|
|
#include <ktxvulkan.h>
|
|
|
|
#define STB_IMAGE_IMPLEMENTATION
|
|
#define STBI_ONLY_JPEG
|
|
#define STBI_ONLY_PNG
|
|
|
|
#ifdef __ARM_NEON
|
|
#define STBI_NEON
|
|
#endif
|
|
|
|
#include "stb_image.h"
|
|
|
|
using stbi_ptr = std::unique_ptr<void, decltype([](void * pixels) { stbi_image_free(pixels); })>;
|
|
|
|
namespace
|
|
{
|
|
struct image_resources
|
|
{
|
|
image_allocation allocation;
|
|
vk::Image image;
|
|
vk::raii::ImageView image_view = nullptr;
|
|
};
|
|
|
|
struct ktx_image_resources
|
|
{
|
|
ktxVulkanTexture ktx_texture;
|
|
VkDevice device;
|
|
vk::Image image;
|
|
vk::raii::ImageView image_view = nullptr;
|
|
|
|
ktx_image_resources() = default;
|
|
ktx_image_resources(const ktx_image_resources&) = delete;
|
|
ktx_image_resources& operator=(const ktx_image_resources&) = delete;
|
|
~ktx_image_resources()
|
|
{
|
|
ktxVulkanTexture_Destruct(&ktx_texture, device, nullptr);
|
|
}
|
|
};
|
|
|
|
template <typename T>
|
|
constexpr vk::Format formats[4] = {vk::Format::eUndefined, vk::Format::eUndefined, vk::Format::eUndefined, vk::Format::eUndefined};
|
|
|
|
template <>
|
|
constexpr vk::Format formats<uint8_t>[] = {
|
|
vk::Format::eR8Unorm,
|
|
vk::Format::eR8G8Unorm,
|
|
vk::Format::eUndefined,
|
|
vk::Format::eR8G8B8A8Unorm,
|
|
};
|
|
|
|
template <>
|
|
constexpr vk::Format formats<uint16_t>[] = {
|
|
vk::Format::eR16Unorm,
|
|
vk::Format::eR16G16Unorm,
|
|
vk::Format::eUndefined,
|
|
vk::Format::eR16G16B16A16Unorm,
|
|
};
|
|
|
|
template <>
|
|
constexpr vk::Format formats<float>[] = {
|
|
vk::Format::eR32Sfloat,
|
|
vk::Format::eR32G32Sfloat,
|
|
vk::Format::eUndefined,
|
|
vk::Format::eR32G32B32A32Sfloat,
|
|
};
|
|
|
|
template <typename T>
|
|
vk::Format get_format(int num_components)
|
|
{
|
|
assert(num_components > 0 && num_components <= 4);
|
|
|
|
return formats<T>[num_components - 1];
|
|
}
|
|
|
|
vk::Format get_format_srgb(int num_components)
|
|
{
|
|
switch (num_components)
|
|
{
|
|
case 1:
|
|
return vk::Format::eR8Srgb;
|
|
case 2:
|
|
return vk::Format::eR8G8Srgb;
|
|
case 3:
|
|
return vk::Format::eUndefined;
|
|
case 4:
|
|
return vk::Format::eR8G8B8A8Srgb;
|
|
|
|
default:
|
|
assert(false);
|
|
}
|
|
|
|
__builtin_unreachable();
|
|
}
|
|
|
|
int bytes_per_pixel(vk::Format format)
|
|
{
|
|
switch (format)
|
|
{
|
|
case vk::Format::eR8Srgb:
|
|
case vk::Format::eR8Unorm:
|
|
return 1;
|
|
case vk::Format::eR8G8Srgb:
|
|
case vk::Format::eR8G8Unorm:
|
|
return 2;
|
|
case vk::Format::eR8G8B8A8Srgb:
|
|
case vk::Format::eR8G8B8A8Unorm:
|
|
return 4;
|
|
|
|
case vk::Format::eR16Unorm:
|
|
return 2;
|
|
case vk::Format::eR16G16Unorm:
|
|
return 4;
|
|
case vk::Format::eR16G16B16A16Unorm:
|
|
return 8;
|
|
|
|
case vk::Format::eR32Sfloat:
|
|
return 4;
|
|
case vk::Format::eR32G32Sfloat:
|
|
return 8;
|
|
case vk::Format::eR32G32B32A32Sfloat:
|
|
return 16;
|
|
|
|
default:
|
|
assert(false);
|
|
}
|
|
|
|
__builtin_unreachable();
|
|
}
|
|
} // namespace
|
|
|
|
image_loader::image_loader(vk::raii::PhysicalDevice physical_device, vk::raii::Device & device, vk::raii::Queue& queue, vk::raii::CommandPool & cb_pool) :
|
|
device(device),
|
|
queue(queue),
|
|
cb_pool(cb_pool)
|
|
{
|
|
vdi = ktxVulkanDeviceInfo_Create(*physical_device, *device, *queue, *cb_pool, nullptr);
|
|
}
|
|
|
|
image_loader::~image_loader()
|
|
{
|
|
if (vdi)
|
|
ktxVulkanDeviceInfo_Destroy(vdi);
|
|
}
|
|
|
|
void image_loader::do_load_raw(const void * pixels, vk::Extent3D extent, vk::Format format)
|
|
{
|
|
auto cb = std::move(device.allocateCommandBuffers({
|
|
.commandPool = *cb_pool,
|
|
.level = vk::CommandBufferLevel::ePrimary,
|
|
.commandBufferCount = 1,
|
|
})[0]);
|
|
|
|
cb.begin(vk::CommandBufferBeginInfo{.flags = vk::CommandBufferUsageFlagBits::eOneTimeSubmit});
|
|
|
|
assert(format != vk::Format::eUndefined);
|
|
|
|
this->format = format;
|
|
this->extent = extent;
|
|
if (extent.depth > 1)
|
|
image_view_type = vk::ImageViewType::e3D;
|
|
else
|
|
image_view_type = vk::ImageViewType::e2D;
|
|
|
|
num_mipmaps = std::floor(std::log2(std::max(extent.width, extent.height))) + 1;
|
|
|
|
size_t byte_size = extent.width * extent.height * extent.depth * bytes_per_pixel(format);
|
|
|
|
// Copy to staging buffer
|
|
staging_buffer = buffer_allocation{
|
|
device,
|
|
vk::BufferCreateInfo{
|
|
.size = byte_size,
|
|
.usage = vk::BufferUsageFlagBits::eTransferSrc},
|
|
VmaAllocationCreateInfo{
|
|
.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT,
|
|
.usage = VMA_MEMORY_USAGE_AUTO,
|
|
},
|
|
"image_loader::do_load (staging)"};
|
|
|
|
memcpy(staging_buffer.map(), pixels, byte_size);
|
|
staging_buffer.unmap();
|
|
|
|
// Allocate image
|
|
auto r = std::make_shared<image_resources>();
|
|
r->allocation = image_allocation{
|
|
device,
|
|
vk::ImageCreateInfo{
|
|
.imageType = vk::ImageType::e2D,
|
|
.format = format,
|
|
.extent = extent,
|
|
.mipLevels = num_mipmaps,
|
|
.arrayLayers = 1,
|
|
.samples = vk::SampleCountFlagBits::e1,
|
|
.tiling = vk::ImageTiling::eOptimal,
|
|
.usage = vk::ImageUsageFlagBits::eTransferDst | vk::ImageUsageFlagBits::eTransferSrc | vk::ImageUsageFlagBits::eSampled,
|
|
.initialLayout = vk::ImageLayout::eUndefined},
|
|
VmaAllocationCreateInfo{
|
|
.flags = 0,
|
|
.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE,
|
|
}, "image_loader::do_load"};
|
|
|
|
r->image = (vk::Image)r->allocation;
|
|
|
|
// Transition all mipmap levels layout to eTransferDstOptimal
|
|
cb.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eTransfer, vk::DependencyFlags{}, {}, {}, vk::ImageMemoryBarrier{
|
|
.srcAccessMask = vk::AccessFlagBits::eNone,
|
|
.dstAccessMask = vk::AccessFlagBits::eTransferWrite,
|
|
.oldLayout = vk::ImageLayout::eUndefined,
|
|
.newLayout = vk::ImageLayout::eTransferDstOptimal,
|
|
.image = r->image,
|
|
.subresourceRange = {
|
|
.aspectMask = vk::ImageAspectFlagBits::eColor,
|
|
.baseMipLevel = 0,
|
|
.levelCount = num_mipmaps,
|
|
.baseArrayLayer = 0,
|
|
.layerCount = 1,
|
|
},
|
|
});
|
|
|
|
// Copy image data
|
|
cb.copyBufferToImage(staging_buffer, r->image, vk::ImageLayout::eTransferDstOptimal, vk::BufferImageCopy{
|
|
.bufferOffset = 0,
|
|
.bufferRowLength = 0,
|
|
.bufferImageHeight = 0,
|
|
.imageSubresource = {
|
|
.aspectMask = vk::ImageAspectFlagBits::eColor,
|
|
.mipLevel = 0,
|
|
.baseArrayLayer = 0,
|
|
.layerCount = 1,
|
|
},
|
|
.imageOffset = {0, 0, 0},
|
|
.imageExtent = extent,
|
|
});
|
|
|
|
// Create mipmaps
|
|
int width = extent.width;
|
|
int height = extent.height;
|
|
for (uint32_t level = 1; level < num_mipmaps; level++)
|
|
{
|
|
int next_width = width > 1 ? width / 2 : 1;
|
|
int next_height = height > 1 ? height / 2 : 1;
|
|
|
|
// Transition source image layout to eTransferSrcOptimal
|
|
cb.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eFragmentShader, vk::DependencyFlags{}, {}, {}, vk::ImageMemoryBarrier{
|
|
.srcAccessMask = vk::AccessFlagBits::eTransferWrite,
|
|
.dstAccessMask = vk::AccessFlagBits::eShaderRead,
|
|
.oldLayout = vk::ImageLayout::eTransferDstOptimal,
|
|
.newLayout = vk::ImageLayout::eTransferSrcOptimal,
|
|
.image = r->image,
|
|
.subresourceRange = {
|
|
.aspectMask = vk::ImageAspectFlagBits::eColor,
|
|
.baseMipLevel = level - 1,
|
|
.levelCount = 1,
|
|
.baseArrayLayer = 0,
|
|
.layerCount = 1,
|
|
},
|
|
});
|
|
|
|
// Blit level n-1 to level n
|
|
cb.blitImage(r->image, vk::ImageLayout::eTransferSrcOptimal, r->image, vk::ImageLayout::eTransferDstOptimal, vk::ImageBlit{
|
|
.srcSubresource = {.aspectMask = vk::ImageAspectFlagBits::eColor, .mipLevel = level - 1, .baseArrayLayer = 0, .layerCount = 1},
|
|
.srcOffsets = std::array{
|
|
vk::Offset3D{0, 0, 0},
|
|
vk::Offset3D{width, height, 1},
|
|
},
|
|
.dstSubresource = {.aspectMask = vk::ImageAspectFlagBits::eColor, .mipLevel = level, .baseArrayLayer = 0, .layerCount = 1},
|
|
.dstOffsets = std::array{
|
|
vk::Offset3D{0, 0, 0},
|
|
vk::Offset3D{next_width, next_height, 1},
|
|
},
|
|
},
|
|
vk::Filter::eLinear);
|
|
|
|
// Transition source image layout to eShaderReadOnlyOptimal
|
|
cb.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eFragmentShader, vk::DependencyFlags{}, {}, {}, vk::ImageMemoryBarrier{
|
|
.srcAccessMask = vk::AccessFlagBits::eTransferRead,
|
|
.dstAccessMask = vk::AccessFlagBits::eShaderRead,
|
|
.oldLayout = vk::ImageLayout::eTransferSrcOptimal,
|
|
.newLayout = vk::ImageLayout::eShaderReadOnlyOptimal,
|
|
.image = r->image,
|
|
.subresourceRange = {
|
|
.aspectMask = vk::ImageAspectFlagBits::eColor,
|
|
.baseMipLevel = level - 1,
|
|
.levelCount = 1,
|
|
.baseArrayLayer = 0,
|
|
.layerCount = 1,
|
|
},
|
|
});
|
|
|
|
width = next_width;
|
|
height = next_height;
|
|
}
|
|
|
|
// Transition the last level to eShaderReadOnlyOptimal
|
|
cb.pipelineBarrier(vk::PipelineStageFlagBits::eTransfer, vk::PipelineStageFlagBits::eFragmentShader, vk::DependencyFlags{}, {}, {}, vk::ImageMemoryBarrier{
|
|
.srcAccessMask = vk::AccessFlagBits::eTransferWrite,
|
|
.dstAccessMask = vk::AccessFlagBits::eShaderRead,
|
|
.oldLayout = vk::ImageLayout::eTransferDstOptimal,
|
|
.newLayout = vk::ImageLayout::eShaderReadOnlyOptimal,
|
|
.image = r->image,
|
|
.subresourceRange = {
|
|
.aspectMask = vk::ImageAspectFlagBits::eColor,
|
|
.baseMipLevel = num_mipmaps - 1,
|
|
.levelCount = 1,
|
|
.baseArrayLayer = 0,
|
|
.layerCount = 1,
|
|
},
|
|
});
|
|
|
|
cb.end();
|
|
vk::SubmitInfo info;
|
|
info.setCommandBuffers(*cb);
|
|
auto fence = device.createFence(vk::FenceCreateInfo{});
|
|
queue.submit(info, *fence);
|
|
if (auto result = device.waitForFences(*fence, true, 1'000'000'000); result != vk::Result::eSuccess)
|
|
throw std::runtime_error("vkWaitForfences: " + vk::to_string(result));
|
|
|
|
r->image_view = vk::raii::ImageView{device, vk::ImageViewCreateInfo{
|
|
.image = r->image,
|
|
.viewType = image_view_type,
|
|
.format = format,
|
|
.subresourceRange = {
|
|
.aspectMask = vk::ImageAspectFlagBits::eColor,
|
|
.baseMipLevel = 0,
|
|
.levelCount = num_mipmaps,
|
|
.baseArrayLayer = 0,
|
|
.layerCount = 1,
|
|
},
|
|
}};
|
|
|
|
image_view = std::shared_ptr<vk::raii::ImageView>(r, &r->image_view);
|
|
}
|
|
|
|
void image_loader::do_load_ktx(std::span<const std::byte> bytes)
|
|
{
|
|
ktxTexture* texture;
|
|
ktxVulkanTexture vk_texture;
|
|
|
|
ktxResult err = ktxTexture_CreateFromMemory(reinterpret_cast<const uint8_t*>(bytes.data()), bytes.size(), /*KTX_TEXTURE_CREATE_CHECK_GLTF_BASISU_BIT*/0, &texture);
|
|
|
|
if (err != KTX_SUCCESS)
|
|
{
|
|
spdlog::info("ktxTexture_CreateFromMemory: error {}", (int)err);
|
|
throw std::runtime_error("ktxTexture_CreateFromMemory");
|
|
}
|
|
|
|
if (ktxTexture_NeedsTranscoding(texture))
|
|
{
|
|
// TODO
|
|
}
|
|
|
|
err = ktxTexture_VkUploadEx(texture, vdi, &vk_texture,
|
|
VK_IMAGE_TILING_OPTIMAL,
|
|
VK_IMAGE_USAGE_SAMPLED_BIT,
|
|
VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
|
|
if (err != KTX_SUCCESS)
|
|
{
|
|
// TODO try uncompressing the texture
|
|
ktxTexture_Destroy(texture);
|
|
spdlog::info("ktxTexture_CreateFromMemory: error {}", (int)err);
|
|
throw std::runtime_error("ktxTexture_VkUploadEx");
|
|
}
|
|
|
|
format = vk::Format(vk_texture.imageFormat);
|
|
extent = vk::Extent3D(texture->baseWidth, texture->baseHeight, texture->baseDepth);
|
|
image_view_type = vk::ImageViewType(vk_texture.viewType);
|
|
num_mipmaps = texture->numLevels;
|
|
|
|
ktxTexture_Destroy(texture);
|
|
|
|
auto r = std::make_shared<ktx_image_resources>();
|
|
|
|
r->ktx_texture = vk_texture;
|
|
r->device = *device;
|
|
r->image = vk::Image(vk_texture.image);
|
|
r->image_view = vk::raii::ImageView{device, vk::ImageViewCreateInfo{
|
|
.image = r->image,
|
|
.viewType = image_view_type,
|
|
.format = format,
|
|
.subresourceRange = {
|
|
.aspectMask = vk::ImageAspectFlagBits::eColor,
|
|
.baseMipLevel = 0,
|
|
.levelCount = num_mipmaps,
|
|
.baseArrayLayer = 0,
|
|
.layerCount = 1,
|
|
},
|
|
}};
|
|
|
|
image_view = std::shared_ptr<vk::raii::ImageView>(r, &r->image_view);
|
|
}
|
|
|
|
static constexpr bool starts_with(std::span<const std::byte> data, std::span<const uint8_t> prefix)
|
|
{
|
|
return data.size() >= prefix.size() && !memcmp(data.data(), prefix.data(), prefix.size());
|
|
}
|
|
|
|
// Load a PNG/JPEG/KTX2 file
|
|
void image_loader::load(std::span<const std::byte> bytes, bool srgb)
|
|
{
|
|
const uint8_t ktx1_magic[] = {0xAB, 0x4B, 0x54, 0x58, 0x20, 0x31, 0x31, 0xBB, 0x0D, 0x0A, 0x1A, 0x0A};
|
|
const uint8_t ktx2_magic[] = {0xAB, 0x4B, 0x54, 0x58, 0x20, 0x32, 0x30, 0xBB, 0x0D, 0x0A, 0x1A, 0x0A};
|
|
|
|
if (starts_with(bytes, ktx1_magic) || starts_with(bytes, ktx2_magic))
|
|
{
|
|
do_load_ktx(bytes);
|
|
}
|
|
else
|
|
{
|
|
const stbi_uc * image_data = (const stbi_uc *)bytes.data();
|
|
size_t image_size = bytes.size();
|
|
|
|
int w, h, num_channels, channels_in_file;
|
|
stbi_ptr pixels;
|
|
|
|
if (!stbi_info_from_memory(image_data, image_size, &w, &h, &num_channels))
|
|
throw std::runtime_error("Unsupported image format");
|
|
|
|
assert(num_channels >= 1 && num_channels <= 4);
|
|
|
|
if (num_channels == 3)
|
|
num_channels = 4;
|
|
|
|
if (stbi_is_hdr_from_memory(image_data, image_size))
|
|
{
|
|
pixels = stbi_ptr(stbi_loadf_from_memory(image_data, image_size, &w, &h, &channels_in_file, num_channels));
|
|
format = get_format<float>(num_channels);
|
|
}
|
|
else if (stbi_is_16_bit_from_memory(image_data, image_size))
|
|
{
|
|
pixels = stbi_ptr(stbi_load_16_from_memory(image_data, image_size, &w, &h, &channels_in_file, num_channels));
|
|
format = get_format<uint16_t>(num_channels);
|
|
}
|
|
else
|
|
{
|
|
pixels = stbi_ptr(stbi_load_from_memory(image_data, image_size, &w, &h, &channels_in_file, num_channels));
|
|
format = srgb ? get_format_srgb(num_channels) : get_format<uint8_t>(num_channels);
|
|
}
|
|
|
|
extent.width = w;
|
|
extent.height = h;
|
|
extent.depth = 1;
|
|
|
|
do_load_raw(pixels.get(), extent, format);
|
|
}
|
|
}
|
|
|
|
// Load raw pixel data
|
|
void image_loader::load(const void * pixels, size_t size, vk::Extent3D extent_, vk::Format format_)
|
|
{
|
|
extent = extent_;
|
|
format = format_;
|
|
image_view_type = vk::ImageViewType::e2D;
|
|
|
|
if (size < extent.width * extent.height * extent.depth * bytes_per_pixel(format))
|
|
throw std::invalid_argument("size");
|
|
|
|
do_load_raw(pixels, extent_, format_);
|
|
}
|