Raytracing API rework, fence changes, utilities, advanced mapping

CMakeLists.txt

@@ -17,21 +17,34 @@ option(PHI_ENABLE_DEBUG_MARKERS "enable API debug markers" ON)
 
 # Enables PIX detection, marker support and capture forcing
 # requires WinPixEventRuntime.dll to be available to the executable (in the same folder)
-# this dll is included in extern/win32_pix_runtime/bin/
-option(PHI_ENABLE_D3D12_PIX "enable DirectX 12 PIX integration" OFF)
+# this dll is included in extern/win32_pix_runtime/bin/ and is automatically copied if enabled
+option(PHI_ENABLE_D3D12_PIX "enable DirectX 12 PIX integration" ON)
 option(PHI_ENABLE_SDL2 "enable SDL2 integration" ON)
 
 # =========================================
 # post-process options
 
+macro(_get_vulkan_sdk_version_number_from_folder path_to_check out_version)
+    string ( REGEX MATCH ".*([0-9]+)\\.([0-9]+)\\.([0-9]+)\\.([0-9]+)" result "${path_to_check}" )
+    if (result)
+        # path has a version number
+        SET (${out_version} "${CMAKE_MATCH_1}.${CMAKE_MATCH_2}.${CMAKE_MATCH_3}.${CMAKE_MATCH_4}" )
+    endif()
+endmacro()
+
 if (PHI_BACKEND_VULKAN)
     # use provided .cmake modules in ./cmake/ with the latest FindVulkan.cmake, which is broken out of the box on older CMake versions
     set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 
     find_package(Vulkan)
 
     if (NOT Vulkan_FOUND)
+        message(WARNING "Failed to find Vulkan SDK, ensure Vulkan SDK is installed, and environment variable VULKAN_SDK is set correctly")
         set(PHI_BACKEND_VULKAN OFF)
+    else()
+        set(phi_found_vulkan_sdk_version "")
+        _get_vulkan_sdk_version_number_from_folder(${Vulkan_INCLUDE_DIRS}, phi_found_vulkan_sdk_version)
+        message(STATUS "[phantasm hardware interface] found Vulkan SDK ${phi_found_vulkan_sdk_version}")
     endif()
 endif()
 

src/phantasm-hardware-interface/Backend.hh

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <clean-core/span.hh>
+
 #include <typed-geometry/types/size.hh>
 
 #include <phantasm-hardware-interface/fwd.hh>
@@ -92,16 +94,19 @@ public:
         = 0;
 
     /// create a buffer with optional element stride on resource_heap::upload (shorthand function)
-    [[nodiscard]] virtual handle::resource createUploadBuffer(unsigned size_bytes, unsigned stride_bytes = 0) = 0;
+    [[nodiscard]] virtual handle::resource createUploadBuffer(unsigned size_bytes, unsigned stride_bytes = 0, char const* debug_name = nullptr) = 0;
 
     /// maps a buffer created on resource_heap::upload or ::readback to CPU-accessible memory and returns a pointer
-    /// multiple (nested) maps are allowed, leaving a resource_heap::upload buffer persistently mapped is validW
-    [[nodiscard]] virtual std::byte* mapBuffer(handle::resource res) = 0;
+    /// multiple (nested) maps are allowed, leaving a resource_heap::upload buffer persistently mapped is valid
+    /// begin and end specify the range of CPU-side read data in bytes, end == -1 being the entire width
+    /// NOTE: begin > 0 does not add an offset to the returned pointer
+    [[nodiscard]] virtual std::byte* mapBuffer(handle::resource res, int invalidate_begin = 0, int invalidate_end = -1) = 0;
 
     /// unmaps a buffer, must have been previously mapped using mapBuffer
     /// it is not necessary to unmap a buffer before destruction
     /// on non-desktop it might be required to unmap upload buffers for the writes to become visible
-    virtual void unmapBuffer(handle::resource res) = 0;
+    /// begin and end specify the range of CPU-side modified data in bytes, end == -1 being the entire width
+    virtual void unmapBuffer(handle::resource res, int flush_begin = 0, int flush_end = -1) = 0;
 
     /// destroy a resource
     virtual void free(handle::resource res) = 0;
@@ -127,6 +132,7 @@ public:
     // Pipeline state interface
     //
 
+    /// create a graphics pipeline state
     [[nodiscard]] virtual handle::pipeline_state createPipelineState(arg::vertex_format vertex_format,
                                                                      arg::framebuffer_config const& framebuffer_conf,
                                                                      arg::shader_arg_shapes shader_arg_shapes,
@@ -135,9 +141,14 @@ public:
                                                                      phi::pipeline_config const& primitive_config)
         = 0;
 
+    /// create a graphics pipeline state from a compact description struct
+    [[nodiscard]] virtual handle::pipeline_state createPipelineState(arg::graphics_pipeline_state_desc const& description) = 0;
+
     [[nodiscard]] virtual handle::pipeline_state createComputePipelineState(arg::shader_arg_shapes shader_arg_shapes, arg::shader_binary shader, bool has_root_constants = false)
         = 0;
 
+    [[nodiscard]] virtual handle::pipeline_state createComputePipelineState(arg::compute_pipeline_state_desc const& description) = 0;
+
     virtual void free(handle::pipeline_state ps) = 0;
 
     //
@@ -150,8 +161,13 @@ public:
     /// destroy the given command list handles
     virtual void discard(cc::span<handle::command_list const> cls) = 0;
 
-    /// submit and destroy the given command list handles
-    virtual void submit(cc::span<handle::command_list const> cls, queue_type queue = queue_type::direct) = 0;
+    /// submit and destroy the given command list handles on a specified queue
+    /// waiting on GPU for given fences before execution, and signalling fences on GPU after the commandlists have completed
+    virtual void submit(cc::span<handle::command_list const> cls,
+                        queue_type queue = queue_type::direct,
+                        cc::span<fence_operation const> fence_waits_before = {},
+                        cc::span<fence_operation const> fence_signals_after = {})
+        = 0;
 
     //
     // Fence interface
@@ -169,12 +185,6 @@ public:
     /// block on CPU until a fence reaches a given value
     virtual void waitFenceCPU(handle::fence fence, uint64_t wait_value) = 0;
 
-    /// signal a fence to a given value from a specified GPU queue
-    virtual void signalFenceGPU(handle::fence fence, uint64_t new_value, queue_type queue) = 0;
-
-    /// block on a specified GPU queue until a fence reaches a given value
-    virtual void waitFenceGPU(handle::fence fence, uint64_t wait_value, queue_type queue) = 0;
-
     virtual void free(cc::span<handle::fence const> fences) = 0;
 
     //
@@ -189,30 +199,33 @@ public:
     // Raytracing interface
     //
 
-    [[nodiscard]] virtual handle::pipeline_state createRaytracingPipelineState(arg::raytracing_shader_libraries libraries,
-                                                                               arg::raytracing_argument_associations arg_assocs,
-                                                                               arg::raytracing_hit_groups hit_groups,
-                                                                               unsigned max_recursion,
-                                                                               unsigned max_payload_size_bytes,
-                                                                               unsigned max_attribute_size_bytes)
-        = 0;
-
-    [[nodiscard]] virtual handle::accel_struct createTopLevelAccelStruct(unsigned num_instances) = 0;
+    [[nodiscard]] virtual handle::pipeline_state createRaytracingPipelineState(arg::raytracing_pipeline_state_desc const& description) = 0;
 
+    /// create a bottom level acceleration structure (BLAS) holding geometry elements
+    /// out_native_handle receives the value to be written to accel_struct_instance::native_bottom_level_as_handle
     [[nodiscard]] virtual handle::accel_struct createBottomLevelAccelStruct(cc::span<arg::blas_element const> elements,
                                                                             accel_struct_build_flags_t flags,
                                                                             uint64_t* out_native_handle = nullptr)
         = 0;
 
-    virtual void uploadTopLevelInstances(handle::accel_struct as, cc::span<accel_struct_geometry_instance const> instances) = 0;
+    /// create a top level acceleration structure (TLAS) holding BLAS instances
+    [[nodiscard]] virtual handle::accel_struct createTopLevelAccelStruct(unsigned num_instances, accel_struct_build_flags_t flags) = 0;
 
+    /// receive the internal accel struct buffer for use in shader arguments and resource views
+    /// NOTE: do not free this resource or transition its resource state
     [[nodiscard]] virtual handle::resource getAccelStructBuffer(handle::accel_struct as) = 0;
 
-    [[nodiscard]] virtual shader_table_sizes calculateShaderTableSize(arg::shader_table_records ray_gen_records,
-                                                                      arg::shader_table_records miss_records,
-                                                                      arg::shader_table_records hit_group_records)
+    /// receive the native acceleration struct handle to be written to accel_struct_instance::native_bottom_level_as_handle
+    [[nodiscard]] virtual uint64_t getAccelStructNativeHandle(handle::accel_struct as) = 0;
+
+    /// calculate the buffer sizes and strides to accomodate the given shader table records
+    [[nodiscard]] virtual shader_table_strides calculateShaderTableStrides(arg::shader_table_record const& ray_gen_record,
+                                                                           arg::shader_table_records miss_records,
+                                                                           arg::shader_table_records hit_group_records,
+                                                                           arg::shader_table_records callable_records = {})
         = 0;
 
+    /// write shader table records to memory - usually a mapped buffer
     virtual void writeShaderTable(std::byte* dest, handle::pipeline_state pso, unsigned stride, arg::shader_table_records records) = 0;
 
     virtual void free(handle::accel_struct as) = 0;

src/phantasm-hardware-interface/arguments.hh

@@ -4,7 +4,7 @@
 
 #include <typed-geometry/types/size.hh>
 
-#include <phantasm-hardware-interface/detail/trivial_capped_vector.hh>
+#include <phantasm-hardware-interface/common/container/flat_vector.hh>
 
 #include "limits.hh"
 #include "types.hh"
@@ -14,13 +14,13 @@ namespace phi::arg
 struct framebuffer_config
 {
     /// configs of the render targets, [0, n]
-    detail::trivial_capped_vector<render_target_config, limits::max_render_targets> render_targets;
+    flat_vector<render_target_config, limits::max_render_targets> render_targets;
 
     bool logic_op_enable = false;
     blend_logic_op logic_op = blend_logic_op::no_op;
 
-    /// format of the depth stencil target, [0, 1]
-    detail::trivial_capped_vector<format, 1> depth_target;
+    /// format of the depth stencil target, or format::none
+    format depth_target = format::none;
 
 public:
     void add_render_target(format fmt)
@@ -30,15 +30,14 @@ public:
         render_targets.push_back(new_rt);
     }
 
-    void add_depth_target(format fmt) { depth_target.push_back(fmt); }
-
-    void remove_depth_target() { depth_target.clear(); }
+    void set_depth_target(format fmt) { depth_target = fmt; }
+    void remove_depth_target() { depth_target = format::none; }
 };
 
 struct vertex_format
 {
     cc::span<vertex_attribute_info const> attributes;
-    unsigned vertex_size_bytes;
+    unsigned vertex_size_bytes = 0;
 };
 
 /// A shader argument consists of SRVs, UAVs, an optional CBV, and an offset into it
@@ -67,32 +66,36 @@ using shader_arg_shapes = cc::span<shader_arg_shape const>;
 
 struct shader_binary
 {
-    std::byte const* data; ///< pointer to the (backend-dependent) shader binary data
-    size_t size;
+    std::byte const* data = nullptr; ///< pointer to the (backend-dependent) shader binary data
+    size_t size = 0;
 };
 
 struct graphics_shader
 {
     shader_binary binary;
-    shader_stage stage;
+    shader_stage stage = shader_stage::none;
 };
 
 /// A graphics shader bundle consists of up to 1 shader per graphics stage
 using graphics_shaders = cc::span<graphics_shader const>;
 
-inline bool operator==(shader_arg_shapes const& lhs, shader_arg_shapes const& rhs) noexcept
+struct graphics_pipeline_state_desc
 {
-    if (lhs.size() != rhs.size())
-        return false;
+    pipeline_config config;
+    framebuffer_config framebuffer;
+    vertex_format vertices;
 
-    for (auto i = 0u; i < lhs.size(); ++i)
-    {
-        if (!(lhs[i] == rhs[i]))
-            return false;
-    }
+    flat_vector<graphics_shader, limits::num_graphics_shader_stages> shader_binaries;
+    flat_vector<shader_arg_shape, limits::max_shader_arguments> shader_arg_shapes;
+    bool has_root_constants = false;
+};
 
-    return true;
-}
+struct compute_pipeline_state_desc
+{
+    shader_binary shader;
+    flat_vector<shader_arg_shape, limits::max_shader_arguments> shader_arg_shapes;
+    bool has_root_constants = false;
+};
 
 /// an element in a bottom-level acceleration strucutre
 struct blas_element
@@ -101,44 +104,110 @@ struct blas_element
     handle::resource index_buffer = handle::null_resource; ///< optional
     unsigned num_vertices = 0;
     unsigned num_indices = 0;
+    handle::resource transform_buffer = handle::null_resource; ///< optional
+    unsigned transform_buffer_offset_bytes = 0;
+    format vertex_pos_format = format::rgb32f;
     bool is_opaque = true;
 };
 
+struct raytracing_library_export
+{
+    shader_stage stage = shader_stage::none;
+    char const* entrypoint = nullptr;
+};
+
 /// a shader library lists the symbol names it exports
 struct raytracing_shader_library
 {
     shader_binary binary;
-    detail::trivial_capped_vector<wchar_t const*, 16> symbols;
+    flat_vector<raytracing_library_export, 16> shader_exports;
 };
 
-/// associates symbols exported from libraries with their argument shapes
+/// associates exports from libraries with their argument shapes
 struct raytracing_argument_association
 {
-    detail::trivial_capped_vector<wchar_t const*, 16> symbols;
-    detail::trivial_capped_vector<shader_arg_shape, limits::max_shader_arguments> argument_shapes;
+    enum e_arg_association_target : uint8_t
+    {
+        e_target_identifiable_shader,
+        e_target_hitgroup
+    };
+
+    /// an argument association targets an identifiable shader (identfiable: ray_gen, ray_miss or ray_callable), or a hitgroup
+    e_arg_association_target target_type = e_target_identifiable_shader;
+    /// order corresponds to the order of exports/hitgroups at PSO creation
+    /// NOTE: identifiable shaders are indexed contiguously across libraries, and non-identifiable shaders are skipped
+    flat_vector<unsigned, 16> target_indices;
+
+    flat_vector<shader_arg_shape, limits::max_shader_arguments> argument_shapes;
     bool has_root_constants = false;
+
+public:
+    void set_target_identifiable() { target_type = e_target_identifiable_shader; }
+    void set_target_hitgroup() { target_type = e_target_hitgroup; }
+
+    void add_shader_arg(unsigned num_srvs, unsigned num_uavs, unsigned num_samplers, bool has_cbv)
+    {
+        argument_shapes.push_back(shader_arg_shape{num_srvs, num_uavs, num_samplers, has_cbv});
+    }
 };
 
+/// a triangle hit group, has a closest hit shader, and optionally an any hit and intersection shader
 struct raytracing_hit_group
 {
-    wchar_t const* name = nullptr;
-    wchar_t const* closest_hit_symbol = nullptr;
-    wchar_t const* any_hit_symbol = nullptr;      ///< optional
-    wchar_t const* intersection_symbol = nullptr; ///< optional
+    char const* name = nullptr;
+
+    /// order corresponds to the order of exports, flat across all libraries
+    int closest_hit_export_index = -1;
+    int any_hit_export_index = -1;      ///< optional
+    int intersection_export_index = -1; ///< optional
+};
+
+struct raytracing_pipeline_state_desc
+{
+    cc::span<raytracing_shader_library const> libraries;
+    cc::span<raytracing_argument_association const> argument_associations;
+    cc::span<raytracing_hit_group const> hit_groups;
+
+    unsigned max_recursion = 0;
+    unsigned max_payload_size_bytes = 0;
+    unsigned max_attribute_size_bytes = 0;
 };
 
 struct shader_table_record
 {
-    wchar_t const* symbol = nullptr;     ///< name of the shader or hit group
+    enum e_table_record_target : uint8_t
+    {
+        e_target_identifiable_shader,
+        e_target_hitgroup
+    };
+
+    /// a shader table record targets an identifiable shader (identfiable: ray_gen, ray_miss or ray_callable), or a hitgroup
+    e_table_record_target target_type = e_target_identifiable_shader;
+    /// order corresponds to the order of exports/hitgroups at PSO creation
+    /// NOTE: identifiable shaders are indexed contiguously across libraries, and non-identifiable shaders are skipped
+    unsigned target_index = 0;
+
     void const* root_arg_data = nullptr; ///< optional, data of the root constant data
-    uint32_t root_arg_size = 0;          ///< size of the root constant data
-    detail::trivial_capped_vector<shader_argument, limits::max_shader_arguments> shader_arguments;
-};
+    unsigned root_arg_size_bytes = 0;    ///< size of the root constant data
+    flat_vector<shader_argument, limits::max_shader_arguments> shader_arguments;
+
+    void set_shader(unsigned index)
+    {
+        target_type = e_target_identifiable_shader;
+        target_index = index;
+    }
+
+    void set_hitgroup(unsigned index)
+    {
+        target_type = e_target_hitgroup;
+        target_index = index;
+    }
 
-using raytracing_shader_libraries = cc::span<raytracing_shader_library const>;
-using raytracing_argument_associations = cc::span<raytracing_argument_association const>;
-using raytracing_hit_groups = cc::span<raytracing_hit_group const>;
-using shader_table_records = cc::span<shader_table_record const>;
+    void add_shader_arg(handle::resource cbv, unsigned cbv_off = 0, handle::shader_view sv = handle::null_shader_view)
+    {
+        shader_arguments.push_back(shader_argument{cbv, sv, cbv_off});
+    }
+};
 
 // resource creation info