Merge pull request #979 from Devsh-Graphics-Programming/depfiles

Depfiles

Author: AnastaZIuk

cmake/common.cmake

@@ -1142,6 +1142,7 @@ option(NSC_DEBUG_EDIF_SOURCE_BIT "Add \"-fspv-debug=source\" to NSC Debug CLI" O
 option(NSC_DEBUG_EDIF_LINE_BIT "Add \"-fspv-debug=line\" to NSC Debug CLI" OFF)
 option(NSC_DEBUG_EDIF_TOOL_BIT "Add \"-fspv-debug=tool\" to NSC Debug CLI" ON)
 option(NSC_DEBUG_EDIF_NON_SEMANTIC_BIT "Add \"-fspv-debug=vulkan-with-source\" to NSC Debug CLI" OFF)
+option(NSC_USE_DEPFILE "Generate depfiles for NSC custom commands" ON)
 
 function(NBL_CREATE_NSC_COMPILE_RULES)
     set(COMMENT "this code has been autogenerated with Nabla CMake NBL_CREATE_HLSL_COMPILE_RULES utility")
@@ -1180,6 +1181,7 @@ struct DeviceConfigCaps
 		-fspv-target-env=vulkan1.3 
 		-Wshadow 
 		-Wconversion 
+		-Wno-local-type-template-args 
 		$<$<CONFIG:Debug>:-O0> 
 		$<$<CONFIG:Release>:-O3> 
 		$<$<CONFIG:RelWithDebInfo>:-O3> 
@@ -1207,6 +1209,7 @@ struct DeviceConfigCaps
 
 	if(NOT NBL_EMBED_BUILTIN_RESOURCES)
 		list(APPEND REQUIRED_OPTIONS
+			-no-nbl-builtins
 			-I "${NBL_ROOT_PATH}/include"
 			-I "${NBL_ROOT_PATH}/3rdparty/dxc/dxc/external/SPIRV-Headers/include"
 			-I "${NBL_ROOT_PATH}/3rdparty/boost/superproject/libs/preprocessor/include"
@@ -1215,9 +1218,20 @@ struct DeviceConfigCaps
 	endif()
 
     set(REQUIRED_SINGLE_ARGS TARGET BINARY_DIR OUTPUT_VAR INPUTS INCLUDE NAMESPACE MOUNT_POINT_DEFINE)
-    cmake_parse_arguments(IMPL "" "${REQUIRED_SINGLE_ARGS};LINK_TO" "COMMON_OPTIONS;DEPENDS" ${ARGV})
+    set(OPTIONAL_SINGLE_ARGS GLOB_DIR)
+    cmake_parse_arguments(IMPL "DISCARD_DEFAULT_GLOB" "${REQUIRED_SINGLE_ARGS};${OPTIONAL_SINGLE_ARGS};LINK_TO" "COMMON_OPTIONS;DEPENDS" ${ARGV})
     NBL_PARSE_REQUIRED(IMPL ${REQUIRED_SINGLE_ARGS})
 
+	set(IMPL_HLSL_GLOB "")
+	if(NOT IMPL_DISCARD_DEFAULT_GLOB)
+		set(GLOB_ROOT "${CMAKE_CURRENT_SOURCE_DIR}")
+		if(IMPL_GLOB_DIR)
+			set(GLOB_ROOT "${IMPL_GLOB_DIR}")
+		endif()
+		get_filename_component(GLOB_ROOT "${GLOB_ROOT}" ABSOLUTE BASE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
+		file(GLOB_RECURSE IMPL_HLSL_GLOB CONFIGURE_DEPENDS "${GLOB_ROOT}/*.hlsl")
+	endif()
+
 	if(NOT TARGET ${IMPL_TARGET})
 		add_library(${IMPL_TARGET} INTERFACE)
 	endif()
@@ -1293,6 +1307,10 @@ namespace @IMPL_NAMESPACE@ {
 	list(APPEND MP_DEFINES ${IMPL_MOUNT_POINT_DEFINE}="${IMPL_BINARY_DIR}")
 	set_target_properties(${IMPL_TARGET} PROPERTIES NBL_MOUNT_POINT_DEFINES "${MP_DEFINES}")
 
+	set(RTE "NSC Rules")
+	set(IN "${RTE}/In")
+	set(OUT "${RTE}/Out")
+
     string(JSON JSON_LENGTH LENGTH "${IMPL_INPUTS}")
     math(EXPR LAST_INDEX "${JSON_LENGTH} - 1")
 
@@ -1479,30 +1497,73 @@ namespace @IMPL_NAMESPACE@ {
 				# generate keys and commands for compiling shaders
 				set(FINAL_KEY_REL_PATH "$<CONFIG>/${FINAL_KEY}")
 				set(TARGET_OUTPUT "${IMPL_BINARY_DIR}/${FINAL_KEY_REL_PATH}")
+				set(DEPFILE_PATH "${TARGET_OUTPUT}.d")
+				set(NBL_NSC_LOG_PATH "${TARGET_OUTPUT}.log")
+
+				set(NBL_NSC_DEPFILE_ARGS "")
+				if(NSC_USE_DEPFILE)
+					set(NBL_NSC_DEPFILE_ARGS -MD -MF "${DEPFILE_PATH}")
+				endif()
 
 				set(NBL_NSC_COMPILE_COMMAND
 					"$<TARGET_FILE:nsc>"
 					-Fc "${TARGET_OUTPUT}"
 					${COMPILE_OPTIONS} ${REQUIRED_OPTIONS} ${IMPL_COMMON_OPTIONS}
+					${NBL_NSC_DEPFILE_ARGS}
 					"${CONFIG_FILE}"
 				)
 
-				add_custom_command(OUTPUT "${TARGET_OUTPUT}"
+				get_filename_component(NBL_NSC_INPUT_NAME "${TARGET_INPUT}" NAME)
+				get_filename_component(NBL_NSC_CONFIG_NAME "${CONFIG_FILE}" NAME)
+				set(NBL_NSC_BYPRODUCTS "${NBL_NSC_LOG_PATH}")
+				if(NSC_USE_DEPFILE)
+					list(APPEND NBL_NSC_BYPRODUCTS "${DEPFILE_PATH}")
+				endif()
+
+				set(NBL_NSC_CUSTOM_COMMAND_ARGS
+					OUTPUT "${TARGET_OUTPUT}"
+					BYPRODUCTS ${NBL_NSC_BYPRODUCTS}
 					COMMAND ${NBL_NSC_COMPILE_COMMAND}
 					DEPENDS ${DEPENDS_ON}
-					COMMENT "Creating \"${TARGET_OUTPUT}\""
+					COMMENT "${NBL_NSC_CONFIG_NAME} (${NBL_NSC_INPUT_NAME})"
 					VERBATIM
 					COMMAND_EXPAND_LISTS
 				)
-				set_source_files_properties("${TARGET_OUTPUT}" PROPERTIES GENERATED TRUE)
+				if(NSC_USE_DEPFILE)
+					list(APPEND NBL_NSC_CUSTOM_COMMAND_ARGS DEPFILE "${DEPFILE_PATH}")
+				endif()
+				add_custom_command(${NBL_NSC_CUSTOM_COMMAND_ARGS})
+				set(NBL_NSC_OUT_FILES "${TARGET_OUTPUT}" "${NBL_NSC_LOG_PATH}")
+				if(NSC_USE_DEPFILE)
+					list(APPEND NBL_NSC_OUT_FILES "${DEPFILE_PATH}")
+				endif()
 
-				set(HEADER_ONLY_LIKE "${CONFIG_FILE}" "${TARGET_INPUT}" "${TARGET_OUTPUT}")
+				set_source_files_properties(${NBL_NSC_OUT_FILES} PROPERTIES GENERATED TRUE)
+
+				set(HEADER_ONLY_LIKE "${CONFIG_FILE}" "${TARGET_INPUT}" ${NBL_NSC_OUT_FILES})
 				target_sources(${IMPL_TARGET} PRIVATE ${HEADER_ONLY_LIKE})
 
 				set_source_files_properties(${HEADER_ONLY_LIKE} PROPERTIES 
 					HEADER_FILE_ONLY ON
 					VS_TOOL_OVERRIDE None
 				)
+				if(CMAKE_CONFIGURATION_TYPES)
+					foreach(_CFG IN LISTS CMAKE_CONFIGURATION_TYPES)
+						set(TARGET_OUTPUT_IDE "${IMPL_BINARY_DIR}/${_CFG}/${FINAL_KEY}")
+						set(NBL_NSC_OUT_FILES_IDE "${TARGET_OUTPUT_IDE}" "${TARGET_OUTPUT_IDE}.log")
+						if(NSC_USE_DEPFILE)
+							list(APPEND NBL_NSC_OUT_FILES_IDE "${TARGET_OUTPUT_IDE}.d")
+						endif()
+						source_group("${OUT}/${_CFG}" FILES ${NBL_NSC_OUT_FILES_IDE})
+					endforeach()
+				else()
+					set(TARGET_OUTPUT_IDE "${IMPL_BINARY_DIR}/${FINAL_KEY}")
+					set(NBL_NSC_OUT_FILES_IDE "${TARGET_OUTPUT_IDE}" "${TARGET_OUTPUT_IDE}.log")
+					if(NSC_USE_DEPFILE)
+						list(APPEND NBL_NSC_OUT_FILES_IDE "${TARGET_OUTPUT_IDE}.d")
+					endif()
+					source_group("${OUT}" FILES ${NBL_NSC_OUT_FILES_IDE})
+				endif()
 
 				set_source_files_properties("${TARGET_OUTPUT}" PROPERTIES
 					NBL_SPIRV_REGISTERED_INPUT "${TARGET_INPUT}"
@@ -1544,12 +1605,15 @@ namespace @IMPL_NAMESPACE@ {
 		list(APPEND KEYS ${ACCESS_KEY})
 	endforeach()
 
-	set(RTE "NSC Rules")
-	set(IN "${RTE}/In")
-	set(OUT "${RTE}/Out")
-
 	source_group("${IN}" FILES ${CONFIGS} ${INPUTS})
-	source_group("${OUT}" FILES ${SPIRVs})
+	if(IMPL_HLSL_GLOB)
+		target_sources(${IMPL_TARGET} PRIVATE ${IMPL_HLSL_GLOB})
+		set_source_files_properties(${IMPL_HLSL_GLOB} PROPERTIES 
+			HEADER_FILE_ONLY ON
+			VS_TOOL_OVERRIDE None
+		)
+		source_group("HLSL Files" FILES ${IMPL_HLSL_GLOB})
+	endif()
 
 	set(${IMPL_OUTPUT_VAR} ${KEYS} PARENT_SCOPE)
 endfunction()

docs/nsc-prebuilds.md

@@ -60,7 +60,7 @@ Keys are strings that match the output layout:
 - `INPUT` (string, required): path to `.hlsl` (relative to `CMAKE_CURRENT_SOURCE_DIR` or absolute).
 - `KEY` (string, required): base key (prefer without `.spv`; it is always appended, so using `foo.spv` will result in `foo.spv.spv`).
 - `COMPILE_OPTIONS` (array of strings, optional): per-input extra options (e.g. `["-T","cs_6_8"]`).
-- `DEPENDS` (array of strings, optional): per-input dependencies (extra files that should trigger rebuild).
+- `DEPENDS` (array of strings, optional): extra per-input dependencies that are not discovered via `#include` (see below).
 - `CAPS` (array, optional): permutation caps (see below).
 
 You can register many rules in a single call, and you can call the function multiple times to append rules to the same `TARGET`.
@@ -87,31 +87,21 @@ The helper also exposes CMake options that append NSC debug flags **only for Deb
 
 ## Source files and rebuild dependencies (important)
 
-Make sure shader inputs and includes are:
+NSC supports depfiles and the CMake custom commands consume them, so **changes in any `#include`d HLSL file automatically trigger recompilation of the affected `.spv` outputs**. In most cases you no longer need to list includes manually.
 
-1. Marked as header-only on your target (so the IDE shows them, but the build system doesn't try to compile them with default HLSL rules like `fxc`):
+Use `DEPENDS` only for **extra** inputs that are not discovered via `#include` (e.g. a generated header that is not included, a config file read by a custom include generator, or any non-HLSL file that should trigger a rebuild). You can register those extra dependencies if you need them, but in most projects `DEPENDS` should stay empty.
 
-```cmake
-target_sources(${EXECUTABLE_NAME} PRIVATE ${DEPENDS})
-set_source_files_properties(${DEPENDS} PROPERTIES HEADER_FILE_ONLY ON)
-```
+By default `NBL_CREATE_NSC_COMPILE_RULES` also collects `*.hlsl` files for IDE visibility. It recursively scans the current source directory (or `GLOB_DIR` if provided), adds those files as header-only, and groups them under `HLSL Files`. If you do not want this behavior, pass `DISCARD_DEFAULT_GLOB`.
 
-2. Listed as dependencies of the NSC custom commands (so editing any of them triggers a rebuild of the `.spv` outputs).
-
-This is what the `DEPENDS` argument of `NBL_CREATE_NSC_COMPILE_RULES` (and/or per-input JSON `DEPENDS`) is for. Always include the main `INPUT` file itself and any files it includes; otherwise the build system might not re-run `nsc` when you change them.
+- `GLOB_DIR` (optional): root directory for the default `*.hlsl` scan.
+- `DISCARD_DEFAULT_GLOB` (flag): disables the default scan and IDE grouping.
 
 ## Minimal usage (no permutations)
 
 Example pattern (as in `examples_tests/27_MPMCScheduler/CMakeLists.txt`):
 
 ```cmake
 set(OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/auto-gen")
-set(DEPENDS
-  app_resources/common.hlsl
-  app_resources/shader.comp.hlsl
-)
-target_sources(${EXECUTABLE_NAME} PRIVATE ${DEPENDS})
-set_source_files_properties(${DEPENDS} PROPERTIES HEADER_FILE_ONLY ON)
 
 set(JSON [=[
 [
@@ -128,7 +118,6 @@ set(JSON [=[
 NBL_CREATE_NSC_COMPILE_RULES(
   TARGET ${EXECUTABLE_NAME}SPIRV
   LINK_TO ${EXECUTABLE_NAME}
-  DEPENDS ${DEPENDS}
   BINARY_DIR ${OUTPUT_DIRECTORY}
   MOUNT_POINT_DEFINE NBL_THIS_EXAMPLE_BUILD_MOUNT_POINT
   COMMON_OPTIONS -I ${CMAKE_CURRENT_SOURCE_DIR}
@@ -298,6 +287,8 @@ If the error looks like a preprocessing issue, note that we use Boost.Wave as th
 <details>
 <summary>NSC rules + archive + runtime key usage</summary>
 
+NSC emits depfiles and the custom commands consume them, so changes in `#include`d HLSL files automatically trigger recompilation of the affected outputs. In most cases you do not need to list includes manually. Use `DEPENDS` only for extra inputs that are not discovered via `#include`.
+
 ### CMake (`CMakeLists.txt`)
 
 ```cmake
@@ -306,20 +297,13 @@ include(common)
 nbl_create_executable_project("" "" "" "")
 
 set(OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/auto-gen")
-set(DEPENDS
-  app_resources/common.hlsl
-  app_resources/shader.hlsl
-)
-target_sources(${EXECUTABLE_NAME} PRIVATE ${DEPENDS})
-set_source_files_properties(${DEPENDS} PROPERTIES HEADER_FILE_ONLY ON)
 
 set(JSON [=[
 [
   {
     "INPUT": "app_resources/shader.hlsl",
     "KEY": "shader",
     "COMPILE_OPTIONS": ["-T", "lib_6_8"],
-    "DEPENDS": [],
     "CAPS": [
       {
         "kind": "limits",
@@ -341,7 +325,6 @@ set(JSON [=[
 NBL_CREATE_NSC_COMPILE_RULES(
   TARGET ${EXECUTABLE_NAME}SPIRV
   LINK_TO ${EXECUTABLE_NAME}
-  DEPENDS ${DEPENDS}
   BINARY_DIR ${OUTPUT_DIRECTORY}
   MOUNT_POINT_DEFINE NBL_THIS_EXAMPLE_BUILD_MOUNT_POINT
   COMMON_OPTIONS -I ${CMAKE_CURRENT_SOURCE_DIR}

examples_tests

@@ -137,6 +137,8 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 			const CIncludeFinder* includeFinder = nullptr;
 			std::span<const SMacroDefinition> extraDefines = {};
 			E_SPIRV_VERSION targetSpirvVersion = E_SPIRV_VERSION::ESV_1_6;
+			bool depfile = false;
+			system::path depfilePath = {};
 		};
 
 		// https://github.com/microsoft/DirectXShaderCompiler/blob/main/docs/SPIR-V.rst#debugging
@@ -215,6 +217,10 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 							// Needed for json vector serialization. Making it private and declaring from_json(_, SEntry&) as friend didn't work
 							inline SPreprocessingDependency() {}
 
+							inline const system::path& getRequestingSourceDir() const { return requestingSourceDir; }
+							inline std::string_view getIdentifier() const { return identifier; }
+							inline bool isStandardInclude() const { return standardInclude; }
+
 						private:
 							friend void to_json(nlohmann::json& j, const SEntry::SPreprocessingDependency& dependency);
 							friend void from_json(const nlohmann::json& j, SEntry::SPreprocessingDependency& dependency);
@@ -447,6 +453,17 @@ class NBL_API2 IShaderCompiler : public core::IReferenceCounted
 				NBL_API2 EntrySet::const_iterator find_impl(const SEntry& mainFile, const CIncludeFinder* finder) const;
 		};
 
+		struct DepfileWriteParams
+		{
+			system::ISystem* system = nullptr;
+			std::string_view depfilePath = {};
+			std::string_view outputPath = {};
+			std::string_view sourceIdentifier = {};
+			system::path workingDirectory = {};
+		};
+
+		static bool writeDepfile(const DepfileWriteParams& params, const CCache::SEntry::dependency_container_t& dependencies, const CIncludeFinder* includeFinder = nullptr, system::logger_opt_ptr logger = nullptr);
+
 		core::smart_refctd_ptr<IShader> compileToSPIRV(const std::string_view code, const SCompilerOptions& options) const;
 
 		inline core::smart_refctd_ptr<IShader> compileToSPIRV(const char* code, const SCompilerOptions& options) const

include/nbl/asset/utils/IShaderCompiler.h

@@ -25,7 +25,9 @@ class IFileBase : public core::IReferenceCounted
 			ECF_READ_WRITE = 0b0011,
 			ECF_MAPPABLE = 0b0100,
 			//! Implies ECF_MAPPABLE
-			ECF_COHERENT = 0b1100
+			ECF_COHERENT = 0b1100,
+			ECF_SHARE_READ_WRITE = 0b100000,
+			ECF_SHARE_DELETE = 0b1000000
 		};
 
 		//! Get size of file.