From 962f83879a7630f1b2b1336a5067faba7ccf3b31 Mon Sep 17 00:00:00 2001 From: Philip Rebohle Date: Sat, 15 Feb 2025 15:35:40 +0100 Subject: [PATCH] wip barrier atomic store stuff --- src/dxvk/dxvk_barrier.cpp | 45 ++++++++++--- src/dxvk/dxvk_barrier.h | 24 +++++-- src/dxvk/dxvk_context.cpp | 133 ++++++++++++++++++++++---------------- src/dxvk/dxvk_context.h | 48 +++++++++----- 4 files changed, 168 insertions(+), 82 deletions(-) diff --git a/src/dxvk/dxvk_barrier.cpp b/src/dxvk/dxvk_barrier.cpp index 287a0ff2fcc4..ec80a4bab199 100644 --- a/src/dxvk/dxvk_barrier.cpp +++ b/src/dxvk/dxvk_barrier.cpp @@ -20,20 +20,36 @@ namespace dxvk { bool DxvkBarrierTracker::findRange( const DxvkAddressRange& range, - DxvkAccess accessType) const { + DxvkAccess accessType, + DxvkAtomicStore storeOp) const { uint32_t rootIndex = computeRootIndex(range, accessType); - return findNode(range, rootIndex); + uint32_t nodeIndex = findNode(range, rootIndex); + + if (!nodeIndex || !storeOp) + return nodeIndex; + + // If we are checking for a specific order-invariant store + // op, report a hazard if either the found range does not + // cover the entire range to look up, or if the store op + // does not match previous stores. + auto& node = m_nodes[nodeIndex]; + + return node.payload.storeOp != storeOp + || !node.addressRange.contains(range); } void DxvkBarrierTracker::insertRange( const DxvkAddressRange& range, - DxvkAccess accessType) { - uint32_t rootIndex = computeRootIndex(range, accessType); + DxvkAccess accessType, + DxvkAtomicStore storeOp) { + DxvkBarrierPayload payload = { }; + payload.storeOp = storeOp; // If we can just insert the node with no conflicts, // we don't have to do anything. - uint32_t nodeIndex = insertNode(range, rootIndex); + uint32_t rootIndex = computeRootIndex(range, accessType); + uint32_t nodeIndex = insertNode(range, rootIndex, payload); if (likely(!nodeIndex)) return; @@ -42,6 +58,9 @@ namespace dxvk { // range we want to add already, also don't do anything. auto& node = m_nodes[nodeIndex]; + if (node.payload.storeOp != payload.storeOp) + node.payload.storeOp = DxvkAtomicStore::None; + if (node.addressRange.contains(range)) return; @@ -82,12 +101,15 @@ namespace dxvk { mergedRange.rangeStart = std::min(mergedRange.rangeStart, node.addressRange.rangeStart); mergedRange.rangeEnd = std::max(mergedRange.rangeEnd, node.addressRange.rangeEnd); + if (node.payload.storeOp != payload.storeOp) + payload.storeOp = DxvkAtomicStore::None; + removeNode(nodeIndex, rootIndex); nodeIndex = findNode(range, rootIndex); } - insertNode(mergedRange, rootIndex); + insertNode(mergedRange, rootIndex, payload); } @@ -166,7 +188,8 @@ namespace dxvk { uint32_t DxvkBarrierTracker::insertNode( const DxvkAddressRange& range, - uint32_t rootIndex) { + uint32_t rootIndex, + DxvkBarrierPayload payload) { // Check if the given root is valid at all uint64_t rootBit = uint64_t(1u) << (rootIndex - 1u); @@ -178,6 +201,7 @@ namespace dxvk { auto& node = m_nodes[rootIndex]; node.header = 0; node.addressRange = range; + node.payload = payload; return 0; } else { // Traverse tree and abort if we find any range @@ -209,6 +233,7 @@ namespace dxvk { node.setRed(true); node.setParent(parentIndex); node.addressRange = range; + node.payload = payload; // Only do the fixup to maintain red-black properties if // we haven't marked the root node as red in a deletion. @@ -238,6 +263,7 @@ namespace dxvk { childIndex = m_nodes[childIndex].child(0); node.addressRange = m_nodes[childIndex].addressRange; + node.payload = m_nodes[childIndex].payload; removeNode(childIndex, rootIndex); } else { // Deletion is expected to be exceptionally rare, to the point of @@ -268,6 +294,7 @@ namespace dxvk { node.setRed(child.isRed()); node.addressRange = child.addressRange; + node.payload = child.payload; if (cl) m_nodes[cl].setParent(nodeIndex); if (cr) m_nodes[cr].setParent(nodeIndex); @@ -378,6 +405,7 @@ namespace dxvk { node.setChild(1, rr); std::swap(node.addressRange, m_nodes[r].addressRange); + std::swap(node.payload, m_nodes[r].payload); } @@ -406,6 +434,7 @@ namespace dxvk { node.setChild(1, l); std::swap(node.addressRange, m_nodes[l].addressRange); + std::swap(node.payload, m_nodes[l].payload); } @@ -498,4 +527,4 @@ namespace dxvk { flush(list); } -} \ No newline at end of file +} diff --git a/src/dxvk/dxvk_barrier.h b/src/dxvk/dxvk_barrier.h index 3b4cdf9c598f..a75a33c50b81 100644 --- a/src/dxvk/dxvk_barrier.h +++ b/src/dxvk/dxvk_barrier.h @@ -42,6 +42,14 @@ namespace dxvk { }; + /** + * \brief Barrier nodepayload + */ + struct DxvkBarrierPayload { + DxvkAtomicStore storeOp = DxvkAtomicStore::None; + }; + + /** * \brief Barrier tree node * @@ -62,6 +70,9 @@ namespace dxvk { // Address range of the node DxvkAddressRange addressRange = { }; + // Node payload + DxvkBarrierPayload payload = { }; + void setRed(bool red) { header &= ~uint64_t(1u); header |= uint64_t(red); @@ -117,21 +128,25 @@ namespace dxvk { * * \param [in] range Resource range * \param [in] accessType Access type + * \param [in] storeOp Invariant store op * \returns \c true if the range has a pending access */ bool findRange( const DxvkAddressRange& range, - DxvkAccess accessType) const; + DxvkAccess accessType, + DxvkAtomicStore storeOp) const; /** * \brief Inserts address range for a given access type * * \param [in] range Resource range * \param [in] accessType Access type + * \param [in] storeOp Invariant store op */ void insertRange( const DxvkAddressRange& range, - DxvkAccess accessType); + DxvkAccess accessType, + DxvkAtomicStore storeOp); /** * \brief Clears the entire structure @@ -166,7 +181,8 @@ namespace dxvk { uint32_t insertNode( const DxvkAddressRange& range, - uint32_t rootIndex); + uint32_t rootIndex, + DxvkBarrierPayload payload); void removeNode( uint32_t nodeIndex, @@ -285,4 +301,4 @@ namespace dxvk { }; -} \ No newline at end of file +} diff --git a/src/dxvk/dxvk_context.cpp b/src/dxvk/dxvk_context.cpp index e8adc0ede3b7..1587c708011b 100644 --- a/src/dxvk/dxvk_context.cpp +++ b/src/dxvk/dxvk_context.cpp @@ -5931,7 +5931,7 @@ namespace dxvk { descriptorInfo.image.imageLayout = res.imageView->image()->info().layout; if (BindPoint == VK_PIPELINE_BIND_POINT_COMPUTE || res.imageView->image()->hasGfxStores()) - accessImage(DxvkCmdBuffer::ExecBuffer, *res.imageView, util::pipelineStages(binding.stage), binding.access); + accessImage(DxvkCmdBuffer::ExecBuffer, *res.imageView, util::pipelineStages(binding.stage), binding.access, binding.storeOp); m_cmd->track(res.imageView->image(), (binding.access & vk::AccessWriteMask) ? DxvkAccess::Write : DxvkAccess::Read); @@ -5989,7 +5989,7 @@ namespace dxvk { descriptorInfo.texelBuffer = res.bufferView->handle(); if (BindPoint == VK_PIPELINE_BIND_POINT_COMPUTE || res.bufferView->buffer()->hasGfxStores()) - accessBuffer(DxvkCmdBuffer::ExecBuffer, *res.bufferView, util::pipelineStages(binding.stage), binding.access); + accessBuffer(DxvkCmdBuffer::ExecBuffer, *res.bufferView, util::pipelineStages(binding.stage), binding.access, binding.storeOp); m_cmd->track(res.bufferView->buffer(), (binding.access & vk::AccessWriteMask) ? DxvkAccess::Write : DxvkAccess::Read); @@ -6022,7 +6022,7 @@ namespace dxvk { descriptorInfo = res.bufferSlice.getDescriptor(); if (BindPoint == VK_PIPELINE_BIND_POINT_COMPUTE || unlikely(res.bufferSlice.buffer()->hasGfxStores())) - accessBuffer(DxvkCmdBuffer::ExecBuffer, res.bufferSlice, util::pipelineStages(binding.stage), binding.access); + accessBuffer(DxvkCmdBuffer::ExecBuffer, res.bufferSlice, util::pipelineStages(binding.stage), binding.access, binding.storeOp); m_cmd->track(res.bufferSlice.buffer(), (binding.access & vk::AccessWriteMask) ? DxvkAccess::Write : DxvkAccess::Read); @@ -6743,7 +6743,7 @@ namespace dxvk { case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER: { if (slot.bufferView) { if (!IsGraphics || slot.bufferView->buffer()->hasGfxStores()) - requiresBarrier |= checkBufferViewBarrier(slot.bufferView, binding.access); + requiresBarrier |= checkBufferViewBarrier(slot.bufferView, binding.access, binding.storeOp); else if (binding.access & vk::AccessWriteMask) requiresBarrier |= !slot.bufferView->buffer()->trackGfxStores(); } @@ -6751,18 +6751,18 @@ namespace dxvk { case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER: { if (slot.bufferView && (!IsGraphics || slot.bufferView->buffer()->hasGfxStores())) - requiresBarrier |= checkBufferViewBarrier(slot.bufferView, binding.access); + requiresBarrier |= checkBufferViewBarrier(slot.bufferView, binding.access, binding.storeOp); } break; case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER: { if (slot.bufferSlice.length() && (!IsGraphics || slot.bufferSlice.buffer()->hasGfxStores())) - requiresBarrier |= checkBufferBarrier(slot.bufferSlice, binding.access); + requiresBarrier |= checkBufferBarrier(slot.bufferSlice, binding.access, binding.storeOp); } break; case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: { if (slot.bufferSlice.length()) { if (!IsGraphics || slot.bufferSlice.buffer()->hasGfxStores()) - requiresBarrier |= checkBufferBarrier(slot.bufferSlice, binding.access); + requiresBarrier |= checkBufferBarrier(slot.bufferSlice, binding.access, binding.storeOp); else if (binding.access & vk::AccessWriteMask) requiresBarrier |= !slot.bufferSlice.buffer()->trackGfxStores(); } @@ -6771,7 +6771,7 @@ namespace dxvk { case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: { if (slot.imageView) { if (!IsGraphics || slot.imageView->image()->hasGfxStores()) - requiresBarrier |= checkImageViewBarrier(slot.imageView, binding.access); + requiresBarrier |= checkImageViewBarrier(slot.imageView, binding.access, binding.storeOp); else if (binding.access & vk::AccessWriteMask) requiresBarrier |= !slot.imageView->image()->trackGfxStores(); } @@ -6780,7 +6780,7 @@ namespace dxvk { case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE: case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER: { if (slot.imageView && (!IsGraphics || slot.imageView->image()->hasGfxStores())) - requiresBarrier |= checkImageViewBarrier(slot.imageView, binding.access); + requiresBarrier |= checkImageViewBarrier(slot.imageView, binding.access, binding.storeOp); } break; default: @@ -6833,13 +6833,14 @@ namespace dxvk { if (xfbBufferSlice.length()) { requiresBarrier |= !xfbBufferSlice.buffer()->trackGfxStores(); requiresBarrier |= checkBufferBarrier( - xfbBufferSlice, VK_ACCESS_TRANSFORM_FEEDBACK_WRITE_BIT_EXT); + xfbBufferSlice, VK_ACCESS_TRANSFORM_FEEDBACK_WRITE_BIT_EXT, DxvkAtomicStore::None); if (xfbCounterSlice.length()) { requiresBarrier |= !xfbCounterSlice.buffer()->trackGfxStores(); requiresBarrier |= checkBufferBarrier(xfbCounterSlice, VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_READ_BIT_EXT | - VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_WRITE_BIT_EXT); + VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_WRITE_BIT_EXT, + DxvkAtomicStore::None); } } } @@ -6859,7 +6860,8 @@ namespace dxvk { for (uint32_t i = 0; i < slices.size(); i++) { if (slices[i]->length() && slices[i]->buffer()->hasGfxStores()) { - if (checkBufferBarrier(*slices[i], VK_ACCESS_INDIRECT_COMMAND_READ_BIT)) + if (checkBufferBarrier(*slices[i], + VK_ACCESS_INDIRECT_COMMAND_READ_BIT, DxvkAtomicStore::None)) return true; } } @@ -6871,7 +6873,8 @@ namespace dxvk { const auto& indexBufferSlice = m_state.vi.indexBuffer; if (indexBufferSlice.length() && indexBufferSlice.buffer()->hasGfxStores()) { - if (checkBufferBarrier(indexBufferSlice, VK_ACCESS_INDEX_READ_BIT)) + if (checkBufferBarrier(indexBufferSlice, + VK_ACCESS_INDEX_READ_BIT, DxvkAtomicStore::None)) return true; } } @@ -6885,7 +6888,8 @@ namespace dxvk { const auto& vertexBufferSlice = m_state.vi.vertexBuffers[binding]; if (vertexBufferSlice.length() && vertexBufferSlice.buffer()->hasGfxStores()) { - if (checkBufferBarrier(vertexBufferSlice, VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT)) + if (checkBufferBarrier(vertexBufferSlice, + VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT, DxvkAtomicStore::None)) return true; } } @@ -6898,10 +6902,11 @@ namespace dxvk { template bool DxvkContext::checkBufferBarrier( const DxvkBufferSlice& bufferSlice, - VkAccessFlags access) { - return checkResourceBarrier([this, &bufferSlice] (DxvkAccess access) { + VkAccessFlags access, + DxvkAtomicStore storeOp) { + return checkResourceBarrier([this, &bufferSlice, storeOp] (DxvkAccess access) { return resourceHasAccess(*bufferSlice.buffer(), - bufferSlice.offset(), bufferSlice.length(), access); + bufferSlice.offset(), bufferSlice.length(), access, storeOp); }, access); } @@ -6909,9 +6914,10 @@ namespace dxvk { template bool DxvkContext::checkBufferViewBarrier( const Rc& bufferView, - VkAccessFlags access) { - return checkResourceBarrier([this, &bufferView] (DxvkAccess access) { - return resourceHasAccess(*bufferView, access); + VkAccessFlags access, + DxvkAtomicStore storeOp) { + return checkResourceBarrier([this, &bufferView, storeOp] (DxvkAccess access) { + return resourceHasAccess(*bufferView, access, storeOp); }, access); } @@ -6919,9 +6925,10 @@ namespace dxvk { template bool DxvkContext::checkImageViewBarrier( const Rc& imageView, - VkAccessFlags access) { - return checkResourceBarrier([this, &imageView] (DxvkAccess access) { - return resourceHasAccess(*imageView, access); + VkAccessFlags access, + DxvkAtomicStore storeOp) { + return checkResourceBarrier([this, &imageView, storeOp] (DxvkAccess access) { + return resourceHasAccess(*imageView, access, storeOp); }, access); } @@ -7646,12 +7653,14 @@ namespace dxvk { const VkImageSubresourceRange& subresources, VkImageLayout srcLayout, VkPipelineStageFlags2 srcStages, - VkAccessFlags2 srcAccess) { + VkAccessFlags2 srcAccess, + DxvkAtomicStore storeOp) { accessImage(cmdBuffer, image, subresources, srcLayout, srcStages, srcAccess, image.info().layout, image.info().stages, - image.info().access); + image.info().access, + storeOp); } @@ -7659,11 +7668,12 @@ namespace dxvk { DxvkCmdBuffer cmdBuffer, const DxvkImageView& imageView, VkPipelineStageFlags2 srcStages, - VkAccessFlags2 srcAccess) { + VkAccessFlags2 srcAccess, + DxvkAtomicStore storeOp) { accessImage(cmdBuffer, *imageView.image(), imageView.imageSubresources(), imageView.image()->info().layout, - srcStages, srcAccess); + srcStages, srcAccess, storeOp); } @@ -7676,7 +7686,8 @@ namespace dxvk { VkAccessFlags2 srcAccess, VkImageLayout dstLayout, VkPipelineStageFlags2 dstStages, - VkAccessFlags2 dstAccess) { + VkAccessFlags2 dstAccess, + DxvkAtomicStore storeOp) { auto& batch = getBarrierBatch(cmdBuffer); if (srcLayout == VK_IMAGE_LAYOUT_UNDEFINED || srcLayout == VK_IMAGE_LAYOUT_PREINITIALIZED) @@ -7710,9 +7721,9 @@ namespace dxvk { + (subresources.baseArrayLayer + subresources.layerCount - 1u); if (hasWrite) - m_barrierTracker.insertRange(range, DxvkAccess::Write); + m_barrierTracker.insertRange(range, DxvkAccess::Write, storeOp); if (hasRead) - m_barrierTracker.insertRange(range, DxvkAccess::Read); + m_barrierTracker.insertRange(range, DxvkAccess::Read, storeOp); } else { DxvkAddressRange range; range.resource = image.getResourceId(); @@ -7722,9 +7733,9 @@ namespace dxvk { range.rangeEnd = range.rangeStart + subresources.layerCount - 1u; if (hasWrite) - m_barrierTracker.insertRange(range, DxvkAccess::Write); + m_barrierTracker.insertRange(range, DxvkAccess::Write, storeOp); if (hasRead) - m_barrierTracker.insertRange(range, DxvkAccess::Read); + m_barrierTracker.insertRange(range, DxvkAccess::Read, storeOp); } } } @@ -7737,11 +7748,13 @@ namespace dxvk { VkDeviceSize offset, VkDeviceSize size, VkPipelineStageFlags2 srcStages, - VkAccessFlags2 srcAccess) { + VkAccessFlags2 srcAccess, + DxvkAtomicStore storeOp) { accessBuffer(cmdBuffer, buffer, offset, size, srcStages, srcAccess, buffer.info().stages, - buffer.info().access); + buffer.info().access, + storeOp); } @@ -7753,7 +7766,8 @@ namespace dxvk { VkPipelineStageFlags2 srcStages, VkAccessFlags2 srcAccess, VkPipelineStageFlags2 dstStages, - VkAccessFlags2 dstAccess) { + VkAccessFlags2 dstAccess, + DxvkAtomicStore storeOp) { if (unlikely(!size)) return; @@ -7774,9 +7788,9 @@ namespace dxvk { range.rangeEnd = offset + size - 1; if (srcAccess & vk::AccessWriteMask) - m_barrierTracker.insertRange(range, DxvkAccess::Write); + m_barrierTracker.insertRange(range, DxvkAccess::Write, storeOp); if (srcAccess & vk::AccessReadMask) - m_barrierTracker.insertRange(range, DxvkAccess::Read); + m_barrierTracker.insertRange(range, DxvkAccess::Read, storeOp); } } @@ -7785,12 +7799,14 @@ namespace dxvk { DxvkCmdBuffer cmdBuffer, const DxvkBufferSlice& bufferSlice, VkPipelineStageFlags2 srcStages, - VkAccessFlags2 srcAccess) { + VkAccessFlags2 srcAccess, + DxvkAtomicStore storeOp) { accessBuffer(cmdBuffer, *bufferSlice.buffer(), bufferSlice.offset(), bufferSlice.length(), - srcStages, srcAccess); + srcStages, srcAccess, + storeOp); } @@ -7800,13 +7816,15 @@ namespace dxvk { VkPipelineStageFlags2 srcStages, VkAccessFlags2 srcAccess, VkPipelineStageFlags2 dstStages, - VkAccessFlags2 dstAccess) { + VkAccessFlags2 dstAccess, + DxvkAtomicStore storeOp) { accessBuffer(cmdBuffer, *bufferSlice.buffer(), bufferSlice.offset(), bufferSlice.length(), srcStages, srcAccess, - dstStages, dstAccess); + dstStages, dstAccess, + storeOp); } @@ -7814,12 +7832,13 @@ namespace dxvk { DxvkCmdBuffer cmdBuffer, DxvkBufferView& bufferView, VkPipelineStageFlags2 srcStages, - VkAccessFlags2 srcAccess) { + VkAccessFlags2 srcAccess, + DxvkAtomicStore storeOp) { accessBuffer(cmdBuffer, *bufferView.buffer(), bufferView.info().offset, bufferView.info().size, - srcStages, srcAccess); + srcStages, srcAccess, storeOp); } @@ -7829,13 +7848,15 @@ namespace dxvk { VkPipelineStageFlags2 srcStages, VkAccessFlags2 srcAccess, VkPipelineStageFlags2 dstStages, - VkAccessFlags2 dstAccess) { + VkAccessFlags2 dstAccess, + DxvkAtomicStore storeOp) { accessBuffer(cmdBuffer, *bufferView.buffer(), bufferView.info().offset, bufferView.info().size, srcStages, srcAccess, - dstStages, dstAccess); + dstStages, dstAccess, + storeOp); } @@ -7921,7 +7942,8 @@ namespace dxvk { DxvkBuffer& buffer, VkDeviceSize offset, VkDeviceSize size, - DxvkAccess access) { + DxvkAccess access, + DxvkAtomicStore storeOp) { if (unlikely(!size)) return false; @@ -7930,23 +7952,25 @@ namespace dxvk { range.rangeStart = offset; range.rangeEnd = offset + size - 1; - return m_barrierTracker.findRange(range, access); + return m_barrierTracker.findRange(range, access, storeOp); } bool DxvkContext::resourceHasAccess( DxvkBufferView& bufferView, - DxvkAccess access) { + DxvkAccess access, + DxvkAtomicStore storeOp) { return resourceHasAccess(*bufferView.buffer(), bufferView.info().offset, - bufferView.info().size, access); + bufferView.info().size, access, storeOp); } bool DxvkContext::resourceHasAccess( DxvkImage& image, const VkImageSubresourceRange& subresources, - DxvkAccess access) { + DxvkAccess access, + DxvkAtomicStore storeOp) { uint32_t layerCount = image.info().numLayers; // Subresources are enumerated in such a way that array layers of @@ -7962,7 +7986,7 @@ namespace dxvk { // Probe all subresources first, only check individual mip levels // if there are overlaps and if we are checking a subset of array // layers of multiple mips. - bool dirty = m_barrierTracker.findRange(range, access); + bool dirty = m_barrierTracker.findRange(range, access, storeOp); if (!dirty || subresources.levelCount == 1u || subresources.layerCount == layerCount) return dirty; @@ -7971,7 +7995,7 @@ namespace dxvk { range.rangeStart = i * layerCount + subresources.baseArrayLayer; range.rangeEnd = range.rangeStart + subresources.layerCount - 1u; - dirty = m_barrierTracker.findRange(range, access); + dirty = m_barrierTracker.findRange(range, access, storeOp); } return dirty; @@ -7980,8 +8004,9 @@ namespace dxvk { bool DxvkContext::resourceHasAccess( DxvkImageView& imageView, - DxvkAccess access) { - return resourceHasAccess(*imageView.image(), imageView.imageSubresources(), access); + DxvkAccess access, + DxvkAtomicStore storeOp) { + return resourceHasAccess(*imageView.image(), imageView.imageSubresources(), access, storeOp); } diff --git a/src/dxvk/dxvk_context.h b/src/dxvk/dxvk_context.h index ace683ebff79..74a0733021ba 100644 --- a/src/dxvk/dxvk_context.h +++ b/src/dxvk/dxvk_context.h @@ -1772,17 +1772,20 @@ namespace dxvk { template bool checkBufferBarrier( const DxvkBufferSlice& bufferSlice, - VkAccessFlags access); + VkAccessFlags access, + DxvkAtomicStore storeOp); template bool checkBufferViewBarrier( const Rc& bufferView, - VkAccessFlags access); + VkAccessFlags access, + DxvkAtomicStore storeOp); template bool checkImageViewBarrier( const Rc& imageView, - VkAccessFlags access); + VkAccessFlags access, + DxvkAtomicStore storeOp); template bool canIgnoreWawHazards() { @@ -1892,13 +1895,15 @@ namespace dxvk { const VkImageSubresourceRange& subresources, VkImageLayout srcLayout, VkPipelineStageFlags2 srcStages, - VkAccessFlags2 srcAccess); + VkAccessFlags2 srcAccess, + DxvkAtomicStore storeOp = DxvkAtomicStore::None); void accessImage( DxvkCmdBuffer cmdBuffer, const DxvkImageView& imageView, VkPipelineStageFlags2 srcStages, - VkAccessFlags2 srcAccess); + VkAccessFlags2 srcAccess, + DxvkAtomicStore storeOp = DxvkAtomicStore::None); void accessImage( DxvkCmdBuffer cmdBuffer, @@ -1909,7 +1914,8 @@ namespace dxvk { VkAccessFlags2 srcAccess, VkImageLayout dstLayout, VkPipelineStageFlags2 dstStages, - VkAccessFlags2 dstAccess); + VkAccessFlags2 dstAccess, + DxvkAtomicStore storeOp = DxvkAtomicStore::None); void accessBuffer( DxvkCmdBuffer cmdBuffer, @@ -1917,7 +1923,8 @@ namespace dxvk { VkDeviceSize offset, VkDeviceSize size, VkPipelineStageFlags2 srcStages, - VkAccessFlags2 srcAccess); + VkAccessFlags2 srcAccess, + DxvkAtomicStore storeOp = DxvkAtomicStore::None); void accessBuffer( DxvkCmdBuffer cmdBuffer, @@ -1927,13 +1934,15 @@ namespace dxvk { VkPipelineStageFlags2 srcStages, VkAccessFlags2 srcAccess, VkPipelineStageFlags2 dstStages, - VkAccessFlags2 dstAccess); + VkAccessFlags2 dstAccess, + DxvkAtomicStore storeOp = DxvkAtomicStore::None); void accessBuffer( DxvkCmdBuffer cmdBuffer, const DxvkBufferSlice& bufferSlice, VkPipelineStageFlags2 srcStages, - VkAccessFlags2 srcAccess); + VkAccessFlags2 srcAccess, + DxvkAtomicStore storeOp = DxvkAtomicStore::None); void accessBuffer( DxvkCmdBuffer cmdBuffer, @@ -1941,13 +1950,15 @@ namespace dxvk { VkPipelineStageFlags2 srcStages, VkAccessFlags2 srcAccess, VkPipelineStageFlags2 dstStages, - VkAccessFlags2 dstAccess); + VkAccessFlags2 dstAccess, + DxvkAtomicStore storeOp = DxvkAtomicStore::None); void accessBuffer( DxvkCmdBuffer cmdBuffer, DxvkBufferView& bufferView, VkPipelineStageFlags2 srcStages, - VkAccessFlags2 srcAccess); + VkAccessFlags2 srcAccess, + DxvkAtomicStore storeOp = DxvkAtomicStore::None); void accessBuffer( DxvkCmdBuffer cmdBuffer, @@ -1955,7 +1966,8 @@ namespace dxvk { VkPipelineStageFlags2 srcStages, VkAccessFlags2 srcAccess, VkPipelineStageFlags2 dstStages, - VkAccessFlags2 dstAccess); + VkAccessFlags2 dstAccess, + DxvkAtomicStore storeOp = DxvkAtomicStore::None); void accessDrawBuffer( VkDeviceSize offset, @@ -1991,20 +2003,24 @@ namespace dxvk { DxvkBuffer& buffer, VkDeviceSize offset, VkDeviceSize size, - DxvkAccess access); + DxvkAccess access, + DxvkAtomicStore storeOp = DxvkAtomicStore::None); bool resourceHasAccess( DxvkBufferView& bufferView, - DxvkAccess access); + DxvkAccess access, + DxvkAtomicStore storeOp = DxvkAtomicStore::None); bool resourceHasAccess( DxvkImage& image, const VkImageSubresourceRange& subresources, - DxvkAccess access); + DxvkAccess access, + DxvkAtomicStore storeOp = DxvkAtomicStore::None); bool resourceHasAccess( DxvkImageView& imageView, - DxvkAccess access); + DxvkAccess access, + DxvkAtomicStore storeOp = DxvkAtomicStore::None); DxvkBarrierBatch& getBarrierBatch( DxvkCmdBuffer cmdBuffer);