src/video_core/buffer_cache/buffer_cache.h

@@ -9,6 +9,8 @@
 #include <algorithm>
 #include <memory>
 #include <numeric>
+#include <type_traits>
+#include <utility>
 
 #include "common/range_sets.inc"
 #include "video_core/buffer_cache/buffer_cache_base.h"
@@ -19,6 +21,43 @@ namespace VideoCommon {
 
 using Core::DEVICE_PAGESIZE;
 
+namespace staging_detail {
+template <typename T, typename = void>
+struct has_flush_range : std::false_type {};
+template <typename T>
+struct has_flush_range<
+    T, std::void_t<decltype(std::declval<T&>().FlushRange(size_t{}, size_t{}))>> : std::true_type {};
+
+template <typename T, typename = void>
+struct has_invalidate_range : std::false_type {};
+template <typename T>
+struct has_invalidate_range<
+    T, std::void_t<decltype(std::declval<T&>().InvalidateRange(size_t{}, size_t{}))>>
+    : std::true_type {};
+} // namespace staging_detail
+
+template <typename Ref>
+inline void StagingFlushRange(Ref& ref, size_t offset, size_t size) {
+    if constexpr (staging_detail::has_flush_range<Ref>::value) {
+        ref.FlushRange(offset, size);
+    } else {
+        (void)ref;
+        (void)offset;
+        (void)size;
+    }
+}
+
+template <typename Ref>
+inline void StagingInvalidateRange(Ref& ref, size_t offset, size_t size) {
+    if constexpr (staging_detail::has_invalidate_range<Ref>::value) {
+        ref.InvalidateRange(offset, size);
+    } else {
+        (void)ref;
+        (void)offset;
+        (void)size;
+    }
+}
+
 template <class P>
 BufferCache<P>::BufferCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, Runtime& runtime_)
     : runtime{runtime_}, device_memory{device_memory_}, memory_tracker{device_memory} {
@@ -633,6 +672,7 @@ void BufferCache<P>::PopAsyncBuffers() {
     u8* base = async_buffer->mapped_span.data();
     const size_t base_offset = async_buffer->offset;
     for (const auto& copy : downloads) {
+        StagingInvalidateRange(*async_buffer, copy.dst_offset, copy.size);
         const DAddr device_addr = static_cast<DAddr>(copy.src_offset);
         const u64 dst_offset = copy.dst_offset - base_offset;
         const u8* read_mapped_memory = base + dst_offset;
@@ -696,6 +736,7 @@ void BufferCache<P>::BindHostIndexBuffer() {
                 {BufferCopy{.src_offset = upload_staging.offset, .dst_offset = 0, .size = size}}};
             std::memcpy(upload_staging.mapped_span.data(),
                         draw_state.inline_index_draw_indexes.data(), size);
+            StagingFlushRange(upload_staging, upload_staging.offset, size);
             runtime.CopyBuffer(buffer, upload_staging.buffer, copies, true);
         } else {
             buffer.ImmediateUpload(0, draw_state.inline_index_draw_indexes);
@@ -1519,7 +1560,7 @@ template <class P>
 void BufferCache<P>::MappedUploadMemory([[maybe_unused]] Buffer& buffer,
                                         [[maybe_unused]] u64 total_size_bytes,
                                         [[maybe_unused]] std::span<BufferCopy> copies) {
-    if constexpr (USE_MEMORY_MAPS) {
+        if constexpr (USE_MEMORY_MAPS) {
         auto upload_staging = runtime.UploadStagingBuffer(total_size_bytes);
         const std::span<u8> staging_pointer = upload_staging.mapped_span;
         for (BufferCopy& copy : copies) {
@@ -1530,6 +1571,7 @@ void BufferCache<P>::MappedUploadMemory([[maybe_unused]] Buffer& buffer,
             // Apply the staging offset
             copy.src_offset += upload_staging.offset;
         }
+        StagingFlushRange(upload_staging, upload_staging.offset, total_size_bytes);
         const bool can_reorder = runtime.CanReorderUpload(buffer, copies);
         runtime.CopyBuffer(buffer, upload_staging.buffer, copies, true, can_reorder);
     }
@@ -1572,6 +1614,7 @@ void BufferCache<P>::InlineMemoryImplementation(DAddr dest_address, size_t copy_
         }};
         u8* const src_pointer = upload_staging.mapped_span.data();
         std::memcpy(src_pointer, inlined_buffer.data(), copy_size);
+        StagingFlushRange(upload_staging, upload_staging.offset, copy_size);
         const bool can_reorder = runtime.CanReorderUpload(buffer, copies);
         runtime.CopyBuffer(buffer, upload_staging.buffer, copies, true, can_reorder);
     } else {
@@ -1626,6 +1669,7 @@ void BufferCache<P>::DownloadBufferMemory(Buffer& buffer, DAddr device_addr, u64
         }
         runtime.CopyBuffer(download_staging.buffer, buffer, copies_span, true);
         runtime.Finish();
+        StagingInvalidateRange(download_staging, download_staging.offset, total_size_bytes);
         for (const BufferCopy& copy : copies) {
             const DAddr copy_device_addr = buffer.CpuAddr() + copy.src_offset;
             // Undo the modified offset

src/video_core/renderer_vulkan/renderer_vulkan.cpp

@@ -166,7 +166,7 @@ try
 
     if (Settings::values.renderer_force_max_clock.GetValue() && device.ShouldBoostClocks()) {
         turbo_mode.emplace(instance, dld);
-        scheduler.RegisterOnSubmit([this] { turbo_mode->QueueSubmitted(); });
+        scheduler.AddOnSubmit([this] { turbo_mode->QueueSubmitted(); });
     }
 
     Report();
@@ -176,7 +176,7 @@ try
 }
 
 RendererVulkan::~RendererVulkan() {
-    scheduler.RegisterOnSubmit([] {});
+    scheduler.RegisterOnSubmit(std::function<void()>{});
     void(device.GetLogical().WaitIdle());
 }
 

src/video_core/renderer_vulkan/vk_buffer_cache.cpp

@@ -179,6 +179,11 @@ public:
         }();
 
         u8* staging_data = host_visible ? buffer.Mapped().data() : staging.mapped_span.data();
+#ifdef YUZU_DEBUG
+        if (!host_visible) {
+            ASSERT(staging.mapped_span.size() >= size_bytes);
+        }
+#endif
         const size_t quad_size = bytes_per_index * 6;
 
         for (u32 first = 0; first < num_first_offset_copies; ++first) {
@@ -189,6 +194,7 @@ public:
         }
 
         if (!host_visible) {
+            staging.FlushRange(staging.offset, static_cast<VkDeviceSize>(size_bytes));
             scheduler.RequestOutsideRenderPassOperationContext();
             scheduler.Record([src_buffer = staging.buffer, src_offset = staging.offset,
                               dst_buffer = *buffer, size_bytes](vk::CommandBuffer cmdbuf) {
@@ -514,6 +520,10 @@ void BufferCacheRuntime::BindIndexBuffer(PrimitiveTopology topology, IndexFormat
         ReserveNullBuffer();
         vk_buffer = *null_buffer;
     }
+#ifdef YUZU_DEBUG
+    const size_t bytes_per_index = BytesPerIndex(vk_index_type);
+    ASSERT(bytes_per_index == 0 || (vk_offset % bytes_per_index) == 0);
+#endif
     scheduler.Record([vk_buffer, vk_offset, vk_index_type](vk::CommandBuffer cmdbuf) {
         cmdbuf.BindIndexBuffer(vk_buffer, vk_offset, vk_index_type);
     });

src/video_core/renderer_vulkan/vk_buffer_cache.h

@@ -127,26 +127,43 @@ public:
     std::span<u8> BindMappedUniformBuffer([[maybe_unused]] size_t stage,
                                           [[maybe_unused]] u32 binding_index, u32 size) {
         const StagingBufferRef ref = staging_pool.Request(size, MemoryUsage::Upload);
-        BindBuffer(ref.buffer, static_cast<u32>(ref.offset), size);
+        BindBuffer(ref.buffer, ref.offset, static_cast<VkDeviceSize>(size));
+#ifdef YUZU_DEBUG
+        ASSERT(ref.mapped_span.size() >= size);
+        const VkDeviceSize ubo_align = device.GetUniformBufferAlignment();
+        ASSERT(ubo_align == 0 || (ref.offset % ubo_align) == 0);
+#endif
         return ref.mapped_span;
     }
 
-    void BindUniformBuffer(VkBuffer buffer, u32 offset, u32 size) {
+    void BindUniformBuffer(VkBuffer buffer, VkDeviceSize offset, VkDeviceSize size) {
+#ifdef YUZU_DEBUG
+        const VkDeviceSize ubo_align = device.GetUniformBufferAlignment();
+        ASSERT(ubo_align == 0 || (offset % ubo_align) == 0);
+#endif
         BindBuffer(buffer, offset, size);
     }
 
-    void BindStorageBuffer(VkBuffer buffer, u32 offset, u32 size,
+    void BindStorageBuffer(VkBuffer buffer, VkDeviceSize offset, VkDeviceSize size,
                            [[maybe_unused]] bool is_written) {
+#ifdef YUZU_DEBUG
+        const VkDeviceSize ssbo_align = device.GetStorageBufferAlignment();
+        ASSERT(ssbo_align == 0 || (offset % ssbo_align) == 0);
+#endif
         BindBuffer(buffer, offset, size);
     }
 
     void BindTextureBuffer(Buffer& buffer, u32 offset, u32 size,
                            VideoCore::Surface::PixelFormat format) {
+#ifdef YUZU_DEBUG
+        const VkDeviceSize texel_align = device.GetTexelBufferAlignment();
+        ASSERT(texel_align == 0 || (offset % texel_align) == 0);
+#endif
         guest_descriptor_queue.AddTexelBuffer(buffer.View(offset, size, format));
     }
 
 private:
-    void BindBuffer(VkBuffer buffer, u32 offset, u32 size) {
+    void BindBuffer(VkBuffer buffer, VkDeviceSize offset, VkDeviceSize size) {
         guest_descriptor_queue.AddBuffer(buffer, offset, size);
     }
 

src/video_core/renderer_vulkan/vk_query_cache.cpp

@@ -850,10 +850,17 @@ public:
             pending_flush_sets.pop_front();
         }
 
+        const VkDeviceSize read_size =
+            static_cast<VkDeviceSize>(flushed_queries.size() * TFBQueryBank::QUERY_SIZE);
+        staging_ref.InvalidateRange(staging_ref.offset, read_size);
+
         size_t offset_base = staging_ref.offset;
         for (auto q : flushed_queries) {
             auto* query = GetQuery(q);
             u32 result = 0;
+#ifdef YUZU_DEBUG
+            ASSERT(staging_ref.mapped_span.size() >= offset_base + sizeof(u32));
+#endif
             std::memcpy(&result, staging_ref.mapped_span.data() + offset_base, sizeof(u32));
             query->value = static_cast<u64>(result);
             query->flags |= VideoCommon::QueryFlagBits::IsFinalValueSynced;
@@ -1567,10 +1574,14 @@ void QueryCacheRuntime::SyncValues(std::span<SyncValuesType> values, VkBuffer ba
                              impl->little_cache[which_copy].first,
                 .size = values[i].size,
             });
+#ifdef YUZU_DEBUG
+            ASSERT(ref.mapped_span.size() >= accumulated_size + values[i].size);
+#endif
             std::memcpy(ref.mapped_span.data() + accumulated_size, &values[i].value,
                         values[i].size);
             accumulated_size += values[i].size;
         }
+        ref.FlushRange(ref.offset, static_cast<VkDeviceSize>(accumulated_size));
         src_buffer = ref.buffer;
     } else {
         for (size_t i = 0; i < values.size(); i++) {

src/video_core/renderer_vulkan/vk_scheduler.cpp

@@ -15,6 +15,7 @@
 #include "video_core/renderer_vulkan/vk_command_pool.h"
 #include "video_core/renderer_vulkan/vk_master_semaphore.h"
 #include "video_core/renderer_vulkan/vk_scheduler.h"
+#include "video_core/renderer_vulkan/vk_staging_buffer_pool.h"
 #include "video_core/renderer_vulkan/vk_state_tracker.h"
 #include "video_core/renderer_vulkan/vk_texture_cache.h"
 #include "video_core/vulkan_common/vulkan_device.h"
@@ -233,8 +234,14 @@ u64 Scheduler::SubmitExecution(VkSemaphore signal_semaphore, VkSemaphore wait_se
         upload_cmdbuf.End();
         cmdbuf.End();
 
-        if (on_submit) {
+        if (staging_buffer_pool) {
-            on_submit();
+            staging_buffer_pool->FlushStream();
+        }
+
+        for (const auto& callback : on_submit_callbacks) {
+            if (callback) {
+                callback();
+            }
         }
 
         std::scoped_lock lock{submit_mutex};

src/video_core/renderer_vulkan/vk_scheduler.h

@@ -10,6 +10,7 @@
 #include <thread>
 #include <utility>
 #include <queue>
+#include <vector>
 
 #include "common/alignment.h"
 #include "common/common_types.h"
@@ -29,6 +30,7 @@ class Device;
 class Framebuffer;
 class GraphicsPipeline;
 class StateTracker;
+class StagingBufferPool;
 
 struct QueryCacheParams;
 
@@ -73,9 +75,23 @@ public:
         query_cache = &query_cache_;
     }
 
-    // Registers a callback to perform on queue submission.
+    void SetStagingBufferPool(StagingBufferPool* pool) {
+        staging_buffer_pool = pool;
+    }
+
+    // Registers a callback to perform on queue submission, replacing existing callbacks.
     void RegisterOnSubmit(std::function<void()>&& func) {
-        on_submit = std::move(func);
+        on_submit_callbacks.clear();
+        if (func) {
+            on_submit_callbacks.emplace_back(std::move(func));
+        }
+    }
+
+    // Adds an additional callback to perform on queue submission.
+    void AddOnSubmit(std::function<void()>&& func) {
+        if (func) {
+            on_submit_callbacks.emplace_back(std::move(func));
+        }
     }
 
     /// Send work to a separate thread.
@@ -237,12 +253,13 @@ private:
     std::unique_ptr<CommandPool> command_pool;
 
     VideoCommon::QueryCacheBase<QueryCacheParams>* query_cache = nullptr;
+    StagingBufferPool* staging_buffer_pool = nullptr;
 
     vk::CommandBuffer current_cmdbuf;
     vk::CommandBuffer current_upload_cmdbuf;
 
     std::unique_ptr<CommandChunk> chunk;
-    std::function<void()> on_submit;
+    std::vector<std::function<void()>> on_submit_callbacks;
 
     State state;
 

src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp

@@ -5,6 +5,7 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 #include <algorithm>
+#include <memory>
 #include <utility>
 #include <vector>
 
@@ -30,6 +31,11 @@ constexpr VkDeviceSize MIN_STREAM_ALIGNMENT = 256;
 // Stream buffer size in bytes
 constexpr VkDeviceSize MAX_STREAM_BUFFER_SIZE = 128_MiB;
 
+VkDeviceSize GetStreamAlignment(const Device& device) {
+    return (std::max)({device.GetUniformBufferAlignment(), device.GetStorageBufferAlignment(),
+                       device.GetTexelBufferAlignment(), MIN_STREAM_ALIGNMENT});
+}
+
 size_t GetStreamBufferSize(const Device& device, VkDeviceSize alignment) {
     VkDeviceSize size{0};
     if (device.HasDebuggingToolAttached()) {
@@ -63,8 +69,7 @@ size_t GetStreamBufferSize(const Device& device, VkDeviceSize alignment) {
 StagingBufferPool::StagingBufferPool(const Device& device_, MemoryAllocator& memory_allocator_,
                                      Scheduler& scheduler_)
     : device{device_}, memory_allocator{memory_allocator_}, scheduler{scheduler_},
-      stream_alignment{std::max<VkDeviceSize>(device_.GetUniformBufferAlignment(),
+      stream_alignment{GetStreamAlignment(device_)},
-                                              MIN_STREAM_ALIGNMENT)},
       stream_buffer_size{GetStreamBufferSize(device_, stream_alignment)},
       region_size{stream_buffer_size / StagingBufferPool::NUM_SYNCS} {
     VkBufferCreateInfo stream_ci = {
@@ -87,9 +92,18 @@ StagingBufferPool::StagingBufferPool(const Device& device_, MemoryAllocator& mem
     }
     stream_pointer = stream_buffer.Mapped();
     ASSERT_MSG(!stream_pointer.empty(), "Stream buffer must be host visible!");
+    stream_is_coherent = stream_buffer.IsHostCoherent();
+    non_coherent_atom_size = std::max<VkDeviceSize>(device.GetNonCoherentAtomSize(),
+                                                    static_cast<VkDeviceSize>(1));
+    dirty_begin = stream_buffer_size;
+    dirty_end = 0;
+    stream_dirty = false;
+    scheduler.SetStagingBufferPool(this);
 }
 
-StagingBufferPool::~StagingBufferPool() = default;
+StagingBufferPool::~StagingBufferPool() {
+    scheduler.SetStagingBufferPool(nullptr);
+}
 
 StagingBufferRef StagingBufferPool::Request(size_t size, MemoryUsage usage, bool deferred) {
     if (!deferred && usage == MemoryUsage::Upload && size <= region_size) {
@@ -121,9 +135,10 @@ void StagingBufferPool::TickFrame() {
 StagingBufferRef StagingBufferPool::GetStreamBuffer(size_t size) {
     const size_t alignment = static_cast<size_t>(stream_alignment);
     const size_t aligned_size = Common::AlignUp(size, alignment);
-    const bool wraps = iterator + size >= stream_buffer_size;
+    const size_t capacity = static_cast<size_t>(stream_buffer_size);
+    const bool wraps = iterator + aligned_size > capacity;
     const size_t new_iterator =
-        wraps ? aligned_size : Common::AlignUp(iterator + size, alignment);
+        wraps ? aligned_size : Common::AlignUp(iterator + aligned_size, alignment);
     const size_t begin_region = wraps ? 0 : Region(iterator);
     const size_t last_byte = new_iterator == 0 ? 0 : new_iterator - 1;
     const size_t end_region = (std::min)(Region(last_byte) + 1, NUM_SYNCS);
@@ -167,16 +182,69 @@ StagingBufferRef StagingBufferPool::GetStreamBuffer(size_t size) {
         free_iterator = (std::max)(free_iterator, offset + aligned_size);
     }
 
+    TrackStreamWrite(static_cast<VkDeviceSize>(offset), static_cast<VkDeviceSize>(aligned_size));
+
     return StagingBufferRef{
         .buffer = *stream_buffer,
         .offset = static_cast<VkDeviceSize>(offset),
         .mapped_span = stream_pointer.subspan(offset, size),
-        .usage{},
+        .usage = MemoryUsage::Upload,
-        .log2_level{},
+        .log2_level = 0,
-        .index{},
+        .index = 0,
+        .owner = &stream_buffer,
+        .atom_size = non_coherent_atom_size,
+        .is_coherent = stream_is_coherent,
+        .is_stream_ring = true,
     };
 }
 
+void StagingBufferPool::TrackStreamWrite(VkDeviceSize offset, VkDeviceSize size) {
+    if (stream_is_coherent || size == 0) {
+        return;
+    }
+    const VkDeviceSize clamped_offset = (std::min)(offset, stream_buffer_size);
+    const VkDeviceSize clamped_end = (std::min)(clamped_offset + size, stream_buffer_size);
+    std::scoped_lock lock{stream_mutex};
+    if (!stream_dirty) {
+        dirty_begin = clamped_offset;
+        dirty_end = clamped_end;
+        stream_dirty = true;
+        return;
+    }
+    dirty_begin = (std::min)(dirty_begin, clamped_offset);
+    dirty_end = (std::max)(dirty_end, clamped_end);
+}
+
+void StagingBufferPool::FlushStream() {
+    if (stream_is_coherent) {
+        return;
+    }
+
+    VkDeviceSize flush_begin = 0;
+    VkDeviceSize flush_end = 0;
+    {
+        std::scoped_lock lock{stream_mutex};
+        if (!stream_dirty) {
+            return;
+        }
+        flush_begin = dirty_begin;
+        flush_end = dirty_end;
+        stream_dirty = false;
+        dirty_begin = stream_buffer_size;
+        dirty_end = 0;
+    }
+
+    if (flush_begin >= flush_end) {
+        return;
+    }
+
+    const VkDeviceSize atom = non_coherent_atom_size;
+    const VkDeviceSize aligned_begin = Common::AlignDown(flush_begin, atom);
+    const VkDeviceSize aligned_end = Common::AlignUp(flush_end, atom);
+    const VkDeviceSize flush_size = aligned_end - aligned_begin;
+    stream_buffer.FlushRange(aligned_begin, flush_size);
+}
+
 bool StagingBufferPool::AreRegionsActive(size_t region_begin, size_t region_end) const {
     const u64 gpu_tick = scheduler.GetMasterSemaphore().KnownGpuTick();
     return std::any_of(sync_ticks.begin() + region_begin, sync_ticks.begin() + region_end,
@@ -238,15 +306,19 @@ StagingBufferRef StagingBufferPool::CreateStagingBuffer(size_t size, MemoryUsage
         ++buffer_index;
         buffer.SetObjectNameEXT(fmt::format("Staging Buffer {}", buffer_index).c_str());
     }
+    const bool is_coherent = buffer.IsHostCoherent();
     const std::span<u8> mapped_span = buffer.Mapped();
+    auto buffer_ptr = std::make_unique<vk::Buffer>(std::move(buffer));
     StagingBuffer& entry = GetCache(usage)[log2].entries.emplace_back(StagingBuffer{
-        .buffer = std::move(buffer),
+        .buffer = std::move(buffer_ptr),
         .mapped_span = mapped_span,
         .usage = usage,
         .log2_level = log2,
         .index = unique_ids++,
         .tick = deferred ? (std::numeric_limits<u64>::max)() : scheduler.CurrentTick(),
         .deferred = deferred,
+        .is_coherent = is_coherent,
+        .atom_size = is_coherent ? 1 : non_coherent_atom_size,
     });
     return entry.Ref();
 }

src/video_core/renderer_vulkan/vk_staging_buffer_pool.h

@@ -7,9 +7,12 @@
 #pragma once
 
 #include <climits>
+#include <mutex>
+#include <memory>
 #include <vector>
 
 #include "common/common_types.h"
+#include "common/alignment.h"
 
 #include "video_core/vulkan_common/vulkan_memory_allocator.h"
 #include "video_core/vulkan_common/vulkan_wrapper.h"
@@ -26,10 +29,54 @@ struct StagingBufferRef {
     MemoryUsage usage;
     u32 log2_level;
     u64 index;
+    const vk::Buffer* owner = nullptr;
+    VkDeviceSize atom_size = 1;
+    bool is_coherent = true;
+    bool is_stream_ring = false;
+
+    void FlushRange(VkDeviceSize range_offset, VkDeviceSize size) const {
+        if (!owner || is_coherent || size == 0 || is_stream_ring) {
+            return;
+        }
+        if (size == VK_WHOLE_SIZE) {
+            owner->FlushRange(range_offset, size);
+            return;
+        }
+        const VkDeviceSize atom = atom_size ? atom_size : 1;
+        const VkDeviceSize range_end = range_offset + size;
+        if (range_end < range_offset) {
+            owner->FlushRange(range_offset, size);
+            return;
+        }
+        const VkDeviceSize aligned_begin = Common::AlignDown(range_offset, atom);
+        const VkDeviceSize aligned_end = Common::AlignUp(range_end, atom);
+        owner->FlushRange(aligned_begin, aligned_end - aligned_begin);
+    }
+
+    void InvalidateRange(VkDeviceSize range_offset, VkDeviceSize size) const {
+        if (!owner || is_coherent || size == 0 || is_stream_ring) {
+            return;
+        }
+        if (size == VK_WHOLE_SIZE) {
+            owner->InvalidateRange(range_offset, size);
+            return;
+        }
+        const VkDeviceSize atom = atom_size ? atom_size : 1;
+        const VkDeviceSize range_end = range_offset + size;
+        if (range_end < range_offset) {
+            owner->InvalidateRange(range_offset, size);
+            return;
+        }
+        const VkDeviceSize aligned_begin = Common::AlignDown(range_offset, atom);
+        const VkDeviceSize aligned_end = Common::AlignUp(range_end, atom);
+        owner->InvalidateRange(aligned_begin, aligned_end - aligned_begin);
+    }
 };
 
 class StagingBufferPool {
 public:
+    friend class Scheduler;
+
     static constexpr size_t NUM_SYNCS = 16;
 
     explicit StagingBufferPool(const Device& device, MemoryAllocator& memory_allocator,
@@ -52,22 +99,28 @@ private:
     };
 
     struct StagingBuffer {
-        vk::Buffer buffer;
+        std::unique_ptr<vk::Buffer> buffer;
         std::span<u8> mapped_span;
         MemoryUsage usage;
         u32 log2_level;
         u64 index;
         u64 tick = 0;
         bool deferred{};
+        bool is_coherent = true;
+        VkDeviceSize atom_size = 1;
 
         StagingBufferRef Ref() const noexcept {
             return {
-                .buffer = *buffer,
+                .buffer = buffer ? **buffer : VkBuffer{},
                 .offset = 0,
                 .mapped_span = mapped_span,
                 .usage = usage,
                 .log2_level = log2_level,
                 .index = index,
+                .owner = buffer.get(),
+                .atom_size = atom_size,
+                .is_coherent = is_coherent,
+                .is_stream_ring = false,
             };
         }
     };
@@ -83,6 +136,9 @@ private:
 
     StagingBufferRef GetStreamBuffer(size_t size);
 
+    void TrackStreamWrite(VkDeviceSize offset, VkDeviceSize size);
+    void FlushStream();
+
     bool AreRegionsActive(size_t region_begin, size_t region_end) const;
 
     StagingBufferRef GetStagingBuffer(size_t size, MemoryUsage usage, bool deferred = false);
@@ -110,6 +166,12 @@ private:
     std::span<u8> stream_pointer;
     VkDeviceSize stream_buffer_size;
     VkDeviceSize region_size;
+    bool stream_is_coherent = true;
+    VkDeviceSize non_coherent_atom_size = 1;
+    VkDeviceSize dirty_begin = 0;
+    VkDeviceSize dirty_end = 0;
+    bool stream_dirty = false;
+    std::mutex stream_mutex;
 
     size_t iterator = 0;
     size_t used_iterator = 0;

src/video_core/texture_cache/texture_cache.h

@@ -7,6 +7,8 @@
 #pragma once
 
 #include <unordered_set>
+#include <type_traits>
+#include <utility>
 #include <boost/container/small_vector.hpp>
 
 #include "common/alignment.h"
@@ -30,6 +32,42 @@ using VideoCore::Surface::PixelFormat;
 using VideoCore::Surface::SurfaceType;
 using namespace Common::Literals;
 
+namespace staging_detail {
+template <typename T, typename = void>
+struct has_flush_range : std::false_type {};
+template <typename T>
+struct has_flush_range<
+    T, std::void_t<decltype(std::declval<T&>().FlushRange(size_t{}, size_t{}))>> : std::true_type {};
+template <typename T, typename = void>
+struct has_invalidate_range : std::false_type {};
+template <typename T>
+struct has_invalidate_range<
+    T, std::void_t<decltype(std::declval<T&>().InvalidateRange(size_t{}, size_t{}))>>
+    : std::true_type {};
+} // namespace staging_detail
+
+template <typename Ref>
+inline void StagingFlushRange(Ref& ref, size_t offset, size_t size) {
+    if constexpr (staging_detail::has_flush_range<Ref>::value) {
+        ref.FlushRange(offset, size);
+    } else {
+        (void)ref;
+        (void)offset;
+        (void)size;
+    }
+}
+
+template <typename Ref>
+inline void StagingInvalidateRange(Ref& ref, size_t offset, size_t size) {
+    if constexpr (staging_detail::has_invalidate_range<Ref>::value) {
+        ref.InvalidateRange(offset, size);
+    } else {
+        (void)ref;
+        (void)offset;
+        (void)size;
+    }
+}
+
 template <class P>
 TextureCache<P>::TextureCache(Runtime& runtime_, Tegra::MaxwellDeviceMemoryManager& device_memory_)
     : runtime{runtime_}, device_memory{device_memory_} {
@@ -111,6 +149,7 @@ void TextureCache<P>::RunGarbageCollector() {
             const auto copies = FullDownloadCopies(image.info);
             image.DownloadMemory(map, copies);
             runtime.Finish();
+            StagingInvalidateRange(map, map.offset, image.unswizzled_size_bytes);
             SwizzleImage(*gpu_memory, image.gpu_addr, image.info, copies, map.mapped_span,
                          swizzle_data_buffer);
         }
@@ -567,6 +606,7 @@ void TextureCache<P>::DownloadMemory(DAddr cpu_addr, size_t size) {
         const auto copies = FullDownloadCopies(image.info);
         image.DownloadMemory(map, copies);
         runtime.Finish();
+        StagingInvalidateRange(map, map.offset, image.unswizzled_size_bytes);
         SwizzleImage(*gpu_memory, image.gpu_addr, image.info, copies, map.mapped_span,
                      swizzle_data_buffer);
     }
@@ -863,13 +903,17 @@ void TextureCache<P>::PopAsyncFlushes() {
             if (download_info.is_swizzle) {
                 const ImageBase& image = slot_images[download_info.object_id];
                 const auto copies = FullDownloadCopies(image.info);
-                download_buffer.offset -= Common::AlignUp(image.unswizzled_size_bytes, 64);
+                const size_t aligned_size =
+                    Common::AlignUp(image.unswizzled_size_bytes, static_cast<size_t>(64));
+                download_buffer.offset -= aligned_size;
+                StagingInvalidateRange(download_buffer, download_buffer.offset, aligned_size);
                 std::span<u8> download_span =
                     download_buffer.mapped_span.subspan(download_buffer.offset);
                 SwizzleImage(*gpu_memory, image.gpu_addr, image.info, copies, download_span,
                              swizzle_data_buffer);
             } else {
                 const BufferDownload& buffer_info = slot_buffer_downloads[download_info.object_id];
+                StagingInvalidateRange(download_buffer, download_buffer.offset, buffer_info.size);
                 std::span<u8> download_span =
                     download_buffer.mapped_span.subspan(download_buffer.offset);
                 gpu_memory->WriteBlockUnsafe(buffer_info.address, download_span.data(),
@@ -907,6 +951,7 @@ void TextureCache<P>::PopAsyncFlushes() {
         }
         // Wait for downloads to finish
         runtime.Finish();
+        StagingInvalidateRange(download_map, original_offset, total_size_bytes);
         download_map.offset = original_offset;
         std::span<u8> download_span = download_map.mapped_span;
         for (const PendingDownload& download_info : download_ids) {
@@ -1081,6 +1126,7 @@ void TextureCache<P>::UploadImageContents(Image& image, StagingBuffer& staging)
     if (True(image.flags & ImageFlagBits::AcceleratedUpload)) {
         gpu_memory->ReadBlock(gpu_addr, mapped_span.data(), mapped_span.size_bytes(),
                               VideoCommon::CacheType::NoTextureCache);
+        StagingFlushRange(staging, staging.offset, mapped_span.size_bytes());
         const auto uploads = FullUploadSwizzles(image.info);
         runtime.AccelerateImageUpload(image, staging, uploads);
         return;
@@ -1094,10 +1140,12 @@ void TextureCache<P>::UploadImageContents(Image& image, StagingBuffer& staging)
         auto copies =
             UnswizzleImage(*gpu_memory, gpu_addr, image.info, swizzle_data, unswizzle_data_buffer);
         ConvertImage(unswizzle_data_buffer, image.info, mapped_span, copies);
+        StagingFlushRange(staging, staging.offset, mapped_span.size_bytes());
         image.UploadMemory(staging, copies);
     } else {
         const auto copies =
             UnswizzleImage(*gpu_memory, gpu_addr, image.info, swizzle_data, mapped_span);
+        StagingFlushRange(staging, staging.offset, mapped_span.size_bytes());
         image.UploadMemory(staging, copies);
     }
 }
@@ -1329,6 +1377,7 @@ void TextureCache<P>::TickAsyncDecode() {
         auto staging = runtime.UploadStagingBuffer(MapSizeBytes(image));
         std::memcpy(staging.mapped_span.data(), async_decode->decoded_data.data(),
                     async_decode->decoded_data.size());
+        StagingFlushRange(staging, staging.offset, async_decode->decoded_data.size());
         image.UploadMemory(staging, async_decode->copies);
         image.flags &= ~ImageFlagBits::IsDecoding;
         has_uploads = true;

src/video_core/vulkan_common/vulkan_device.h

@@ -295,6 +295,16 @@ public:
         return properties.properties.limits.minStorageBufferOffsetAlignment;
     }
 
+    /// Returns texel buffer alignment requirement.
+    VkDeviceSize GetTexelBufferAlignment() const {
+        return properties.properties.limits.minTexelBufferOffsetAlignment;
+    }
+
+    /// Returns the non-coherent atom size for memory flushes.
+    VkDeviceSize GetNonCoherentAtomSize() const {
+        return properties.properties.limits.nonCoherentAtomSize;
+    }
+
     /// Returns the maximum range for storage buffers.
     VkDeviceSize GetMaxStorageBufferRange() const {
         return properties.properties.limits.maxStorageBufferRange;

src/video_core/vulkan_common/vulkan_wrapper.cpp

@@ -499,14 +499,22 @@ void Image::Release() const noexcept {
 }
 
 void Buffer::Flush() const {
+    FlushRange(0, VK_WHOLE_SIZE);
+}
+
+void Buffer::FlushRange(VkDeviceSize offset, VkDeviceSize size) const {
     if (!is_coherent) {
-        vmaFlushAllocation(allocator, allocation, 0, VK_WHOLE_SIZE);
+        vmaFlushAllocation(allocator, allocation, offset, size);
     }
 }
 
 void Buffer::Invalidate() const {
+    InvalidateRange(0, VK_WHOLE_SIZE);
+}
+
+void Buffer::InvalidateRange(VkDeviceSize offset, VkDeviceSize size) const {
     if (!is_coherent) {
-        vmaInvalidateAllocation(allocator, allocation, 0, VK_WHOLE_SIZE);
+        vmaInvalidateAllocation(allocator, allocation, offset, size);
     }
 }
 

src/video_core/vulkan_common/vulkan_wrapper.h

@@ -772,10 +772,19 @@ public:
         return !mapped.empty();
     }
 
+    /// Returns true if the buffer memory is host coherent.
+    bool IsHostCoherent() const noexcept {
+        return is_coherent;
+    }
+
     void Flush() const;
 
+    void FlushRange(VkDeviceSize offset, VkDeviceSize size) const;
+
     void Invalidate() const;
 
+    void InvalidateRange(VkDeviceSize offset, VkDeviceSize size) const;
+
     void SetObjectNameEXT(const char* name) const;
 
 private: