diff --git a/CMakeLists.txt b/CMakeLists.txt
index 0cb2af854c..112341ad09 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -8,6 +8,8 @@ project(yuzu)
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules")
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/externals/cmake-modules")
 
+set(CPM_SOURCE_CACHE ${CMAKE_SOURCE_DIR}/.cache/cpm)
+
 include(DownloadExternals)
 include(CMakeDependentOption)
 include(CTest)
@@ -98,6 +100,8 @@ endif()
 
 option(YUZU_DOWNLOAD_ANDROID_VVL "Download validation layer binary for android" ON)
 
+option(FORCE_DOWNLOAD_WIN_BUNDLES "Forcefully download bundled Windows dependencies (useful for CI)" OFF)
+
 CMAKE_DEPENDENT_OPTION(YUZU_ROOM "Enable dedicated room functionality" ON "NOT ANDROID" OFF)
 
 CMAKE_DEPENDENT_OPTION(YUZU_ROOM_STANDALONE "Enable standalone room executable" ON "YUZU_ROOM" OFF)
@@ -164,8 +168,6 @@ if (ANDROID)
     set(CMAKE_POLICY_VERSION_MINIMUM 3.5) # Workaround for Oboe
 endif()
 
-set(CPM_SOURCE_CACHE ${CMAKE_SOURCE_DIR}/.cache/cpm)
-
 if (YUZU_USE_BUNDLED_VCPKG)
     if (ANDROID)
         set(ENV{ANDROID_NDK_HOME} "${ANDROID_NDK}")
@@ -447,7 +449,7 @@ if (ENABLE_SDL2)
         endif()
 
         if (DEFINED SDL2_VER)
-            download_bundled_external("sdl2/" ${SDL2_VER} SDL2_PREFIX)
+            download_bundled_external("sdl2/" ${SDL2_VER} "sdl2-bundled" SDL2_PREFIX)
         endif()
 
         set(SDL2_FOUND YES)
@@ -578,7 +580,7 @@ endif()
 
 if (WIN32 AND YUZU_CRASH_DUMPS)
     set(BREAKPAD_VER "breakpad-c89f9dd")
-    download_bundled_external("breakpad/" ${BREAKPAD_VER} BREAKPAD_PREFIX)
+    download_bundled_external("breakpad/" ${BREAKPAD_VER} "breakpad-win" BREAKPAD_PREFIX)
 
     set(BREAKPAD_CLIENT_INCLUDE_DIR "${BREAKPAD_PREFIX}/include")
     set(BREAKPAD_CLIENT_LIBRARY "${BREAKPAD_PREFIX}/lib/libbreakpad_client.lib")
diff --git a/CMakeModules/DownloadExternals.cmake b/CMakeModules/DownloadExternals.cmake
index 66c1b2b946..d4c1042740 100644
--- a/CMakeModules/DownloadExternals.cmake
+++ b/CMakeModules/DownloadExternals.cmake
@@ -6,37 +6,52 @@
 #   remote_path: path to the file to download, relative to the remote repository root
 #   prefix_var: name of a variable which will be set with the path to the extracted contents
 set(CURRENT_MODULE_DIR ${CMAKE_CURRENT_LIST_DIR})
-function(download_bundled_external remote_path lib_name prefix_var)
+function(download_bundled_external remote_path lib_name cpm_key prefix_var)
     set(package_base_url "https://github.com/eden-emulator/")
     set(package_repo "no_platform")
     set(package_extension "no_platform")
 
-    # TODO(crueter): Windows should store ffmpeg 7z within cpm source cache
-    if (WIN32)
+    if (WIN32 OR FORCE_WIN_ARCHIVES)
+        set(CACHE_KEY "windows")
         set(package_repo "ext-windows-bin/raw/master/")
         set(package_extension ".7z")
     elseif (${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
+        set(CACHE_KEY "linux")
         set(package_repo "ext-linux-bin/raw/master/")
         set(package_extension ".tar.xz")
     elseif (ANDROID)
+        set(CACHE_KEY "android")
         set(package_repo "ext-android-bin/raw/master/")
         set(package_extension ".tar.xz")
     else()
         message(FATAL_ERROR "No package available for this platform")
     endif()
     set(package_url "${package_base_url}${package_repo}")
+    set(full_url ${package_url}${remote_path}${lib_name}${package_extension})
 
-    set(prefix "${CMAKE_BINARY_DIR}/externals/${lib_name}")
-    if (NOT EXISTS "${prefix}")
-        message(STATUS "Downloading binaries for ${lib_name}...")
-        file(DOWNLOAD
-            ${package_url}${remote_path}${lib_name}${package_extension}
-            "${CMAKE_BINARY_DIR}/externals/${lib_name}${package_extension}" SHOW_PROGRESS)
-        execute_process(COMMAND ${CMAKE_COMMAND} -E tar xf "${CMAKE_BINARY_DIR}/externals/${lib_name}${package_extension}"
-            WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/externals")
-    endif()
-    message(STATUS "Using bundled binaries at ${prefix}")
-    set(${prefix_var} "${prefix}" PARENT_SCOPE)
+    set(CPM_USE_LOCAL_PACKAGES OFF)
+
+    CPMAddPackage(
+        NAME ${cpm_key}
+        URL ${full_url}
+        DOWNLOAD_ONLY YES
+        CUSTOM_CACHE_KEY ${CACHE_KEY}
+    )
+
+    set(${prefix_var} "${${cpm_key}_SOURCE_DIR}" PARENT_SCOPE)
+    message(STATUS "Using bundled binaries at ${${cpm_key}_SOURCE_DIR}")
+endfunction()
+
+function(download_win_archives)
+    set(FORCE_WIN_ARCHIVES ON)
+    set(FFmpeg_EXT_NAME "ffmpeg-7.1.1")
+
+    download_bundled_external("ffmpeg/" ${FFmpeg_EXT_NAME} "ffmpeg-bundled" "")
+
+    # TODO(crueter): separate handling for arm64
+    set(SDL2_VER "SDL2-2.32.8")
+    download_bundled_external("sdl2/" ${SDL2_VER} "sdl2-bundled" "")
+    set(FORCE_WIN_ARCHIVES OFF)
 endfunction()
 
 function(download_moltenvk_external platform version)
diff --git a/externals/CMakeLists.txt b/externals/CMakeLists.txt
index 16787387db..b0bce136d3 100644
--- a/externals/CMakeLists.txt
+++ b/externals/CMakeLists.txt
@@ -23,6 +23,10 @@ set(BUILD_SHARED_LIBS OFF)
 # Skip install rules for all externals
 set_directory_properties(PROPERTIES EXCLUDE_FROM_ALL ON)
 
+if (FORCE_DOWNLOAD_WIN_BUNDLES)
+    download_win_archives()
+endif()
+
 # Xbyak (also used by Dynarmic, so needs to be added first)
 if (ARCHITECTURE_x86 OR ARCHITECTURE_x86_64)
     set(CPM_USE_LOCAL_PACKAGES OFF)
diff --git a/externals/ffmpeg/CMakeLists.txt b/externals/ffmpeg/CMakeLists.txt
index b26581a5b8..0bad66f19c 100644
--- a/externals/ffmpeg/CMakeLists.txt
+++ b/externals/ffmpeg/CMakeLists.txt
@@ -241,8 +241,8 @@ elseif(ANDROID)
     else()
         message(FATAL_ERROR "Unsupported architecture for Android FFmpeg")
     endif()
-    set(FFmpeg_PATH "${CMAKE_BINARY_DIR}/externals/${FFmpeg_EXT_NAME}")
-    download_bundled_external("ffmpeg/" ${FFmpeg_EXT_NAME} "")
+
+    download_bundled_external("ffmpeg/" ${FFmpeg_EXT_NAME} "ffmpeg-bundled" FFmpeg_PATH)
     set(FFmpeg_FOUND YES)
     set(FFmpeg_INCLUDE_DIR "${FFmpeg_PATH}/include" CACHE PATH "Path to FFmpeg headers" FORCE)
     set(FFmpeg_LIBRARY_DIR "${FFmpeg_PATH}/lib" CACHE PATH "Path to FFmpeg library directory" FORCE)
@@ -266,8 +266,8 @@ elseif(ANDROID)
 elseif(WIN32)
     # Use yuzu FFmpeg binaries
     set(FFmpeg_EXT_NAME "ffmpeg-7.1.1")
-    set(FFmpeg_PATH "${CMAKE_BINARY_DIR}/externals/${FFmpeg_EXT_NAME}")
-    download_bundled_external("ffmpeg/" ${FFmpeg_EXT_NAME} "")
+
+    download_bundled_external("ffmpeg/" ${FFmpeg_EXT_NAME} "ffmpeg-bundled" FFmpeg_PATH)
     set(FFmpeg_FOUND YES)
     set(FFmpeg_INCLUDE_DIR "${FFmpeg_PATH}/include" CACHE PATH "Path to FFmpeg headers" FORCE)
     set(FFmpeg_LIBRARY_DIR "${FFmpeg_PATH}/bin" CACHE PATH "Path to FFmpeg library directory" FORCE)