diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3599105020..51e2698096 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -270,6 +270,13 @@ if (ANDROID)
     set(CMAKE_POLICY_VERSION_MINIMUM 3.5) # Workaround for Oboe
 endif()
 
+# We need to downgrade debug info (/Zi -> /Z7) to use an older but more cacheable format
+# See https://github.com/nanoant/CMakePCHCompiler/issues/21
+if(WIN32 AND (CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo"))
+    string(REPLACE "/Zi" "/Z7" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}")
+    string(REPLACE "/Zi" "/Z7" CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}")
+endif()
+
 # Default to a Release build
 get_property(IS_MULTI_CONFIG GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
 if (NOT IS_MULTI_CONFIG AND NOT CMAKE_BUILD_TYPE)
diff --git a/externals/CMakeLists.txt b/externals/CMakeLists.txt
index 754ba61a0b..972c150aa1 100644
--- a/externals/CMakeLists.txt
+++ b/externals/CMakeLists.txt
@@ -70,12 +70,15 @@ endif()
 # Sirit
 AddJsonPackage(sirit)
 
-if(MSVC AND USE_CCACHE AND sirit_ADDED)
-    get_target_property(_opts sirit COMPILE_OPTIONS)
-    list(FILTER _opts EXCLUDE REGEX "/Zi")
-    list(APPEND _opts "/Z7")
-    set_target_properties(sirit PROPERTIES COMPILE_OPTIONS "${_opts}")
-elseif(MSVC AND CXX_CLANG)
+# Change to old-but-more-cacheable debug info on Windows
+if (WIN32 AND (CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo"))
+    get_target_property(sirit_opts sirit COMPILE_OPTIONS)
+    list(FILTER sirit_opts EXCLUDE REGEX "/Zi")
+    list(APPEND sirit_opts "/Z7")
+    set_target_properties(sirit PROPERTIES COMPILE_OPTIONS "${sirit_opts}")
+endif()
+
+if(MSVC AND CXX_CLANG)
     target_compile_options(sirit PRIVATE -Wno-error=unused-command-line-argument)
 endif()
 
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 88470c4c42..0f3c5cfd4b 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -101,15 +101,9 @@ if (MSVC AND NOT CXX_CLANG)
         )
     endif()
 
-    if (USE_CCACHE OR YUZU_USE_PRECOMPILED_HEADERS)
-        # when caching, we need to use /Z7 to downgrade debug info to use an older but more cacheable format
-        # Precompiled headers are deleted if not using /Z7. See https://github.com/nanoant/CMakePCHCompiler/issues/21
-        add_compile_options(/Z7)
-        # Avoid D9025 warning
-        string(REPLACE "/Zi" "" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}")
-        string(REPLACE "/Zi" "" CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}")
-    else()
-        add_compile_options(/Zi)
+    if (WIN32 AND (CMAKE_BUILD_TYPE STREQUAL "Debug" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo"))
+        string(REPLACE "/Zi" "/Z7" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}")
+        string(REPLACE "/Zi" "/Z7" CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}")
     endif()
 
     if (ARCHITECTURE_x86_64)