[cmake, core] refactor: update mbedtls and make YUZU_USE_CPM better (#485)

update mbedtls functors to support mbedtls3 signatures

moved some vulkan stuff from externals to root, yuzu_use_cpm manages
them now

needs testing:
- all key/derivation related things
- nca verification
- game loading/updates/stuff

Reviewed-on: #485
Reviewed-by: MaranBr <maranbr@eden-emu.dev>

src/core/CMakeLists.txt

@@ -1200,7 +1200,7 @@ else()
     target_link_libraries(core PUBLIC Boost::headers)
 endif()
 
-target_link_libraries(core PRIVATE fmt::fmt nlohmann_json::nlohmann_json RenderDoc::API mbedtls mbedcrypto)
+target_link_libraries(core PRIVATE fmt::fmt nlohmann_json::nlohmann_json RenderDoc::API MbedTLS::mbedcrypto MbedTLS::mbedtls)
 if (MINGW)
     target_link_libraries(core PRIVATE ${MSWSOCK_LIBRARY})
 endif()

src/core/crypto/aes_util.h

@@ -16,7 +16,7 @@ struct CipherContext;
 enum class Mode {
     CTR = 11,
     ECB = 2,
-    XTS = 70,
+    XTS = 74,
 };
 
 enum class Op {

src/core/crypto/key_manager.cpp

@@ -539,7 +539,7 @@ static std::array<u8, target_size> MGF1(const std::array<u8, in_size>& seed) {
     while (out.size() < target_size) {
         out.resize(out.size() + 0x20);
         seed_exp[in_size + 3] = static_cast<u8>(i);
-        mbedtls_sha256_ret(seed_exp.data(), seed_exp.size(), out.data() + out.size() - 0x20, 0);
+        mbedtls_sha256(seed_exp.data(), seed_exp.size(), out.data() + out.size() - 0x20, 0);
         ++i;
     }
 

src/core/crypto/partition_data_manager.cpp

@@ -178,7 +178,7 @@ std::array<u8, key_size> FindKeyFromHex(const std::vector<u8>& binary,
 
     std::array<u8, 0x20> temp{};
     for (size_t i = 0; i < binary.size() - key_size; ++i) {
-        mbedtls_sha256_ret(binary.data() + i, key_size, temp.data(), 0);
+        mbedtls_sha256(binary.data() + i, key_size, temp.data(), 0);
 
         if (temp != hash)
             continue;
@@ -206,7 +206,7 @@ static std::array<Key128, 0x20> FindEncryptedMasterKeyFromHex(const std::vector<
     AESCipher<Key128> cipher(key, Mode::ECB);
     for (size_t i = 0; i < binary.size() - 0x10; ++i) {
         cipher.Transcode(binary.data() + i, dec_temp.size(), dec_temp.data(), Op::Decrypt);
-        mbedtls_sha256_ret(dec_temp.data(), dec_temp.size(), temp.data(), 0);
+        mbedtls_sha256(dec_temp.data(), dec_temp.size(), temp.data(), 0);
 
         for (size_t k = 0; k < out.size(); ++k) {
             if (temp == master_key_hashes[k]) {

src/core/file_sys/registered_cache.cpp

@@ -64,7 +64,7 @@ static std::string GetRelativePathFromNcaID(const std::array<u8, 16>& nca_id, bo
     }
 
     Core::Crypto::SHA256Hash hash{};
-    mbedtls_sha256_ret(nca_id.data(), nca_id.size(), hash.data(), 0);
+    mbedtls_sha256(nca_id.data(), nca_id.size(), hash.data(), 0);
 
     const auto format_str =
         fmt::runtime(cnmt_suffix ? "/000000{:02X}/{}.cnmt.nca" : "/000000{:02X}/{}.nca");
@@ -146,7 +146,7 @@ bool PlaceholderCache::Create(const NcaID& id, u64 size) const {
     }
 
     Core::Crypto::SHA256Hash hash{};
-    mbedtls_sha256_ret(id.data(), id.size(), hash.data(), 0);
+    mbedtls_sha256(id.data(), id.size(), hash.data(), 0);
     const auto dirname = fmt::format("000000{:02X}", hash[0]);
 
     const auto dir2 = GetOrCreateDirectoryRelative(dir, dirname);
@@ -170,7 +170,7 @@ bool PlaceholderCache::Delete(const NcaID& id) const {
     }
 
     Core::Crypto::SHA256Hash hash{};
-    mbedtls_sha256_ret(id.data(), id.size(), hash.data(), 0);
+    mbedtls_sha256(id.data(), id.size(), hash.data(), 0);
     const auto dirname = fmt::format("000000{:02X}", hash[0]);
 
     const auto dir2 = GetOrCreateDirectoryRelative(dir, dirname);
@@ -665,7 +665,7 @@ InstallResult RegisteredCache::InstallEntry(const NCA& nca, TitleType type,
     const OptionalHeader opt_header{0, 0};
     ContentRecord c_rec{{}, {}, {}, GetCRTypeFromNCAType(nca.GetType()), {}};
     const auto& data = nca.GetBaseFile()->ReadBytes(0x100000);
-    mbedtls_sha256_ret(data.data(), data.size(), c_rec.hash.data(), 0);
+    mbedtls_sha256(data.data(), data.size(), c_rec.hash.data(), 0);
     std::memcpy(&c_rec.nca_id, &c_rec.hash, 16);
     const CNMT new_cnmt(header, opt_header, {c_rec}, {});
     if (!RawInstallYuzuMeta(new_cnmt)) {
@@ -776,7 +776,7 @@ InstallResult RegisteredCache::RawInstallNCA(const NCA& nca, const VfsCopyFuncti
         id = *override_id;
     } else {
         const auto& data = in->ReadBytes(0x100000);
-        mbedtls_sha256_ret(data.data(), data.size(), hash.data(), 0);
+        mbedtls_sha256(data.data(), data.size(), hash.data(), 0);
         memcpy(id.data(), hash.data(), 16);
     }
 

src/core/file_sys/xts_archive.cpp

@@ -68,7 +68,7 @@ NAX::NAX(VirtualFile file_, std::array<u8, 0x10> nca_id)
     : header(std::make_unique<NAXHeader>()),
       file(std::move(file_)), keys{Core::Crypto::KeyManager::Instance()} {
     Core::Crypto::SHA256Hash hash{};
-    mbedtls_sha256_ret(nca_id.data(), nca_id.size(), hash.data(), 0);
+    mbedtls_sha256(nca_id.data(), nca_id.size(), hash.data(), 0);
     status = Parse(fmt::format("/registered/000000{:02X}/{}.nca", hash[0],
                                Common::HexToString(nca_id, false)));
 }

src/core/hle/service/bcat/delivery_cache_directory_service.cpp

@@ -15,7 +15,7 @@ namespace Service::BCAT {
 static BcatDigest DigestFile(const FileSys::VirtualFile& file) {
     BcatDigest out{};
     const auto bytes = file->ReadAllBytes();
-    mbedtls_md5_ret(bytes.data(), bytes.size(), out.data());
+    mbedtls_md5(bytes.data(), bytes.size(), out.data());
     return out;
 }
 

src/core/hle/service/ro/ro.cpp

@@ -178,7 +178,7 @@ struct ProcessContext {
             std::vector<u8> nro_data(size);
             m_process->GetMemory().ReadBlock(base_address, nro_data.data(), size);
 
-            mbedtls_sha256_ret(nro_data.data(), size, hash.data(), 0);
+            mbedtls_sha256(nro_data.data(), size, hash.data(), 0);
         }
 
         for (size_t i = 0; i < MaxNrrInfos; i++) {

src/core/loader/nca.cpp

@@ -150,7 +150,7 @@ ResultStatus AppLoader_NCA::VerifyIntegrity(std::function<bool(size_t, size_t)>
     // Initialize sha256 verification context.
     mbedtls_sha256_context ctx;
     mbedtls_sha256_init(&ctx);
-    mbedtls_sha256_starts_ret(&ctx, 0);
+    mbedtls_sha256_starts(&ctx, 0);
 
     // Ensure we maintain a clean state on exit.
     SCOPE_EXIT {
@@ -168,7 +168,7 @@ ResultStatus AppLoader_NCA::VerifyIntegrity(std::function<bool(size_t, size_t)>
         const size_t read_size = file->Read(buffer.data(), intended_read_size, processed_size);
 
         // Update the hash function with the buffer contents.
-        mbedtls_sha256_update_ret(&ctx, buffer.data(), read_size);
+        mbedtls_sha256_update(&ctx, buffer.data(), read_size);
 
         // Update counters.
         processed_size += read_size;
@@ -181,7 +181,7 @@ ResultStatus AppLoader_NCA::VerifyIntegrity(std::function<bool(size_t, size_t)>
 
     // Finalize context and compute the output hash.
     std::array<u8, NcaSha256HashLength> output_hash;
-    mbedtls_sha256_finish_ret(&ctx, output_hash.data());
+    mbedtls_sha256_finish(&ctx, output_hash.data());
 
     // Compare to expected.
     if (std::memcmp(input_hash.data(), output_hash.data(), NcaSha256HalfHashLength) != 0) {

src/dedicated_room/CMakeLists.txt

@@ -20,7 +20,7 @@ if (ENABLE_WEB_SERVICE)
     target_link_libraries(yuzu-room PRIVATE web_service)
 endif()
 
-target_link_libraries(yuzu-room PRIVATE mbedtls mbedcrypto)
+target_link_libraries(yuzu-room PRIVATE MbedTLS::mbedcrypto MbedTLS::mbedtls)
 if (MSVC)
     target_link_libraries(yuzu-room PRIVATE getopt)
 endif()