From 8cbb3597dc59e0024199a1bb6b9a5606a1ab2b9a Mon Sep 17 00:00:00 2001 From: crueter Date: Fri, 3 Oct 2025 14:39:58 -0400 Subject: [PATCH 1/9] format json script Signed-off-by: crueter --- externals/ffmpeg/cpmfile.json | 10 +++++----- externals/libusb/cpmfile.json | 14 +++++++------- tools/cpm-hash.sh | 2 +- tools/cpm/format-json.sh | 8 ++++++++ 4 files changed, 21 insertions(+), 13 deletions(-) create mode 100755 tools/cpm/format-json.sh diff --git a/externals/ffmpeg/cpmfile.json b/externals/ffmpeg/cpmfile.json index 9b9efaadde..b5b8150fd0 100644 --- a/externals/ffmpeg/cpmfile.json +++ b/externals/ffmpeg/cpmfile.json @@ -5,16 +5,16 @@ "hash": "2a89d664119debbb3c006ab1c48d5d7f26e889f4a65ad2e25c8b0503308295123d5a9c5c78bf683aef5ff09acef8c3fc2837f22d3e8c611528b933bf03bcdd97", "bundled": true }, - "ffmpeg-ci": { + "ffmpeg-ci": { "ci": true, "package": "FFmpeg", "name": "ffmpeg", "repo": "crueter-ci/FFmpeg", "version": "8.0", "min_version": "4.1", - "disabled_platforms": [ - "freebsd", - "solaris" - ] + "disabled_platforms": [ + "freebsd", + "solaris" + ] } } diff --git a/externals/libusb/cpmfile.json b/externals/libusb/cpmfile.json index 0bfa0d7a86..80a0b917db 100644 --- a/externals/libusb/cpmfile.json +++ b/externals/libusb/cpmfile.json @@ -1,8 +1,8 @@ { - "libusb": { - "repo": "libusb/libusb", - "sha": "c060e9ce30", - "hash": "44647357ba1179020cfa6674d809fc35cf6f89bff1c57252fe3a610110f5013ad678fc6eb5918e751d4384c30e2fe678868dbffc5f85736157e546cb9d10accc", - "find_args": "MODULE" - } -} \ No newline at end of file + "libusb": { + "repo": "libusb/libusb", + "sha": "c060e9ce30", + "hash": "44647357ba1179020cfa6674d809fc35cf6f89bff1c57252fe3a610110f5013ad678fc6eb5918e751d4384c30e2fe678868dbffc5f85736157e546cb9d10accc", + "find_args": "MODULE" + } +} diff --git a/tools/cpm-hash.sh b/tools/cpm-hash.sh index da0fb395db..b740936f4d 100755 --- a/tools/cpm-hash.sh +++ b/tools/cpm-hash.sh @@ -1,4 +1,4 @@ #!/bin/sh -SUM=`wget -q https://github.com/$1/archive/$2.zip -O - | sha512sum` +SUM=$(wget -q https://github.com/$1/archive/$2.zip -O - | sha512sum) echo "$SUM" | cut -d " " -f1 diff --git a/tools/cpm/format-json.sh b/tools/cpm/format-json.sh new file mode 100755 index 0000000000..6059077fe1 --- /dev/null +++ b/tools/cpm/format-json.sh @@ -0,0 +1,8 @@ +#!/bin/sh -e + +FILES=$(find . src -maxdepth 3 -name cpmfile.json) + +for file in $FILES; do + jq --indent 4 < $file > $file.new + mv $file.new $file +done From 67f3bf79c12def74a0c5c6ee79038aac6735b90c Mon Sep 17 00:00:00 2001 From: crueter Date: Fri, 3 Oct 2025 16:56:46 -0400 Subject: [PATCH 2/9] [tools] refactor: update/hash checking scripts, use tags for more deps Uses tags for a bunch of deps that can use them Also adds a bunmch of scripts to tools/cpm, notably for checking hashes and checking for updates. TODO: - Documentation - CI target to check hashes - Script to update the json directly (dedup) - Function to filter tags (dedup) - Weekly CI to check for updates Need to get that other CI runner up Signed-off-by: crueter --- cpmfile.json | 69 ++-- externals/cpmfile.json | 37 ++- externals/libusb/cpmfile.json | 7 +- src/dynarmic/externals/cpmfile.json | 7 +- tools/cpm-fetch-all.sh | 7 +- tools/cpm-fetch.sh | 410 ++++++++++++------------ tools/cpm-hash.sh | 4 - tools/cpm/check-all-hashes.sh | 14 + tools/cpm/check-all-updates.sh | 11 + tools/cpm/check-hash.sh | 110 +++++++ tools/cpm/check-updates.sh | 97 ++++++ tools/cpm/{format-json.sh => format.sh} | 0 tools/cpm/hash.sh | 22 ++ tools/{ => cpm}/url-hash.sh | 2 +- tools/cpm/which.sh | 9 + 15 files changed, 541 insertions(+), 265 deletions(-) delete mode 100755 tools/cpm-hash.sh create mode 100755 tools/cpm/check-all-hashes.sh create mode 100755 tools/cpm/check-all-updates.sh create mode 100755 tools/cpm/check-hash.sh create mode 100755 tools/cpm/check-updates.sh rename tools/cpm/{format-json.sh => format.sh} (100%) create mode 100755 tools/cpm/hash.sh rename tools/{ => cpm}/url-hash.sh (53%) create mode 100755 tools/cpm/which.sh diff --git a/cpmfile.json b/cpmfile.json index f1fd5ce1cf..53d0346245 100644 --- a/cpmfile.json +++ b/cpmfile.json @@ -23,12 +23,9 @@ }, "fmt": { "repo": "fmtlib/fmt", - "sha": "40626af88b", - "hash": "d59f06c24339f223de4ec2afeba1c67b5835a0f350a1ffa86242a72fc3e616a6b8b21798355428d4200c75287308b66634619ffa0b52ba5bd74cc01772ea1a8a", - "version": "8", - "options": [ - "FMT_INSTALL OFF" - ] + "tag": "12.0.0", + "hash": "c4ab814c20fbad7e3f0ae169125a4988a2795631194703251481dc36b18da65c886c4faa9acd046b0a295005217b3689eb0126108a9ba5aac2ca909aae263c2f", + "version": "8" }, "lz4": { "name": "lz4", @@ -40,16 +37,18 @@ "nlohmann": { "package": "nlohmann_json", "repo": "nlohmann/json", - "sha": "55f93686c0", - "hash": "b739749b066800e21154506ea150d2c5cbce8a45344177f46f884547a1399d26753166fd0df8135269ce28cf223552b1b65cd625b88c844d54753f2434900486", - "version": "3.8" + "tag": "v%VERSION%", + "hash": "6cc1e86261f8fac21cc17a33da3b6b3c3cd5c116755651642af3c9e99bb3538fd42c1bd50397a77c8fb6821bc62d90e6b91bcdde77a78f58f2416c62fc53b97d", + "version": "3.8", + "git_version": "3.12.0" }, "zlib": { "package": "ZLIB", "repo": "madler/zlib", - "sha": "51b7f2abda", - "hash": "16eaf1f3752489d12fd9ce30f7b5f7cbd5cb8ff53d617005a9847ae72d937f65e01e68be747f62d7ac19fd0c9aeba9956e60f16d6b465c5fdc2f3d08b4db2e6c", + "tag": "v%VERSION%", + "hash": "8c9642495bafd6fad4ab9fb67f09b268c69ff9af0f4f20cf15dfc18852ff1f312bd8ca41de761b3f8d8e90e77d79f2ccacd3d4c5b19e475ecf09d021fdfe9088", "version": "1.2", + "git_version": "1.3.1", "options": [ "ZLIB_BUILD_SHARED OFF", "ZLIB_INSTALL OFF" @@ -57,8 +56,8 @@ }, "zstd": { "repo": "facebook/zstd", - "sha": "f8745da6ff", - "hash": "3037007f990040fe32573b46f9bef8762fd5dbeeb07ffffcbfeba51ec98167edae39bb9c87f9299efcd61c4e467c5e84f7c19f0df7799bc1fc04864a278792ee", + "sha": "b8d6101fba", + "hash": "a6c8e5272214fd3e65e03ae4fc375f452bd2f646623886664ee23e239e35751cfc842db4d34a84a8039d89fc8f76556121f2a4ae350d017bdff5e22150f9c3de", "version": "1.5", "source_subdir": "build/cmake", "find_args": "MODULE", @@ -69,15 +68,17 @@ "catch2": { "package": "Catch2", "repo": "catchorg/Catch2", - "sha": "644821ce28", - "hash": "f8795f98acf2c02c0db8e734cc866d5caebab4b4a306e93598b97cb3c0c728dafe8283dce27ffe8d42460e5ae7302f3f32e7e274a7f991b73511ac88eef21b1f", - "version": "3.0.1" + "tag": "v%VERSION%", + "hash": "a95495142f915d6e9c2a23e80fe360343e9097680066a2f9d3037a070ba5f81ee5559a0407cc9e972dc2afae325873f1fc7ea07a64012c0f01aac6e549f03e3f", + "version": "3.0.1", + "git_version": "3.11.0" }, "enet": { "repo": "lsalzman/enet", - "sha": "2662c0de09", - "hash": "3de1beb4fa3d6b1e03eda8dd1e7580694f854af3ed3975dcdabfdcdf76b97f322b9734d35ea7f185855bb490d957842b938b26da4dd2dfded509390f8d2794dd", + "tag": "v%VERSION%", + "hash": "a0d2fa8c957704dd49e00a726284ac5ca034b50b00d2b20a94fa1bbfbb80841467834bfdc84aa0ed0d6aab894608fd6c86c3b94eee46343f0e6d9c22e391dbf9", "version": "1.3", + "git_version": "1.3.18", "find_args": "MODULE" }, "opus": { @@ -94,17 +95,18 @@ "vulkan-utility-headers": { "package": "VulkanUtilityLibraries", "repo": "scripts/VulkanUtilityHeaders", - "tag": "1.4.326", + "tag": "1.4.328", "artifact": "VulkanUtilityHeaders.tar.zst", "git_host": "git.crueter.xyz", - "hash": "5924629755cb1605c4aa4eee20ef7957a9dd8d61e4df548be656d98054f2730c4109693c1bd35811f401f4705d2ccff9fc849be32b0d8480bc3f73541a5e0964" + "hash": "9922217b39faf73cd4fc1510f2fdba14a49aa5c0d77f9ee24ee0512cef16b234d0cabc83c1fec861fa5df1d43e7f086ca9b6501753899119f39c5ca530cb0dae" }, "spirv-tools": { "package": "SPIRV-Tools", "repo": "KhronosGroup/SPIRV-Tools", - "sha": "40eb301f32", - "hash": "58d0fb1047d69373cf24c73e6f78c73a72a6cca3b4df1d9f083b9dcc0962745ef154abf3dbe9b3623b835be20c6ec769431cf11733349f45e7568b3525f707aa", + "tag": "v%VERSION%", + "hash": "b17940433ced72e004c5eeffd7dd411b6afcc6a52ee31de6427d88edceb8172369be8ec8bf5b65708a78bf41fdae264d554aa7750b2209831679ab36bc867567", "find_args": "MODULE", + "git_version": "2025.4", "options": [ "SPIRV_SKIP_EXECUTABLES ON" ] @@ -125,7 +127,8 @@ "hash": "6671fb8fcaa832e5b115dfdce8f78baa6a4aea71f5c89a640583634cdee27aefe3bf4be075744da91f7c3ae5ea4e0c765c8fc3937b5cfd9ea73d87ef496524da", "version": "3", "git_version": "3.6.4", - "artifact": "%TAG%.tar.bz2" + "artifact": "%TAG%.tar.bz2", + "skip_updates": true }, "cubeb": { "repo": "mozilla/cubeb", @@ -153,17 +156,20 @@ "simpleini": { "package": "SimpleIni", "repo": "brofield/simpleini", - "sha": "09c21bda1d", - "hash": "99779ca9b6e040d36558cadf484f9ffdab5b47bcc8fc72e4d33639d1d60c0ceb4410d335ba445d72a4324e455167fd6769d99b459943aa135bec085dff2d4b7c", - "find_args": "MODULE" + "tag": "v%VERSION%", + "hash": "6c198636816a0018adbf7f735d402c64245c6fcd540b7360d4388d46f007f3a520686cdaec4705cb8cb31401b2cb4797a80b42ea5d08a6a5807c0848386f7ca1", + "find_args": "MODULE", + "git_version": "4.22" }, "sdl2_generic": { "package": "SDL2", "repo": "libsdl-org/SDL", - "sha": "54772f345a", - "hash": "2a68a0e01c390043aa9d9df63d8a20a52076c88bb460ac4e0f33194ca7d9bc8fadbbcc04e7506872ac4b6354a73fbc267c036f82200da59465789b87c7d9e3a4", + "tag": "release-%VERSION%", + "hash": "d58b5b9964fea3c04d13aae646b92e9646125f269088fc6698089f0dbb9506091739cb8a72bfead08aa3952f8b4df0a7886396eeee54a28388a6b89be01f44cf", "key": "generic", - "bundled": true + "bundled": true, + "git_version": "2.32.8", + "skip_updates": true }, "sdl2_steamdeck": { "package": "SDL2", @@ -171,7 +177,8 @@ "sha": "cc016b0046", "hash": "34d5ef58da6a4f9efa6689c82f67badcbd741f5a4f562a9c2c30828fa839830fb07681c5dc6a7851520e261c8405a416ac0a2c2513b51984fb3b4fa4dcb3e20b", "key": "steamdeck", - "bundled": true + "bundled": true, + "skip_updates": "true" }, "sdl2": { "ci": true, @@ -185,7 +192,7 @@ "llvm-mingw": { "repo": "misc/llvm-mingw", "git_host": "git.crueter.xyz", - "tag": "20250828", + "tag": "%VERSION%", "version": "20250828", "artifact": "clang-rt-builtins.tar.zst", "hash": "d902392caf94e84f223766e2cc51ca5fab6cae36ab8dc6ef9ef6a683ab1c483bfcfe291ef0bd38ab16a4ecc4078344fa8af72da2f225ab4c378dee23f6186181" diff --git a/externals/cpmfile.json b/externals/cpmfile.json index dcafc8f97d..4d0b3af9f4 100644 --- a/externals/cpmfile.json +++ b/externals/cpmfile.json @@ -2,9 +2,10 @@ "vulkan-memory-allocator": { "package": "VulkanMemoryAllocator", "repo": "GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator", - "sha": "1076b348ab", - "hash": "a46b44e4286d08cffda058e856c47f44c7fed3da55fe9555976eb3907fdcc20ead0b1860b0c38319cda01dbf9b1aa5d4b4038c7f1f8fbd97283d837fa9af9772", - "find_args": "CONFIG" + "tag": "v%VERSION%", + "hash": "deb5902ef8db0e329fbd5f3f4385eb0e26bdd9f14f3a2334823fb3fe18f36bc5d235d620d6e5f6fe3551ec3ea7038638899db8778c09f6d5c278f5ff95c3344b", + "find_args": "CONFIG", + "git_version": "3.3.0" }, "sirit": { "repo": "eden-emulator/sirit", @@ -17,8 +18,9 @@ }, "httplib": { "repo": "yhirose/cpp-httplib", - "sha": "a609330e4c", - "hash": "dd3fd0572f8367d8549e1319fd98368b3e75801a293b0c3ac9b4adb806473a4506a484b3d389dc5bee5acc460cb90af7a20e5df705a1696b56496b30b9ce7ed2" + "tag": "v%VERSION%", + "hash": "b364500f76e2ecb0fe21b032d831272e3f1dfeea71af74e325f8fc4ce9dcdb3c941b97a5b422bdeafb9facd058597b90f8bfc284fb9afe3c33fefa15dd5a010b", + "git_version": "0.26.0" }, "cpp-jwt": { "version": "1.4", @@ -33,16 +35,18 @@ "xbyak_sun": { "package": "xbyak", "repo": "herumi/xbyak", - "sha": "9bb219333a", - "hash": "303165d45c8c19387ec49d9fda7d7a4e0d86d4c0153898c23f25ce2d58ece567f44c0bbbfe348239b933edb6e1a1e34f4bc1c0ab3a285bee5da0e548879387b0", - "bundled": true + "tag": "v%VERSION%", + "hash": "e84992c65ad62c577e2746ec5180132fd2875166d1e6b1521a0ff619787e1645792fe5f6a858fe94ed66f297912b6a6b89a509b5d5f5e81a2db1dd7e6790b1f5", + "bundled": true, + "git_version": "7.30" }, "xbyak": { "package": "xbyak", "repo": "herumi/xbyak", - "sha": "4e44f4614d", - "hash": "5824e92159e07fa36a774aedd3b3ef3541d0241371d522cffa4ab3e1f215fa5097b1b77865b47b2481376c704fa079875557ea463ca63d0a7fd6a8a20a589e70", - "bundled": true + "sha": "v7.22", + "hash": "ff4a461b35ced007adbe9b388a90e4b53c46f9250260a559c04a757ad34ac482baf92700f113c106af6512d6894e849c4409efaf1c003207bf245a5910d848a1", + "bundled": true, + "skip_updates": true }, "oaknut": { "version": "2.0.1", @@ -60,15 +64,16 @@ }, "oboe": { "repo": "google/oboe", - "sha": "2bc873e53c", - "hash": "02329058a7f9cf7d5039afaae5ab170d9f42f60f4c01e21eaf4f46073886922b057a9ae30eeac040b3ac182f51b9c1bfe9fe1050a2c9f6ce567a1a9a0ec2c768", + "tag": "1.10.0", + "hash": "ce4011afe7345370d4ead3b891cd69a5ef224b129535783586c0ca75051d303ed446e6c7f10bde8da31fff58d6e307f1732a3ffd03b249f9ef1fd48fd4132715", "bundled": true }, "unordered-dense": { "package": "unordered_dense", "repo": "martinus/unordered_dense", - "sha": "73f3cbb237", - "hash": "c08c03063938339d61392b687562909c1a92615b6ef39ec8df19ea472aa6b6478e70d7d5e33d4a27b5d23f7806daf57fe1bacb8124c8a945c918c7663a9e8532", - "find_args": "CONFIG" + "tag": "v%VERSION%", + "hash": "f9c819e28e1c1a387acfee09277d6af5e366597a0d39acf1c687acf0608a941ba966af8aaebdb8fba0126c7360269c4a51754ef4cab17c35c01a30215f953368", + "find_args": "CONFIG", + "git_version": "4.5.0" } } diff --git a/externals/libusb/cpmfile.json b/externals/libusb/cpmfile.json index 80a0b917db..dc69841ab7 100644 --- a/externals/libusb/cpmfile.json +++ b/externals/libusb/cpmfile.json @@ -1,8 +1,9 @@ { "libusb": { "repo": "libusb/libusb", - "sha": "c060e9ce30", - "hash": "44647357ba1179020cfa6674d809fc35cf6f89bff1c57252fe3a610110f5013ad678fc6eb5918e751d4384c30e2fe678868dbffc5f85736157e546cb9d10accc", - "find_args": "MODULE" + "tag": "v%VERSION%", + "hash": "98c5f7940ff06b25c9aa65aa98e23de4c79a4c1067595f4c73cc145af23a1c286639e1ba11185cd91bab702081f307b973f08a4c9746576dc8d01b3620a3aeb5", + "find_args": "MODULE", + "git_version": "1.0.29" } } diff --git a/src/dynarmic/externals/cpmfile.json b/src/dynarmic/externals/cpmfile.json index 718163baf5..099ded57a4 100644 --- a/src/dynarmic/externals/cpmfile.json +++ b/src/dynarmic/externals/cpmfile.json @@ -1,9 +1,10 @@ { "biscuit": { - "version": "0.9.1", "repo": "lioncash/biscuit", - "sha": "76b0be8dae", - "hash": "47d55ed02d032d6cf3dc107c6c0a9aea686d5f25aefb81d1af91db027b6815bd5add1755505e19d76625feeb17aa2db6cd1668fe0dad2e6a411519bde6ca4489" + "tag": "v%VERSION%", + "hash": "1229f345b014f7ca544dedb4edb3311e41ba736f9aa9a67f88b5f26f3c983288c6bb6cdedcfb0b8a02c63088a37e6a0d7ba97d9c2a4d721b213916327cffe28a", + "version": "0.9.1", + "git_version": "0.19.0" }, "mcl": { "version": "0.1.12", diff --git a/tools/cpm-fetch-all.sh b/tools/cpm-fetch-all.sh index 9d5005ec44..9dc1ac2a55 100755 --- a/tools/cpm-fetch-all.sh +++ b/tools/cpm-fetch-all.sh @@ -1,4 +1,4 @@ -#!/bin/bash -ex +#!/bin/bash -e # SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project # SPDX-License-Identifier: GPL-3.0-or-later @@ -6,6 +6,9 @@ # SPDX-FileCopyrightText: 2025 crueter # SPDX-License-Identifier: GPL-3.0-or-later -LIBS=$(find . src -maxdepth 3 -name cpmfile.json -exec jq -j 'keys_unsorted | join(" ")' {} \; -printf " ") +# shellcheck disable=SC2038 +# shellcheck disable=SC2016 +[ -z "$PACKAGES" ] && PACKAGES=$(find . src -maxdepth 3 -name cpmfile.json | xargs jq -s 'reduce .[] as $item ({}; . * $item)') +LIBS=$(echo "$PACKAGES" | jq -j 'keys_unsorted | join(" ")') tools/cpm-fetch.sh $LIBS \ No newline at end of file diff --git a/tools/cpm-fetch.sh b/tools/cpm-fetch.sh index 088df8464e..bdb125601a 100755 --- a/tools/cpm-fetch.sh +++ b/tools/cpm-fetch.sh @@ -10,227 +10,227 @@ mkdir -p $CPM_SOURCE_CACHE +# fetch json once to make lives easier +# shellcheck disable=SC2038 +[ -z "$PACKAGES" ] && PACKAGES=$(find . src -maxdepth 3 -name cpmfile.json | xargs jq -s 'reduce .[] as $item ({}; . * $item)') + ROOTDIR="$PWD" TMP=$(mktemp -d) download_package() { - FILENAME=$(basename "$DOWNLOAD") + FILENAME=$(basename "$DOWNLOAD") - OUTFILE="$TMP/$FILENAME" - - LOWER_PACKAGE=$(tr '[:upper:]' '[:lower:]' <<< "$PACKAGE_NAME") - OUTDIR="${CPM_SOURCE_CACHE}/${LOWER_PACKAGE}/${KEY}" - [ -d "$OUTDIR" ] && return - - curl "$DOWNLOAD" -sS -L -o "$OUTFILE" - - ACTUAL_HASH=$(${HASH_ALGO}sum "$OUTFILE" | cut -d" " -f1) - [ "$ACTUAL_HASH" != "$HASH" ] && echo "!! $FILENAME did not match expected hash; expected $HASH but got $ACTUAL_HASH" && exit 1 - - mkdir -p "$OUTDIR" - - pushd "$OUTDIR" > /dev/null - - case "$FILENAME" in - (*.7z) - 7z x "$OUTFILE" > /dev/null - ;; - (*.tar*) - tar xf "$OUTFILE" > /dev/null - ;; - (*.zip) - unzip "$OUTFILE" > /dev/null - ;; - esac - - # basically if only one real item exists at the top we just move everything from there - # since github and some vendors hate me - DIRS=$(find -maxdepth 1 -type d -o -type f) - - # thanks gnu - if [ $(wc -l <<< "$DIRS") -eq 2 ]; then - SUBDIR=$(find . -maxdepth 1 -type d -not -name ".") - mv "$SUBDIR"/* . - mv "$SUBDIR"/.* . 2>/dev/null || true - rmdir "$SUBDIR" - fi - - if grep -e "patches" <<< "$JSON" > /dev/null; then - PATCHES=$(jq -r '.patches | join(" ")' <<< "$JSON") - for patch in $PATCHES; do - patch --binary -p1 < "$ROOTDIR"/.patch/$package/$patch - done - fi - - popd > /dev/null -} - -ci_package() { - REPO=$(jq -r ".repo" <<< "$JSON") - EXT=$(jq -r '.extension' <<< "$JSON") - [ "$EXT" = null ] && EXT="tar.zst" - - VERSION=$(jq -r ".version" <<< "$JSON") - - NAME=$(jq -r ".name" <<< "$JSON") - [ "$NAME" = null ] && NAME="$PACKAGE" - - PACKAGE=$(jq -r ".package | \"$package\"" <<< "$JSON") - - DISABLED=$(jq -j '.disabled_platforms' <<< "$JSON") - - [ "$REPO" = null ] && echo "No repo defined for CI package $package" && return - - echo "-- CI package $PACKAGE" - - for platform in windows-amd64 windows-arm64 android solaris freebsd linux linux-aarch64; do - echo "-- * platform $platform" - - case $DISABLED in - (*"$platform"*) - echo "-- * -- disabled" - continue - ;; - (*) ;; - esac - - FILENAME="${NAME}-${platform}-${VERSION}.${EXT}" - DOWNLOAD="https://$GIT_HOST/${REPO}/releases/download/v${VERSION}/${FILENAME}" - PACKAGE_NAME="$PACKAGE" - KEY=$platform + OUTFILE="$TMP/$FILENAME" LOWER_PACKAGE=$(tr '[:upper:]' '[:lower:]' <<< "$PACKAGE_NAME") OUTDIR="${CPM_SOURCE_CACHE}/${LOWER_PACKAGE}/${KEY}" - [ -d "$OUTDIR" ] && continue + [ -d "$OUTDIR" ] && return + curl "$DOWNLOAD" -sS -L -o "$OUTFILE" + + ACTUAL_HASH=$(${HASH_ALGO}sum "$OUTFILE" | cut -d" " -f1) + [ "$ACTUAL_HASH" != "$HASH" ] && echo "!! $FILENAME did not match expected hash; expected $HASH but got $ACTUAL_HASH" && exit 1 + + mkdir -p "$OUTDIR" + + pushd "$OUTDIR" > /dev/null + + case "$FILENAME" in + (*.7z) + 7z x "$OUTFILE" > /dev/null + ;; + (*.tar*) + tar xf "$OUTFILE" > /dev/null + ;; + (*.zip) + unzip "$OUTFILE" > /dev/null + ;; + esac + + # basically if only one real item exists at the top we just move everything from there + # since github and some vendors hate me + DIRS=$(find -maxdepth 1 -type d -o -type f) + + # thanks gnu + if [ "$(wc -l <<< "$DIRS")" -eq 2 ]; then + SUBDIR=$(find . -maxdepth 1 -type d -not -name ".") + mv "$SUBDIR"/* . + mv "$SUBDIR"/.* . 2>/dev/null || true + rmdir "$SUBDIR" + fi + + if grep -e "patches" <<< "$JSON" > /dev/null; then + PATCHES=$(jq -r '.patches | join(" ")' <<< "$JSON") + for patch in $PATCHES; do + patch --binary -p1 < "$ROOTDIR"/.patch/$package/$patch + done + fi + + popd > /dev/null +} + +ci_package() { + EXT=$(jq -r '.extension' <<< "$JSON") + [ "$EXT" = null ] && EXT="tar.zst" + + VERSION=$(jq -r ".version" <<< "$JSON") + + NAME=$(jq -r ".name" <<< "$JSON") + [ "$NAME" = null ] && NAME="$PACKAGE" + + PACKAGE=$(jq -r ".package | \"$package\"" <<< "$JSON") + + DISABLED=$(jq -j '.disabled_platforms' <<< "$JSON") + + [ "$REPO" = null ] && echo "No repo defined for CI package $package" && return + + echo "-- CI package $PACKAGE" + + for platform in windows-amd64 windows-arm64 android solaris freebsd linux linux-aarch64; do + echo "-- * platform $platform" + + case $DISABLED in + (*"$platform"*) + echo "-- * -- disabled" + continue + ;; + (*) ;; + esac + + FILENAME="${NAME}-${platform}-${VERSION}.${EXT}" + DOWNLOAD="https://$GIT_HOST/${REPO}/releases/download/v${VERSION}/${FILENAME}" + PACKAGE_NAME="$PACKAGE" + KEY=$platform + + LOWER_PACKAGE=$(tr '[:upper:]' '[:lower:]' <<< "$PACKAGE_NAME") + OUTDIR="${CPM_SOURCE_CACHE}/${LOWER_PACKAGE}/${KEY}" + [ -d "$OUTDIR" ] && continue + + HASH_ALGO=$(jq -r ".hash_algo" <<< "$JSON") + [ "$HASH_ALGO" = null ] && HASH_ALGO=sha512 + + HASH_SUFFIX="${HASH_ALGO}sum" + HASH_URL="${DOWNLOAD}.${HASH_SUFFIX}" + + HASH=$(curl "$HASH_URL" -sS -q -L -o -) + + download_package + done +} + +for package in "$@" +do + # prepare for cancer + JSON=$(echo "$PACKAGES" | jq -r ".\"$package\" | select( . != null )") + + [ -z "$JSON" ] && echo "!! No cpmfile definition for $package" && continue + + PACKAGE_NAME=$(jq -r ".package" <<< "$JSON") + [ "$PACKAGE_NAME" = null ] && PACKAGE_NAME="$package" + + GIT_HOST=$(jq -r ".git_host" <<< "$JSON") + [ "$GIT_HOST" = null ] && GIT_HOST=github.com + + REPO=$(jq -r ".repo" <<< "$JSON") + + CI=$(jq -r ".ci" <<< "$JSON") + if [ "$CI" != null ]; then + ci_package + continue + fi + + VERSION=$(jq -r ".version" <<< "$JSON") + GIT_VERSION=$(jq -r ".git_version" <<< "$JSON") + + [ "$GIT_VERSION" = null ] && GIT_VERSION="$VERSION" + + # url parsing + URL=$(jq -r ".url" <<< "$JSON") + SHA=$(jq -r ".sha" <<< "$JSON") + + VERSION=$(jq -r ".version" <<< "$JSON") + GIT_VERSION=$(jq -r ".git_version" <<< "$JSON") + + if [ "$GIT_VERSION" != null ]; then + VERSION_REPLACE="$GIT_VERSION" + else + VERSION_REPLACE="$VERSION" + fi + + TAG=$(jq -r ".tag" <<< "$JSON") + + TAG=$(sed "s/%VERSION%/$VERSION_REPLACE/" <<< $TAG) + + ARTIFACT=$(jq -r ".artifact" <<< "$JSON") + ARTIFACT=$(sed "s/%VERSION%/$VERSION_REPLACE/" <<< $ARTIFACT) + ARTIFACT=$(sed "s/%TAG%/$TAG/" <<< $ARTIFACT) + + if [ "$URL" != "null" ]; then + DOWNLOAD="$URL" + elif [ "$REPO" != "null" ]; then + GIT_URL="https://$GIT_HOST/$REPO" + + BRANCH=$(jq -r ".branch" <<< "$JSON") + + if [ "$TAG" != "null" ]; then + if [ "$ARTIFACT" != "null" ]; then + DOWNLOAD="${GIT_URL}/releases/download/${TAG}/${ARTIFACT}" + else + DOWNLOAD="${GIT_URL}/archive/refs/tags/${TAG}.tar.gz" + fi + elif [ "$SHA" != "null" ]; then + DOWNLOAD="${GIT_URL}/archive/${SHA}.zip" + else + if [ "$BRANCH" = null ]; then + BRANCH=master + fi + + DOWNLOAD="${GIT_URL}/archive/refs/heads/${BRANCH}.zip" + fi + else + echo "!! No repo or URL defined for $package" + continue + fi + + # key parsing + KEY=$(jq -r ".key" <<< "$JSON") + + if [ "$KEY" = null ]; then + if [ "$SHA" != null ]; then + KEY=$(cut -c1-4 - <<< "$SHA") + elif [ "$GIT_VERSION" != null ]; then + KEY="$GIT_VERSION" + elif [ "$TAG" != null ]; then + KEY="$TAG" + elif [ "$VERSION" != null ]; then + KEY="$VERSION" + else + echo "!! No valid key could be determined for $package. Must define one of: key, sha, tag, version, git_version" + continue + fi + fi + + echo "-- Downloading regular package $package, with key $KEY, from $DOWNLOAD" + + # hash parsing HASH_ALGO=$(jq -r ".hash_algo" <<< "$JSON") [ "$HASH_ALGO" = null ] && HASH_ALGO=sha512 - HASH_SUFFIX="${HASH_ALGO}sum" - HASH_URL="${DOWNLOAD}.${HASH_SUFFIX}" + HASH=$(jq -r ".hash" <<< "$JSON") - HASH=$(curl "$HASH_URL" -sS -q -L -o -) + if [ "$HASH" = null ]; then + HASH_SUFFIX="${HASH_ALGO}sum" + HASH_URL=$(jq -r ".hash_url" <<< "$JSON") + + if [ "$HASH_URL" = null ]; then + HASH_URL="${DOWNLOAD}.${HASH_SUFFIX}" + fi + + HASH=$(curl "$HASH_URL" -L -o -) + fi download_package - done -} - -for package in $@ -do - # prepare for cancer - # TODO(crueter): Fetch json once? - JSON=$(find . src -maxdepth 3 -name cpmfile.json -exec jq -r ".\"$package\" | select( . != null )" {} \;) - - [ -z "$JSON" ] && echo "!! No cpmfile definition for $package" && continue - - PACKAGE_NAME=$(jq -r ".package" <<< "$JSON") - [ "$PACKAGE_NAME" = null ] && PACKAGE_NAME="$package" - - GIT_HOST=$(jq -r ".git_host" <<< "$JSON") - [ "$GIT_HOST" = null ] && GIT_HOST=github.com - REPO=$(jq -r ".repo" <<< "$JSON") - - CI=$(jq -r ".ci" <<< "$JSON") - if [ "$CI" != null ]; then - ci_package - continue - fi - - VERSION=$(jq -r ".version" <<< "$JSON") - GIT_VERSION=$(jq -r ".git_version" <<< "$JSON") - TAG=$(jq -r ".tag" <<< "$JSON") - SHA=$(jq -r ".sha" <<< "$JSON") - - [ "$GIT_VERSION" = null ] && GIT_VERSION="$VERSION" - [ "$GIT_VERSION" = null ] && GIT_VERSION="$TAG" - - # url parsing WOOOHOOHOHOOHOHOH - URL=$(jq -r ".url" <<< "$JSON") - SHA=$(jq -r ".sha" <<< "$JSON") - - VERSION=$(jq -r ".version" <<< "$JSON") - GIT_VERSION=$(jq -r ".git_version" <<< "$JSON") - - if [ "$GIT_VERSION" != null ]; then - VERSION_REPLACE="$GIT_VERSION" - else - VERSION_REPLACE="$VERSION" - fi - - TAG=$(jq -r ".tag" <<< "$JSON") - - TAG=$(sed "s/%VERSION%/$VERSION_REPLACE/" <<< $TAG) - - ARTIFACT=$(jq -r ".artifact" <<< "$JSON") - ARTIFACT=$(sed "s/%VERSION%/$VERSION_REPLACE/" <<< $ARTIFACT) - ARTIFACT=$(sed "s/%TAG%/$TAG/" <<< $ARTIFACT) - - if [ "$URL" != "null" ]; then - DOWNLOAD="$URL" - elif [ "$REPO" != "null" ]; then - GIT_URL="https://$GIT_HOST/$REPO" - - BRANCH=$(jq -r ".branch" <<< "$JSON") - - if [ "$TAG" != "null" ]; then - if [ "$ARTIFACT" != "null" ]; then - DOWNLOAD="${GIT_URL}/releases/download/${TAG}/${ARTIFACT}" - else - DOWNLOAD="${GIT_URL}/archive/refs/tags/${TAG}.tar.gz" - fi - elif [ "$SHA" != "null" ]; then - DOWNLOAD="${GIT_URL}/archive/${SHA}.zip" - else - if [ "$BRANCH" = null ]; then - BRANCH=master - fi - - DOWNLOAD="${GIT_URL}/archive/refs/heads/${BRANCH}.zip" - fi - else - echo "!! No repo or URL defined for $package" - continue - fi - - # key parsing - KEY=$(jq -r ".key" <<< "$JSON") - - if [ "$KEY" = null ]; then - if [ "$SHA" != null ]; then - KEY=$(cut -c1-4 - <<< "$SHA") - elif [ "$GIT_VERSION" != null ]; then - KEY="$GIT_VERSION" - elif [ "$TAG" != null ]; then - KEY="$TAG" - elif [ "$VERSION" != null ]; then - KEY="$VERSION" - else - echo "!! No valid key could be determined for $package. Must define one of: key, sha, tag, version, git_version" - continue - fi - fi - - echo "-- Downloading regular package $package, with key $KEY, from $DOWNLOAD" - - # hash parsing - HASH_ALGO=$(jq -r ".hash_algo" <<< "$JSON") - [ "$HASH_ALGO" = null ] && HASH_ALGO=sha512 - - HASH=$(jq -r ".hash" <<< "$JSON") - - if [ "$HASH" = null ]; then - HASH_SUFFIX="${HASH_ALGO}sum" - HASH_URL=$(jq -r ".hash_url" <<< "$JSON") - - if [ "$HASH_URL" = null ]; then - HASH_URL="${DOWNLOAD}.${HASH_SUFFIX}" - fi - - HASH=$(curl "$HASH_URL" -L -o -) - fi - - download_package done rm -rf $TMP \ No newline at end of file diff --git a/tools/cpm-hash.sh b/tools/cpm-hash.sh deleted file mode 100755 index b740936f4d..0000000000 --- a/tools/cpm-hash.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -SUM=$(wget -q https://github.com/$1/archive/$2.zip -O - | sha512sum) -echo "$SUM" | cut -d " " -f1 diff --git a/tools/cpm/check-all-hashes.sh b/tools/cpm/check-all-hashes.sh new file mode 100755 index 0000000000..a270c368db --- /dev/null +++ b/tools/cpm/check-all-hashes.sh @@ -0,0 +1,14 @@ +#!/bin/bash -e + +# SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +# SPDX-License-Identifier: GPL-3.0-or-later + +# SPDX-FileCopyrightText: 2025 crueter +# SPDX-License-Identifier: GPL-3.0-or-later + +# shellcheck disable=SC2038 +# shellcheck disable=SC2016 +[ -z "$PACKAGES" ] && PACKAGES=$(find . src -maxdepth 3 -name cpmfile.json | xargs jq -s 'reduce .[] as $item ({}; . * $item)') +LIBS=$(echo "$PACKAGES" | jq -j 'keys_unsorted | join(" ")') + +tools/cpm/check-hash.sh $LIBS \ No newline at end of file diff --git a/tools/cpm/check-all-updates.sh b/tools/cpm/check-all-updates.sh new file mode 100755 index 0000000000..e0c744abf0 --- /dev/null +++ b/tools/cpm/check-all-updates.sh @@ -0,0 +1,11 @@ +#!/bin/bash -e + +# SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +# SPDX-License-Identifier: GPL-3.0-or-later + +# SPDX-FileCopyrightText: 2025 crueter +# SPDX-License-Identifier: GPL-3.0-or-later + +LIBS=$(find . src -maxdepth 3 -name cpmfile.json -exec jq -j 'keys_unsorted | join(" ")' {} \; -printf " ") + +tools/cpm/check-updates.sh $LIBS \ No newline at end of file diff --git a/tools/cpm/check-hash.sh b/tools/cpm/check-hash.sh new file mode 100755 index 0000000000..bf7c70ffcb --- /dev/null +++ b/tools/cpm/check-hash.sh @@ -0,0 +1,110 @@ +#!/bin/bash -e + +# SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +# SPDX-License-Identifier: GPL-3.0-or-later + +# SPDX-FileCopyrightText: 2025 crueter +# SPDX-License-Identifier: GPL-3.0-or-later + +# env vars: +# - UPDATE: update if available +# - FORCE: forcefully update + +# shellcheck disable=SC2016 +# shellcheck disable=SC2038 +[ -z "$PACKAGES" ] && PACKAGES=$(find . src -maxdepth 3 -name cpmfile.json | xargs jq -s 'reduce .[] as $item ({}; . * $item)') + +for package in "$@" +do + JSON=$(echo "$PACKAGES" | jq -r ".\"$package\" | select( . != null )") + + [ -z "$JSON" ] && echo "!! No cpmfile definition for $package" && continue + + GIT_HOST=$(jq -r ".git_host" <<< "$JSON") + [ "$GIT_HOST" = null ] && GIT_HOST=github.com + + CI=$(jq -r ".ci" <<< "$JSON") + if [ "$CI" != null ]; then + continue + fi + + HASH_URL=$(jq -r ".hash_url" <<< "$JSON") + [ "$HASH_URL" != null ] && continue + + REPO=$(jq -r ".repo" <<< "$JSON") + TAG=$(jq -r ".tag" <<< "$JSON") + + URL=$(jq -r ".url" <<< "$JSON") + SHA=$(jq -r ".sha" <<< "$JSON") + + echo "-- Package $package" + + GIT_VERSION=$(jq -r ".git_version" <<< "$JSON") + + VERSION=$(jq -r ".version" <<< "$JSON") + GIT_VERSION=$(jq -r ".git_version" <<< "$JSON") + + if [ "$GIT_VERSION" != null ]; then + VERSION_REPLACE="$GIT_VERSION" + else + VERSION_REPLACE="$VERSION" + fi + + TAG=${TAG//%VERSION%/$VERSION_REPLACE} + + ARTIFACT=$(jq -r ".artifact" <<< "$JSON") + ARTIFACT=${ARTIFACT//%VERSION%/$VERSION_REPLACE} + ARTIFACT=${ARTIFACT//%TAG%/$TAG} + + if [ "$URL" != "null" ]; then + DOWNLOAD="$URL" + elif [ "$REPO" != "null" ]; then + GIT_URL="https://$GIT_HOST/$REPO" + + BRANCH=$(jq -r ".branch" <<< "$JSON") + + if [ "$TAG" != "null" ]; then + if [ "$ARTIFACT" != "null" ]; then + DOWNLOAD="${GIT_URL}/releases/download/${TAG}/${ARTIFACT}" + else + DOWNLOAD="${GIT_URL}/archive/refs/tags/${TAG}.tar.gz" + fi + elif [ "$SHA" != "null" ]; then + DOWNLOAD="${GIT_URL}/archive/${SHA}.zip" + else + if [ "$BRANCH" = null ]; then + BRANCH=master + fi + + DOWNLOAD="${GIT_URL}/archive/refs/heads/${BRANCH}.zip" + fi + else + echo "!! No repo or URL defined for $package" + continue + fi + + # hash parsing + HASH_ALGO=$(jq -r ".hash_algo" <<< "$JSON") + [ "$HASH_ALGO" = null ] && HASH_ALGO=sha512 + + HASH=$(jq -r ".hash" <<< "$JSON") + + [ "$HASH" = null ] && echo "-- * Warning: no hash specified" && continue + + export USE_TAG=true + ACTUAL=$(tools/cpm/url-hash.sh "$DOWNLOAD") + + # shellcheck disable=SC2028 + [ "$ACTUAL" != "$HASH" ] && echo "-- * Expected $HASH" && echo "-- * Got $ACTUAL" + + if [ "$UPDATE" = "true" ] && [ "$ACTUAL" != "$HASH" ]; then + NEW_JSON=$(echo "$JSON" | jq ".hash = \"$ACTUAL\"") + + FILE=$(tools/cpm/which.sh "$package") + + jq --indent 4 --argjson repl "$NEW_JSON" ".\"$package\" *= \$repl" "$FILE" > "$FILE".new + mv "$FILE".new "$FILE" + + echo "-- * -- Updated $FILE" + fi +done \ No newline at end of file diff --git a/tools/cpm/check-updates.sh b/tools/cpm/check-updates.sh new file mode 100755 index 0000000000..8aa46b3924 --- /dev/null +++ b/tools/cpm/check-updates.sh @@ -0,0 +1,97 @@ +#!/bin/bash -e + +# SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +# SPDX-License-Identifier: GPL-3.0-or-later + +# SPDX-FileCopyrightText: 2025 crueter +# SPDX-License-Identifier: GPL-3.0-or-later + +# env vars: +# - UPDATE: update if available +# - FORCE: forcefully update + +# shellcheck disable=SC2016 +# shellcheck disable=SC2038 +[ -z "$PACKAGES" ] && PACKAGES=$(find . src -maxdepth 3 -name cpmfile.json | xargs jq -s 'reduce .[] as $item ({}; . * $item)') + +for package in "$@" +do + JSON=$(echo "$PACKAGES" | jq -r ".\"$package\" | select( . != null )") + + [ -z "$JSON" ] && echo "!! No cpmfile definition for $package" && continue + + GIT_HOST=$(jq -r ".git_host" <<< "$JSON") + [ "$GIT_HOST" = null ] && GIT_HOST=github.com + + REPO=$(jq -r ".repo" <<< "$JSON") + TAG=$(jq -r ".tag" <<< "$JSON") + SKIP=$(jq -r ".skip_updates" <<< "$JSON") + + [ "$SKIP" = "true" ] && continue + + [ "$REPO" = null ] && continue # echo "No repo defined for $package, skipping" && continue + [ "$GIT_HOST" != "github.com" ] && continue # echo "Unsupported host $GIT_HOST for $package, skipping" && continue + [ "$TAG" = null ] && continue # echo "No tag defined for $package, skipping" && continue + + echo "-- Package $package" + + GIT_VERSION=$(jq -r ".git_version" <<< "$JSON") + + echo "$TAG" | grep -e "%VERSION%" > /dev/null && HAS_REPLACE=true || HAS_REPLACE=false + ORIGINAL_TAG="$TAG" + + TAG=${TAG//%VERSION%/$GIT_VERSION} + + ARTIFACT=$(jq -r ".artifact" <<< "$JSON") + ARTIFACT=${ARTIFACT//%VERSION%/$GIT_VERSION} + ARTIFACT=${ARTIFACT//%TAG%/$TAG} + + # Use gh-cli to avoid ratelimits lmao + TAGS=$(gh api --method GET "/repos/$REPO/tags") + + # filter out some commonly known annoyances + TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("vulkan-sdk"; "i") | not)]') + TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("yotta"; "i") | not)]') + + # ignore betas/alphas + TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("alpha"; "i") | not)]') + TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("beta"; "i") | not)]') + TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("rc"; "i") | not)]') + + if [ "$HAS_REPLACE" = "true" ]; then + # this just extracts the tag prefix + VERSION_PREFIX=$(echo "$ORIGINAL_TAG" | cut -d"%" -f1) + + # then we strip out the prefix from the new tag, and make that our new git_version + NEW_GIT_VERSION=${LATEST//$VERSION_PREFIX/} + fi + + # thanks fmt + [ "$package" = fmt ] && TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("v0.11"; "i") | not)]') + + LATEST=$(echo "$TAGS" | jq -r '.[0].name') + + [ "$LATEST" = "$TAG" ] && [ "$FORCE" != "true" ] && echo "-- * Up-to-date" && continue + + echo "-- * Version $LATEST available, current is $TAG" + + export USE_TAG=true + HASH=$(tools/cpm/hash.sh "$REPO" "$LATEST") + + echo "-- * New hash: $HASH" + + if [ "$UPDATE" = "true" ]; then + if [ "$HAS_REPLACE" = "true" ]; then + NEW_JSON=$(echo "$JSON" | jq ".hash = \"$HASH\" | .git_version = \"$NEW_GIT_VERSION\"") + else + NEW_JSON=$(echo "$JSON" | jq ".hash = \"$HASH\" | .tag = \"$LATEST\"") + fi + + FILE=$(tools/cpm/which.sh "$package") + + jq --indent 4 --argjson repl "$NEW_JSON" ".\"$package\" *= \$repl" "$FILE" > "$FILE".new + mv "$FILE".new "$FILE" + + echo "-- * -- Updated $FILE" + fi +done diff --git a/tools/cpm/format-json.sh b/tools/cpm/format.sh similarity index 100% rename from tools/cpm/format-json.sh rename to tools/cpm/format.sh diff --git a/tools/cpm/hash.sh b/tools/cpm/hash.sh new file mode 100755 index 0000000000..d9476e19c4 --- /dev/null +++ b/tools/cpm/hash.sh @@ -0,0 +1,22 @@ +#!/bin/sh -e + +# usage: hash.sh repo tag-or-sha +# env vars: GIT_HOST, USE_TAG (use tag instead of sha), ARTIFACT (download artifact with that name instead of src archive) + +REPO="$1" +[ -z "$GIT_HOST" ] && GIT_HOST=github.com +GIT_URL="https://$GIT_HOST/$REPO" + +if [ "$USE_TAG" = "true" ]; then + if [ -z "$ARTIFACT" ]; then + URL="${GIT_URL}/archive/refs/tags/$2.tar.gz" + else + URL="${GIT_URL}/releases/download/$2/${ARTIFACT}" + fi +else + URL="${GIT_URL}/archive/$2.zip" +fi + +SUM=$(wget -q "$URL" -O - | sha512sum) + +echo "$SUM" | cut -d " " -f1 diff --git a/tools/url-hash.sh b/tools/cpm/url-hash.sh similarity index 53% rename from tools/url-hash.sh rename to tools/cpm/url-hash.sh index a54dec8bb2..6709bb4a03 100755 --- a/tools/url-hash.sh +++ b/tools/cpm/url-hash.sh @@ -1,4 +1,4 @@ #!/bin/sh -SUM=`wget -q $1 -O - | sha512sum` +SUM=$(wget -q $1 -O - | sha512sum) echo "$SUM" | cut -d " " -f1 diff --git a/tools/cpm/which.sh b/tools/cpm/which.sh new file mode 100755 index 0000000000..d547bf879d --- /dev/null +++ b/tools/cpm/which.sh @@ -0,0 +1,9 @@ +#!/bin/sh -e + +# check which file a package is in + +JSON=$(find . src -maxdepth 3 -name cpmfile.json -exec grep -l "$1" {} \;) + +[ -z "$JSON" ] && echo "!! No cpmfile definition for $1" + +echo $JSON \ No newline at end of file From 3cf01f06b3cb75f47f558a4aa4fa9fa8ff009ea5 Mon Sep 17 00:00:00 2001 From: crueter Date: Fri, 3 Oct 2025 16:59:53 -0400 Subject: [PATCH 3/9] license headers Signed-off-by: crueter --- tools/cpm/format.sh | 3 +++ tools/cpm/hash.sh | 3 +++ tools/cpm/url-hash.sh | 3 +++ tools/cpm/which.sh | 3 +++ 4 files changed, 12 insertions(+) diff --git a/tools/cpm/format.sh b/tools/cpm/format.sh index 6059077fe1..e636c402b4 100755 --- a/tools/cpm/format.sh +++ b/tools/cpm/format.sh @@ -1,5 +1,8 @@ #!/bin/sh -e +# SPDX-FileCopyrightText: 2025 crueter +# SPDX-License-Identifier: GPL-3.0-or-later + FILES=$(find . src -maxdepth 3 -name cpmfile.json) for file in $FILES; do diff --git a/tools/cpm/hash.sh b/tools/cpm/hash.sh index d9476e19c4..0c653e75c2 100755 --- a/tools/cpm/hash.sh +++ b/tools/cpm/hash.sh @@ -1,5 +1,8 @@ #!/bin/sh -e +# SPDX-FileCopyrightText: 2025 crueter +# SPDX-License-Identifier: GPL-3.0-or-later + # usage: hash.sh repo tag-or-sha # env vars: GIT_HOST, USE_TAG (use tag instead of sha), ARTIFACT (download artifact with that name instead of src archive) diff --git a/tools/cpm/url-hash.sh b/tools/cpm/url-hash.sh index 6709bb4a03..444b14b174 100755 --- a/tools/cpm/url-hash.sh +++ b/tools/cpm/url-hash.sh @@ -1,4 +1,7 @@ #!/bin/sh +# SPDX-FileCopyrightText: 2025 crueter +# SPDX-License-Identifier: GPL-3.0-or-later + SUM=$(wget -q $1 -O - | sha512sum) echo "$SUM" | cut -d " " -f1 diff --git a/tools/cpm/which.sh b/tools/cpm/which.sh index d547bf879d..af12deb1b4 100755 --- a/tools/cpm/which.sh +++ b/tools/cpm/which.sh @@ -1,5 +1,8 @@ #!/bin/sh -e +# SPDX-FileCopyrightText: 2025 crueter +# SPDX-License-Identifier: GPL-3.0-or-later + # check which file a package is in JSON=$(find . src -maxdepth 3 -name cpmfile.json -exec grep -l "$1" {} \;) From 71a87b2c55eaf85582f0cc052e50861778790548 Mon Sep 17 00:00:00 2001 From: MaranBr Date: Fri, 3 Oct 2025 23:08:20 +0200 Subject: [PATCH 4/9] [video_core] Fix stutters and freezes when playing FMV content in some games (#2650) This fixes stutters and freezes when playing FMV content in some games. Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/2650 Co-authored-by: MaranBr Co-committed-by: MaranBr --- src/video_core/engines/maxwell_dma.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/video_core/engines/maxwell_dma.cpp b/src/video_core/engines/maxwell_dma.cpp index 1a8a7c8dce..52ca9bbdb6 100644 --- a/src/video_core/engines/maxwell_dma.cpp +++ b/src/video_core/engines/maxwell_dma.cpp @@ -64,7 +64,6 @@ void MaxwellDMA::Launch() { // TODO(Subv): Perform more research and implement all features of this engine. const LaunchDMA& launch = regs.launch_dma; ASSERT(launch.interrupt_type == LaunchDMA::InterruptType::NONE); - ASSERT(launch.data_transfer_type == LaunchDMA::DataTransferType::NON_PIPELINED); if (launch.multi_line_enable) { const bool is_src_pitch = launch.src_memory_layout == LaunchDMA::MemoryLayout::PITCH; @@ -157,8 +156,6 @@ void MaxwellDMA::Launch() { } void MaxwellDMA::CopyBlockLinearToPitch() { - UNIMPLEMENTED_IF(regs.launch_dma.remap_enable != 0); - u32 bytes_per_pixel = 1; DMA::ImageOperand src_operand; src_operand.bytes_per_pixel = bytes_per_pixel; From 272df1fa837909ab514c8c36a24485fb4bb99add Mon Sep 17 00:00:00 2001 From: lizzie Date: Sat, 4 Oct 2025 02:48:39 +0200 Subject: [PATCH 5/9] [settings] default to opengl on solaris (#2659) Vulkan support still wonky on most distros. Signed-off-by: lizzie Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/2659 Reviewed-by: crueter Reviewed-by: MaranBr Co-authored-by: lizzie Co-committed-by: lizzie --- src/common/settings.h | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/common/settings.h b/src/common/settings.h index 891bde608c..59e75d3ee0 100644 --- a/src/common/settings.h +++ b/src/common/settings.h @@ -320,11 +320,19 @@ struct Values { linkage, true, "cpuopt_unsafe_ignore_global_monitor", Category::CpuUnsafe}; // Renderer - SwitchableSetting renderer_backend{ - linkage, RendererBackend::Vulkan, + SwitchableSetting renderer_backend{linkage, +#if defined(__sun__) || defined(__managarm__) + RendererBackend::OpenGL, +#else + RendererBackend::Vulkan, +#endif "backend", Category::Renderer}; - SwitchableSetting shader_backend{ - linkage, ShaderBackend::SpirV, + SwitchableSetting shader_backend{linkage, +#if defined(__sun__) || defined(__managarm__) + ShaderBackend::Glsl, +#else + ShaderBackend::SpirV, +#endif "shader_backend", Category::Renderer, Specialization::RuntimeList}; SwitchableSetting vulkan_device{linkage, 0, "vulkan_device", Category::Renderer, Specialization::RuntimeList}; From 83730cd4c151bf88066bae0a3af846ae1f4eaa4c Mon Sep 17 00:00:00 2001 From: crueter Date: Sat, 4 Oct 2025 09:27:13 +0200 Subject: [PATCH 6/9] [cmake] update CI deps, feat: sirit CI + new CI spec (#2655) Updates sirit to our fork's latest version w/ SPIRV Headers included (end goal is to remove spirv-headers entirely, as spirv-tools-ci should include them inline as well) Adds a sirit CI on our fork for all platforms (saves a bit of compile time) My CI spec has changed a little bit, and now there is no need for an additional CMake file after the initial CMakeLists.txt (since targets are now global imported). Plus, UNIX amd64 now has the amd64 suffix like aarch64 and windows Updates SDL2 to 2.32.10 and OpenSSL to 3.6.0 Finally, on Solaris all CI packages (sans FFmpeg) are now built with OmniOS, which should in theory be fully compatible with OpenIndiana (our recommended Sun-based target) but obviously will need testing Need testing: - [ ] Make sure I didn't nuke shader emission - [ ] Make sure FreeBSD, OpenBSD, and OpenIndiana work fine with bundled sirit (check linking especially) - [ ] Make sure SDL2, OpenSSL work with OpenIndiana now - [ ] SDL2 on all platforms (input, etc) Signed-off-by: crueter Reviewed-on: https://git.eden-emu.dev/eden-emu/eden/pulls/2655 --- CMakeLists.txt | 181 +++++-------------------- CMakeModules/CPMUtil.cmake | 24 ++-- cpmfile.json | 106 +-------------- docs/CPM.md | 24 ++-- externals/CMakeLists.txt | 159 ++++++++++++++++++++-- externals/cpmfile.json | 120 +++++++++++++++- externals/ffmpeg/cpmfile.json | 5 +- src/qt_common/CMakeLists.txt | 2 +- src/qt_common/externals/CMakeLists.txt | 2 +- src/shader_recompiler/CMakeLists.txt | 2 +- src/video_core/CMakeLists.txt | 2 +- src/yuzu/CMakeLists.txt | 2 - tools/cpm-fetch.sh | 4 +- 13 files changed, 340 insertions(+), 293 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a9ff2e9458..b397ba01aa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -147,6 +147,7 @@ if (ENABLE_SDL2) option(YUZU_USE_BUNDLED_SDL2 "Download bundled SDL2 build" "${MSVC}") endif() +# qt stuff option(ENABLE_QT "Enable the Qt frontend" ON) option(ENABLE_QT_TRANSLATION "Enable translations for the Qt frontend" OFF) option(ENABLE_QT_UPDATE_CHECKER "Enable update checker for the Qt frontend" OFF) @@ -163,8 +164,12 @@ if (MSVC OR ANDROID) endif() option(YUZU_USE_CPM "Use CPM to fetch system dependencies (fmt, boost, etc) if needed. Externals will still be fetched." ${EXT_DEFAULT}) +# ffmpeg option(YUZU_USE_BUNDLED_FFMPEG "Download bundled FFmpeg" ${EXT_DEFAULT}) -cmake_dependent_option(YUZU_USE_EXTERNAL_FFMPEG "Build FFmpeg from source" OFF "NOT WIN32 AND NOT ANDROID" OFF) +cmake_dependent_option(YUZU_USE_EXTERNAL_FFMPEG "Build FFmpeg from source" "${PLATFORM_SUN}" "NOT WIN32 AND NOT ANDROID" OFF) + +# sirit +option(YUZU_USE_BUNDLED_SIRIT "Download bundled sirit" ${EXT_DEFAULT}) cmake_dependent_option(ENABLE_LIBUSB "Enable the use of LibUSB" ON "NOT ANDROID" OFF) @@ -312,7 +317,6 @@ endif() if (ARCHITECTURE_arm64 AND (ANDROID OR PLATFORM_LINUX)) set(HAS_NCE 1) add_compile_definitions(HAS_NCE=1) - find_package(oaknut 2.0.1) endif() if (YUZU_ROOM) @@ -452,22 +456,6 @@ if (YUZU_USE_CPM) add_library(zstd::zstd ALIAS libzstd_static) endif() - # Catch2 - if (YUZU_TESTS OR DYNARMIC_TESTS) - AddJsonPackage(catch2) - endif() - - # ENet - AddJsonPackage(enet) - - if (enet_ADDED) - target_include_directories(enet INTERFACE ${enet_SOURCE_DIR}/include) - endif() - - if (NOT TARGET enet::enet) - add_library(enet::enet ALIAS enet) - endif() - # Opus AddJsonPackage(opus) @@ -482,31 +470,10 @@ if (YUZU_USE_CPM) if (NOT TARGET Opus::opus) add_library(Opus::opus ALIAS opus) endif() - - # VulkanUtilityHeaders - pulls in headers and utility libs - AddJsonPackage(vulkan-utility-headers) - - # small hack - if (NOT VulkanUtilityLibraries_ADDED) - find_package(VulkanHeaders 1.3.274 REQUIRED) - endif() - - # SPIRV Headers - AddJsonPackage(spirv-headers) - - # SPIRV Tools - AddJsonPackage(spirv-tools) - - if (SPIRV-Tools_ADDED) - add_library(SPIRV-Tools::SPIRV-Tools ALIAS SPIRV-Tools-static) - target_link_libraries(SPIRV-Tools-static PRIVATE SPIRV-Tools-opt SPIRV-Tools-link) - endif() - - # mbedtls - AddJsonPackage(mbedtls) else() # Enforce the search mode of non-required packages for better and shorter failure messages find_package(fmt 8 REQUIRED) + if (NOT YUZU_DISABLE_LLVM) find_package(LLVM MODULE COMPONENTS Demangle) endif() @@ -515,7 +482,7 @@ else() find_package(lz4 REQUIRED) find_package(RenderDoc MODULE) find_package(stb MODULE) - find_package(enet 1.3 MODULE REQUIRED) + find_package(Opus 1.3 MODULE REQUIRED) find_package(ZLIB 1.2 REQUIRED) find_package(zstd 1.5 REQUIRED MODULE) @@ -527,29 +494,6 @@ else() find_package(Boost 1.57.0 REQUIRED) endif() - # OpenBSD does not package mbedtls3 (only 2) - if (PLATFORM_OPENBSD) - AddJsonPackage(mbedtls) - else() - find_package(MbedTLS 3 REQUIRED) - endif() - - find_package(VulkanUtilityLibraries REQUIRED) - find_package(VulkanHeaders 1.3.274 REQUIRED) - - # FreeBSD does not package spirv-headers - if (PLATFORM_FREEBSD) - AddJsonPackage(spirv-headers) - else() - find_package(SPIRV-Headers 1.3.274 REQUIRED) - endif() - - find_package(SPIRV-Tools MODULE REQUIRED) - - if (YUZU_TESTS) - find_package(Catch2 3.0.1 REQUIRED) - endif() - if (CMAKE_SYSTEM_NAME STREQUAL "Linux" OR ANDROID) find_package(gamemode 1.7 MODULE) endif() @@ -563,90 +507,6 @@ if(NOT TARGET Boost::headers) AddJsonPackage(boost_headers) endif() -# DiscordRPC -if (USE_DISCORD_PRESENCE) - if (ARCHITECTURE_arm64) - add_compile_definitions(RAPIDJSON_ENDIAN=RAPIDJSON_LITTLEENDIAN) - endif() - - AddJsonPackage(discord-rpc) - - target_include_directories(discord-rpc INTERFACE ${discord-rpc_SOURCE_DIR}/include) - add_library(DiscordRPC::discord-rpc ALIAS discord-rpc) -endif() - -# SimpleIni -AddJsonPackage(simpleini) - -# Most linux distros don't package cubeb, so enable regardless of cpm settings -if(ENABLE_CUBEB) - AddJsonPackage(cubeb) - - if (cubeb_ADDED) - if (NOT MSVC) - if (TARGET speex) - target_compile_options(speex PRIVATE -Wno-sign-compare) - endif() - - set_target_properties(cubeb PROPERTIES COMPILE_OPTIONS "") - target_compile_options(cubeb INTERFACE - -Wno-implicit-const-int-float-conversion - -Wno-shadow - -Wno-missing-declarations - -Wno-return-type - -Wno-uninitialized - ) - else() - target_compile_options(cubeb PRIVATE - /wd4456 - /wd4458 - ) - endif() - endif() - - if (NOT TARGET cubeb::cubeb) - add_library(cubeb::cubeb ALIAS cubeb) - endif() -endif() - -# find SDL2 exports a bunch of variables that are needed, so its easier to do this outside of the YUZU_find_package -if (ENABLE_SDL2) - if (YUZU_USE_EXTERNAL_SDL2) - message(STATUS "Using SDL2 from externals.") - if (NOT WIN32) - # Yuzu itself needs: Atomic Audio Events Joystick Haptic Sensor Threads Timers - # Since 2.0.18 Atomic+Threads required for HIDAPI/libusb (see https://github.com/libsdl-org/SDL/issues/5095) - # Yuzu-cmd also needs: Video (depends on Loadso/Dlopen) - # CPUinfo also required for SDL Audio, at least until 2.28.0 (see https://github.com/libsdl-org/SDL/issues/7809) - set(SDL_UNUSED_SUBSYSTEMS - File Filesystem - Locale Power Render) - foreach(_SUB ${SDL_UNUSED_SUBSYSTEMS}) - string(TOUPPER ${_SUB} _OPT) - set(SDL_${_OPT} OFF) - endforeach() - - set(HIDAPI ON) - endif() - - if (APPLE) - set(SDL_FILE ON) - endif() - - if ("${YUZU_SYSTEM_PROFILE}" STREQUAL "steamdeck") - set(SDL_PIPEWIRE OFF) # build errors out with this on - AddJsonPackage("sdl2_steamdeck") - else() - AddJsonPackage("sdl2_generic") - endif() - elseif (YUZU_USE_BUNDLED_SDL2) - message(STATUS "Using bundled SDL2") - AddJsonPackage(sdl2) - endif() - - find_package(SDL2 2.26.4 REQUIRED) -endif() - # List of all FFmpeg components required set(FFmpeg_COMPONENTS avcodec @@ -677,6 +537,11 @@ add_subdirectory(externals) # pass targets from externals find_package(libusb) find_package(VulkanMemoryAllocator) +find_package(enet) +find_package(MbedTLS) +find_package(VulkanUtilityLibraries) +find_package(SimpleIni) +find_package(SPIRV-Tools) if (ARCHITECTURE_x86 OR ARCHITECTURE_x86_64) find_package(xbyak) @@ -690,6 +555,26 @@ if (ENABLE_WEB_SERVICE OR ENABLE_QT_UPDATE_CHECKER) find_package(cpp-jwt) endif() +if (ARCHITECTURE_arm64 OR DYNARMIC_TESTS) + find_package(oaknut) +endif() + +if (ENABLE_SDL2) + find_package(SDL2) +endif() + +if (USE_DISCORD_PRESENCE) + find_package(DiscordRPC) +endif() + +if (ENABLE_CUBEB) + find_package(cubeb) +endif() + +if (YUZU_TESTS OR DYNARMIC_TESTS) + find_package(Catch2) +endif() + if (ENABLE_QT) if (YUZU_USE_BUNDLED_QT) download_qt(6.8.3) diff --git a/CMakeModules/CPMUtil.cmake b/CMakeModules/CPMUtil.cmake index d84c069399..3347c7e884 100644 --- a/CMakeModules/CPMUtil.cmake +++ b/CMakeModules/CPMUtil.cmake @@ -107,7 +107,6 @@ function(AddJsonPackage) get_json_element("${object}" name name "${JSON_NAME}") get_json_element("${object}" extension extension "tar.zst") get_json_element("${object}" min_version min_version "") - get_json_element("${object}" cmake_filename cmake_filename "") get_json_element("${object}" raw_disabled disabled_platforms "") if (raw_disabled) @@ -124,7 +123,6 @@ function(AddJsonPackage) EXTENSION ${extension} MIN_VERSION ${min_version} DISABLED_PLATFORMS ${disabled_platforms} - CMAKE_FILENAME ${cmake_filename} ) # pass stuff to parent scope @@ -139,6 +137,7 @@ function(AddJsonPackage) endif() get_json_element("${object}" hash hash "") + get_json_element("${object}" hash_suffix hash_suffix "") get_json_element("${object}" sha sha "") get_json_element("${object}" url url "") get_json_element("${object}" key key "") @@ -208,6 +207,7 @@ function(AddJsonPackage) VERSION "${version}" URL "${url}" HASH "${hash}" + HASH_SUFFIX "${hash_suffix}" SHA "${sha}" REPO "${repo}" KEY "${key}" @@ -533,7 +533,6 @@ function(AddCIPackage) EXTENSION MIN_VERSION DISABLED_PLATFORMS - CMAKE_FILENAME ) cmake_parse_arguments(PKG_ARGS "" "${oneValueArgs}" "" ${ARGN}) @@ -589,25 +588,28 @@ function(AddCIPackage) add_ci_package(android) endif() - if(PLATFORM_SUN AND NOT "solaris" IN_LIST DISABLED_PLATFORMS) - add_ci_package(solaris) + if(PLATFORM_SUN AND NOT "solaris-amd64" IN_LIST DISABLED_PLATFORMS) + add_ci_package(solaris-amd64) endif() - if(PLATFORM_FREEBSD AND NOT "freebsd" IN_LIST DISABLED_PLATFORMS) - add_ci_package(freebsd) + if(PLATFORM_FREEBSD AND NOT "freebsd-amd64" IN_LIST DISABLED_PLATFORMS) + add_ci_package(freebsd-amd64) endif() - if((PLATFORM_LINUX AND ARCHITECTURE_x86_64) AND NOT "linux" IN_LIST DISABLED_PLATFORMS) - add_ci_package(linux) + if((PLATFORM_LINUX AND ARCHITECTURE_x86_64) AND NOT "linux-amd64" IN_LIST DISABLED_PLATFORMS) + add_ci_package(linux-amd64) endif() if((PLATFORM_LINUX AND ARCHITECTURE_arm64) AND NOT "linux-aarch64" IN_LIST DISABLED_PLATFORMS) add_ci_package(linux-aarch64) endif() - if (DEFINED ARTIFACT_DIR) - include(${ARTIFACT_DIR}/${ARTIFACT_CMAKE}.cmake) + # TODO(crueter): macOS amd64/aarch64 split mayhaps + if (APPLE AND NOT "macos-universal" IN_LIST DISABLED_PLATFORMS) + add_ci_package(macos-universal) + endif() + if (DEFINED ARTIFACT_DIR) set(${ARTIFACT_PACKAGE}_ADDED TRUE PARENT_SCOPE) set(${ARTIFACT_PACKAGE}_SOURCE_DIR "${ARTIFACT_DIR}" PARENT_SCOPE) else() diff --git a/cpmfile.json b/cpmfile.json index f1fd5ce1cf..4cf61f0ffc 100644 --- a/cpmfile.json +++ b/cpmfile.json @@ -4,8 +4,11 @@ "package": "OpenSSL", "name": "openssl", "repo": "crueter-ci/OpenSSL", - "version": "3.5.3", - "min_version": "1.1.1" + "version": "3.6.0", + "min_version": "1.1.1", + "disabled_platforms": [ + "macos-universal" + ] }, "boost": { "package": "Boost", @@ -66,20 +69,6 @@ "ZSTD_BUILD_SHARED OFF" ] }, - "catch2": { - "package": "Catch2", - "repo": "catchorg/Catch2", - "sha": "644821ce28", - "hash": "f8795f98acf2c02c0db8e734cc866d5caebab4b4a306e93598b97cb3c0c728dafe8283dce27ffe8d42460e5ae7302f3f32e7e274a7f991b73511ac88eef21b1f", - "version": "3.0.1" - }, - "enet": { - "repo": "lsalzman/enet", - "sha": "2662c0de09", - "hash": "3de1beb4fa3d6b1e03eda8dd1e7580694f854af3ed3975dcdabfdcdf76b97f322b9734d35ea7f185855bb490d957842b938b26da4dd2dfded509390f8d2794dd", - "version": "1.3", - "find_args": "MODULE" - }, "opus": { "package": "Opus", "repo": "crueter/opus", @@ -91,97 +80,12 @@ "OPUS_PRESUME_NEON ON" ] }, - "vulkan-utility-headers": { - "package": "VulkanUtilityLibraries", - "repo": "scripts/VulkanUtilityHeaders", - "tag": "1.4.326", - "artifact": "VulkanUtilityHeaders.tar.zst", - "git_host": "git.crueter.xyz", - "hash": "5924629755cb1605c4aa4eee20ef7957a9dd8d61e4df548be656d98054f2730c4109693c1bd35811f401f4705d2ccff9fc849be32b0d8480bc3f73541a5e0964" - }, - "spirv-tools": { - "package": "SPIRV-Tools", - "repo": "KhronosGroup/SPIRV-Tools", - "sha": "40eb301f32", - "hash": "58d0fb1047d69373cf24c73e6f78c73a72a6cca3b4df1d9f083b9dcc0962745ef154abf3dbe9b3623b835be20c6ec769431cf11733349f45e7568b3525f707aa", - "find_args": "MODULE", - "options": [ - "SPIRV_SKIP_EXECUTABLES ON" - ] - }, - "spirv-headers": { - "package": "SPIRV-Headers", - "repo": "KhronosGroup/SPIRV-Headers", - "sha": "4e209d3d7e", - "hash": "f48bbe18341ed55ea0fe280dbbbc0a44bf222278de6e716e143ca1e95ca320b06d4d23d6583fbf8d03e1428f3dac8fa00e5b82ddcd6b425e6236d85af09550a4", - "options": [ - "SPIRV_WERROR OFF" - ] - }, - "mbedtls": { - "package": "MbedTLS", - "repo": "Mbed-TLS/mbedtls", - "tag": "mbedtls-%VERSION%", - "hash": "6671fb8fcaa832e5b115dfdce8f78baa6a4aea71f5c89a640583634cdee27aefe3bf4be075744da91f7c3ae5ea4e0c765c8fc3937b5cfd9ea73d87ef496524da", - "version": "3", - "git_version": "3.6.4", - "artifact": "%TAG%.tar.bz2" - }, - "cubeb": { - "repo": "mozilla/cubeb", - "sha": "fa02160712", - "hash": "82d808356752e4064de48c8fecbe7856715ade1e76b53937116bf07129fc1cc5b3de5e4b408de3cd000187ba8dc32ca4109661cb7e0355a52e54bd81b9be1c61", - "find_args": "CONFIG", - "options": [ - "USE_SANITIZERS OFF", - "BUILD_TESTS OFF", - "BUILD_TOOLS OFF", - "BUNDLE_SPEEX ON" - ] - }, "boost_headers": { "repo": "boostorg/headers", "sha": "95930ca8f5", "hash": "d1dece16f3b209109de02123c537bfe1adf07a62b16c166367e7e5d62e0f7c323bf804c89b3192dd6871bc58a9d879d25a1cc3f7b9da0e497cf266f165816e2a", "bundled": true }, - "discord-rpc": { - "repo": "eden-emulator/discord-rpc", - "sha": "1cf7772bb6", - "hash": "e9b35e6f2c075823257bcd59f06fe7bb2ccce1976f44818d2e28810435ef79c712a3c4f20f40da41f691342a4058cf86b078eb7f9d9e4dae83c0547c21ec4f97" - }, - "simpleini": { - "package": "SimpleIni", - "repo": "brofield/simpleini", - "sha": "09c21bda1d", - "hash": "99779ca9b6e040d36558cadf484f9ffdab5b47bcc8fc72e4d33639d1d60c0ceb4410d335ba445d72a4324e455167fd6769d99b459943aa135bec085dff2d4b7c", - "find_args": "MODULE" - }, - "sdl2_generic": { - "package": "SDL2", - "repo": "libsdl-org/SDL", - "sha": "54772f345a", - "hash": "2a68a0e01c390043aa9d9df63d8a20a52076c88bb460ac4e0f33194ca7d9bc8fadbbcc04e7506872ac4b6354a73fbc267c036f82200da59465789b87c7d9e3a4", - "key": "generic", - "bundled": true - }, - "sdl2_steamdeck": { - "package": "SDL2", - "repo": "libsdl-org/SDL", - "sha": "cc016b0046", - "hash": "34d5ef58da6a4f9efa6689c82f67badcbd741f5a4f562a9c2c30828fa839830fb07681c5dc6a7851520e261c8405a416ac0a2c2513b51984fb3b4fa4dcb3e20b", - "key": "steamdeck", - "bundled": true - }, - "sdl2": { - "ci": true, - "package": "SDL2", - "name": "SDL2", - "repo": "crueter-ci/SDL2", - "version": "2.32.8", - "min_version": "2.26.4", - "cmake_filename": "sdl2" - }, "llvm-mingw": { "repo": "misc/llvm-mingw", "git_host": "git.crueter.xyz", diff --git a/docs/CPM.md b/docs/CPM.md index 03d8a039f9..c01376469d 100644 --- a/docs/CPM.md +++ b/docs/CPM.md @@ -108,7 +108,9 @@ All dependencies must be identifiable in some way for usage in the dependency vi URLs: - `GIT_URL` -- `REPO` as a GitHub repository +- `REPO` as a Git repository + * You may optionally specify `GIT_HOST` to use a custom host, e.g. `GIT_HOST git.crueter.xyz`. Note that the git host MUST be GitHub-like in its artifact/archive downloads, e.g. Forgejo + * If `GIT_HOST` is unspecified, defaults to `github.com` - `URL` Versions (bundled): @@ -135,11 +137,11 @@ Adds a package that follows crueter's CI repository spec. * `windows-amd64` * `windows-arm64` * `android` - * `solaris` - * `freebsd` - * `linux` + * `solaris-amd64` + * `freebsd-amd64` + * `linux-amd64` * `linux-aarch64` -- `CMAKE_FILENAME`: Custom CMake filename, relative to the package root (default `${PACKAGE_ROOT}/${NAME}.cmake`) + * `macos-universal` ### AddJsonPackage @@ -155,10 +157,17 @@ The cpmfile is an object of objects, with each sub-object being named according If `ci` is `false`: - `hash` -> `HASH` +- `hash_suffix` -> `HASH_SUFFIX` - `sha` -> `SHA` +- `key` -> `KEY` - `tag` -> `TAG` + * If the tag contains `%VERSION%`, that part will be replaced by the `git_version`, OR `version` if `git_version` is not specified +- `url` -> `URL` - `artifact` -> `ARTIFACT` + * If the artifact contains `%VERSION%`, that part will be replaced by the `git_version`, OR `version` if `git_version` is not specified + * If the artifact contains `%TAG%`, that part will be replaced by the `tag` (with its replacement already done) - `git_version` -> `GIT_VERSION` +- `git_host` -> `GIT_HOST` - `source_subdir` -> `SOURCE_SUBDIR` - `bundled` -> `BUNDLED_PACKAGE` - `find_args` -> `FIND_PACKAGE_ARGUMENTS` @@ -172,7 +181,6 @@ If `ci` is `true`: - `name` -> `NAME`, defaults to the object key - `extension` -> `EXTENSION`, defaults to `tar.zst` - `min_version` -> `MIN_VERSION` -- `cmake_filename` -> `CMAKE_FILENAME` - `extension` -> `EXTENSION` ### Examples @@ -192,8 +200,8 @@ In order: OpenSSL CI, Boost (tag + artifact), Opus (options + find_args), discor "boost": { "package": "Boost", "repo": "boostorg/boost", - "tag": "boost-1.88.0", - "artifact": "boost-1.88.0-cmake.7z", + "tag": "boost-%VERSION%", + "artifact": "%TAG%-cmake.7z", "hash": "e5b049e5b61964480ca816395f63f95621e66cb9bcf616a8b10e441e0e69f129e22443acb11e77bc1e8170f8e4171b9b7719891efc43699782bfcd4b3a365f01", "git_version": "1.88.0", "version": "1.57" diff --git a/externals/CMakeLists.txt b/externals/CMakeLists.txt index 754ba61a0b..f22b8753f8 100644 --- a/externals/CMakeLists.txt +++ b/externals/CMakeLists.txt @@ -39,6 +39,144 @@ if (ARCHITECTURE_arm64 OR DYNARMIC_TESTS) AddJsonPackage(oaknut) endif() +# enet +AddJsonPackage(enet) + +if (enet_ADDED) + target_include_directories(enet INTERFACE ${enet_SOURCE_DIR}/include) +endif() + +if (NOT TARGET enet::enet) + add_library(enet::enet ALIAS enet) +endif() + +# mbedtls +AddJsonPackage(mbedtls) + +# VulkanUtilityHeaders - pulls in headers and utility libs +AddJsonPackage(vulkan-utility-headers) + +# small hack +if (NOT VulkanUtilityLibraries_ADDED) + find_package(VulkanHeaders 1.3.274 REQUIRED) +endif() + +# DiscordRPC +if (USE_DISCORD_PRESENCE) + if (ARCHITECTURE_arm64) + add_compile_definitions(RAPIDJSON_ENDIAN=RAPIDJSON_LITTLEENDIAN) + endif() + + AddJsonPackage(discord-rpc) + + if (DiscordRPC_ADDED) + target_include_directories(discord-rpc INTERFACE ${DiscordRPC_SOURCE_DIR}/include) + add_library(DiscordRPC::discord-rpc ALIAS discord-rpc) + endif() +endif() + +# SimpleIni +AddJsonPackage(simpleini) + +# Most linux distros don't package cubeb, so enable regardless of cpm settings +if(ENABLE_CUBEB) + AddJsonPackage(cubeb) + + if (cubeb_ADDED) + if (NOT MSVC) + if (TARGET speex) + target_compile_options(speex PRIVATE -Wno-sign-compare) + endif() + + set_target_properties(cubeb PROPERTIES COMPILE_OPTIONS "") + target_compile_options(cubeb INTERFACE + -Wno-implicit-const-int-float-conversion + -Wno-shadow + -Wno-missing-declarations + -Wno-return-type + -Wno-uninitialized + ) + else() + target_compile_options(cubeb PRIVATE + /wd4456 + /wd4458 + ) + endif() + endif() + + if (NOT TARGET cubeb::cubeb) + add_library(cubeb::cubeb ALIAS cubeb) + endif() +endif() + +# find SDL2 exports a bunch of variables that are needed, so its easier to do this outside of the YUZU_find_package +if (ENABLE_SDL2) + if (YUZU_USE_EXTERNAL_SDL2) + message(STATUS "Using SDL2 from externals.") + if (NOT WIN32) + # Yuzu itself needs: Atomic Audio Events Joystick Haptic Sensor Threads Timers + # Since 2.0.18 Atomic+Threads required for HIDAPI/libusb (see https://github.com/libsdl-org/SDL/issues/5095) + # Yuzu-cmd also needs: Video (depends on Loadso/Dlopen) + # CPUinfo also required for SDL Audio, at least until 2.28.0 (see https://github.com/libsdl-org/SDL/issues/7809) + set(SDL_UNUSED_SUBSYSTEMS + File Filesystem + Locale Power Render) + foreach(_SUB ${SDL_UNUSED_SUBSYSTEMS}) + string(TOUPPER ${_SUB} _OPT) + set(SDL_${_OPT} OFF) + endforeach() + + set(HIDAPI ON) + endif() + + if (APPLE) + set(SDL_FILE ON) + endif() + + if ("${YUZU_SYSTEM_PROFILE}" STREQUAL "steamdeck") + set(SDL_PIPEWIRE OFF) # build errors out with this on + AddJsonPackage("sdl2_steamdeck") + else() + AddJsonPackage("sdl2_generic") + endif() + elseif (YUZU_USE_BUNDLED_SDL2) + message(STATUS "Using bundled SDL2") + AddJsonPackage(sdl2) + endif() + + find_package(SDL2 2.26.4 REQUIRED) +endif() + +# SPIRV Headers +# We only need SPIRV-Headers iff spirv-tools is bundled +if (SPIRV-Tools_FORCE_BUNDLED OR CPMUTIL_FORCE_BUNDLED) + set(NEED_SPIRV_HEADERS ON) +else() + find_package(SPIRV-Tools MODULE QUIET) + if (NOT SPIRV-Tools_FOUND) + set(NEED_SPIRV_HEADERS ON) + else() + set(NEED_SPIRV_HEADERS OFF) + endif() +endif() + +if (NEED_SPIRV_HEADERS) + AddJsonPackage(spirv-headers) +endif() + +# SPIRV Tools +AddJsonPackage(spirv-tools) + +if (SPIRV-Tools_ADDED) + add_library(SPIRV-Tools::SPIRV-Tools ALIAS SPIRV-Tools-static) + target_link_libraries(SPIRV-Tools-static PRIVATE SPIRV-Tools-opt SPIRV-Tools-link) +endif() + +# Catch2 +if (YUZU_TESTS OR DYNARMIC_TESTS) + AddJsonPackage(catch2) +endif() + # getopt if (MSVC) add_subdirectory(getopt) @@ -68,15 +206,18 @@ if (VulkanMemoryAllocator_ADDED) endif() # Sirit -AddJsonPackage(sirit) - -if(MSVC AND USE_CCACHE AND sirit_ADDED) - get_target_property(_opts sirit COMPILE_OPTIONS) - list(FILTER _opts EXCLUDE REGEX "/Zi") - list(APPEND _opts "/Z7") - set_target_properties(sirit PROPERTIES COMPILE_OPTIONS "${_opts}") -elseif(MSVC AND CXX_CLANG) - target_compile_options(sirit PRIVATE -Wno-error=unused-command-line-argument) +if (YUZU_USE_BUNDLED_SIRIT) + AddJsonPackage(sirit-ci) +else() + AddJsonPackage(sirit) + if(MSVC AND USE_CCACHE AND sirit_ADDED) + get_target_property(_opts sirit COMPILE_OPTIONS) + list(FILTER _opts EXCLUDE REGEX "/Zi") + list(APPEND _opts "/Z7") + set_target_properties(siritobj PROPERTIES COMPILE_OPTIONS "${_opts}") + elseif(MSVC AND CXX_CLANG) + target_compile_options(siritobj PRIVATE -Wno-error=unused-command-line-argument) + endif() endif() # httplib diff --git a/externals/cpmfile.json b/externals/cpmfile.json index dcafc8f97d..cb32a75ed5 100644 --- a/externals/cpmfile.json +++ b/externals/cpmfile.json @@ -8,12 +8,18 @@ }, "sirit": { "repo": "eden-emulator/sirit", - "sha": "db1f1e8ab5", - "hash": "73eb3a042848c63a10656545797e85f40d142009dfb7827384548a385e1e28e1ac72f42b25924ce530d58275f8638554281e884d72f9c7aaf4ed08690a414b05", - "find_args": "CONFIG", - "options": [ - "SIRIT_USE_SYSTEM_SPIRV_HEADERS ON" - ] + "version": "1.0.1", + "tag": "v%VERSION%", + "artifact": "sirit-source-%VERSION%.tar.zst", + "hash_suffix": "sha512sum", + "find_args": "CONFIG" + }, + "sirit-ci": { + "ci": true, + "package": "sirit", + "name": "sirit", + "repo": "eden-emulator/sirit", + "version": "1.0.0" }, "httplib": { "repo": "yhirose/cpp-httplib", @@ -70,5 +76,107 @@ "sha": "73f3cbb237", "hash": "c08c03063938339d61392b687562909c1a92615b6ef39ec8df19ea472aa6b6478e70d7d5e33d4a27b5d23f7806daf57fe1bacb8124c8a945c918c7663a9e8532", "find_args": "CONFIG" + }, + "mbedtls": { + "package": "MbedTLS", + "repo": "Mbed-TLS/mbedtls", + "tag": "mbedtls-%VERSION%", + "hash": "6671fb8fcaa832e5b115dfdce8f78baa6a4aea71f5c89a640583634cdee27aefe3bf4be075744da91f7c3ae5ea4e0c765c8fc3937b5cfd9ea73d87ef496524da", + "version": "3", + "git_version": "3.6.4", + "artifact": "%TAG%.tar.bz2" + }, + "enet": { + "repo": "lsalzman/enet", + "sha": "2662c0de09", + "hash": "3de1beb4fa3d6b1e03eda8dd1e7580694f854af3ed3975dcdabfdcdf76b97f322b9734d35ea7f185855bb490d957842b938b26da4dd2dfded509390f8d2794dd", + "version": "1.3", + "find_args": "MODULE" + }, + "vulkan-utility-headers": { + "package": "VulkanUtilityLibraries", + "repo": "scripts/VulkanUtilityHeaders", + "tag": "1.4.326", + "artifact": "VulkanUtilityHeaders.tar.zst", + "git_host": "git.crueter.xyz", + "hash": "5924629755cb1605c4aa4eee20ef7957a9dd8d61e4df548be656d98054f2730c4109693c1bd35811f401f4705d2ccff9fc849be32b0d8480bc3f73541a5e0964" + }, + "spirv-tools": { + "package": "SPIRV-Tools", + "repo": "KhronosGroup/SPIRV-Tools", + "sha": "40eb301f32", + "hash": "58d0fb1047d69373cf24c73e6f78c73a72a6cca3b4df1d9f083b9dcc0962745ef154abf3dbe9b3623b835be20c6ec769431cf11733349f45e7568b3525f707aa", + "find_args": "MODULE", + "options": [ + "SPIRV_SKIP_EXECUTABLES ON" + ] + }, + "spirv-headers": { + "package": "SPIRV-Headers", + "repo": "KhronosGroup/SPIRV-Headers", + "sha": "4e209d3d7e", + "hash": "f48bbe18341ed55ea0fe280dbbbc0a44bf222278de6e716e143ca1e95ca320b06d4d23d6583fbf8d03e1428f3dac8fa00e5b82ddcd6b425e6236d85af09550a4", + "options": [ + "SPIRV_WERROR OFF" + ] + }, + "cubeb": { + "repo": "mozilla/cubeb", + "sha": "fa02160712", + "hash": "82d808356752e4064de48c8fecbe7856715ade1e76b53937116bf07129fc1cc5b3de5e4b408de3cd000187ba8dc32ca4109661cb7e0355a52e54bd81b9be1c61", + "find_args": "CONFIG", + "options": [ + "USE_SANITIZERS OFF", + "BUILD_TESTS OFF", + "BUILD_TOOLS OFF", + "BUNDLE_SPEEX ON" + ] + }, + "discord-rpc": { + "package": "DiscordRPC", + "repo": "eden-emulator/discord-rpc", + "sha": "1cf7772bb6", + "hash": "e9b35e6f2c075823257bcd59f06fe7bb2ccce1976f44818d2e28810435ef79c712a3c4f20f40da41f691342a4058cf86b078eb7f9d9e4dae83c0547c21ec4f97" + }, + "simpleini": { + "package": "SimpleIni", + "repo": "brofield/simpleini", + "sha": "09c21bda1d", + "hash": "99779ca9b6e040d36558cadf484f9ffdab5b47bcc8fc72e4d33639d1d60c0ceb4410d335ba445d72a4324e455167fd6769d99b459943aa135bec085dff2d4b7c", + "find_args": "MODULE" + }, + "sdl2_generic": { + "package": "SDL2", + "repo": "libsdl-org/SDL", + "sha": "54772f345a", + "hash": "2a68a0e01c390043aa9d9df63d8a20a52076c88bb460ac4e0f33194ca7d9bc8fadbbcc04e7506872ac4b6354a73fbc267c036f82200da59465789b87c7d9e3a4", + "key": "generic", + "bundled": true + }, + "sdl2_steamdeck": { + "package": "SDL2", + "repo": "libsdl-org/SDL", + "sha": "cc016b0046", + "hash": "34d5ef58da6a4f9efa6689c82f67badcbd741f5a4f562a9c2c30828fa839830fb07681c5dc6a7851520e261c8405a416ac0a2c2513b51984fb3b4fa4dcb3e20b", + "key": "steamdeck", + "bundled": true + }, + "sdl2": { + "ci": true, + "package": "SDL2", + "name": "SDL2", + "repo": "crueter-ci/SDL2", + "version": "2.32.10", + "min_version": "2.26.4", + "disabled_platforms": [ + "macos-universal" + ] + }, + "catch2": { + "package": "Catch2", + "repo": "catchorg/Catch2", + "sha": "644821ce28", + "hash": "f8795f98acf2c02c0db8e734cc866d5caebab4b4a306e93598b97cb3c0c728dafe8283dce27ffe8d42460e5ae7302f3f32e7e274a7f991b73511ac88eef21b1f", + "version": "3.0.1" } } diff --git a/externals/ffmpeg/cpmfile.json b/externals/ffmpeg/cpmfile.json index 9b9efaadde..ff48ce742d 100644 --- a/externals/ffmpeg/cpmfile.json +++ b/externals/ffmpeg/cpmfile.json @@ -13,8 +13,9 @@ "version": "8.0", "min_version": "4.1", "disabled_platforms": [ - "freebsd", - "solaris" + "freebsd-amd64", + "solaris-amd64", + "macos-universal" ] } } diff --git a/src/qt_common/CMakeLists.txt b/src/qt_common/CMakeLists.txt index eb36de4cf2..aa931f113e 100644 --- a/src/qt_common/CMakeLists.txt +++ b/src/qt_common/CMakeLists.txt @@ -39,7 +39,7 @@ endif() add_subdirectory(externals) -target_link_libraries(qt_common PRIVATE core Qt6::Core SimpleIni::SimpleIni QuaZip::QuaZip frozen::frozen) +target_link_libraries(qt_common PRIVATE core Qt6::Core SimpleIni::SimpleIni QuaZip::QuaZip) if (NOT APPLE AND ENABLE_OPENGL) target_compile_definitions(qt_common PUBLIC HAS_OPENGL) diff --git a/src/qt_common/externals/CMakeLists.txt b/src/qt_common/externals/CMakeLists.txt index 189a52c0a6..e7b2e7b3e6 100644 --- a/src/qt_common/externals/CMakeLists.txt +++ b/src/qt_common/externals/CMakeLists.txt @@ -17,4 +17,4 @@ AddJsonPackage(quazip) # frozen # TODO(crueter): Qt String Lookup -AddJsonPackage(frozen) +# AddJsonPackage(frozen) diff --git a/src/shader_recompiler/CMakeLists.txt b/src/shader_recompiler/CMakeLists.txt index 55cdc17c1f..79a4bf4fd2 100644 --- a/src/shader_recompiler/CMakeLists.txt +++ b/src/shader_recompiler/CMakeLists.txt @@ -246,7 +246,7 @@ add_library(shader_recompiler STATIC ) -target_link_libraries(shader_recompiler PUBLIC common fmt::fmt sirit SPIRV-Tools::SPIRV-Tools) +target_link_libraries(shader_recompiler PUBLIC common fmt::fmt sirit::sirit SPIRV-Tools::SPIRV-Tools) if (MSVC) target_compile_options(shader_recompiler PRIVATE diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index 27c8ed9c1d..4a168241a4 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -333,7 +333,7 @@ target_link_options(video_core PRIVATE ${FFmpeg_LDFLAGS}) add_dependencies(video_core host_shaders) target_include_directories(video_core PRIVATE ${HOST_SHADERS_INCLUDE}) -target_link_libraries(video_core PRIVATE sirit) +target_link_libraries(video_core PRIVATE sirit::sirit) # Header-only stuff needed by all dependent targets target_link_libraries(video_core PUBLIC Vulkan::UtilityHeaders GPUOpen::VulkanMemoryAllocator) diff --git a/src/yuzu/CMakeLists.txt b/src/yuzu/CMakeLists.txt index 00e03bd935..c03f7a3abf 100644 --- a/src/yuzu/CMakeLists.txt +++ b/src/yuzu/CMakeLists.txt @@ -397,8 +397,6 @@ if (NOT WIN32) target_include_directories(yuzu PRIVATE ${Qt6Gui_PRIVATE_INCLUDE_DIRS}) endif() -target_link_libraries(yuzu PRIVATE Vulkan::Headers) - if (UNIX AND NOT APPLE) target_link_libraries(yuzu PRIVATE Qt6::DBus) diff --git a/tools/cpm-fetch.sh b/tools/cpm-fetch.sh index 088df8464e..0219845707 100755 --- a/tools/cpm-fetch.sh +++ b/tools/cpm-fetch.sh @@ -84,7 +84,7 @@ ci_package() { echo "-- CI package $PACKAGE" - for platform in windows-amd64 windows-arm64 android solaris freebsd linux linux-aarch64; do + for platform in windows-amd64 windows-arm64 android solaris-amd64 freebsd-amd64 linux-amd64 linux-aarch64 macos-universal; do echo "-- * platform $platform" case $DISABLED in @@ -227,7 +227,7 @@ do HASH_URL="${DOWNLOAD}.${HASH_SUFFIX}" fi - HASH=$(curl "$HASH_URL" -L -o -) + HASH=$(curl "$HASH_URL" -sS -L -o -) fi download_package From 13437a360f63d6efe71449886ada13db95a1af49 Mon Sep 17 00:00:00 2001 From: crueter Date: Fri, 3 Oct 2025 14:39:58 -0400 Subject: [PATCH 7/9] format json script Signed-off-by: crueter --- externals/ffmpeg/cpmfile.json | 2 +- externals/libusb/cpmfile.json | 14 +++++++------- tools/cpm-hash.sh | 2 +- tools/cpm/format-json.sh | 8 ++++++++ 4 files changed, 17 insertions(+), 9 deletions(-) create mode 100755 tools/cpm/format-json.sh diff --git a/externals/ffmpeg/cpmfile.json b/externals/ffmpeg/cpmfile.json index ff48ce742d..97ca227939 100644 --- a/externals/ffmpeg/cpmfile.json +++ b/externals/ffmpeg/cpmfile.json @@ -5,7 +5,7 @@ "hash": "2a89d664119debbb3c006ab1c48d5d7f26e889f4a65ad2e25c8b0503308295123d5a9c5c78bf683aef5ff09acef8c3fc2837f22d3e8c611528b933bf03bcdd97", "bundled": true }, - "ffmpeg-ci": { + "ffmpeg-ci": { "ci": true, "package": "FFmpeg", "name": "ffmpeg", diff --git a/externals/libusb/cpmfile.json b/externals/libusb/cpmfile.json index 0bfa0d7a86..80a0b917db 100644 --- a/externals/libusb/cpmfile.json +++ b/externals/libusb/cpmfile.json @@ -1,8 +1,8 @@ { - "libusb": { - "repo": "libusb/libusb", - "sha": "c060e9ce30", - "hash": "44647357ba1179020cfa6674d809fc35cf6f89bff1c57252fe3a610110f5013ad678fc6eb5918e751d4384c30e2fe678868dbffc5f85736157e546cb9d10accc", - "find_args": "MODULE" - } -} \ No newline at end of file + "libusb": { + "repo": "libusb/libusb", + "sha": "c060e9ce30", + "hash": "44647357ba1179020cfa6674d809fc35cf6f89bff1c57252fe3a610110f5013ad678fc6eb5918e751d4384c30e2fe678868dbffc5f85736157e546cb9d10accc", + "find_args": "MODULE" + } +} diff --git a/tools/cpm-hash.sh b/tools/cpm-hash.sh index da0fb395db..b740936f4d 100755 --- a/tools/cpm-hash.sh +++ b/tools/cpm-hash.sh @@ -1,4 +1,4 @@ #!/bin/sh -SUM=`wget -q https://github.com/$1/archive/$2.zip -O - | sha512sum` +SUM=$(wget -q https://github.com/$1/archive/$2.zip -O - | sha512sum) echo "$SUM" | cut -d " " -f1 diff --git a/tools/cpm/format-json.sh b/tools/cpm/format-json.sh new file mode 100755 index 0000000000..6059077fe1 --- /dev/null +++ b/tools/cpm/format-json.sh @@ -0,0 +1,8 @@ +#!/bin/sh -e + +FILES=$(find . src -maxdepth 3 -name cpmfile.json) + +for file in $FILES; do + jq --indent 4 < $file > $file.new + mv $file.new $file +done From d359415f1dbb82d6858266eadf2ae33e9a72ce0b Mon Sep 17 00:00:00 2001 From: crueter Date: Fri, 3 Oct 2025 16:56:46 -0400 Subject: [PATCH 8/9] [tools] refactor: update/hash checking scripts, use tags for more deps Uses tags for a bunch of deps that can use them Also adds a bunmch of scripts to tools/cpm, notably for checking hashes and checking for updates. TODO: - Documentation - CI target to check hashes - Script to update the json directly (dedup) - Function to filter tags (dedup) - Weekly CI to check for updates Need to get that other CI runner up Signed-off-by: crueter --- cpmfile.json | 27 +- externals/cpmfile.json | 128 ++++---- externals/libusb/cpmfile.json | 7 +- src/dynarmic/externals/cpmfile.json | 7 +- tools/cpm-fetch-all.sh | 7 +- tools/cpm-fetch.sh | 410 ++++++++++++------------ tools/cpm-hash.sh | 4 - tools/cpm/check-all-hashes.sh | 14 + tools/cpm/check-all-updates.sh | 11 + tools/cpm/check-hash.sh | 110 +++++++ tools/cpm/check-updates.sh | 97 ++++++ tools/cpm/{format-json.sh => format.sh} | 0 tools/cpm/hash.sh | 22 ++ tools/{ => cpm}/url-hash.sh | 2 +- tools/cpm/which.sh | 9 + 15 files changed, 566 insertions(+), 289 deletions(-) delete mode 100755 tools/cpm-hash.sh create mode 100755 tools/cpm/check-all-hashes.sh create mode 100755 tools/cpm/check-all-updates.sh create mode 100755 tools/cpm/check-hash.sh create mode 100755 tools/cpm/check-updates.sh rename tools/cpm/{format-json.sh => format.sh} (100%) create mode 100755 tools/cpm/hash.sh rename tools/{ => cpm}/url-hash.sh (53%) create mode 100755 tools/cpm/which.sh diff --git a/cpmfile.json b/cpmfile.json index 4cf61f0ffc..385d10cd81 100644 --- a/cpmfile.json +++ b/cpmfile.json @@ -26,12 +26,9 @@ }, "fmt": { "repo": "fmtlib/fmt", - "sha": "40626af88b", - "hash": "d59f06c24339f223de4ec2afeba1c67b5835a0f350a1ffa86242a72fc3e616a6b8b21798355428d4200c75287308b66634619ffa0b52ba5bd74cc01772ea1a8a", - "version": "8", - "options": [ - "FMT_INSTALL OFF" - ] + "tag": "12.0.0", + "hash": "c4ab814c20fbad7e3f0ae169125a4988a2795631194703251481dc36b18da65c886c4faa9acd046b0a295005217b3689eb0126108a9ba5aac2ca909aae263c2f", + "version": "8" }, "lz4": { "name": "lz4", @@ -43,16 +40,18 @@ "nlohmann": { "package": "nlohmann_json", "repo": "nlohmann/json", - "sha": "55f93686c0", - "hash": "b739749b066800e21154506ea150d2c5cbce8a45344177f46f884547a1399d26753166fd0df8135269ce28cf223552b1b65cd625b88c844d54753f2434900486", - "version": "3.8" + "tag": "v%VERSION%", + "hash": "6cc1e86261f8fac21cc17a33da3b6b3c3cd5c116755651642af3c9e99bb3538fd42c1bd50397a77c8fb6821bc62d90e6b91bcdde77a78f58f2416c62fc53b97d", + "version": "3.8", + "git_version": "3.12.0" }, "zlib": { "package": "ZLIB", "repo": "madler/zlib", - "sha": "51b7f2abda", - "hash": "16eaf1f3752489d12fd9ce30f7b5f7cbd5cb8ff53d617005a9847ae72d937f65e01e68be747f62d7ac19fd0c9aeba9956e60f16d6b465c5fdc2f3d08b4db2e6c", + "tag": "v%VERSION%", + "hash": "8c9642495bafd6fad4ab9fb67f09b268c69ff9af0f4f20cf15dfc18852ff1f312bd8ca41de761b3f8d8e90e77d79f2ccacd3d4c5b19e475ecf09d021fdfe9088", "version": "1.2", + "git_version": "1.3.1", "options": [ "ZLIB_BUILD_SHARED OFF", "ZLIB_INSTALL OFF" @@ -60,8 +59,8 @@ }, "zstd": { "repo": "facebook/zstd", - "sha": "f8745da6ff", - "hash": "3037007f990040fe32573b46f9bef8762fd5dbeeb07ffffcbfeba51ec98167edae39bb9c87f9299efcd61c4e467c5e84f7c19f0df7799bc1fc04864a278792ee", + "sha": "b8d6101fba", + "hash": "a6c8e5272214fd3e65e03ae4fc375f452bd2f646623886664ee23e239e35751cfc842db4d34a84a8039d89fc8f76556121f2a4ae350d017bdff5e22150f9c3de", "version": "1.5", "source_subdir": "build/cmake", "find_args": "MODULE", @@ -89,7 +88,7 @@ "llvm-mingw": { "repo": "misc/llvm-mingw", "git_host": "git.crueter.xyz", - "tag": "20250828", + "tag": "%VERSION%", "version": "20250828", "artifact": "clang-rt-builtins.tar.zst", "hash": "d902392caf94e84f223766e2cc51ca5fab6cae36ab8dc6ef9ef6a683ab1c483bfcfe291ef0bd38ab16a4ecc4078344fa8af72da2f225ab4c378dee23f6186181" diff --git a/externals/cpmfile.json b/externals/cpmfile.json index cb32a75ed5..1ff392923f 100644 --- a/externals/cpmfile.json +++ b/externals/cpmfile.json @@ -2,13 +2,14 @@ "vulkan-memory-allocator": { "package": "VulkanMemoryAllocator", "repo": "GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator", - "sha": "1076b348ab", - "hash": "a46b44e4286d08cffda058e856c47f44c7fed3da55fe9555976eb3907fdcc20ead0b1860b0c38319cda01dbf9b1aa5d4b4038c7f1f8fbd97283d837fa9af9772", - "find_args": "CONFIG" + "tag": "v%VERSION%", + "hash": "deb5902ef8db0e329fbd5f3f4385eb0e26bdd9f14f3a2334823fb3fe18f36bc5d235d620d6e5f6fe3551ec3ea7038638899db8778c09f6d5c278f5ff95c3344b", + "find_args": "CONFIG", + "git_version": "3.3.0" }, "sirit": { "repo": "eden-emulator/sirit", - "version": "1.0.1", + "git_version": "1.0.1", "tag": "v%VERSION%", "artifact": "sirit-source-%VERSION%.tar.zst", "hash_suffix": "sha512sum", @@ -23,8 +24,9 @@ }, "httplib": { "repo": "yhirose/cpp-httplib", - "sha": "a609330e4c", - "hash": "dd3fd0572f8367d8549e1319fd98368b3e75801a293b0c3ac9b4adb806473a4506a484b3d389dc5bee5acc460cb90af7a20e5df705a1696b56496b30b9ce7ed2" + "tag": "v%VERSION%", + "hash": "b364500f76e2ecb0fe21b032d831272e3f1dfeea71af74e325f8fc4ce9dcdb3c941b97a5b422bdeafb9facd058597b90f8bfc284fb9afe3c33fefa15dd5a010b", + "git_version": "0.26.0" }, "cpp-jwt": { "version": "1.4", @@ -39,16 +41,18 @@ "xbyak_sun": { "package": "xbyak", "repo": "herumi/xbyak", - "sha": "9bb219333a", - "hash": "303165d45c8c19387ec49d9fda7d7a4e0d86d4c0153898c23f25ce2d58ece567f44c0bbbfe348239b933edb6e1a1e34f4bc1c0ab3a285bee5da0e548879387b0", - "bundled": true + "tag": "v%VERSION%", + "hash": "e84992c65ad62c577e2746ec5180132fd2875166d1e6b1521a0ff619787e1645792fe5f6a858fe94ed66f297912b6a6b89a509b5d5f5e81a2db1dd7e6790b1f5", + "bundled": true, + "git_version": "7.30" }, "xbyak": { "package": "xbyak", "repo": "herumi/xbyak", - "sha": "4e44f4614d", - "hash": "5824e92159e07fa36a774aedd3b3ef3541d0241371d522cffa4ab3e1f215fa5097b1b77865b47b2481376c704fa079875557ea463ca63d0a7fd6a8a20a589e70", - "bundled": true + "sha": "v7.22", + "hash": "ff4a461b35ced007adbe9b388a90e4b53c46f9250260a559c04a757ad34ac482baf92700f113c106af6512d6894e849c4409efaf1c003207bf245a5910d848a1", + "bundled": true, + "skip_updates": true }, "oaknut": { "version": "2.0.1", @@ -66,16 +70,17 @@ }, "oboe": { "repo": "google/oboe", - "sha": "2bc873e53c", - "hash": "02329058a7f9cf7d5039afaae5ab170d9f42f60f4c01e21eaf4f46073886922b057a9ae30eeac040b3ac182f51b9c1bfe9fe1050a2c9f6ce567a1a9a0ec2c768", + "tag": "1.10.0", + "hash": "ce4011afe7345370d4ead3b891cd69a5ef224b129535783586c0ca75051d303ed446e6c7f10bde8da31fff58d6e307f1732a3ffd03b249f9ef1fd48fd4132715", "bundled": true }, "unordered-dense": { "package": "unordered_dense", "repo": "martinus/unordered_dense", - "sha": "73f3cbb237", - "hash": "c08c03063938339d61392b687562909c1a92615b6ef39ec8df19ea472aa6b6478e70d7d5e33d4a27b5d23f7806daf57fe1bacb8124c8a945c918c7663a9e8532", - "find_args": "CONFIG" + "tag": "v%VERSION%", + "hash": "f9c819e28e1c1a387acfee09277d6af5e366597a0d39acf1c687acf0608a941ba966af8aaebdb8fba0126c7360269c4a51754ef4cab17c35c01a30215f953368", + "find_args": "CONFIG", + "git_version": "4.5.0" }, "mbedtls": { "package": "MbedTLS", @@ -84,29 +89,32 @@ "hash": "6671fb8fcaa832e5b115dfdce8f78baa6a4aea71f5c89a640583634cdee27aefe3bf4be075744da91f7c3ae5ea4e0c765c8fc3937b5cfd9ea73d87ef496524da", "version": "3", "git_version": "3.6.4", - "artifact": "%TAG%.tar.bz2" + "artifact": "%TAG%.tar.bz2", + "skip_updates": true }, "enet": { "repo": "lsalzman/enet", - "sha": "2662c0de09", - "hash": "3de1beb4fa3d6b1e03eda8dd1e7580694f854af3ed3975dcdabfdcdf76b97f322b9734d35ea7f185855bb490d957842b938b26da4dd2dfded509390f8d2794dd", + "tag": "v%VERSION%", + "hash": "a0d2fa8c957704dd49e00a726284ac5ca034b50b00d2b20a94fa1bbfbb80841467834bfdc84aa0ed0d6aab894608fd6c86c3b94eee46343f0e6d9c22e391dbf9", "version": "1.3", + "git_version": "1.3.18", "find_args": "MODULE" }, "vulkan-utility-headers": { "package": "VulkanUtilityLibraries", "repo": "scripts/VulkanUtilityHeaders", - "tag": "1.4.326", + "tag": "1.4.328", "artifact": "VulkanUtilityHeaders.tar.zst", "git_host": "git.crueter.xyz", - "hash": "5924629755cb1605c4aa4eee20ef7957a9dd8d61e4df548be656d98054f2730c4109693c1bd35811f401f4705d2ccff9fc849be32b0d8480bc3f73541a5e0964" + "hash": "9922217b39faf73cd4fc1510f2fdba14a49aa5c0d77f9ee24ee0512cef16b234d0cabc83c1fec861fa5df1d43e7f086ca9b6501753899119f39c5ca530cb0dae" }, - "spirv-tools": { + "spirv-tools": { "package": "SPIRV-Tools", "repo": "KhronosGroup/SPIRV-Tools", - "sha": "40eb301f32", - "hash": "58d0fb1047d69373cf24c73e6f78c73a72a6cca3b4df1d9f083b9dcc0962745ef154abf3dbe9b3623b835be20c6ec769431cf11733349f45e7568b3525f707aa", + "tag": "v%VERSION%", + "hash": "b17940433ced72e004c5eeffd7dd411b6afcc6a52ee31de6427d88edceb8172369be8ec8bf5b65708a78bf41fdae264d554aa7750b2209831679ab36bc867567", "find_args": "MODULE", + "git_version": "2025.4", "options": [ "SPIRV_SKIP_EXECUTABLES ON" ] @@ -132,35 +140,6 @@ "BUNDLE_SPEEX ON" ] }, - "discord-rpc": { - "package": "DiscordRPC", - "repo": "eden-emulator/discord-rpc", - "sha": "1cf7772bb6", - "hash": "e9b35e6f2c075823257bcd59f06fe7bb2ccce1976f44818d2e28810435ef79c712a3c4f20f40da41f691342a4058cf86b078eb7f9d9e4dae83c0547c21ec4f97" - }, - "simpleini": { - "package": "SimpleIni", - "repo": "brofield/simpleini", - "sha": "09c21bda1d", - "hash": "99779ca9b6e040d36558cadf484f9ffdab5b47bcc8fc72e4d33639d1d60c0ceb4410d335ba445d72a4324e455167fd6769d99b459943aa135bec085dff2d4b7c", - "find_args": "MODULE" - }, - "sdl2_generic": { - "package": "SDL2", - "repo": "libsdl-org/SDL", - "sha": "54772f345a", - "hash": "2a68a0e01c390043aa9d9df63d8a20a52076c88bb460ac4e0f33194ca7d9bc8fadbbcc04e7506872ac4b6354a73fbc267c036f82200da59465789b87c7d9e3a4", - "key": "generic", - "bundled": true - }, - "sdl2_steamdeck": { - "package": "SDL2", - "repo": "libsdl-org/SDL", - "sha": "cc016b0046", - "hash": "34d5ef58da6a4f9efa6689c82f67badcbd741f5a4f562a9c2c30828fa839830fb07681c5dc6a7851520e261c8405a416ac0a2c2513b51984fb3b4fa4dcb3e20b", - "key": "steamdeck", - "bundled": true - }, "sdl2": { "ci": true, "package": "SDL2", @@ -175,8 +154,43 @@ "catch2": { "package": "Catch2", "repo": "catchorg/Catch2", - "sha": "644821ce28", - "hash": "f8795f98acf2c02c0db8e734cc866d5caebab4b4a306e93598b97cb3c0c728dafe8283dce27ffe8d42460e5ae7302f3f32e7e274a7f991b73511ac88eef21b1f", - "version": "3.0.1" + "tag": "v%VERSION%", + "hash": "a95495142f915d6e9c2a23e80fe360343e9097680066a2f9d3037a070ba5f81ee5559a0407cc9e972dc2afae325873f1fc7ea07a64012c0f01aac6e549f03e3f", + "version": "3.0.1", + "git_version": "3.11.0" + }, + "discord-rpc": { + "package": "DiscordRPC", + "repo": "eden-emulator/discord-rpc", + "sha": "1cf7772bb6", + "hash": "e9b35e6f2c075823257bcd59f06fe7bb2ccce1976f44818d2e28810435ef79c712a3c4f20f40da41f691342a4058cf86b078eb7f9d9e4dae83c0547c21ec4f97", + "find_args": "MODULE" + }, + "simpleini": { + "package": "SimpleIni", + "repo": "brofield/simpleini", + "tag": "v%VERSION%", + "hash": "6c198636816a0018adbf7f735d402c64245c6fcd540b7360d4388d46f007f3a520686cdaec4705cb8cb31401b2cb4797a80b42ea5d08a6a5807c0848386f7ca1", + "find_args": "MODULE", + "git_version": "4.22" + }, + "sdl2_generic": { + "package": "SDL2", + "repo": "libsdl-org/SDL", + "tag": "release-%VERSION%", + "hash": "d5622d6bb7266f7942a7b8ad43e8a22524893bf0c2ea1af91204838d9b78d32768843f6faa248757427b8404b8c6443776d4afa6b672cd8571a4e0c03a829383", + "key": "generic", + "bundled": true, + "git_version": "2.32.10", + "skip_updates": true + }, + "sdl2_steamdeck": { + "package": "SDL2", + "repo": "libsdl-org/SDL", + "sha": "cc016b0046", + "hash": "34d5ef58da6a4f9efa6689c82f67badcbd741f5a4f562a9c2c30828fa839830fb07681c5dc6a7851520e261c8405a416ac0a2c2513b51984fb3b4fa4dcb3e20b", + "key": "steamdeck", + "bundled": true, + "skip_updates": "true" } } diff --git a/externals/libusb/cpmfile.json b/externals/libusb/cpmfile.json index 80a0b917db..dc69841ab7 100644 --- a/externals/libusb/cpmfile.json +++ b/externals/libusb/cpmfile.json @@ -1,8 +1,9 @@ { "libusb": { "repo": "libusb/libusb", - "sha": "c060e9ce30", - "hash": "44647357ba1179020cfa6674d809fc35cf6f89bff1c57252fe3a610110f5013ad678fc6eb5918e751d4384c30e2fe678868dbffc5f85736157e546cb9d10accc", - "find_args": "MODULE" + "tag": "v%VERSION%", + "hash": "98c5f7940ff06b25c9aa65aa98e23de4c79a4c1067595f4c73cc145af23a1c286639e1ba11185cd91bab702081f307b973f08a4c9746576dc8d01b3620a3aeb5", + "find_args": "MODULE", + "git_version": "1.0.29" } } diff --git a/src/dynarmic/externals/cpmfile.json b/src/dynarmic/externals/cpmfile.json index 718163baf5..099ded57a4 100644 --- a/src/dynarmic/externals/cpmfile.json +++ b/src/dynarmic/externals/cpmfile.json @@ -1,9 +1,10 @@ { "biscuit": { - "version": "0.9.1", "repo": "lioncash/biscuit", - "sha": "76b0be8dae", - "hash": "47d55ed02d032d6cf3dc107c6c0a9aea686d5f25aefb81d1af91db027b6815bd5add1755505e19d76625feeb17aa2db6cd1668fe0dad2e6a411519bde6ca4489" + "tag": "v%VERSION%", + "hash": "1229f345b014f7ca544dedb4edb3311e41ba736f9aa9a67f88b5f26f3c983288c6bb6cdedcfb0b8a02c63088a37e6a0d7ba97d9c2a4d721b213916327cffe28a", + "version": "0.9.1", + "git_version": "0.19.0" }, "mcl": { "version": "0.1.12", diff --git a/tools/cpm-fetch-all.sh b/tools/cpm-fetch-all.sh index 9d5005ec44..9dc1ac2a55 100755 --- a/tools/cpm-fetch-all.sh +++ b/tools/cpm-fetch-all.sh @@ -1,4 +1,4 @@ -#!/bin/bash -ex +#!/bin/bash -e # SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project # SPDX-License-Identifier: GPL-3.0-or-later @@ -6,6 +6,9 @@ # SPDX-FileCopyrightText: 2025 crueter # SPDX-License-Identifier: GPL-3.0-or-later -LIBS=$(find . src -maxdepth 3 -name cpmfile.json -exec jq -j 'keys_unsorted | join(" ")' {} \; -printf " ") +# shellcheck disable=SC2038 +# shellcheck disable=SC2016 +[ -z "$PACKAGES" ] && PACKAGES=$(find . src -maxdepth 3 -name cpmfile.json | xargs jq -s 'reduce .[] as $item ({}; . * $item)') +LIBS=$(echo "$PACKAGES" | jq -j 'keys_unsorted | join(" ")') tools/cpm-fetch.sh $LIBS \ No newline at end of file diff --git a/tools/cpm-fetch.sh b/tools/cpm-fetch.sh index 0219845707..e9829aa482 100755 --- a/tools/cpm-fetch.sh +++ b/tools/cpm-fetch.sh @@ -10,227 +10,227 @@ mkdir -p $CPM_SOURCE_CACHE +# fetch json once to make lives easier +# shellcheck disable=SC2038 +[ -z "$PACKAGES" ] && PACKAGES=$(find . src -maxdepth 3 -name cpmfile.json | xargs jq -s 'reduce .[] as $item ({}; . * $item)') + ROOTDIR="$PWD" TMP=$(mktemp -d) download_package() { - FILENAME=$(basename "$DOWNLOAD") + FILENAME=$(basename "$DOWNLOAD") - OUTFILE="$TMP/$FILENAME" - - LOWER_PACKAGE=$(tr '[:upper:]' '[:lower:]' <<< "$PACKAGE_NAME") - OUTDIR="${CPM_SOURCE_CACHE}/${LOWER_PACKAGE}/${KEY}" - [ -d "$OUTDIR" ] && return - - curl "$DOWNLOAD" -sS -L -o "$OUTFILE" - - ACTUAL_HASH=$(${HASH_ALGO}sum "$OUTFILE" | cut -d" " -f1) - [ "$ACTUAL_HASH" != "$HASH" ] && echo "!! $FILENAME did not match expected hash; expected $HASH but got $ACTUAL_HASH" && exit 1 - - mkdir -p "$OUTDIR" - - pushd "$OUTDIR" > /dev/null - - case "$FILENAME" in - (*.7z) - 7z x "$OUTFILE" > /dev/null - ;; - (*.tar*) - tar xf "$OUTFILE" > /dev/null - ;; - (*.zip) - unzip "$OUTFILE" > /dev/null - ;; - esac - - # basically if only one real item exists at the top we just move everything from there - # since github and some vendors hate me - DIRS=$(find -maxdepth 1 -type d -o -type f) - - # thanks gnu - if [ $(wc -l <<< "$DIRS") -eq 2 ]; then - SUBDIR=$(find . -maxdepth 1 -type d -not -name ".") - mv "$SUBDIR"/* . - mv "$SUBDIR"/.* . 2>/dev/null || true - rmdir "$SUBDIR" - fi - - if grep -e "patches" <<< "$JSON" > /dev/null; then - PATCHES=$(jq -r '.patches | join(" ")' <<< "$JSON") - for patch in $PATCHES; do - patch --binary -p1 < "$ROOTDIR"/.patch/$package/$patch - done - fi - - popd > /dev/null -} - -ci_package() { - REPO=$(jq -r ".repo" <<< "$JSON") - EXT=$(jq -r '.extension' <<< "$JSON") - [ "$EXT" = null ] && EXT="tar.zst" - - VERSION=$(jq -r ".version" <<< "$JSON") - - NAME=$(jq -r ".name" <<< "$JSON") - [ "$NAME" = null ] && NAME="$PACKAGE" - - PACKAGE=$(jq -r ".package | \"$package\"" <<< "$JSON") - - DISABLED=$(jq -j '.disabled_platforms' <<< "$JSON") - - [ "$REPO" = null ] && echo "No repo defined for CI package $package" && return - - echo "-- CI package $PACKAGE" - - for platform in windows-amd64 windows-arm64 android solaris-amd64 freebsd-amd64 linux-amd64 linux-aarch64 macos-universal; do - echo "-- * platform $platform" - - case $DISABLED in - (*"$platform"*) - echo "-- * -- disabled" - continue - ;; - (*) ;; - esac - - FILENAME="${NAME}-${platform}-${VERSION}.${EXT}" - DOWNLOAD="https://$GIT_HOST/${REPO}/releases/download/v${VERSION}/${FILENAME}" - PACKAGE_NAME="$PACKAGE" - KEY=$platform + OUTFILE="$TMP/$FILENAME" LOWER_PACKAGE=$(tr '[:upper:]' '[:lower:]' <<< "$PACKAGE_NAME") OUTDIR="${CPM_SOURCE_CACHE}/${LOWER_PACKAGE}/${KEY}" - [ -d "$OUTDIR" ] && continue + [ -d "$OUTDIR" ] && return + curl "$DOWNLOAD" -sS -L -o "$OUTFILE" + + ACTUAL_HASH=$(${HASH_ALGO}sum "$OUTFILE" | cut -d" " -f1) + [ "$ACTUAL_HASH" != "$HASH" ] && echo "!! $FILENAME did not match expected hash; expected $HASH but got $ACTUAL_HASH" && exit 1 + + mkdir -p "$OUTDIR" + + pushd "$OUTDIR" > /dev/null + + case "$FILENAME" in + (*.7z) + 7z x "$OUTFILE" > /dev/null + ;; + (*.tar*) + tar xf "$OUTFILE" > /dev/null + ;; + (*.zip) + unzip "$OUTFILE" > /dev/null + ;; + esac + + # basically if only one real item exists at the top we just move everything from there + # since github and some vendors hate me + DIRS=$(find -maxdepth 1 -type d -o -type f) + + # thanks gnu + if [ "$(wc -l <<< "$DIRS")" -eq 2 ]; then + SUBDIR=$(find . -maxdepth 1 -type d -not -name ".") + mv "$SUBDIR"/* . + mv "$SUBDIR"/.* . 2>/dev/null || true + rmdir "$SUBDIR" + fi + + if grep -e "patches" <<< "$JSON" > /dev/null; then + PATCHES=$(jq -r '.patches | join(" ")' <<< "$JSON") + for patch in $PATCHES; do + patch --binary -p1 < "$ROOTDIR"/.patch/$package/$patch + done + fi + + popd > /dev/null +} + +ci_package() { + EXT=$(jq -r '.extension' <<< "$JSON") + [ "$EXT" = null ] && EXT="tar.zst" + + VERSION=$(jq -r ".version" <<< "$JSON") + + NAME=$(jq -r ".name" <<< "$JSON") + [ "$NAME" = null ] && NAME="$PACKAGE" + + PACKAGE=$(jq -r ".package | \"$package\"" <<< "$JSON") + + DISABLED=$(jq -j '.disabled_platforms' <<< "$JSON") + + [ "$REPO" = null ] && echo "No repo defined for CI package $package" && return + + echo "-- CI package $PACKAGE" + + for platform in windows-amd64 windows-arm64 android solaris-amd64 freebsd-amd64 linux-amd64 linux-aarch64 macos-universal; do + echo "-- * platform $platform" + + case $DISABLED in + (*"$platform"*) + echo "-- * -- disabled" + continue + ;; + (*) ;; + esac + + FILENAME="${NAME}-${platform}-${VERSION}.${EXT}" + DOWNLOAD="https://$GIT_HOST/${REPO}/releases/download/v${VERSION}/${FILENAME}" + PACKAGE_NAME="$PACKAGE" + KEY=$platform + + LOWER_PACKAGE=$(tr '[:upper:]' '[:lower:]' <<< "$PACKAGE_NAME") + OUTDIR="${CPM_SOURCE_CACHE}/${LOWER_PACKAGE}/${KEY}" + [ -d "$OUTDIR" ] && continue + + HASH_ALGO=$(jq -r ".hash_algo" <<< "$JSON") + [ "$HASH_ALGO" = null ] && HASH_ALGO=sha512 + + HASH_SUFFIX="${HASH_ALGO}sum" + HASH_URL="${DOWNLOAD}.${HASH_SUFFIX}" + + HASH=$(curl "$HASH_URL" -sS -q -L -o -) + + download_package + done +} + +for package in "$@" +do + # prepare for cancer + JSON=$(echo "$PACKAGES" | jq -r ".\"$package\" | select( . != null )") + + [ -z "$JSON" ] && echo "!! No cpmfile definition for $package" && continue + + PACKAGE_NAME=$(jq -r ".package" <<< "$JSON") + [ "$PACKAGE_NAME" = null ] && PACKAGE_NAME="$package" + + GIT_HOST=$(jq -r ".git_host" <<< "$JSON") + [ "$GIT_HOST" = null ] && GIT_HOST=github.com + + REPO=$(jq -r ".repo" <<< "$JSON") + + CI=$(jq -r ".ci" <<< "$JSON") + if [ "$CI" != null ]; then + ci_package + continue + fi + + VERSION=$(jq -r ".version" <<< "$JSON") + GIT_VERSION=$(jq -r ".git_version" <<< "$JSON") + + [ "$GIT_VERSION" = null ] && GIT_VERSION="$VERSION" + + # url parsing + URL=$(jq -r ".url" <<< "$JSON") + SHA=$(jq -r ".sha" <<< "$JSON") + + VERSION=$(jq -r ".version" <<< "$JSON") + GIT_VERSION=$(jq -r ".git_version" <<< "$JSON") + + if [ "$GIT_VERSION" != null ]; then + VERSION_REPLACE="$GIT_VERSION" + else + VERSION_REPLACE="$VERSION" + fi + + TAG=$(jq -r ".tag" <<< "$JSON") + + TAG=$(sed "s/%VERSION%/$VERSION_REPLACE/" <<< $TAG) + + ARTIFACT=$(jq -r ".artifact" <<< "$JSON") + ARTIFACT=$(sed "s/%VERSION%/$VERSION_REPLACE/" <<< $ARTIFACT) + ARTIFACT=$(sed "s/%TAG%/$TAG/" <<< $ARTIFACT) + + if [ "$URL" != "null" ]; then + DOWNLOAD="$URL" + elif [ "$REPO" != "null" ]; then + GIT_URL="https://$GIT_HOST/$REPO" + + BRANCH=$(jq -r ".branch" <<< "$JSON") + + if [ "$TAG" != "null" ]; then + if [ "$ARTIFACT" != "null" ]; then + DOWNLOAD="${GIT_URL}/releases/download/${TAG}/${ARTIFACT}" + else + DOWNLOAD="${GIT_URL}/archive/refs/tags/${TAG}.tar.gz" + fi + elif [ "$SHA" != "null" ]; then + DOWNLOAD="${GIT_URL}/archive/${SHA}.zip" + else + if [ "$BRANCH" = null ]; then + BRANCH=master + fi + + DOWNLOAD="${GIT_URL}/archive/refs/heads/${BRANCH}.zip" + fi + else + echo "!! No repo or URL defined for $package" + continue + fi + + # key parsing + KEY=$(jq -r ".key" <<< "$JSON") + + if [ "$KEY" = null ]; then + if [ "$SHA" != null ]; then + KEY=$(cut -c1-4 - <<< "$SHA") + elif [ "$GIT_VERSION" != null ]; then + KEY="$GIT_VERSION" + elif [ "$TAG" != null ]; then + KEY="$TAG" + elif [ "$VERSION" != null ]; then + KEY="$VERSION" + else + echo "!! No valid key could be determined for $package. Must define one of: key, sha, tag, version, git_version" + continue + fi + fi + + echo "-- Downloading regular package $package, with key $KEY, from $DOWNLOAD" + + # hash parsing HASH_ALGO=$(jq -r ".hash_algo" <<< "$JSON") [ "$HASH_ALGO" = null ] && HASH_ALGO=sha512 - HASH_SUFFIX="${HASH_ALGO}sum" - HASH_URL="${DOWNLOAD}.${HASH_SUFFIX}" + HASH=$(jq -r ".hash" <<< "$JSON") - HASH=$(curl "$HASH_URL" -sS -q -L -o -) + if [ "$HASH" = null ]; then + HASH_SUFFIX="${HASH_ALGO}sum" + HASH_URL=$(jq -r ".hash_url" <<< "$JSON") + + if [ "$HASH_URL" = null ]; then + HASH_URL="${DOWNLOAD}.${HASH_SUFFIX}" + fi + + HASH=$(curl "$HASH_URL" -Ss -L -o -) + fi download_package - done -} - -for package in $@ -do - # prepare for cancer - # TODO(crueter): Fetch json once? - JSON=$(find . src -maxdepth 3 -name cpmfile.json -exec jq -r ".\"$package\" | select( . != null )" {} \;) - - [ -z "$JSON" ] && echo "!! No cpmfile definition for $package" && continue - - PACKAGE_NAME=$(jq -r ".package" <<< "$JSON") - [ "$PACKAGE_NAME" = null ] && PACKAGE_NAME="$package" - - GIT_HOST=$(jq -r ".git_host" <<< "$JSON") - [ "$GIT_HOST" = null ] && GIT_HOST=github.com - REPO=$(jq -r ".repo" <<< "$JSON") - - CI=$(jq -r ".ci" <<< "$JSON") - if [ "$CI" != null ]; then - ci_package - continue - fi - - VERSION=$(jq -r ".version" <<< "$JSON") - GIT_VERSION=$(jq -r ".git_version" <<< "$JSON") - TAG=$(jq -r ".tag" <<< "$JSON") - SHA=$(jq -r ".sha" <<< "$JSON") - - [ "$GIT_VERSION" = null ] && GIT_VERSION="$VERSION" - [ "$GIT_VERSION" = null ] && GIT_VERSION="$TAG" - - # url parsing WOOOHOOHOHOOHOHOH - URL=$(jq -r ".url" <<< "$JSON") - SHA=$(jq -r ".sha" <<< "$JSON") - - VERSION=$(jq -r ".version" <<< "$JSON") - GIT_VERSION=$(jq -r ".git_version" <<< "$JSON") - - if [ "$GIT_VERSION" != null ]; then - VERSION_REPLACE="$GIT_VERSION" - else - VERSION_REPLACE="$VERSION" - fi - - TAG=$(jq -r ".tag" <<< "$JSON") - - TAG=$(sed "s/%VERSION%/$VERSION_REPLACE/" <<< $TAG) - - ARTIFACT=$(jq -r ".artifact" <<< "$JSON") - ARTIFACT=$(sed "s/%VERSION%/$VERSION_REPLACE/" <<< $ARTIFACT) - ARTIFACT=$(sed "s/%TAG%/$TAG/" <<< $ARTIFACT) - - if [ "$URL" != "null" ]; then - DOWNLOAD="$URL" - elif [ "$REPO" != "null" ]; then - GIT_URL="https://$GIT_HOST/$REPO" - - BRANCH=$(jq -r ".branch" <<< "$JSON") - - if [ "$TAG" != "null" ]; then - if [ "$ARTIFACT" != "null" ]; then - DOWNLOAD="${GIT_URL}/releases/download/${TAG}/${ARTIFACT}" - else - DOWNLOAD="${GIT_URL}/archive/refs/tags/${TAG}.tar.gz" - fi - elif [ "$SHA" != "null" ]; then - DOWNLOAD="${GIT_URL}/archive/${SHA}.zip" - else - if [ "$BRANCH" = null ]; then - BRANCH=master - fi - - DOWNLOAD="${GIT_URL}/archive/refs/heads/${BRANCH}.zip" - fi - else - echo "!! No repo or URL defined for $package" - continue - fi - - # key parsing - KEY=$(jq -r ".key" <<< "$JSON") - - if [ "$KEY" = null ]; then - if [ "$SHA" != null ]; then - KEY=$(cut -c1-4 - <<< "$SHA") - elif [ "$GIT_VERSION" != null ]; then - KEY="$GIT_VERSION" - elif [ "$TAG" != null ]; then - KEY="$TAG" - elif [ "$VERSION" != null ]; then - KEY="$VERSION" - else - echo "!! No valid key could be determined for $package. Must define one of: key, sha, tag, version, git_version" - continue - fi - fi - - echo "-- Downloading regular package $package, with key $KEY, from $DOWNLOAD" - - # hash parsing - HASH_ALGO=$(jq -r ".hash_algo" <<< "$JSON") - [ "$HASH_ALGO" = null ] && HASH_ALGO=sha512 - - HASH=$(jq -r ".hash" <<< "$JSON") - - if [ "$HASH" = null ]; then - HASH_SUFFIX="${HASH_ALGO}sum" - HASH_URL=$(jq -r ".hash_url" <<< "$JSON") - - if [ "$HASH_URL" = null ]; then - HASH_URL="${DOWNLOAD}.${HASH_SUFFIX}" - fi - - HASH=$(curl "$HASH_URL" -sS -L -o -) - fi - - download_package done rm -rf $TMP \ No newline at end of file diff --git a/tools/cpm-hash.sh b/tools/cpm-hash.sh deleted file mode 100755 index b740936f4d..0000000000 --- a/tools/cpm-hash.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -SUM=$(wget -q https://github.com/$1/archive/$2.zip -O - | sha512sum) -echo "$SUM" | cut -d " " -f1 diff --git a/tools/cpm/check-all-hashes.sh b/tools/cpm/check-all-hashes.sh new file mode 100755 index 0000000000..a270c368db --- /dev/null +++ b/tools/cpm/check-all-hashes.sh @@ -0,0 +1,14 @@ +#!/bin/bash -e + +# SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +# SPDX-License-Identifier: GPL-3.0-or-later + +# SPDX-FileCopyrightText: 2025 crueter +# SPDX-License-Identifier: GPL-3.0-or-later + +# shellcheck disable=SC2038 +# shellcheck disable=SC2016 +[ -z "$PACKAGES" ] && PACKAGES=$(find . src -maxdepth 3 -name cpmfile.json | xargs jq -s 'reduce .[] as $item ({}; . * $item)') +LIBS=$(echo "$PACKAGES" | jq -j 'keys_unsorted | join(" ")') + +tools/cpm/check-hash.sh $LIBS \ No newline at end of file diff --git a/tools/cpm/check-all-updates.sh b/tools/cpm/check-all-updates.sh new file mode 100755 index 0000000000..e0c744abf0 --- /dev/null +++ b/tools/cpm/check-all-updates.sh @@ -0,0 +1,11 @@ +#!/bin/bash -e + +# SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +# SPDX-License-Identifier: GPL-3.0-or-later + +# SPDX-FileCopyrightText: 2025 crueter +# SPDX-License-Identifier: GPL-3.0-or-later + +LIBS=$(find . src -maxdepth 3 -name cpmfile.json -exec jq -j 'keys_unsorted | join(" ")' {} \; -printf " ") + +tools/cpm/check-updates.sh $LIBS \ No newline at end of file diff --git a/tools/cpm/check-hash.sh b/tools/cpm/check-hash.sh new file mode 100755 index 0000000000..bf7c70ffcb --- /dev/null +++ b/tools/cpm/check-hash.sh @@ -0,0 +1,110 @@ +#!/bin/bash -e + +# SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +# SPDX-License-Identifier: GPL-3.0-or-later + +# SPDX-FileCopyrightText: 2025 crueter +# SPDX-License-Identifier: GPL-3.0-or-later + +# env vars: +# - UPDATE: update if available +# - FORCE: forcefully update + +# shellcheck disable=SC2016 +# shellcheck disable=SC2038 +[ -z "$PACKAGES" ] && PACKAGES=$(find . src -maxdepth 3 -name cpmfile.json | xargs jq -s 'reduce .[] as $item ({}; . * $item)') + +for package in "$@" +do + JSON=$(echo "$PACKAGES" | jq -r ".\"$package\" | select( . != null )") + + [ -z "$JSON" ] && echo "!! No cpmfile definition for $package" && continue + + GIT_HOST=$(jq -r ".git_host" <<< "$JSON") + [ "$GIT_HOST" = null ] && GIT_HOST=github.com + + CI=$(jq -r ".ci" <<< "$JSON") + if [ "$CI" != null ]; then + continue + fi + + HASH_URL=$(jq -r ".hash_url" <<< "$JSON") + [ "$HASH_URL" != null ] && continue + + REPO=$(jq -r ".repo" <<< "$JSON") + TAG=$(jq -r ".tag" <<< "$JSON") + + URL=$(jq -r ".url" <<< "$JSON") + SHA=$(jq -r ".sha" <<< "$JSON") + + echo "-- Package $package" + + GIT_VERSION=$(jq -r ".git_version" <<< "$JSON") + + VERSION=$(jq -r ".version" <<< "$JSON") + GIT_VERSION=$(jq -r ".git_version" <<< "$JSON") + + if [ "$GIT_VERSION" != null ]; then + VERSION_REPLACE="$GIT_VERSION" + else + VERSION_REPLACE="$VERSION" + fi + + TAG=${TAG//%VERSION%/$VERSION_REPLACE} + + ARTIFACT=$(jq -r ".artifact" <<< "$JSON") + ARTIFACT=${ARTIFACT//%VERSION%/$VERSION_REPLACE} + ARTIFACT=${ARTIFACT//%TAG%/$TAG} + + if [ "$URL" != "null" ]; then + DOWNLOAD="$URL" + elif [ "$REPO" != "null" ]; then + GIT_URL="https://$GIT_HOST/$REPO" + + BRANCH=$(jq -r ".branch" <<< "$JSON") + + if [ "$TAG" != "null" ]; then + if [ "$ARTIFACT" != "null" ]; then + DOWNLOAD="${GIT_URL}/releases/download/${TAG}/${ARTIFACT}" + else + DOWNLOAD="${GIT_URL}/archive/refs/tags/${TAG}.tar.gz" + fi + elif [ "$SHA" != "null" ]; then + DOWNLOAD="${GIT_URL}/archive/${SHA}.zip" + else + if [ "$BRANCH" = null ]; then + BRANCH=master + fi + + DOWNLOAD="${GIT_URL}/archive/refs/heads/${BRANCH}.zip" + fi + else + echo "!! No repo or URL defined for $package" + continue + fi + + # hash parsing + HASH_ALGO=$(jq -r ".hash_algo" <<< "$JSON") + [ "$HASH_ALGO" = null ] && HASH_ALGO=sha512 + + HASH=$(jq -r ".hash" <<< "$JSON") + + [ "$HASH" = null ] && echo "-- * Warning: no hash specified" && continue + + export USE_TAG=true + ACTUAL=$(tools/cpm/url-hash.sh "$DOWNLOAD") + + # shellcheck disable=SC2028 + [ "$ACTUAL" != "$HASH" ] && echo "-- * Expected $HASH" && echo "-- * Got $ACTUAL" + + if [ "$UPDATE" = "true" ] && [ "$ACTUAL" != "$HASH" ]; then + NEW_JSON=$(echo "$JSON" | jq ".hash = \"$ACTUAL\"") + + FILE=$(tools/cpm/which.sh "$package") + + jq --indent 4 --argjson repl "$NEW_JSON" ".\"$package\" *= \$repl" "$FILE" > "$FILE".new + mv "$FILE".new "$FILE" + + echo "-- * -- Updated $FILE" + fi +done \ No newline at end of file diff --git a/tools/cpm/check-updates.sh b/tools/cpm/check-updates.sh new file mode 100755 index 0000000000..8aa46b3924 --- /dev/null +++ b/tools/cpm/check-updates.sh @@ -0,0 +1,97 @@ +#!/bin/bash -e + +# SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +# SPDX-License-Identifier: GPL-3.0-or-later + +# SPDX-FileCopyrightText: 2025 crueter +# SPDX-License-Identifier: GPL-3.0-or-later + +# env vars: +# - UPDATE: update if available +# - FORCE: forcefully update + +# shellcheck disable=SC2016 +# shellcheck disable=SC2038 +[ -z "$PACKAGES" ] && PACKAGES=$(find . src -maxdepth 3 -name cpmfile.json | xargs jq -s 'reduce .[] as $item ({}; . * $item)') + +for package in "$@" +do + JSON=$(echo "$PACKAGES" | jq -r ".\"$package\" | select( . != null )") + + [ -z "$JSON" ] && echo "!! No cpmfile definition for $package" && continue + + GIT_HOST=$(jq -r ".git_host" <<< "$JSON") + [ "$GIT_HOST" = null ] && GIT_HOST=github.com + + REPO=$(jq -r ".repo" <<< "$JSON") + TAG=$(jq -r ".tag" <<< "$JSON") + SKIP=$(jq -r ".skip_updates" <<< "$JSON") + + [ "$SKIP" = "true" ] && continue + + [ "$REPO" = null ] && continue # echo "No repo defined for $package, skipping" && continue + [ "$GIT_HOST" != "github.com" ] && continue # echo "Unsupported host $GIT_HOST for $package, skipping" && continue + [ "$TAG" = null ] && continue # echo "No tag defined for $package, skipping" && continue + + echo "-- Package $package" + + GIT_VERSION=$(jq -r ".git_version" <<< "$JSON") + + echo "$TAG" | grep -e "%VERSION%" > /dev/null && HAS_REPLACE=true || HAS_REPLACE=false + ORIGINAL_TAG="$TAG" + + TAG=${TAG//%VERSION%/$GIT_VERSION} + + ARTIFACT=$(jq -r ".artifact" <<< "$JSON") + ARTIFACT=${ARTIFACT//%VERSION%/$GIT_VERSION} + ARTIFACT=${ARTIFACT//%TAG%/$TAG} + + # Use gh-cli to avoid ratelimits lmao + TAGS=$(gh api --method GET "/repos/$REPO/tags") + + # filter out some commonly known annoyances + TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("vulkan-sdk"; "i") | not)]') + TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("yotta"; "i") | not)]') + + # ignore betas/alphas + TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("alpha"; "i") | not)]') + TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("beta"; "i") | not)]') + TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("rc"; "i") | not)]') + + if [ "$HAS_REPLACE" = "true" ]; then + # this just extracts the tag prefix + VERSION_PREFIX=$(echo "$ORIGINAL_TAG" | cut -d"%" -f1) + + # then we strip out the prefix from the new tag, and make that our new git_version + NEW_GIT_VERSION=${LATEST//$VERSION_PREFIX/} + fi + + # thanks fmt + [ "$package" = fmt ] && TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("v0.11"; "i") | not)]') + + LATEST=$(echo "$TAGS" | jq -r '.[0].name') + + [ "$LATEST" = "$TAG" ] && [ "$FORCE" != "true" ] && echo "-- * Up-to-date" && continue + + echo "-- * Version $LATEST available, current is $TAG" + + export USE_TAG=true + HASH=$(tools/cpm/hash.sh "$REPO" "$LATEST") + + echo "-- * New hash: $HASH" + + if [ "$UPDATE" = "true" ]; then + if [ "$HAS_REPLACE" = "true" ]; then + NEW_JSON=$(echo "$JSON" | jq ".hash = \"$HASH\" | .git_version = \"$NEW_GIT_VERSION\"") + else + NEW_JSON=$(echo "$JSON" | jq ".hash = \"$HASH\" | .tag = \"$LATEST\"") + fi + + FILE=$(tools/cpm/which.sh "$package") + + jq --indent 4 --argjson repl "$NEW_JSON" ".\"$package\" *= \$repl" "$FILE" > "$FILE".new + mv "$FILE".new "$FILE" + + echo "-- * -- Updated $FILE" + fi +done diff --git a/tools/cpm/format-json.sh b/tools/cpm/format.sh similarity index 100% rename from tools/cpm/format-json.sh rename to tools/cpm/format.sh diff --git a/tools/cpm/hash.sh b/tools/cpm/hash.sh new file mode 100755 index 0000000000..d9476e19c4 --- /dev/null +++ b/tools/cpm/hash.sh @@ -0,0 +1,22 @@ +#!/bin/sh -e + +# usage: hash.sh repo tag-or-sha +# env vars: GIT_HOST, USE_TAG (use tag instead of sha), ARTIFACT (download artifact with that name instead of src archive) + +REPO="$1" +[ -z "$GIT_HOST" ] && GIT_HOST=github.com +GIT_URL="https://$GIT_HOST/$REPO" + +if [ "$USE_TAG" = "true" ]; then + if [ -z "$ARTIFACT" ]; then + URL="${GIT_URL}/archive/refs/tags/$2.tar.gz" + else + URL="${GIT_URL}/releases/download/$2/${ARTIFACT}" + fi +else + URL="${GIT_URL}/archive/$2.zip" +fi + +SUM=$(wget -q "$URL" -O - | sha512sum) + +echo "$SUM" | cut -d " " -f1 diff --git a/tools/url-hash.sh b/tools/cpm/url-hash.sh similarity index 53% rename from tools/url-hash.sh rename to tools/cpm/url-hash.sh index a54dec8bb2..6709bb4a03 100755 --- a/tools/url-hash.sh +++ b/tools/cpm/url-hash.sh @@ -1,4 +1,4 @@ #!/bin/sh -SUM=`wget -q $1 -O - | sha512sum` +SUM=$(wget -q $1 -O - | sha512sum) echo "$SUM" | cut -d " " -f1 diff --git a/tools/cpm/which.sh b/tools/cpm/which.sh new file mode 100755 index 0000000000..d547bf879d --- /dev/null +++ b/tools/cpm/which.sh @@ -0,0 +1,9 @@ +#!/bin/sh -e + +# check which file a package is in + +JSON=$(find . src -maxdepth 3 -name cpmfile.json -exec grep -l "$1" {} \;) + +[ -z "$JSON" ] && echo "!! No cpmfile definition for $1" + +echo $JSON \ No newline at end of file From a65eea69c899f6564f2e01b3a710cf1b8a354bbc Mon Sep 17 00:00:00 2001 From: crueter Date: Fri, 3 Oct 2025 16:59:53 -0400 Subject: [PATCH 9/9] license headers Signed-off-by: crueter --- tools/cpm/format.sh | 3 +++ tools/cpm/hash.sh | 3 +++ tools/cpm/url-hash.sh | 3 +++ tools/cpm/which.sh | 3 +++ 4 files changed, 12 insertions(+) diff --git a/tools/cpm/format.sh b/tools/cpm/format.sh index 6059077fe1..e636c402b4 100755 --- a/tools/cpm/format.sh +++ b/tools/cpm/format.sh @@ -1,5 +1,8 @@ #!/bin/sh -e +# SPDX-FileCopyrightText: 2025 crueter +# SPDX-License-Identifier: GPL-3.0-or-later + FILES=$(find . src -maxdepth 3 -name cpmfile.json) for file in $FILES; do diff --git a/tools/cpm/hash.sh b/tools/cpm/hash.sh index d9476e19c4..0c653e75c2 100755 --- a/tools/cpm/hash.sh +++ b/tools/cpm/hash.sh @@ -1,5 +1,8 @@ #!/bin/sh -e +# SPDX-FileCopyrightText: 2025 crueter +# SPDX-License-Identifier: GPL-3.0-or-later + # usage: hash.sh repo tag-or-sha # env vars: GIT_HOST, USE_TAG (use tag instead of sha), ARTIFACT (download artifact with that name instead of src archive) diff --git a/tools/cpm/url-hash.sh b/tools/cpm/url-hash.sh index 6709bb4a03..444b14b174 100755 --- a/tools/cpm/url-hash.sh +++ b/tools/cpm/url-hash.sh @@ -1,4 +1,7 @@ #!/bin/sh +# SPDX-FileCopyrightText: 2025 crueter +# SPDX-License-Identifier: GPL-3.0-or-later + SUM=$(wget -q $1 -O - | sha512sum) echo "$SUM" | cut -d " " -f1 diff --git a/tools/cpm/which.sh b/tools/cpm/which.sh index d547bf879d..af12deb1b4 100755 --- a/tools/cpm/which.sh +++ b/tools/cpm/which.sh @@ -1,5 +1,8 @@ #!/bin/sh -e +# SPDX-FileCopyrightText: 2025 crueter +# SPDX-License-Identifier: GPL-3.0-or-later + # check which file a package is in JSON=$(find . src -maxdepth 3 -name cpmfile.json -exec grep -l "$1" {} \;)