From 67f3bf79c12def74a0c5c6ee79038aac6735b90c Mon Sep 17 00:00:00 2001 From: crueter Date: Fri, 3 Oct 2025 16:56:46 -0400 Subject: [PATCH] [tools] refactor: update/hash checking scripts, use tags for more deps Uses tags for a bunch of deps that can use them Also adds a bunmch of scripts to tools/cpm, notably for checking hashes and checking for updates. TODO: - Documentation - CI target to check hashes - Script to update the json directly (dedup) - Function to filter tags (dedup) - Weekly CI to check for updates Need to get that other CI runner up Signed-off-by: crueter --- cpmfile.json | 69 ++-- externals/cpmfile.json | 37 ++- externals/libusb/cpmfile.json | 7 +- src/dynarmic/externals/cpmfile.json | 7 +- tools/cpm-fetch-all.sh | 7 +- tools/cpm-fetch.sh | 410 ++++++++++++------------ tools/cpm-hash.sh | 4 - tools/cpm/check-all-hashes.sh | 14 + tools/cpm/check-all-updates.sh | 11 + tools/cpm/check-hash.sh | 110 +++++++ tools/cpm/check-updates.sh | 97 ++++++ tools/cpm/{format-json.sh => format.sh} | 0 tools/cpm/hash.sh | 22 ++ tools/{ => cpm}/url-hash.sh | 2 +- tools/cpm/which.sh | 9 + 15 files changed, 541 insertions(+), 265 deletions(-) delete mode 100755 tools/cpm-hash.sh create mode 100755 tools/cpm/check-all-hashes.sh create mode 100755 tools/cpm/check-all-updates.sh create mode 100755 tools/cpm/check-hash.sh create mode 100755 tools/cpm/check-updates.sh rename tools/cpm/{format-json.sh => format.sh} (100%) create mode 100755 tools/cpm/hash.sh rename tools/{ => cpm}/url-hash.sh (53%) create mode 100755 tools/cpm/which.sh diff --git a/cpmfile.json b/cpmfile.json index f1fd5ce1cf..53d0346245 100644 --- a/cpmfile.json +++ b/cpmfile.json @@ -23,12 +23,9 @@ }, "fmt": { "repo": "fmtlib/fmt", - "sha": "40626af88b", - "hash": "d59f06c24339f223de4ec2afeba1c67b5835a0f350a1ffa86242a72fc3e616a6b8b21798355428d4200c75287308b66634619ffa0b52ba5bd74cc01772ea1a8a", - "version": "8", - "options": [ - "FMT_INSTALL OFF" - ] + "tag": "12.0.0", + "hash": "c4ab814c20fbad7e3f0ae169125a4988a2795631194703251481dc36b18da65c886c4faa9acd046b0a295005217b3689eb0126108a9ba5aac2ca909aae263c2f", + "version": "8" }, "lz4": { "name": "lz4", @@ -40,16 +37,18 @@ "nlohmann": { "package": "nlohmann_json", "repo": "nlohmann/json", - "sha": "55f93686c0", - "hash": "b739749b066800e21154506ea150d2c5cbce8a45344177f46f884547a1399d26753166fd0df8135269ce28cf223552b1b65cd625b88c844d54753f2434900486", - "version": "3.8" + "tag": "v%VERSION%", + "hash": "6cc1e86261f8fac21cc17a33da3b6b3c3cd5c116755651642af3c9e99bb3538fd42c1bd50397a77c8fb6821bc62d90e6b91bcdde77a78f58f2416c62fc53b97d", + "version": "3.8", + "git_version": "3.12.0" }, "zlib": { "package": "ZLIB", "repo": "madler/zlib", - "sha": "51b7f2abda", - "hash": "16eaf1f3752489d12fd9ce30f7b5f7cbd5cb8ff53d617005a9847ae72d937f65e01e68be747f62d7ac19fd0c9aeba9956e60f16d6b465c5fdc2f3d08b4db2e6c", + "tag": "v%VERSION%", + "hash": "8c9642495bafd6fad4ab9fb67f09b268c69ff9af0f4f20cf15dfc18852ff1f312bd8ca41de761b3f8d8e90e77d79f2ccacd3d4c5b19e475ecf09d021fdfe9088", "version": "1.2", + "git_version": "1.3.1", "options": [ "ZLIB_BUILD_SHARED OFF", "ZLIB_INSTALL OFF" @@ -57,8 +56,8 @@ }, "zstd": { "repo": "facebook/zstd", - "sha": "f8745da6ff", - "hash": "3037007f990040fe32573b46f9bef8762fd5dbeeb07ffffcbfeba51ec98167edae39bb9c87f9299efcd61c4e467c5e84f7c19f0df7799bc1fc04864a278792ee", + "sha": "b8d6101fba", + "hash": "a6c8e5272214fd3e65e03ae4fc375f452bd2f646623886664ee23e239e35751cfc842db4d34a84a8039d89fc8f76556121f2a4ae350d017bdff5e22150f9c3de", "version": "1.5", "source_subdir": "build/cmake", "find_args": "MODULE", @@ -69,15 +68,17 @@ "catch2": { "package": "Catch2", "repo": "catchorg/Catch2", - "sha": "644821ce28", - "hash": "f8795f98acf2c02c0db8e734cc866d5caebab4b4a306e93598b97cb3c0c728dafe8283dce27ffe8d42460e5ae7302f3f32e7e274a7f991b73511ac88eef21b1f", - "version": "3.0.1" + "tag": "v%VERSION%", + "hash": "a95495142f915d6e9c2a23e80fe360343e9097680066a2f9d3037a070ba5f81ee5559a0407cc9e972dc2afae325873f1fc7ea07a64012c0f01aac6e549f03e3f", + "version": "3.0.1", + "git_version": "3.11.0" }, "enet": { "repo": "lsalzman/enet", - "sha": "2662c0de09", - "hash": "3de1beb4fa3d6b1e03eda8dd1e7580694f854af3ed3975dcdabfdcdf76b97f322b9734d35ea7f185855bb490d957842b938b26da4dd2dfded509390f8d2794dd", + "tag": "v%VERSION%", + "hash": "a0d2fa8c957704dd49e00a726284ac5ca034b50b00d2b20a94fa1bbfbb80841467834bfdc84aa0ed0d6aab894608fd6c86c3b94eee46343f0e6d9c22e391dbf9", "version": "1.3", + "git_version": "1.3.18", "find_args": "MODULE" }, "opus": { @@ -94,17 +95,18 @@ "vulkan-utility-headers": { "package": "VulkanUtilityLibraries", "repo": "scripts/VulkanUtilityHeaders", - "tag": "1.4.326", + "tag": "1.4.328", "artifact": "VulkanUtilityHeaders.tar.zst", "git_host": "git.crueter.xyz", - "hash": "5924629755cb1605c4aa4eee20ef7957a9dd8d61e4df548be656d98054f2730c4109693c1bd35811f401f4705d2ccff9fc849be32b0d8480bc3f73541a5e0964" + "hash": "9922217b39faf73cd4fc1510f2fdba14a49aa5c0d77f9ee24ee0512cef16b234d0cabc83c1fec861fa5df1d43e7f086ca9b6501753899119f39c5ca530cb0dae" }, "spirv-tools": { "package": "SPIRV-Tools", "repo": "KhronosGroup/SPIRV-Tools", - "sha": "40eb301f32", - "hash": "58d0fb1047d69373cf24c73e6f78c73a72a6cca3b4df1d9f083b9dcc0962745ef154abf3dbe9b3623b835be20c6ec769431cf11733349f45e7568b3525f707aa", + "tag": "v%VERSION%", + "hash": "b17940433ced72e004c5eeffd7dd411b6afcc6a52ee31de6427d88edceb8172369be8ec8bf5b65708a78bf41fdae264d554aa7750b2209831679ab36bc867567", "find_args": "MODULE", + "git_version": "2025.4", "options": [ "SPIRV_SKIP_EXECUTABLES ON" ] @@ -125,7 +127,8 @@ "hash": "6671fb8fcaa832e5b115dfdce8f78baa6a4aea71f5c89a640583634cdee27aefe3bf4be075744da91f7c3ae5ea4e0c765c8fc3937b5cfd9ea73d87ef496524da", "version": "3", "git_version": "3.6.4", - "artifact": "%TAG%.tar.bz2" + "artifact": "%TAG%.tar.bz2", + "skip_updates": true }, "cubeb": { "repo": "mozilla/cubeb", @@ -153,17 +156,20 @@ "simpleini": { "package": "SimpleIni", "repo": "brofield/simpleini", - "sha": "09c21bda1d", - "hash": "99779ca9b6e040d36558cadf484f9ffdab5b47bcc8fc72e4d33639d1d60c0ceb4410d335ba445d72a4324e455167fd6769d99b459943aa135bec085dff2d4b7c", - "find_args": "MODULE" + "tag": "v%VERSION%", + "hash": "6c198636816a0018adbf7f735d402c64245c6fcd540b7360d4388d46f007f3a520686cdaec4705cb8cb31401b2cb4797a80b42ea5d08a6a5807c0848386f7ca1", + "find_args": "MODULE", + "git_version": "4.22" }, "sdl2_generic": { "package": "SDL2", "repo": "libsdl-org/SDL", - "sha": "54772f345a", - "hash": "2a68a0e01c390043aa9d9df63d8a20a52076c88bb460ac4e0f33194ca7d9bc8fadbbcc04e7506872ac4b6354a73fbc267c036f82200da59465789b87c7d9e3a4", + "tag": "release-%VERSION%", + "hash": "d58b5b9964fea3c04d13aae646b92e9646125f269088fc6698089f0dbb9506091739cb8a72bfead08aa3952f8b4df0a7886396eeee54a28388a6b89be01f44cf", "key": "generic", - "bundled": true + "bundled": true, + "git_version": "2.32.8", + "skip_updates": true }, "sdl2_steamdeck": { "package": "SDL2", @@ -171,7 +177,8 @@ "sha": "cc016b0046", "hash": "34d5ef58da6a4f9efa6689c82f67badcbd741f5a4f562a9c2c30828fa839830fb07681c5dc6a7851520e261c8405a416ac0a2c2513b51984fb3b4fa4dcb3e20b", "key": "steamdeck", - "bundled": true + "bundled": true, + "skip_updates": "true" }, "sdl2": { "ci": true, @@ -185,7 +192,7 @@ "llvm-mingw": { "repo": "misc/llvm-mingw", "git_host": "git.crueter.xyz", - "tag": "20250828", + "tag": "%VERSION%", "version": "20250828", "artifact": "clang-rt-builtins.tar.zst", "hash": "d902392caf94e84f223766e2cc51ca5fab6cae36ab8dc6ef9ef6a683ab1c483bfcfe291ef0bd38ab16a4ecc4078344fa8af72da2f225ab4c378dee23f6186181" diff --git a/externals/cpmfile.json b/externals/cpmfile.json index dcafc8f97d..4d0b3af9f4 100644 --- a/externals/cpmfile.json +++ b/externals/cpmfile.json @@ -2,9 +2,10 @@ "vulkan-memory-allocator": { "package": "VulkanMemoryAllocator", "repo": "GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator", - "sha": "1076b348ab", - "hash": "a46b44e4286d08cffda058e856c47f44c7fed3da55fe9555976eb3907fdcc20ead0b1860b0c38319cda01dbf9b1aa5d4b4038c7f1f8fbd97283d837fa9af9772", - "find_args": "CONFIG" + "tag": "v%VERSION%", + "hash": "deb5902ef8db0e329fbd5f3f4385eb0e26bdd9f14f3a2334823fb3fe18f36bc5d235d620d6e5f6fe3551ec3ea7038638899db8778c09f6d5c278f5ff95c3344b", + "find_args": "CONFIG", + "git_version": "3.3.0" }, "sirit": { "repo": "eden-emulator/sirit", @@ -17,8 +18,9 @@ }, "httplib": { "repo": "yhirose/cpp-httplib", - "sha": "a609330e4c", - "hash": "dd3fd0572f8367d8549e1319fd98368b3e75801a293b0c3ac9b4adb806473a4506a484b3d389dc5bee5acc460cb90af7a20e5df705a1696b56496b30b9ce7ed2" + "tag": "v%VERSION%", + "hash": "b364500f76e2ecb0fe21b032d831272e3f1dfeea71af74e325f8fc4ce9dcdb3c941b97a5b422bdeafb9facd058597b90f8bfc284fb9afe3c33fefa15dd5a010b", + "git_version": "0.26.0" }, "cpp-jwt": { "version": "1.4", @@ -33,16 +35,18 @@ "xbyak_sun": { "package": "xbyak", "repo": "herumi/xbyak", - "sha": "9bb219333a", - "hash": "303165d45c8c19387ec49d9fda7d7a4e0d86d4c0153898c23f25ce2d58ece567f44c0bbbfe348239b933edb6e1a1e34f4bc1c0ab3a285bee5da0e548879387b0", - "bundled": true + "tag": "v%VERSION%", + "hash": "e84992c65ad62c577e2746ec5180132fd2875166d1e6b1521a0ff619787e1645792fe5f6a858fe94ed66f297912b6a6b89a509b5d5f5e81a2db1dd7e6790b1f5", + "bundled": true, + "git_version": "7.30" }, "xbyak": { "package": "xbyak", "repo": "herumi/xbyak", - "sha": "4e44f4614d", - "hash": "5824e92159e07fa36a774aedd3b3ef3541d0241371d522cffa4ab3e1f215fa5097b1b77865b47b2481376c704fa079875557ea463ca63d0a7fd6a8a20a589e70", - "bundled": true + "sha": "v7.22", + "hash": "ff4a461b35ced007adbe9b388a90e4b53c46f9250260a559c04a757ad34ac482baf92700f113c106af6512d6894e849c4409efaf1c003207bf245a5910d848a1", + "bundled": true, + "skip_updates": true }, "oaknut": { "version": "2.0.1", @@ -60,15 +64,16 @@ }, "oboe": { "repo": "google/oboe", - "sha": "2bc873e53c", - "hash": "02329058a7f9cf7d5039afaae5ab170d9f42f60f4c01e21eaf4f46073886922b057a9ae30eeac040b3ac182f51b9c1bfe9fe1050a2c9f6ce567a1a9a0ec2c768", + "tag": "1.10.0", + "hash": "ce4011afe7345370d4ead3b891cd69a5ef224b129535783586c0ca75051d303ed446e6c7f10bde8da31fff58d6e307f1732a3ffd03b249f9ef1fd48fd4132715", "bundled": true }, "unordered-dense": { "package": "unordered_dense", "repo": "martinus/unordered_dense", - "sha": "73f3cbb237", - "hash": "c08c03063938339d61392b687562909c1a92615b6ef39ec8df19ea472aa6b6478e70d7d5e33d4a27b5d23f7806daf57fe1bacb8124c8a945c918c7663a9e8532", - "find_args": "CONFIG" + "tag": "v%VERSION%", + "hash": "f9c819e28e1c1a387acfee09277d6af5e366597a0d39acf1c687acf0608a941ba966af8aaebdb8fba0126c7360269c4a51754ef4cab17c35c01a30215f953368", + "find_args": "CONFIG", + "git_version": "4.5.0" } } diff --git a/externals/libusb/cpmfile.json b/externals/libusb/cpmfile.json index 80a0b917db..dc69841ab7 100644 --- a/externals/libusb/cpmfile.json +++ b/externals/libusb/cpmfile.json @@ -1,8 +1,9 @@ { "libusb": { "repo": "libusb/libusb", - "sha": "c060e9ce30", - "hash": "44647357ba1179020cfa6674d809fc35cf6f89bff1c57252fe3a610110f5013ad678fc6eb5918e751d4384c30e2fe678868dbffc5f85736157e546cb9d10accc", - "find_args": "MODULE" + "tag": "v%VERSION%", + "hash": "98c5f7940ff06b25c9aa65aa98e23de4c79a4c1067595f4c73cc145af23a1c286639e1ba11185cd91bab702081f307b973f08a4c9746576dc8d01b3620a3aeb5", + "find_args": "MODULE", + "git_version": "1.0.29" } } diff --git a/src/dynarmic/externals/cpmfile.json b/src/dynarmic/externals/cpmfile.json index 718163baf5..099ded57a4 100644 --- a/src/dynarmic/externals/cpmfile.json +++ b/src/dynarmic/externals/cpmfile.json @@ -1,9 +1,10 @@ { "biscuit": { - "version": "0.9.1", "repo": "lioncash/biscuit", - "sha": "76b0be8dae", - "hash": "47d55ed02d032d6cf3dc107c6c0a9aea686d5f25aefb81d1af91db027b6815bd5add1755505e19d76625feeb17aa2db6cd1668fe0dad2e6a411519bde6ca4489" + "tag": "v%VERSION%", + "hash": "1229f345b014f7ca544dedb4edb3311e41ba736f9aa9a67f88b5f26f3c983288c6bb6cdedcfb0b8a02c63088a37e6a0d7ba97d9c2a4d721b213916327cffe28a", + "version": "0.9.1", + "git_version": "0.19.0" }, "mcl": { "version": "0.1.12", diff --git a/tools/cpm-fetch-all.sh b/tools/cpm-fetch-all.sh index 9d5005ec44..9dc1ac2a55 100755 --- a/tools/cpm-fetch-all.sh +++ b/tools/cpm-fetch-all.sh @@ -1,4 +1,4 @@ -#!/bin/bash -ex +#!/bin/bash -e # SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project # SPDX-License-Identifier: GPL-3.0-or-later @@ -6,6 +6,9 @@ # SPDX-FileCopyrightText: 2025 crueter # SPDX-License-Identifier: GPL-3.0-or-later -LIBS=$(find . src -maxdepth 3 -name cpmfile.json -exec jq -j 'keys_unsorted | join(" ")' {} \; -printf " ") +# shellcheck disable=SC2038 +# shellcheck disable=SC2016 +[ -z "$PACKAGES" ] && PACKAGES=$(find . src -maxdepth 3 -name cpmfile.json | xargs jq -s 'reduce .[] as $item ({}; . * $item)') +LIBS=$(echo "$PACKAGES" | jq -j 'keys_unsorted | join(" ")') tools/cpm-fetch.sh $LIBS \ No newline at end of file diff --git a/tools/cpm-fetch.sh b/tools/cpm-fetch.sh index 088df8464e..bdb125601a 100755 --- a/tools/cpm-fetch.sh +++ b/tools/cpm-fetch.sh @@ -10,227 +10,227 @@ mkdir -p $CPM_SOURCE_CACHE +# fetch json once to make lives easier +# shellcheck disable=SC2038 +[ -z "$PACKAGES" ] && PACKAGES=$(find . src -maxdepth 3 -name cpmfile.json | xargs jq -s 'reduce .[] as $item ({}; . * $item)') + ROOTDIR="$PWD" TMP=$(mktemp -d) download_package() { - FILENAME=$(basename "$DOWNLOAD") + FILENAME=$(basename "$DOWNLOAD") - OUTFILE="$TMP/$FILENAME" - - LOWER_PACKAGE=$(tr '[:upper:]' '[:lower:]' <<< "$PACKAGE_NAME") - OUTDIR="${CPM_SOURCE_CACHE}/${LOWER_PACKAGE}/${KEY}" - [ -d "$OUTDIR" ] && return - - curl "$DOWNLOAD" -sS -L -o "$OUTFILE" - - ACTUAL_HASH=$(${HASH_ALGO}sum "$OUTFILE" | cut -d" " -f1) - [ "$ACTUAL_HASH" != "$HASH" ] && echo "!! $FILENAME did not match expected hash; expected $HASH but got $ACTUAL_HASH" && exit 1 - - mkdir -p "$OUTDIR" - - pushd "$OUTDIR" > /dev/null - - case "$FILENAME" in - (*.7z) - 7z x "$OUTFILE" > /dev/null - ;; - (*.tar*) - tar xf "$OUTFILE" > /dev/null - ;; - (*.zip) - unzip "$OUTFILE" > /dev/null - ;; - esac - - # basically if only one real item exists at the top we just move everything from there - # since github and some vendors hate me - DIRS=$(find -maxdepth 1 -type d -o -type f) - - # thanks gnu - if [ $(wc -l <<< "$DIRS") -eq 2 ]; then - SUBDIR=$(find . -maxdepth 1 -type d -not -name ".") - mv "$SUBDIR"/* . - mv "$SUBDIR"/.* . 2>/dev/null || true - rmdir "$SUBDIR" - fi - - if grep -e "patches" <<< "$JSON" > /dev/null; then - PATCHES=$(jq -r '.patches | join(" ")' <<< "$JSON") - for patch in $PATCHES; do - patch --binary -p1 < "$ROOTDIR"/.patch/$package/$patch - done - fi - - popd > /dev/null -} - -ci_package() { - REPO=$(jq -r ".repo" <<< "$JSON") - EXT=$(jq -r '.extension' <<< "$JSON") - [ "$EXT" = null ] && EXT="tar.zst" - - VERSION=$(jq -r ".version" <<< "$JSON") - - NAME=$(jq -r ".name" <<< "$JSON") - [ "$NAME" = null ] && NAME="$PACKAGE" - - PACKAGE=$(jq -r ".package | \"$package\"" <<< "$JSON") - - DISABLED=$(jq -j '.disabled_platforms' <<< "$JSON") - - [ "$REPO" = null ] && echo "No repo defined for CI package $package" && return - - echo "-- CI package $PACKAGE" - - for platform in windows-amd64 windows-arm64 android solaris freebsd linux linux-aarch64; do - echo "-- * platform $platform" - - case $DISABLED in - (*"$platform"*) - echo "-- * -- disabled" - continue - ;; - (*) ;; - esac - - FILENAME="${NAME}-${platform}-${VERSION}.${EXT}" - DOWNLOAD="https://$GIT_HOST/${REPO}/releases/download/v${VERSION}/${FILENAME}" - PACKAGE_NAME="$PACKAGE" - KEY=$platform + OUTFILE="$TMP/$FILENAME" LOWER_PACKAGE=$(tr '[:upper:]' '[:lower:]' <<< "$PACKAGE_NAME") OUTDIR="${CPM_SOURCE_CACHE}/${LOWER_PACKAGE}/${KEY}" - [ -d "$OUTDIR" ] && continue + [ -d "$OUTDIR" ] && return + curl "$DOWNLOAD" -sS -L -o "$OUTFILE" + + ACTUAL_HASH=$(${HASH_ALGO}sum "$OUTFILE" | cut -d" " -f1) + [ "$ACTUAL_HASH" != "$HASH" ] && echo "!! $FILENAME did not match expected hash; expected $HASH but got $ACTUAL_HASH" && exit 1 + + mkdir -p "$OUTDIR" + + pushd "$OUTDIR" > /dev/null + + case "$FILENAME" in + (*.7z) + 7z x "$OUTFILE" > /dev/null + ;; + (*.tar*) + tar xf "$OUTFILE" > /dev/null + ;; + (*.zip) + unzip "$OUTFILE" > /dev/null + ;; + esac + + # basically if only one real item exists at the top we just move everything from there + # since github and some vendors hate me + DIRS=$(find -maxdepth 1 -type d -o -type f) + + # thanks gnu + if [ "$(wc -l <<< "$DIRS")" -eq 2 ]; then + SUBDIR=$(find . -maxdepth 1 -type d -not -name ".") + mv "$SUBDIR"/* . + mv "$SUBDIR"/.* . 2>/dev/null || true + rmdir "$SUBDIR" + fi + + if grep -e "patches" <<< "$JSON" > /dev/null; then + PATCHES=$(jq -r '.patches | join(" ")' <<< "$JSON") + for patch in $PATCHES; do + patch --binary -p1 < "$ROOTDIR"/.patch/$package/$patch + done + fi + + popd > /dev/null +} + +ci_package() { + EXT=$(jq -r '.extension' <<< "$JSON") + [ "$EXT" = null ] && EXT="tar.zst" + + VERSION=$(jq -r ".version" <<< "$JSON") + + NAME=$(jq -r ".name" <<< "$JSON") + [ "$NAME" = null ] && NAME="$PACKAGE" + + PACKAGE=$(jq -r ".package | \"$package\"" <<< "$JSON") + + DISABLED=$(jq -j '.disabled_platforms' <<< "$JSON") + + [ "$REPO" = null ] && echo "No repo defined for CI package $package" && return + + echo "-- CI package $PACKAGE" + + for platform in windows-amd64 windows-arm64 android solaris freebsd linux linux-aarch64; do + echo "-- * platform $platform" + + case $DISABLED in + (*"$platform"*) + echo "-- * -- disabled" + continue + ;; + (*) ;; + esac + + FILENAME="${NAME}-${platform}-${VERSION}.${EXT}" + DOWNLOAD="https://$GIT_HOST/${REPO}/releases/download/v${VERSION}/${FILENAME}" + PACKAGE_NAME="$PACKAGE" + KEY=$platform + + LOWER_PACKAGE=$(tr '[:upper:]' '[:lower:]' <<< "$PACKAGE_NAME") + OUTDIR="${CPM_SOURCE_CACHE}/${LOWER_PACKAGE}/${KEY}" + [ -d "$OUTDIR" ] && continue + + HASH_ALGO=$(jq -r ".hash_algo" <<< "$JSON") + [ "$HASH_ALGO" = null ] && HASH_ALGO=sha512 + + HASH_SUFFIX="${HASH_ALGO}sum" + HASH_URL="${DOWNLOAD}.${HASH_SUFFIX}" + + HASH=$(curl "$HASH_URL" -sS -q -L -o -) + + download_package + done +} + +for package in "$@" +do + # prepare for cancer + JSON=$(echo "$PACKAGES" | jq -r ".\"$package\" | select( . != null )") + + [ -z "$JSON" ] && echo "!! No cpmfile definition for $package" && continue + + PACKAGE_NAME=$(jq -r ".package" <<< "$JSON") + [ "$PACKAGE_NAME" = null ] && PACKAGE_NAME="$package" + + GIT_HOST=$(jq -r ".git_host" <<< "$JSON") + [ "$GIT_HOST" = null ] && GIT_HOST=github.com + + REPO=$(jq -r ".repo" <<< "$JSON") + + CI=$(jq -r ".ci" <<< "$JSON") + if [ "$CI" != null ]; then + ci_package + continue + fi + + VERSION=$(jq -r ".version" <<< "$JSON") + GIT_VERSION=$(jq -r ".git_version" <<< "$JSON") + + [ "$GIT_VERSION" = null ] && GIT_VERSION="$VERSION" + + # url parsing + URL=$(jq -r ".url" <<< "$JSON") + SHA=$(jq -r ".sha" <<< "$JSON") + + VERSION=$(jq -r ".version" <<< "$JSON") + GIT_VERSION=$(jq -r ".git_version" <<< "$JSON") + + if [ "$GIT_VERSION" != null ]; then + VERSION_REPLACE="$GIT_VERSION" + else + VERSION_REPLACE="$VERSION" + fi + + TAG=$(jq -r ".tag" <<< "$JSON") + + TAG=$(sed "s/%VERSION%/$VERSION_REPLACE/" <<< $TAG) + + ARTIFACT=$(jq -r ".artifact" <<< "$JSON") + ARTIFACT=$(sed "s/%VERSION%/$VERSION_REPLACE/" <<< $ARTIFACT) + ARTIFACT=$(sed "s/%TAG%/$TAG/" <<< $ARTIFACT) + + if [ "$URL" != "null" ]; then + DOWNLOAD="$URL" + elif [ "$REPO" != "null" ]; then + GIT_URL="https://$GIT_HOST/$REPO" + + BRANCH=$(jq -r ".branch" <<< "$JSON") + + if [ "$TAG" != "null" ]; then + if [ "$ARTIFACT" != "null" ]; then + DOWNLOAD="${GIT_URL}/releases/download/${TAG}/${ARTIFACT}" + else + DOWNLOAD="${GIT_URL}/archive/refs/tags/${TAG}.tar.gz" + fi + elif [ "$SHA" != "null" ]; then + DOWNLOAD="${GIT_URL}/archive/${SHA}.zip" + else + if [ "$BRANCH" = null ]; then + BRANCH=master + fi + + DOWNLOAD="${GIT_URL}/archive/refs/heads/${BRANCH}.zip" + fi + else + echo "!! No repo or URL defined for $package" + continue + fi + + # key parsing + KEY=$(jq -r ".key" <<< "$JSON") + + if [ "$KEY" = null ]; then + if [ "$SHA" != null ]; then + KEY=$(cut -c1-4 - <<< "$SHA") + elif [ "$GIT_VERSION" != null ]; then + KEY="$GIT_VERSION" + elif [ "$TAG" != null ]; then + KEY="$TAG" + elif [ "$VERSION" != null ]; then + KEY="$VERSION" + else + echo "!! No valid key could be determined for $package. Must define one of: key, sha, tag, version, git_version" + continue + fi + fi + + echo "-- Downloading regular package $package, with key $KEY, from $DOWNLOAD" + + # hash parsing HASH_ALGO=$(jq -r ".hash_algo" <<< "$JSON") [ "$HASH_ALGO" = null ] && HASH_ALGO=sha512 - HASH_SUFFIX="${HASH_ALGO}sum" - HASH_URL="${DOWNLOAD}.${HASH_SUFFIX}" + HASH=$(jq -r ".hash" <<< "$JSON") - HASH=$(curl "$HASH_URL" -sS -q -L -o -) + if [ "$HASH" = null ]; then + HASH_SUFFIX="${HASH_ALGO}sum" + HASH_URL=$(jq -r ".hash_url" <<< "$JSON") + + if [ "$HASH_URL" = null ]; then + HASH_URL="${DOWNLOAD}.${HASH_SUFFIX}" + fi + + HASH=$(curl "$HASH_URL" -L -o -) + fi download_package - done -} - -for package in $@ -do - # prepare for cancer - # TODO(crueter): Fetch json once? - JSON=$(find . src -maxdepth 3 -name cpmfile.json -exec jq -r ".\"$package\" | select( . != null )" {} \;) - - [ -z "$JSON" ] && echo "!! No cpmfile definition for $package" && continue - - PACKAGE_NAME=$(jq -r ".package" <<< "$JSON") - [ "$PACKAGE_NAME" = null ] && PACKAGE_NAME="$package" - - GIT_HOST=$(jq -r ".git_host" <<< "$JSON") - [ "$GIT_HOST" = null ] && GIT_HOST=github.com - REPO=$(jq -r ".repo" <<< "$JSON") - - CI=$(jq -r ".ci" <<< "$JSON") - if [ "$CI" != null ]; then - ci_package - continue - fi - - VERSION=$(jq -r ".version" <<< "$JSON") - GIT_VERSION=$(jq -r ".git_version" <<< "$JSON") - TAG=$(jq -r ".tag" <<< "$JSON") - SHA=$(jq -r ".sha" <<< "$JSON") - - [ "$GIT_VERSION" = null ] && GIT_VERSION="$VERSION" - [ "$GIT_VERSION" = null ] && GIT_VERSION="$TAG" - - # url parsing WOOOHOOHOHOOHOHOH - URL=$(jq -r ".url" <<< "$JSON") - SHA=$(jq -r ".sha" <<< "$JSON") - - VERSION=$(jq -r ".version" <<< "$JSON") - GIT_VERSION=$(jq -r ".git_version" <<< "$JSON") - - if [ "$GIT_VERSION" != null ]; then - VERSION_REPLACE="$GIT_VERSION" - else - VERSION_REPLACE="$VERSION" - fi - - TAG=$(jq -r ".tag" <<< "$JSON") - - TAG=$(sed "s/%VERSION%/$VERSION_REPLACE/" <<< $TAG) - - ARTIFACT=$(jq -r ".artifact" <<< "$JSON") - ARTIFACT=$(sed "s/%VERSION%/$VERSION_REPLACE/" <<< $ARTIFACT) - ARTIFACT=$(sed "s/%TAG%/$TAG/" <<< $ARTIFACT) - - if [ "$URL" != "null" ]; then - DOWNLOAD="$URL" - elif [ "$REPO" != "null" ]; then - GIT_URL="https://$GIT_HOST/$REPO" - - BRANCH=$(jq -r ".branch" <<< "$JSON") - - if [ "$TAG" != "null" ]; then - if [ "$ARTIFACT" != "null" ]; then - DOWNLOAD="${GIT_URL}/releases/download/${TAG}/${ARTIFACT}" - else - DOWNLOAD="${GIT_URL}/archive/refs/tags/${TAG}.tar.gz" - fi - elif [ "$SHA" != "null" ]; then - DOWNLOAD="${GIT_URL}/archive/${SHA}.zip" - else - if [ "$BRANCH" = null ]; then - BRANCH=master - fi - - DOWNLOAD="${GIT_URL}/archive/refs/heads/${BRANCH}.zip" - fi - else - echo "!! No repo or URL defined for $package" - continue - fi - - # key parsing - KEY=$(jq -r ".key" <<< "$JSON") - - if [ "$KEY" = null ]; then - if [ "$SHA" != null ]; then - KEY=$(cut -c1-4 - <<< "$SHA") - elif [ "$GIT_VERSION" != null ]; then - KEY="$GIT_VERSION" - elif [ "$TAG" != null ]; then - KEY="$TAG" - elif [ "$VERSION" != null ]; then - KEY="$VERSION" - else - echo "!! No valid key could be determined for $package. Must define one of: key, sha, tag, version, git_version" - continue - fi - fi - - echo "-- Downloading regular package $package, with key $KEY, from $DOWNLOAD" - - # hash parsing - HASH_ALGO=$(jq -r ".hash_algo" <<< "$JSON") - [ "$HASH_ALGO" = null ] && HASH_ALGO=sha512 - - HASH=$(jq -r ".hash" <<< "$JSON") - - if [ "$HASH" = null ]; then - HASH_SUFFIX="${HASH_ALGO}sum" - HASH_URL=$(jq -r ".hash_url" <<< "$JSON") - - if [ "$HASH_URL" = null ]; then - HASH_URL="${DOWNLOAD}.${HASH_SUFFIX}" - fi - - HASH=$(curl "$HASH_URL" -L -o -) - fi - - download_package done rm -rf $TMP \ No newline at end of file diff --git a/tools/cpm-hash.sh b/tools/cpm-hash.sh deleted file mode 100755 index b740936f4d..0000000000 --- a/tools/cpm-hash.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -SUM=$(wget -q https://github.com/$1/archive/$2.zip -O - | sha512sum) -echo "$SUM" | cut -d " " -f1 diff --git a/tools/cpm/check-all-hashes.sh b/tools/cpm/check-all-hashes.sh new file mode 100755 index 0000000000..a270c368db --- /dev/null +++ b/tools/cpm/check-all-hashes.sh @@ -0,0 +1,14 @@ +#!/bin/bash -e + +# SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +# SPDX-License-Identifier: GPL-3.0-or-later + +# SPDX-FileCopyrightText: 2025 crueter +# SPDX-License-Identifier: GPL-3.0-or-later + +# shellcheck disable=SC2038 +# shellcheck disable=SC2016 +[ -z "$PACKAGES" ] && PACKAGES=$(find . src -maxdepth 3 -name cpmfile.json | xargs jq -s 'reduce .[] as $item ({}; . * $item)') +LIBS=$(echo "$PACKAGES" | jq -j 'keys_unsorted | join(" ")') + +tools/cpm/check-hash.sh $LIBS \ No newline at end of file diff --git a/tools/cpm/check-all-updates.sh b/tools/cpm/check-all-updates.sh new file mode 100755 index 0000000000..e0c744abf0 --- /dev/null +++ b/tools/cpm/check-all-updates.sh @@ -0,0 +1,11 @@ +#!/bin/bash -e + +# SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +# SPDX-License-Identifier: GPL-3.0-or-later + +# SPDX-FileCopyrightText: 2025 crueter +# SPDX-License-Identifier: GPL-3.0-or-later + +LIBS=$(find . src -maxdepth 3 -name cpmfile.json -exec jq -j 'keys_unsorted | join(" ")' {} \; -printf " ") + +tools/cpm/check-updates.sh $LIBS \ No newline at end of file diff --git a/tools/cpm/check-hash.sh b/tools/cpm/check-hash.sh new file mode 100755 index 0000000000..bf7c70ffcb --- /dev/null +++ b/tools/cpm/check-hash.sh @@ -0,0 +1,110 @@ +#!/bin/bash -e + +# SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +# SPDX-License-Identifier: GPL-3.0-or-later + +# SPDX-FileCopyrightText: 2025 crueter +# SPDX-License-Identifier: GPL-3.0-or-later + +# env vars: +# - UPDATE: update if available +# - FORCE: forcefully update + +# shellcheck disable=SC2016 +# shellcheck disable=SC2038 +[ -z "$PACKAGES" ] && PACKAGES=$(find . src -maxdepth 3 -name cpmfile.json | xargs jq -s 'reduce .[] as $item ({}; . * $item)') + +for package in "$@" +do + JSON=$(echo "$PACKAGES" | jq -r ".\"$package\" | select( . != null )") + + [ -z "$JSON" ] && echo "!! No cpmfile definition for $package" && continue + + GIT_HOST=$(jq -r ".git_host" <<< "$JSON") + [ "$GIT_HOST" = null ] && GIT_HOST=github.com + + CI=$(jq -r ".ci" <<< "$JSON") + if [ "$CI" != null ]; then + continue + fi + + HASH_URL=$(jq -r ".hash_url" <<< "$JSON") + [ "$HASH_URL" != null ] && continue + + REPO=$(jq -r ".repo" <<< "$JSON") + TAG=$(jq -r ".tag" <<< "$JSON") + + URL=$(jq -r ".url" <<< "$JSON") + SHA=$(jq -r ".sha" <<< "$JSON") + + echo "-- Package $package" + + GIT_VERSION=$(jq -r ".git_version" <<< "$JSON") + + VERSION=$(jq -r ".version" <<< "$JSON") + GIT_VERSION=$(jq -r ".git_version" <<< "$JSON") + + if [ "$GIT_VERSION" != null ]; then + VERSION_REPLACE="$GIT_VERSION" + else + VERSION_REPLACE="$VERSION" + fi + + TAG=${TAG//%VERSION%/$VERSION_REPLACE} + + ARTIFACT=$(jq -r ".artifact" <<< "$JSON") + ARTIFACT=${ARTIFACT//%VERSION%/$VERSION_REPLACE} + ARTIFACT=${ARTIFACT//%TAG%/$TAG} + + if [ "$URL" != "null" ]; then + DOWNLOAD="$URL" + elif [ "$REPO" != "null" ]; then + GIT_URL="https://$GIT_HOST/$REPO" + + BRANCH=$(jq -r ".branch" <<< "$JSON") + + if [ "$TAG" != "null" ]; then + if [ "$ARTIFACT" != "null" ]; then + DOWNLOAD="${GIT_URL}/releases/download/${TAG}/${ARTIFACT}" + else + DOWNLOAD="${GIT_URL}/archive/refs/tags/${TAG}.tar.gz" + fi + elif [ "$SHA" != "null" ]; then + DOWNLOAD="${GIT_URL}/archive/${SHA}.zip" + else + if [ "$BRANCH" = null ]; then + BRANCH=master + fi + + DOWNLOAD="${GIT_URL}/archive/refs/heads/${BRANCH}.zip" + fi + else + echo "!! No repo or URL defined for $package" + continue + fi + + # hash parsing + HASH_ALGO=$(jq -r ".hash_algo" <<< "$JSON") + [ "$HASH_ALGO" = null ] && HASH_ALGO=sha512 + + HASH=$(jq -r ".hash" <<< "$JSON") + + [ "$HASH" = null ] && echo "-- * Warning: no hash specified" && continue + + export USE_TAG=true + ACTUAL=$(tools/cpm/url-hash.sh "$DOWNLOAD") + + # shellcheck disable=SC2028 + [ "$ACTUAL" != "$HASH" ] && echo "-- * Expected $HASH" && echo "-- * Got $ACTUAL" + + if [ "$UPDATE" = "true" ] && [ "$ACTUAL" != "$HASH" ]; then + NEW_JSON=$(echo "$JSON" | jq ".hash = \"$ACTUAL\"") + + FILE=$(tools/cpm/which.sh "$package") + + jq --indent 4 --argjson repl "$NEW_JSON" ".\"$package\" *= \$repl" "$FILE" > "$FILE".new + mv "$FILE".new "$FILE" + + echo "-- * -- Updated $FILE" + fi +done \ No newline at end of file diff --git a/tools/cpm/check-updates.sh b/tools/cpm/check-updates.sh new file mode 100755 index 0000000000..8aa46b3924 --- /dev/null +++ b/tools/cpm/check-updates.sh @@ -0,0 +1,97 @@ +#!/bin/bash -e + +# SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +# SPDX-License-Identifier: GPL-3.0-or-later + +# SPDX-FileCopyrightText: 2025 crueter +# SPDX-License-Identifier: GPL-3.0-or-later + +# env vars: +# - UPDATE: update if available +# - FORCE: forcefully update + +# shellcheck disable=SC2016 +# shellcheck disable=SC2038 +[ -z "$PACKAGES" ] && PACKAGES=$(find . src -maxdepth 3 -name cpmfile.json | xargs jq -s 'reduce .[] as $item ({}; . * $item)') + +for package in "$@" +do + JSON=$(echo "$PACKAGES" | jq -r ".\"$package\" | select( . != null )") + + [ -z "$JSON" ] && echo "!! No cpmfile definition for $package" && continue + + GIT_HOST=$(jq -r ".git_host" <<< "$JSON") + [ "$GIT_HOST" = null ] && GIT_HOST=github.com + + REPO=$(jq -r ".repo" <<< "$JSON") + TAG=$(jq -r ".tag" <<< "$JSON") + SKIP=$(jq -r ".skip_updates" <<< "$JSON") + + [ "$SKIP" = "true" ] && continue + + [ "$REPO" = null ] && continue # echo "No repo defined for $package, skipping" && continue + [ "$GIT_HOST" != "github.com" ] && continue # echo "Unsupported host $GIT_HOST for $package, skipping" && continue + [ "$TAG" = null ] && continue # echo "No tag defined for $package, skipping" && continue + + echo "-- Package $package" + + GIT_VERSION=$(jq -r ".git_version" <<< "$JSON") + + echo "$TAG" | grep -e "%VERSION%" > /dev/null && HAS_REPLACE=true || HAS_REPLACE=false + ORIGINAL_TAG="$TAG" + + TAG=${TAG//%VERSION%/$GIT_VERSION} + + ARTIFACT=$(jq -r ".artifact" <<< "$JSON") + ARTIFACT=${ARTIFACT//%VERSION%/$GIT_VERSION} + ARTIFACT=${ARTIFACT//%TAG%/$TAG} + + # Use gh-cli to avoid ratelimits lmao + TAGS=$(gh api --method GET "/repos/$REPO/tags") + + # filter out some commonly known annoyances + TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("vulkan-sdk"; "i") | not)]') + TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("yotta"; "i") | not)]') + + # ignore betas/alphas + TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("alpha"; "i") | not)]') + TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("beta"; "i") | not)]') + TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("rc"; "i") | not)]') + + if [ "$HAS_REPLACE" = "true" ]; then + # this just extracts the tag prefix + VERSION_PREFIX=$(echo "$ORIGINAL_TAG" | cut -d"%" -f1) + + # then we strip out the prefix from the new tag, and make that our new git_version + NEW_GIT_VERSION=${LATEST//$VERSION_PREFIX/} + fi + + # thanks fmt + [ "$package" = fmt ] && TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("v0.11"; "i") | not)]') + + LATEST=$(echo "$TAGS" | jq -r '.[0].name') + + [ "$LATEST" = "$TAG" ] && [ "$FORCE" != "true" ] && echo "-- * Up-to-date" && continue + + echo "-- * Version $LATEST available, current is $TAG" + + export USE_TAG=true + HASH=$(tools/cpm/hash.sh "$REPO" "$LATEST") + + echo "-- * New hash: $HASH" + + if [ "$UPDATE" = "true" ]; then + if [ "$HAS_REPLACE" = "true" ]; then + NEW_JSON=$(echo "$JSON" | jq ".hash = \"$HASH\" | .git_version = \"$NEW_GIT_VERSION\"") + else + NEW_JSON=$(echo "$JSON" | jq ".hash = \"$HASH\" | .tag = \"$LATEST\"") + fi + + FILE=$(tools/cpm/which.sh "$package") + + jq --indent 4 --argjson repl "$NEW_JSON" ".\"$package\" *= \$repl" "$FILE" > "$FILE".new + mv "$FILE".new "$FILE" + + echo "-- * -- Updated $FILE" + fi +done diff --git a/tools/cpm/format-json.sh b/tools/cpm/format.sh similarity index 100% rename from tools/cpm/format-json.sh rename to tools/cpm/format.sh diff --git a/tools/cpm/hash.sh b/tools/cpm/hash.sh new file mode 100755 index 0000000000..d9476e19c4 --- /dev/null +++ b/tools/cpm/hash.sh @@ -0,0 +1,22 @@ +#!/bin/sh -e + +# usage: hash.sh repo tag-or-sha +# env vars: GIT_HOST, USE_TAG (use tag instead of sha), ARTIFACT (download artifact with that name instead of src archive) + +REPO="$1" +[ -z "$GIT_HOST" ] && GIT_HOST=github.com +GIT_URL="https://$GIT_HOST/$REPO" + +if [ "$USE_TAG" = "true" ]; then + if [ -z "$ARTIFACT" ]; then + URL="${GIT_URL}/archive/refs/tags/$2.tar.gz" + else + URL="${GIT_URL}/releases/download/$2/${ARTIFACT}" + fi +else + URL="${GIT_URL}/archive/$2.zip" +fi + +SUM=$(wget -q "$URL" -O - | sha512sum) + +echo "$SUM" | cut -d " " -f1 diff --git a/tools/url-hash.sh b/tools/cpm/url-hash.sh similarity index 53% rename from tools/url-hash.sh rename to tools/cpm/url-hash.sh index a54dec8bb2..6709bb4a03 100755 --- a/tools/url-hash.sh +++ b/tools/cpm/url-hash.sh @@ -1,4 +1,4 @@ #!/bin/sh -SUM=`wget -q $1 -O - | sha512sum` +SUM=$(wget -q $1 -O - | sha512sum) echo "$SUM" | cut -d " " -f1 diff --git a/tools/cpm/which.sh b/tools/cpm/which.sh new file mode 100755 index 0000000000..d547bf879d --- /dev/null +++ b/tools/cpm/which.sh @@ -0,0 +1,9 @@ +#!/bin/sh -e + +# check which file a package is in + +JSON=$(find . src -maxdepth 3 -name cpmfile.json -exec grep -l "$1" {} \;) + +[ -z "$JSON" ] && echo "!! No cpmfile definition for $1" + +echo $JSON \ No newline at end of file