From d9e641157e5665293e33d6b1feb72fe628d48deb Mon Sep 17 00:00:00 2001 From: crueter Date: Sat, 4 Oct 2025 04:59:04 -0400 Subject: [PATCH] [tools] refactor: reorganize, commonalities, moar tags, posix compat all scripts are fully posix compliant Signed-off-by: crueter --- .shellcheckrc | 1 + cpmfile.json | 5 +- externals/cpmfile.json | 16 ++- tools/cpm-fetch-all.sh | 15 +-- tools/cpm-fetch.sh | 236 --------------------------------- tools/cpm/check-all-hashes.sh | 11 +- tools/cpm/check-all-updates.sh | 8 +- tools/cpm/check-hash.sh | 78 ++--------- tools/cpm/check-updates.sh | 54 +++----- tools/cpm/common.sh | 19 +++ tools/cpm/download.sh | 103 ++++++++++++++ tools/cpm/fetch-all.sh | 13 ++ tools/cpm/fetch.sh | 39 ++++++ tools/cpm/format.sh | 4 +- tools/cpm/hash.sh | 2 +- tools/cpm/package.sh | 225 +++++++++++++++++++++++++++++++ tools/cpm/url-hash.sh | 2 +- tools/cpm/which.sh | 2 +- 18 files changed, 458 insertions(+), 375 deletions(-) create mode 100644 .shellcheckrc mode change 100755 => 120000 tools/cpm-fetch-all.sh delete mode 100755 tools/cpm-fetch.sh create mode 100755 tools/cpm/common.sh create mode 100755 tools/cpm/download.sh create mode 100755 tools/cpm/fetch-all.sh create mode 100755 tools/cpm/fetch.sh create mode 100755 tools/cpm/package.sh diff --git a/.shellcheckrc b/.shellcheckrc new file mode 100644 index 0000000000..96e22629de --- /dev/null +++ b/.shellcheckrc @@ -0,0 +1 @@ +shell=sh diff --git a/cpmfile.json b/cpmfile.json index 385d10cd81..aab9b4dd3d 100644 --- a/cpmfile.json +++ b/cpmfile.json @@ -26,9 +26,10 @@ }, "fmt": { "repo": "fmtlib/fmt", - "tag": "12.0.0", + "tag": "%VERSION%", "hash": "c4ab814c20fbad7e3f0ae169125a4988a2795631194703251481dc36b18da65c886c4faa9acd046b0a295005217b3689eb0126108a9ba5aac2ca909aae263c2f", - "version": "8" + "version": "8", + "git_version": "12.0.0" }, "lz4": { "name": "lz4", diff --git a/externals/cpmfile.json b/externals/cpmfile.json index 1ff392923f..a3f7540e6d 100644 --- a/externals/cpmfile.json +++ b/externals/cpmfile.json @@ -49,16 +49,18 @@ "xbyak": { "package": "xbyak", "repo": "herumi/xbyak", - "sha": "v7.22", + "tag": "v%VERSION%", "hash": "ff4a461b35ced007adbe9b388a90e4b53c46f9250260a559c04a757ad34ac482baf92700f113c106af6512d6894e849c4409efaf1c003207bf245a5910d848a1", + "git_version": "7.22", "bundled": true, "skip_updates": true }, "oaknut": { + "repo": "eden-emulator/oaknut", "version": "2.0.1", - "repo": "merryhime/oaknut", - "sha": "94c726ce03", - "hash": "d8d082242fa1881abce3c82f8dafa002c4e561e66a69e7fc038af67faa5eff2630f082d3d19579c88c4c9f9488e54552accc8cb90e7ce743efe043b6230c08ac" + "git_version": "2.0.3", + "tag": "v%VERSION%", + "hash": "9697e80a7d5d9bcb3ce51051a9a24962fb90ca79d215f1f03ae6b58da8ba13a63b5dda1b4dde3d26ac6445029696b8ef2883f4e5a777b342bba01283ed293856" }, "libadrenotools": { "repo": "bylaws/libadrenotools", @@ -70,8 +72,9 @@ }, "oboe": { "repo": "google/oboe", - "tag": "1.10.0", + "tag": "%VERSION%", "hash": "ce4011afe7345370d4ead3b891cd69a5ef224b129535783586c0ca75051d303ed446e6c7f10bde8da31fff58d6e307f1732a3ffd03b249f9ef1fd48fd4132715", + "git_version": "1.10.0", "bundled": true }, "unordered-dense": { @@ -103,7 +106,8 @@ "vulkan-utility-headers": { "package": "VulkanUtilityLibraries", "repo": "scripts/VulkanUtilityHeaders", - "tag": "1.4.328", + "tag": "%VERSION%", + "git_version": "1.4.328", "artifact": "VulkanUtilityHeaders.tar.zst", "git_host": "git.crueter.xyz", "hash": "9922217b39faf73cd4fc1510f2fdba14a49aa5c0d77f9ee24ee0512cef16b234d0cabc83c1fec861fa5df1d43e7f086ca9b6501753899119f39c5ca530cb0dae" diff --git a/tools/cpm-fetch-all.sh b/tools/cpm-fetch-all.sh deleted file mode 100755 index 9dc1ac2a55..0000000000 --- a/tools/cpm-fetch-all.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -e - -# SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project -# SPDX-License-Identifier: GPL-3.0-or-later - -# SPDX-FileCopyrightText: 2025 crueter -# SPDX-License-Identifier: GPL-3.0-or-later - -# shellcheck disable=SC2038 -# shellcheck disable=SC2016 -[ -z "$PACKAGES" ] && PACKAGES=$(find . src -maxdepth 3 -name cpmfile.json | xargs jq -s 'reduce .[] as $item ({}; . * $item)') -LIBS=$(echo "$PACKAGES" | jq -j 'keys_unsorted | join(" ")') - -tools/cpm-fetch.sh $LIBS \ No newline at end of file diff --git a/tools/cpm-fetch-all.sh b/tools/cpm-fetch-all.sh new file mode 120000 index 0000000000..ea773fac18 --- /dev/null +++ b/tools/cpm-fetch-all.sh @@ -0,0 +1 @@ +cpm/fetch-all.sh \ No newline at end of file diff --git a/tools/cpm-fetch.sh b/tools/cpm-fetch.sh deleted file mode 100755 index e9829aa482..0000000000 --- a/tools/cpm-fetch.sh +++ /dev/null @@ -1,236 +0,0 @@ -#!/bin/bash -e - -# SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project -# SPDX-License-Identifier: GPL-3.0-or-later - -# SPDX-FileCopyrightText: 2025 crueter -# SPDX-License-Identifier: GPL-3.0-or-later - -[ -z "$CPM_SOURCE_CACHE" ] && CPM_SOURCE_CACHE=$PWD/.cache/cpm - -mkdir -p $CPM_SOURCE_CACHE - -# fetch json once to make lives easier -# shellcheck disable=SC2038 -[ -z "$PACKAGES" ] && PACKAGES=$(find . src -maxdepth 3 -name cpmfile.json | xargs jq -s 'reduce .[] as $item ({}; . * $item)') - -ROOTDIR="$PWD" - -TMP=$(mktemp -d) - -download_package() { - FILENAME=$(basename "$DOWNLOAD") - - OUTFILE="$TMP/$FILENAME" - - LOWER_PACKAGE=$(tr '[:upper:]' '[:lower:]' <<< "$PACKAGE_NAME") - OUTDIR="${CPM_SOURCE_CACHE}/${LOWER_PACKAGE}/${KEY}" - [ -d "$OUTDIR" ] && return - - curl "$DOWNLOAD" -sS -L -o "$OUTFILE" - - ACTUAL_HASH=$(${HASH_ALGO}sum "$OUTFILE" | cut -d" " -f1) - [ "$ACTUAL_HASH" != "$HASH" ] && echo "!! $FILENAME did not match expected hash; expected $HASH but got $ACTUAL_HASH" && exit 1 - - mkdir -p "$OUTDIR" - - pushd "$OUTDIR" > /dev/null - - case "$FILENAME" in - (*.7z) - 7z x "$OUTFILE" > /dev/null - ;; - (*.tar*) - tar xf "$OUTFILE" > /dev/null - ;; - (*.zip) - unzip "$OUTFILE" > /dev/null - ;; - esac - - # basically if only one real item exists at the top we just move everything from there - # since github and some vendors hate me - DIRS=$(find -maxdepth 1 -type d -o -type f) - - # thanks gnu - if [ "$(wc -l <<< "$DIRS")" -eq 2 ]; then - SUBDIR=$(find . -maxdepth 1 -type d -not -name ".") - mv "$SUBDIR"/* . - mv "$SUBDIR"/.* . 2>/dev/null || true - rmdir "$SUBDIR" - fi - - if grep -e "patches" <<< "$JSON" > /dev/null; then - PATCHES=$(jq -r '.patches | join(" ")' <<< "$JSON") - for patch in $PATCHES; do - patch --binary -p1 < "$ROOTDIR"/.patch/$package/$patch - done - fi - - popd > /dev/null -} - -ci_package() { - EXT=$(jq -r '.extension' <<< "$JSON") - [ "$EXT" = null ] && EXT="tar.zst" - - VERSION=$(jq -r ".version" <<< "$JSON") - - NAME=$(jq -r ".name" <<< "$JSON") - [ "$NAME" = null ] && NAME="$PACKAGE" - - PACKAGE=$(jq -r ".package | \"$package\"" <<< "$JSON") - - DISABLED=$(jq -j '.disabled_platforms' <<< "$JSON") - - [ "$REPO" = null ] && echo "No repo defined for CI package $package" && return - - echo "-- CI package $PACKAGE" - - for platform in windows-amd64 windows-arm64 android solaris-amd64 freebsd-amd64 linux-amd64 linux-aarch64 macos-universal; do - echo "-- * platform $platform" - - case $DISABLED in - (*"$platform"*) - echo "-- * -- disabled" - continue - ;; - (*) ;; - esac - - FILENAME="${NAME}-${platform}-${VERSION}.${EXT}" - DOWNLOAD="https://$GIT_HOST/${REPO}/releases/download/v${VERSION}/${FILENAME}" - PACKAGE_NAME="$PACKAGE" - KEY=$platform - - LOWER_PACKAGE=$(tr '[:upper:]' '[:lower:]' <<< "$PACKAGE_NAME") - OUTDIR="${CPM_SOURCE_CACHE}/${LOWER_PACKAGE}/${KEY}" - [ -d "$OUTDIR" ] && continue - - HASH_ALGO=$(jq -r ".hash_algo" <<< "$JSON") - [ "$HASH_ALGO" = null ] && HASH_ALGO=sha512 - - HASH_SUFFIX="${HASH_ALGO}sum" - HASH_URL="${DOWNLOAD}.${HASH_SUFFIX}" - - HASH=$(curl "$HASH_URL" -sS -q -L -o -) - - download_package - done -} - -for package in "$@" -do - # prepare for cancer - JSON=$(echo "$PACKAGES" | jq -r ".\"$package\" | select( . != null )") - - [ -z "$JSON" ] && echo "!! No cpmfile definition for $package" && continue - - PACKAGE_NAME=$(jq -r ".package" <<< "$JSON") - [ "$PACKAGE_NAME" = null ] && PACKAGE_NAME="$package" - - GIT_HOST=$(jq -r ".git_host" <<< "$JSON") - [ "$GIT_HOST" = null ] && GIT_HOST=github.com - - REPO=$(jq -r ".repo" <<< "$JSON") - - CI=$(jq -r ".ci" <<< "$JSON") - if [ "$CI" != null ]; then - ci_package - continue - fi - - VERSION=$(jq -r ".version" <<< "$JSON") - GIT_VERSION=$(jq -r ".git_version" <<< "$JSON") - - [ "$GIT_VERSION" = null ] && GIT_VERSION="$VERSION" - - # url parsing - URL=$(jq -r ".url" <<< "$JSON") - SHA=$(jq -r ".sha" <<< "$JSON") - - VERSION=$(jq -r ".version" <<< "$JSON") - GIT_VERSION=$(jq -r ".git_version" <<< "$JSON") - - if [ "$GIT_VERSION" != null ]; then - VERSION_REPLACE="$GIT_VERSION" - else - VERSION_REPLACE="$VERSION" - fi - - TAG=$(jq -r ".tag" <<< "$JSON") - - TAG=$(sed "s/%VERSION%/$VERSION_REPLACE/" <<< $TAG) - - ARTIFACT=$(jq -r ".artifact" <<< "$JSON") - ARTIFACT=$(sed "s/%VERSION%/$VERSION_REPLACE/" <<< $ARTIFACT) - ARTIFACT=$(sed "s/%TAG%/$TAG/" <<< $ARTIFACT) - - if [ "$URL" != "null" ]; then - DOWNLOAD="$URL" - elif [ "$REPO" != "null" ]; then - GIT_URL="https://$GIT_HOST/$REPO" - - BRANCH=$(jq -r ".branch" <<< "$JSON") - - if [ "$TAG" != "null" ]; then - if [ "$ARTIFACT" != "null" ]; then - DOWNLOAD="${GIT_URL}/releases/download/${TAG}/${ARTIFACT}" - else - DOWNLOAD="${GIT_URL}/archive/refs/tags/${TAG}.tar.gz" - fi - elif [ "$SHA" != "null" ]; then - DOWNLOAD="${GIT_URL}/archive/${SHA}.zip" - else - if [ "$BRANCH" = null ]; then - BRANCH=master - fi - - DOWNLOAD="${GIT_URL}/archive/refs/heads/${BRANCH}.zip" - fi - else - echo "!! No repo or URL defined for $package" - continue - fi - - # key parsing - KEY=$(jq -r ".key" <<< "$JSON") - - if [ "$KEY" = null ]; then - if [ "$SHA" != null ]; then - KEY=$(cut -c1-4 - <<< "$SHA") - elif [ "$GIT_VERSION" != null ]; then - KEY="$GIT_VERSION" - elif [ "$TAG" != null ]; then - KEY="$TAG" - elif [ "$VERSION" != null ]; then - KEY="$VERSION" - else - echo "!! No valid key could be determined for $package. Must define one of: key, sha, tag, version, git_version" - continue - fi - fi - - echo "-- Downloading regular package $package, with key $KEY, from $DOWNLOAD" - - # hash parsing - HASH_ALGO=$(jq -r ".hash_algo" <<< "$JSON") - [ "$HASH_ALGO" = null ] && HASH_ALGO=sha512 - - HASH=$(jq -r ".hash" <<< "$JSON") - - if [ "$HASH" = null ]; then - HASH_SUFFIX="${HASH_ALGO}sum" - HASH_URL=$(jq -r ".hash_url" <<< "$JSON") - - if [ "$HASH_URL" = null ]; then - HASH_URL="${DOWNLOAD}.${HASH_SUFFIX}" - fi - - HASH=$(curl "$HASH_URL" -Ss -L -o -) - fi - - download_package -done - -rm -rf $TMP \ No newline at end of file diff --git a/tools/cpm/check-all-hashes.sh b/tools/cpm/check-all-hashes.sh index a270c368db..730288b4bf 100755 --- a/tools/cpm/check-all-hashes.sh +++ b/tools/cpm/check-all-hashes.sh @@ -6,9 +6,10 @@ # SPDX-FileCopyrightText: 2025 crueter # SPDX-License-Identifier: GPL-3.0-or-later -# shellcheck disable=SC2038 -# shellcheck disable=SC2016 -[ -z "$PACKAGES" ] && PACKAGES=$(find . src -maxdepth 3 -name cpmfile.json | xargs jq -s 'reduce .[] as $item ({}; . * $item)') -LIBS=$(echo "$PACKAGES" | jq -j 'keys_unsorted | join(" ")') +SCRIPT_DIR=$(dirname -- "$0") -tools/cpm/check-hash.sh $LIBS \ No newline at end of file +# shellcheck disable=SC1091 +. tools/cpm/common.sh + +# shellcheck disable=SC2086 +"$SCRIPT_DIR"/check-hash.sh $LIBS \ No newline at end of file diff --git a/tools/cpm/check-all-updates.sh b/tools/cpm/check-all-updates.sh index e0c744abf0..09f1ec08ea 100755 --- a/tools/cpm/check-all-updates.sh +++ b/tools/cpm/check-all-updates.sh @@ -6,6 +6,10 @@ # SPDX-FileCopyrightText: 2025 crueter # SPDX-License-Identifier: GPL-3.0-or-later -LIBS=$(find . src -maxdepth 3 -name cpmfile.json -exec jq -j 'keys_unsorted | join(" ")' {} \; -printf " ") +SCRIPT_DIR=$(dirname -- "$0") -tools/cpm/check-updates.sh $LIBS \ No newline at end of file +# shellcheck disable=SC1091 +. tools/cpm/common.sh + +# shellcheck disable=SC2086 +"$SCRIPT_DIR"/check-updates.sh $LIBS \ No newline at end of file diff --git a/tools/cpm/check-hash.sh b/tools/cpm/check-hash.sh index bf7c70ffcb..e1081f67ba 100755 --- a/tools/cpm/check-hash.sh +++ b/tools/cpm/check-hash.sh @@ -1,4 +1,4 @@ -#!/bin/bash -e +#!/bin/sh -e # SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project # SPDX-License-Identifier: GPL-3.0-or-later @@ -7,88 +7,26 @@ # SPDX-License-Identifier: GPL-3.0-or-later # env vars: -# - UPDATE: update if available -# - FORCE: forcefully update +# - UPDATE: fix hashes if needed -# shellcheck disable=SC2016 -# shellcheck disable=SC2038 -[ -z "$PACKAGES" ] && PACKAGES=$(find . src -maxdepth 3 -name cpmfile.json | xargs jq -s 'reduce .[] as $item ({}; . * $item)') +# shellcheck disable=SC1091 +. tools/cpm/common.sh for package in "$@" do - JSON=$(echo "$PACKAGES" | jq -r ".\"$package\" | select( . != null )") + export package + # shellcheck disable=SC1091 + . tools/cpm/package.sh - [ -z "$JSON" ] && echo "!! No cpmfile definition for $package" && continue - - GIT_HOST=$(jq -r ".git_host" <<< "$JSON") - [ "$GIT_HOST" = null ] && GIT_HOST=github.com - - CI=$(jq -r ".ci" <<< "$JSON") if [ "$CI" != null ]; then continue fi - HASH_URL=$(jq -r ".hash_url" <<< "$JSON") [ "$HASH_URL" != null ] && continue - - REPO=$(jq -r ".repo" <<< "$JSON") - TAG=$(jq -r ".tag" <<< "$JSON") - - URL=$(jq -r ".url" <<< "$JSON") - SHA=$(jq -r ".sha" <<< "$JSON") + [ "$HASH_SUFFIX" != null ] && continue echo "-- Package $package" - GIT_VERSION=$(jq -r ".git_version" <<< "$JSON") - - VERSION=$(jq -r ".version" <<< "$JSON") - GIT_VERSION=$(jq -r ".git_version" <<< "$JSON") - - if [ "$GIT_VERSION" != null ]; then - VERSION_REPLACE="$GIT_VERSION" - else - VERSION_REPLACE="$VERSION" - fi - - TAG=${TAG//%VERSION%/$VERSION_REPLACE} - - ARTIFACT=$(jq -r ".artifact" <<< "$JSON") - ARTIFACT=${ARTIFACT//%VERSION%/$VERSION_REPLACE} - ARTIFACT=${ARTIFACT//%TAG%/$TAG} - - if [ "$URL" != "null" ]; then - DOWNLOAD="$URL" - elif [ "$REPO" != "null" ]; then - GIT_URL="https://$GIT_HOST/$REPO" - - BRANCH=$(jq -r ".branch" <<< "$JSON") - - if [ "$TAG" != "null" ]; then - if [ "$ARTIFACT" != "null" ]; then - DOWNLOAD="${GIT_URL}/releases/download/${TAG}/${ARTIFACT}" - else - DOWNLOAD="${GIT_URL}/archive/refs/tags/${TAG}.tar.gz" - fi - elif [ "$SHA" != "null" ]; then - DOWNLOAD="${GIT_URL}/archive/${SHA}.zip" - else - if [ "$BRANCH" = null ]; then - BRANCH=master - fi - - DOWNLOAD="${GIT_URL}/archive/refs/heads/${BRANCH}.zip" - fi - else - echo "!! No repo or URL defined for $package" - continue - fi - - # hash parsing - HASH_ALGO=$(jq -r ".hash_algo" <<< "$JSON") - [ "$HASH_ALGO" = null ] && HASH_ALGO=sha512 - - HASH=$(jq -r ".hash" <<< "$JSON") - [ "$HASH" = null ] && echo "-- * Warning: no hash specified" && continue export USE_TAG=true diff --git a/tools/cpm/check-updates.sh b/tools/cpm/check-updates.sh index 8aa46b3924..3d37d4cbb0 100755 --- a/tools/cpm/check-updates.sh +++ b/tools/cpm/check-updates.sh @@ -1,4 +1,4 @@ -#!/bin/bash -e +#!/bin/sh -e # SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project # SPDX-License-Identifier: GPL-3.0-or-later @@ -10,42 +10,26 @@ # - UPDATE: update if available # - FORCE: forcefully update -# shellcheck disable=SC2016 -# shellcheck disable=SC2038 -[ -z "$PACKAGES" ] && PACKAGES=$(find . src -maxdepth 3 -name cpmfile.json | xargs jq -s 'reduce .[] as $item ({}; . * $item)') +# shellcheck disable=SC1091 +. tools/cpm/common.sh for package in "$@" do - JSON=$(echo "$PACKAGES" | jq -r ".\"$package\" | select( . != null )") + export package + # shellcheck disable=SC1091 + . tools/cpm/package.sh - [ -z "$JSON" ] && echo "!! No cpmfile definition for $package" && continue - - GIT_HOST=$(jq -r ".git_host" <<< "$JSON") - [ "$GIT_HOST" = null ] && GIT_HOST=github.com - - REPO=$(jq -r ".repo" <<< "$JSON") - TAG=$(jq -r ".tag" <<< "$JSON") - SKIP=$(jq -r ".skip_updates" <<< "$JSON") + SKIP=$(value "skip_updates") [ "$SKIP" = "true" ] && continue - [ "$REPO" = null ] && continue # echo "No repo defined for $package, skipping" && continue - [ "$GIT_HOST" != "github.com" ] && continue # echo "Unsupported host $GIT_HOST for $package, skipping" && continue - [ "$TAG" = null ] && continue # echo "No tag defined for $package, skipping" && continue + [ "$REPO" = null ] && continue + [ "$GIT_HOST" != "github.com" ] && continue # TODO + [ "$TAG" = null ] && continue echo "-- Package $package" - GIT_VERSION=$(jq -r ".git_version" <<< "$JSON") - - echo "$TAG" | grep -e "%VERSION%" > /dev/null && HAS_REPLACE=true || HAS_REPLACE=false - ORIGINAL_TAG="$TAG" - - TAG=${TAG//%VERSION%/$GIT_VERSION} - - ARTIFACT=$(jq -r ".artifact" <<< "$JSON") - ARTIFACT=${ARTIFACT//%VERSION%/$GIT_VERSION} - ARTIFACT=${ARTIFACT//%TAG%/$TAG} - + # TODO(crueter): Support for Forgejo updates w/ forgejo_token # Use gh-cli to avoid ratelimits lmao TAGS=$(gh api --method GET "/repos/$REPO/tags") @@ -58,14 +42,6 @@ do TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("beta"; "i") | not)]') TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("rc"; "i") | not)]') - if [ "$HAS_REPLACE" = "true" ]; then - # this just extracts the tag prefix - VERSION_PREFIX=$(echo "$ORIGINAL_TAG" | cut -d"%" -f1) - - # then we strip out the prefix from the new tag, and make that our new git_version - NEW_GIT_VERSION=${LATEST//$VERSION_PREFIX/} - fi - # thanks fmt [ "$package" = fmt ] && TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("v0.11"; "i") | not)]') @@ -73,6 +49,14 @@ do [ "$LATEST" = "$TAG" ] && [ "$FORCE" != "true" ] && echo "-- * Up-to-date" && continue + if [ "$HAS_REPLACE" = "true" ]; then + # this just extracts the tag prefix + VERSION_PREFIX=$(echo "$ORIGINAL_TAG" | cut -d"%" -f1) + + # then we strip out the prefix from the new tag, and make that our new git_version + NEW_GIT_VERSION=$(echo "$LATEST" | sed "s/$VERSION_PREFIX//g") + fi + echo "-- * Version $LATEST available, current is $TAG" export USE_TAG=true diff --git a/tools/cpm/common.sh b/tools/cpm/common.sh new file mode 100755 index 0000000000..5f9b2e42c7 --- /dev/null +++ b/tools/cpm/common.sh @@ -0,0 +1,19 @@ +#!/bin/sh -e + +# SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +# SPDX-License-Identifier: GPL-3.0-or-later + +# SPDX-FileCopyrightText: 2025 crueter +# SPDX-License-Identifier: GPL-3.0-or-later + +# shellcheck disable=SC2038 +# shellcheck disable=SC2016 +[ -z "$PACKAGES" ] && PACKAGES=$(find . src -maxdepth 3 -name cpmfile.json | xargs jq -s 'reduce .[] as $item ({}; . * $item)') +LIBS=$(echo "$PACKAGES" | jq -j 'keys_unsorted | join(" ")') + +export PACKAGES +export LIBS + +value() { + echo "$JSON" | jq -r ".$1" +} \ No newline at end of file diff --git a/tools/cpm/download.sh b/tools/cpm/download.sh new file mode 100755 index 0000000000..b2f7e3c5c9 --- /dev/null +++ b/tools/cpm/download.sh @@ -0,0 +1,103 @@ +#!/bin/sh -e + +# SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +# SPDX-License-Identifier: GPL-3.0-or-later + +# SPDX-FileCopyrightText: 2025 crueter +# SPDX-License-Identifier: GPL-3.0-or-later + +# env vars: +# - UPDATE: fix hashes if needed + +# shellcheck disable=SC1091 +. tools/cpm/common.sh + +download_package() { + FILENAME=$(basename "$DOWNLOAD") + + OUTFILE="$TMP/$FILENAME" + + LOWER_PACKAGE=$(echo "$PACKAGE_NAME" | tr '[:upper:]' '[:lower:]') + OUTDIR="${CPM_SOURCE_CACHE}/${LOWER_PACKAGE}/${KEY}" + [ -d "$OUTDIR" ] && return + + curl "$DOWNLOAD" -sS -L -o "$OUTFILE" + + ACTUAL_HASH=$("${HASH_ALGO}"sum "$OUTFILE" | cut -d" " -f1) + [ "$ACTUAL_HASH" != "$HASH" ] && echo "!! $FILENAME did not match expected hash; expected $HASH but got $ACTUAL_HASH" && exit 1 + + mkdir -p "$OUTDIR" + + PREVDIR="$PWD" + cd "$OUTDIR" + + case "$FILENAME" in + (*.7z) + 7z x "$OUTFILE" > /dev/null + ;; + (*.tar*) + tar xf "$OUTFILE" > /dev/null + ;; + (*.zip) + unzip "$OUTFILE" > /dev/null + ;; + esac + + # basically if only one real item exists at the top we just move everything from there + # since github and some vendors hate me + DIRS=$(find . -maxdepth 1 -type d -o -type f) + + # thanks gnu + if [ "$(echo "$DIRS" | wc -l)" -eq 2 ]; then + SUBDIR=$(find . -maxdepth 1 -type d -not -name ".") + mv "$SUBDIR"/* . + mv "$SUBDIR"/.* . 2>/dev/null || true + rmdir "$SUBDIR" + fi + + if echo "$JSON" | grep -e "patches" > /dev/null; then + PATCHES=$(echo "$JSON" | jq -r '.patches | join(" ")') + for patch in $PATCHES; do + # shellcheck disable=SC2154 + patch --binary -p1 < "$ROOTDIR/.patch/$package/$patch" + done + fi + + cd "$PREVDIR" +} + +ci_package() { + [ "$REPO" = null ] && echo "-- ! No repo defined" && return + + echo "-- CI package $PACKAGE_NAME" + + for platform in windows-amd64 windows-arm64 android solaris-amd64 freebsd-amd64 linux-amd64 linux-aarch64 macos-universal; do + echo "-- * platform $platform" + + case $DISABLED in + (*"$platform"*) + echo "-- * -- disabled" + continue + ;; + (*) ;; + esac + + FILENAME="${NAME}-${platform}-${VERSION}.${EXT}" + DOWNLOAD="https://$GIT_HOST/${REPO}/releases/download/v${VERSION}/${FILENAME}" + KEY=$platform + + LOWER_PACKAGE=$(echo "$PACKAGE_NAME" | tr '[:upper:]' '[:lower:]') + OUTDIR="${CPM_SOURCE_CACHE}/${LOWER_PACKAGE}/${KEY}" + [ -d "$OUTDIR" ] && continue + + HASH_ALGO=$(value "hash_algo") + [ "$HASH_ALGO" = null ] && HASH_ALGO=sha512 + + HASH_SUFFIX="${HASH_ALGO}sum" + HASH_URL="${DOWNLOAD}.${HASH_SUFFIX}" + + HASH=$(curl "$HASH_URL" -sS -q -L -o -) + + download_package + done +} diff --git a/tools/cpm/fetch-all.sh b/tools/cpm/fetch-all.sh new file mode 100755 index 0000000000..16c898023f --- /dev/null +++ b/tools/cpm/fetch-all.sh @@ -0,0 +1,13 @@ +#!/bin/sh -e + +# SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +# SPDX-License-Identifier: GPL-3.0-or-later + +# SPDX-FileCopyrightText: 2025 crueter +# SPDX-License-Identifier: GPL-3.0-or-later + +# shellcheck disable=SC1091 +. tools/cpm/common.sh + +# shellcheck disable=SC2086 +tools/cpm/fetch.sh $LIBS \ No newline at end of file diff --git a/tools/cpm/fetch.sh b/tools/cpm/fetch.sh new file mode 100755 index 0000000000..26a397d6ae --- /dev/null +++ b/tools/cpm/fetch.sh @@ -0,0 +1,39 @@ +#!/bin/sh -e + +# SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +# SPDX-License-Identifier: GPL-3.0-or-later + +# SPDX-FileCopyrightText: 2025 crueter +# SPDX-License-Identifier: GPL-3.0-or-later + +[ -z "$CPM_SOURCE_CACHE" ] && CPM_SOURCE_CACHE=$PWD/.cache/cpm + +mkdir -p "$CPM_SOURCE_CACHE" + +# shellcheck disable=SC1091 +. tools/cpm/common.sh + +# shellcheck disable=SC1091 +. tools/cpm/download.sh + +# shellcheck disable=SC2034 +ROOTDIR="$PWD" + +TMP=$(mktemp -d) + +# shellcheck disable=SC2034 +for package in "$@" +do + export package + # shellcheck disable=SC1091 + . tools/cpm/package.sh + + if [ "$CI" = "true" ]; then + ci_package + else + echo "-- Downloading regular package $package, with key $KEY, from $DOWNLOAD" + download_package + fi +done + +rm -rf "$TMP" \ No newline at end of file diff --git a/tools/cpm/format.sh b/tools/cpm/format.sh index e636c402b4..7af66dffa3 100755 --- a/tools/cpm/format.sh +++ b/tools/cpm/format.sh @@ -6,6 +6,6 @@ FILES=$(find . src -maxdepth 3 -name cpmfile.json) for file in $FILES; do - jq --indent 4 < $file > $file.new - mv $file.new $file + jq --indent 4 < "$file" > "$file".new + mv "$file".new "$file" done diff --git a/tools/cpm/hash.sh b/tools/cpm/hash.sh index 0c653e75c2..27061bd9a4 100755 --- a/tools/cpm/hash.sh +++ b/tools/cpm/hash.sh @@ -11,7 +11,7 @@ REPO="$1" GIT_URL="https://$GIT_HOST/$REPO" if [ "$USE_TAG" = "true" ]; then - if [ -z "$ARTIFACT" ]; then + if [ -z "$ARTIFACT" ] || [ "$ARTIFACT" = "null" ]; then URL="${GIT_URL}/archive/refs/tags/$2.tar.gz" else URL="${GIT_URL}/releases/download/$2/${ARTIFACT}" diff --git a/tools/cpm/package.sh b/tools/cpm/package.sh new file mode 100755 index 0000000000..89e01089b0 --- /dev/null +++ b/tools/cpm/package.sh @@ -0,0 +1,225 @@ +#!/bin/sh -e + +# SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +# SPDX-License-Identifier: GPL-3.0-or-later + +# SPDX-FileCopyrightText: 2025 crueter +# SPDX-License-Identifier: GPL-3.0-or-later + +# env vars: +# - UPDATE: fix hashes if needed + +# shellcheck disable=SC1091 +. tools/cpm/common.sh + +[ -z "$package" ] && echo "Package was not specified" && exit 0 + +# shellcheck disable=SC2153 +JSON=$(echo "$PACKAGES" | jq -r ".\"$package\" | select( . != null )") + +[ -z "$JSON" ] && echo "!! No cpmfile definition for $package" && exit 1 + +# unset stuff +unset PACKAGE_NAME +unset REPO +unset CI +unset GIT_HOST +unset EXT +unset NAME +unset DISABLED +unset TAG +unset ARTIFACT +unset SHA +unset VERSION +unset GIT_VERSION +unset DOWNLOAD +unset URL +unset KEY +unset HASH + +unset ORIGINAL_TAG +unset HAS_REPLACE +unset VERSION_REPLACE + +unset HASH_URL +unset HASH_SUFFIX +unset HASH_ALGO + +######## +# Meta # +######## + +REPO=$(value "repo") +CI=$(value "ci") + +PACKAGE_NAME=$(value "package") +[ "$PACKAGE_NAME" = null ] && PACKAGE_NAME="$package" + +GIT_HOST=$(value "git_host") +[ "$GIT_HOST" = null ] && GIT_HOST=github.com + +export PACKAGE_NAME +export REPO +export CI +export GIT_HOST + +###################### +# CI Package Parsing # +###################### + +VERSION=$(value "version") + +if [ "$CI" = "true" ]; then + EXT=$(value "extension") + [ "$EXT" = null ] && EXT="tar.zst" + + NAME=$(value "name") + DISABLED=$(echo "$JSON" | jq -j '.disabled_platforms') + + [ "$NAME" = null ] && NAME="$PACKAGE" + + export EXT + export NAME + export DISABLED + export VERSION + + # slight annoyance + TAG=null + ARTIFACT=null + SHA=null + VERSION=null + GIT_VERSION=null + DOWNLOAD=null + URL=null + KEY=null + HASH=null + ORIGINAL_TAG=null + HAS_REPLACE=null + VERSION_REPLACE=null + HASH_URL=null + HASH_SUFFIX=null + HASH_ALGO=null + + return 0 +fi + +############## +# Versioning # +############## + +TAG=$(value "tag") +ARTIFACT=$(value "artifact") +SHA=$(value "sha") +GIT_VERSION=$(value "git_version") + +[ "$GIT_VERSION" = null ] && GIT_VERSION="$VERSION" + +if [ "$GIT_VERSION" != null ]; then + VERSION_REPLACE="$GIT_VERSION" +else + VERSION_REPLACE="$VERSION" +fi + +echo "$TAG" | grep -e "%VERSION%" > /dev/null && HAS_REPLACE=true || HAS_REPLACE=false +ORIGINAL_TAG="$TAG" + +TAG=$(echo "$TAG" | sed "s/%VERSION%/$VERSION_REPLACE/g") +ARTIFACT=$(echo "$ARTIFACT" | sed "s/%VERSION%/$VERSION_REPLACE/g") +ARTIFACT=$(echo "$ARTIFACT" | sed "s/%TAG%/$TAG/g") + +export TAG +export ARTIFACT +export SHA +export VERSION +export GIT_VERSION +export ORIGINAL_TAG +export HAS_REPLACE +export VERSION_REPLACE + +############### +# URL Parsing # +############### + +URL=$(value "url") + +if [ "$URL" != "null" ]; then + DOWNLOAD="$URL" +elif [ "$REPO" != "null" ]; then + GIT_URL="https://$GIT_HOST/$REPO" + + BRANCH=$(value "branch") + + if [ "$TAG" != "null" ]; then + if [ "$ARTIFACT" != "null" ]; then + DOWNLOAD="${GIT_URL}/releases/download/${TAG}/${ARTIFACT}" + else + DOWNLOAD="${GIT_URL}/archive/refs/tags/${TAG}.tar.gz" + fi + elif [ "$SHA" != "null" ]; then + DOWNLOAD="${GIT_URL}/archive/${SHA}.zip" + else + if [ "$BRANCH" = null ]; then + BRANCH=master + fi + + DOWNLOAD="${GIT_URL}/archive/refs/heads/${BRANCH}.zip" + fi +else + echo "!! No repo or URL defined for $package" + exit 1 +fi + +export DOWNLOAD +export URL + +############### +# Key Parsing # +############### + +KEY=$(value "key") + +if [ "$KEY" = null ]; then + if [ "$SHA" != null ]; then + KEY=$(echo "$SHA" | cut -c1-4) + elif [ "$GIT_VERSION" != null ]; then + KEY="$GIT_VERSION" + elif [ "$TAG" != null ]; then + KEY="$TAG" + elif [ "$VERSION" != null ]; then + KEY="$VERSION" + else + echo "!! No valid key could be determined for $package. Must define one of: key, sha, tag, version, git_version" + exit 1 + fi +fi + +export KEY + +################ +# Hash Parsing # +################ + +HASH_ALGO=$(value "hash_algo") +[ "$HASH_ALGO" = null ] && HASH_ALGO=sha512 + +HASH=$(value "hash") + +if [ "$HASH" = null ]; then + HASH_SUFFIX="${HASH_ALGO}sum" + HASH_URL=$(value "hash_url") + + if [ "$HASH_URL" = null ]; then + HASH_URL="${DOWNLOAD}.${HASH_SUFFIX}" + fi + + HASH=$(curl "$HASH_URL" -Ss -L -o -) +else + HASH_URL=null + HASH_SUFFIX=null +fi + +export HASH_URL +export HASH_SUFFIX +export HASH +export HASH_ALGO +export JSON \ No newline at end of file diff --git a/tools/cpm/url-hash.sh b/tools/cpm/url-hash.sh index 444b14b174..c911dacb37 100755 --- a/tools/cpm/url-hash.sh +++ b/tools/cpm/url-hash.sh @@ -3,5 +3,5 @@ # SPDX-FileCopyrightText: 2025 crueter # SPDX-License-Identifier: GPL-3.0-or-later -SUM=$(wget -q $1 -O - | sha512sum) +SUM=$(wget -q "$1" -O - | sha512sum) echo "$SUM" | cut -d " " -f1 diff --git a/tools/cpm/which.sh b/tools/cpm/which.sh index af12deb1b4..f1ca316a3e 100755 --- a/tools/cpm/which.sh +++ b/tools/cpm/which.sh @@ -9,4 +9,4 @@ JSON=$(find . src -maxdepth 3 -name cpmfile.json -exec grep -l "$1" {} \;) [ -z "$JSON" ] && echo "!! No cpmfile definition for $1" -echo $JSON \ No newline at end of file +echo "$JSON" \ No newline at end of file