[tools, cmake] refactor: update/hash check scripts, use tags for some more deps, proper CPMUtil separation #2666

Merged
crueter merged 19 commits from update-fmt into master 2025-10-05 03:04:54 +02:00
24 changed files with 126 additions and 63 deletions
Showing only changes of commit bf63e6e91e - Show all commits

View file

@ -454,6 +454,7 @@ if (YUZU_USE_CPM)
if (zstd_ADDED)
add_library(zstd::zstd ALIAS libzstd_static)
add_library(zstd::libzstd ALIAS libzstd_static)
endif()
# Opus
@ -489,9 +490,9 @@ else()
# wow
if (PLATFORM_LINUX)
find_package(Boost 1.57.0 REQUIRED headers context system fiber)
find_package(Boost 1.57.0 CONFIG REQUIRED headers context system fiber)
else()
find_package(Boost 1.57.0 REQUIRED)
find_package(Boost 1.57.0 CONFIG REQUIRED)
endif()
if (CMAKE_SYSTEM_NAME STREQUAL "Linux" OR ANDROID)
@ -542,6 +543,7 @@ find_package(MbedTLS)
find_package(VulkanUtilityLibraries)
find_package(SimpleIni)
find_package(SPIRV-Tools)
find_package(sirit)
if (ARCHITECTURE_x86 OR ARCHITECTURE_x86_64)
find_package(xbyak)
@ -593,6 +595,8 @@ if (ENABLE_QT)
endif()
if (CMAKE_SYSTEM_NAME STREQUAL "Linux")
# yes Qt, we get it
set(QT_NO_PRIVATE_MODULE_WARNING ON)
find_package(Qt6 REQUIRED COMPONENTS DBus OPTIONAL_COMPONENTS GuiPrivate)
elseif (UNIX AND NOT APPLE)
find_package(Qt6 REQUIRED COMPONENTS DBus Gui)

View file

@ -743,9 +743,11 @@ function(CPMAddPackage)
if(NOT DEFINED CPM_ARGS_NAME)
set(CPM_ARGS_NAME ${nameFromUrl})
endif()
if(NOT DEFINED CPM_ARGS_VERSION)
set(CPM_ARGS_VERSION ${verFromUrl})
endif()
# this is dumb and should not be done
# if(NOT DEFINED CPM_ARGS_VERSION)
# set(CPM_ARGS_VERSION ${verFromUrl})
# endif()
list(APPEND CPM_ARGS_UNPARSED_ARGUMENTS URL "${CPM_ARGS_URL}")
endif()

View file

@ -277,6 +277,7 @@ function(AddPackage)
KEY
BUNDLED_PACKAGE
FORCE_BUNDLED_PACKAGE
FIND_PACKAGE_ARGUMENTS
)
@ -426,7 +427,9 @@ function(AddPackage)
- BUNDLED_PACKAGE
- default to allow local
]]#
if (${PKG_ARGS_NAME}_FORCE_SYSTEM)
if (PKG_ARGS_FORCE_BUNDLED_PACKAGE)
set_precedence(OFF OFF)
elseif (${PKG_ARGS_NAME}_FORCE_SYSTEM)
set_precedence(ON ON)
elseif (${PKG_ARGS_NAME}_FORCE_BUNDLED)
set_precedence(OFF OFF)
@ -446,9 +449,14 @@ function(AddPackage)
set_precedence(ON OFF)
endif()
if (DEFINED PKG_ARGS_VERSION)
list(APPEND EXTRA_ARGS
VERSION ${PKG_ARGS_VERSION}
)
endif()
CPMAddPackage(
NAME ${PKG_ARGS_NAME}
VERSION ${PKG_ARGS_VERSION}
URL ${pkg_url}
URL_HASH ${pkg_hash}
CUSTOM_CACHE_KEY ${pkg_key}
@ -459,6 +467,8 @@ function(AddPackage)
PATCHES ${PKG_ARGS_PATCHES}
EXCLUDE_FROM_ALL ON
${EXTRA_ARGS}
${PKG_ARGS_UNPARSED_ARGUMENTS}
)
@ -511,12 +521,12 @@ function(add_ci_package key)
NAME ${ARTIFACT_PACKAGE}
REPO ${ARTIFACT_REPO}
TAG v${ARTIFACT_VERSION}
VERSION ${ARTIFACT_VERSION}
GIT_VERSION ${ARTIFACT_VERSION}
ARTIFACT ${ARTIFACT}
KEY ${key}
HASH_SUFFIX sha512sum
BUNDLED_PACKAGE ON
FORCE_BUNDLED_PACKAGE ON
)
set(ARTIFACT_DIR ${${ARTIFACT_PACKAGE}_SOURCE_DIR} PARENT_SCOPE)

View file

@ -18,6 +18,7 @@
"hash": "4fb7f6fde92762305aad8754d7643cd918dd1f3f67e104e9ab385b18c73178d72a17321354eb203b790b6702f2cf6d725a5d6e2dfbc63b1e35f9eb59fb42ece9",
"git_version": "1.89.0",
"version": "1.57",
"find_args": "CONFIG",
"patches": [
"0001-clang-cl.patch",
"0002-use-marmasm.patch",

View file

@ -147,12 +147,29 @@ if (ENABLE_SDL2)
find_package(SDL2 2.26.4 REQUIRED)
endif()
# Sirit
if (YUZU_USE_BUNDLED_SIRIT)
AddJsonPackage(sirit-ci)
else()
AddJsonPackage(sirit)
if(MSVC AND USE_CCACHE AND sirit_ADDED)
get_target_property(_opts sirit COMPILE_OPTIONS)
list(FILTER _opts EXCLUDE REGEX "/Zi")
list(APPEND _opts "/Z7")
set_target_properties(siritobj PROPERTIES COMPILE_OPTIONS "${_opts}")
elseif(MSVC AND CXX_CLANG)
target_compile_options(siritobj PRIVATE -Wno-error=unused-command-line-argument)
endif()
endif()
# SPIRV Headers
# We only need SPIRV-Headers iff spirv-tools is bundled
if (SPIRV-Tools_FORCE_BUNDLED OR CPMUTIL_FORCE_BUNDLED)
if (TARGET SPIRV-Headers::SPIRV-Headers)
set(NEED_SPIRV_HEADERS OFF)
elseif (SPIRV-Tools_FORCE_BUNDLED OR CPMUTIL_FORCE_BUNDLED)
set(NEED_SPIRV_HEADERS ON)
else()
find_package(SPIRV-Tools MODULE QUIET)
find_package(SPIRV-Tools QUIET)
if (NOT SPIRV-Tools_FOUND)
set(NEED_SPIRV_HEADERS ON)
else()
@ -205,21 +222,6 @@ if (VulkanMemoryAllocator_ADDED)
endif()
endif()
# Sirit
if (YUZU_USE_BUNDLED_SIRIT)
AddJsonPackage(sirit-ci)
else()
AddJsonPackage(sirit)
if(MSVC AND USE_CCACHE AND sirit_ADDED)
get_target_property(_opts sirit COMPILE_OPTIONS)
list(FILTER _opts EXCLUDE REGEX "/Zi")
list(APPEND _opts "/Z7")
set_target_properties(siritobj PROPERTIES COMPILE_OPTIONS "${_opts}")
elseif(MSVC AND CXX_CLANG)
target_compile_options(siritobj PRIVATE -Wno-error=unused-command-line-argument)
endif()
endif()
# httplib
if (ENABLE_WEB_SERVICE OR ENABLE_QT_UPDATE_CHECKER)
AddJsonPackage(httplib)

View file

@ -117,7 +117,6 @@
"repo": "KhronosGroup/SPIRV-Tools",
"tag": "v%VERSION%",
"hash": "b17940433ced72e004c5eeffd7dd411b6afcc6a52ee31de6427d88edceb8172369be8ec8bf5b65708a78bf41fdae264d554aa7750b2209831679ab36bc867567",
"find_args": "MODULE",
"git_version": "2025.4",
"options": [
"SPIRV_SKIP_EXECUTABLES ON"

View file

@ -30,7 +30,6 @@
#include <array>
#include <cstdint>
#include <memory>
using u8 = std::uint8_t; ///< 8-bit unsigned byte
using u16 = std::uint16_t; ///< 16-bit unsigned short

View file

@ -4,6 +4,8 @@
#pragma once
#include <functional>
#include <memory>
#include "core/frontend/applets/applet.h"
#include "core/hle/service/nfp/nfp_types.h"

View file

@ -5,6 +5,8 @@
#include <variant>
#include <vector>
#include <memory>
#include <fmt/printf.h>
#include "common/common_types.h"
#include "core/memory/dmnt_cheat_types.h"

View file

@ -2,7 +2,7 @@
# SPDX-License-Identifier: GPL-3.0-or-later
include(TargetArchitectureSpecificSources)
add_library(dynarmic
add_library(dynarmic STATIC
backend/block_range_information.cpp
backend/block_range_information.h
backend/exception_handler.h

View file

@ -1,9 +1,12 @@
// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
// TODO(crueter): This is identical to root common_types.h
#pragma once
#include <cstdint>
#include <cstdlib>
#include <array>
using u8 = std::uint8_t; ///< 8-bit unsigned byte

View file

@ -5,6 +5,7 @@
#include <array>
#include <mutex>
#include <memory>
#include "common/bit_field.h"
#include "common/common_types.h"

View file

@ -4,6 +4,7 @@
#pragma once
#include <mutex>
#include <memory>
#include "common/common_types.h"
#include "core/hle/result.h"

View file

@ -4,6 +4,7 @@
#pragma once
#include <mutex>
#include <memory>
#include "common/common_types.h"
#include "core/hle/result.h"

View file

@ -4,6 +4,7 @@
#pragma once
#include <mutex>
#include <memory>
#include "common/common_types.h"
#include "core/hle/result.h"

View file

@ -95,9 +95,10 @@ static FileSys::VirtualFile VfsDirectoryCreateFileWrapper(const FileSys::Virtual
#include <QUrl>
#include <QtConcurrent/QtConcurrent>
#ifdef HAVE_SDL2
#include <QCheckBox>
#include <QStringLiteral>
#ifdef HAVE_SDL2
#include <SDL.h> // For SDL ScreenSaver functions
#endif

View file

@ -1 +0,0 @@
cpm/fetch-all.sh

17
tools/cpm-fetch-all.sh Normal file
View file

@ -0,0 +1,17 @@
#!/bin/sh -e
# SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
# SPDX-License-Identifier: GPL-3.0-or-later
# SPDX-FileCopyrightText: 2025 crueter
# SPDX-License-Identifier: GPL-3.0-or-later
# provided for workflow compat
# shellcheck disable=SC1091
. tools/cpm/common.sh
chmod +x tools/cpm/fetch.sh
# shellcheck disable=SC2086
tools/cpm/fetch.sh $LIBS

7
tools/cpm/README.md Normal file → Executable file
View file

@ -17,12 +17,17 @@ These scripts are generally reserved for internal use.
* Generally only used by the fetch scripts.
- `package.sh`: The actual package parser.
* Inputs:
- `package`: The package key
- `PACKAGE`: The package key
* Outputs:
- Basically everything. You're best off reading the code rather than me poorly explaining it.
- `which.sh`: Find which cpmfile a package is located in.
* Inputs:
- The package key
- `replace.sh`: Replace a package's cpmfile definition.
* Inputs:
- `PACKAGE`: The package key
- `NEW_JSON`: All keys to replace/add
* Keys not found in the new json are not touched. Keys cannot currently be deleted.
## Simple Utilities

View file

@ -9,9 +9,9 @@
# shellcheck disable=SC1091
. tools/cpm/common.sh
for package in "$@"
for PACKAGE in "$@"
do
export package
export PACKAGE
# shellcheck disable=SC1091
. tools/cpm/package.sh
@ -22,7 +22,7 @@ do
[ "$HASH_URL" != null ] && continue
[ "$HASH_SUFFIX" != null ] && continue
echo "-- Package $package"
echo "-- Package $PACKAGE"
[ "$HASH" = null ] && echo "-- * Warning: no hash specified" && continue
@ -33,13 +33,9 @@ do
[ "$ACTUAL" != "$HASH" ] && echo "-- * Expected $HASH" && echo "-- * Got $ACTUAL"
if [ "$UPDATE" = "true" ] && [ "$ACTUAL" != "$HASH" ]; then
# shellcheck disable=SC2034
NEW_JSON=$(echo "$JSON" | jq ".hash = \"$ACTUAL\"")
FILE=$(tools/cpm/which.sh "$package")
jq --indent 4 --argjson repl "$NEW_JSON" ".\"$package\" *= \$repl" "$FILE" > "$FILE".new
mv "$FILE".new "$FILE"
echo "-- * -- Updated $FILE"
tools/cpm/replace.sh
fi
done

View file

@ -13,9 +13,9 @@
# shellcheck disable=SC1091
. tools/cpm/common.sh
for package in "$@"
for PACKAGE in "$@"
do
export package
export PACKAGE
# shellcheck disable=SC1091
. tools/cpm/package.sh
@ -28,7 +28,7 @@ do
# shellcheck disable=SC2153
[ "$TAG" = null ] && continue
echo "-- Package $package"
echo "-- Package $PACKAGE"
# TODO(crueter): Support for Forgejo updates w/ forgejo_token
# Use gh-cli to avoid ratelimits lmao
@ -36,8 +36,8 @@ do
# filter out some commonly known annoyances
# TODO add more
TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("vulkan-sdk"; "i") | not)]')
TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("yotta"; "i") | not)]')
TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("vulkan-sdk"; "i") | not)]') # vulkan
TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("yotta"; "i") | not)]') # mbedtls
# ignore betas/alphas
TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("alpha"; "i") | not)]')
@ -45,7 +45,7 @@ do
TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("rc"; "i") | not)]')
# Add package-specific overrides here, e.g. here for fmt:
[ "$package" = fmt ] && TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("v0.11"; "i") | not)]')
[ "$PACKAGE" = fmt ] && TAGS=$(echo "$TAGS" | jq '[.[] | select(.name | test("v0.11"; "i") | not)]')
LATEST=$(echo "$TAGS" | jq -r '.[0].name')
@ -68,16 +68,13 @@ do
if [ "$UPDATE" = "true" ]; then
if [ "$HAS_REPLACE" = "true" ]; then
# shellcheck disable=SC2034
NEW_JSON=$(echo "$JSON" | jq ".hash = \"$HASH\" | .git_version = \"$NEW_GIT_VERSION\"")
else
# shellcheck disable=SC2034
NEW_JSON=$(echo "$JSON" | jq ".hash = \"$HASH\" | .tag = \"$LATEST\"")
fi
FILE=$(tools/cpm/which.sh "$package")
jq --indent 4 --argjson repl "$NEW_JSON" ".\"$package\" *= \$repl" "$FILE" > "$FILE".new
mv "$FILE".new "$FILE"
echo "-- * -- Updated $FILE"
tools/cpm/replace.sh
fi
done

View file

@ -56,7 +56,7 @@ download_package() {
PATCHES=$(echo "$JSON" | jq -r '.patches | join(" ")')
for patch in $PATCHES; do
# shellcheck disable=SC2154
patch --binary -p1 < "$ROOTDIR/.patch/$package/$patch"
patch --binary -p1 < "$ROOTDIR/.patch/$PACKAGE/$patch"
done
fi

View file

@ -19,16 +19,16 @@ ROOTDIR="$PWD"
TMP=$(mktemp -d)
# shellcheck disable=SC2034
for package in "$@"
for PACKAGE in "$@"
do
export package
export PACKAGE
# shellcheck disable=SC1091
. tools/cpm/package.sh
if [ "$CI" = "true" ]; then
ci_package
else
echo "-- Downloading regular package $package, with key $KEY, from $DOWNLOAD"
echo "-- Downloading regular package $PACKAGE, with key $KEY, from $DOWNLOAD"
download_package
fi
done

View file

@ -9,12 +9,12 @@
# shellcheck disable=SC1091
. tools/cpm/common.sh
[ -z "$package" ] && echo "Package was not specified" && exit 0
[ -z "$PACKAGE" ] && echo "Package was not specified" && exit 0
# shellcheck disable=SC2153
JSON=$(echo "$PACKAGES" | jq -r ".\"$package\" | select( . != null )")
JSON=$(echo "$PACKAGES" | jq -r ".\"$PACKAGE\" | select( . != null )")
[ -z "$JSON" ] && echo "!! No cpmfile definition for $package" && exit 1
[ -z "$JSON" ] && echo "!! No cpmfile definition for $PACKAGE" && exit 1
# unset stuff
export PACKAGE_NAME="null"
@ -48,7 +48,7 @@ REPO=$(value "repo")
CI=$(value "ci")
PACKAGE_NAME=$(value "package")
[ "$PACKAGE_NAME" = null ] && PACKAGE_NAME="$package"
[ "$PACKAGE_NAME" = null ] && PACKAGE_NAME="$PACKAGE"
GIT_HOST=$(value "git_host")
[ "$GIT_HOST" = null ] && GIT_HOST=github.com
@ -143,7 +143,7 @@ elif [ "$REPO" != "null" ]; then
DOWNLOAD="${GIT_URL}/archive/refs/heads/${BRANCH}.zip"
fi
else
echo "!! No repo or URL defined for $package"
echo "!! No repo or URL defined for $PACKAGE"
exit 1
fi
@ -166,7 +166,7 @@ if [ "$KEY" = null ]; then
elif [ "$VERSION" != null ]; then
KEY="$VERSION"
else
echo "!! No valid key could be determined for $package. Must define one of: key, sha, tag, version, git_version"
echo "!! No valid key could be determined for $PACKAGE. Must define one of: key, sha, tag, version, git_version"
exit 1
fi
fi

20
tools/cpm/replace.sh Executable file
View file

@ -0,0 +1,20 @@
#!/bin/sh -e
# SPDX-FileCopyrightText: 2025 crueter
# SPDX-License-Identifier: GPL-3.0-or-later
# Replace a specified package with a modified json.
# env vars:
# - PACKAGE: The package key to act on
# - NEW_JSON: The new json to use
[ -z "$PACKAGE" ] && echo "You must provide the PACKAGE environment variable." && return 1
[ -z "$NEW_JSON" ] && echo "You must provide the NEW_JSON environment variable." && return 1
FILE=$(tools/cpm/which.sh "$PACKAGE")
jq --indent 4 --argjson repl "$NEW_JSON" ".\"$PACKAGE\" *= \$repl" "$FILE" > "$FILE".new
mv "$FILE".new "$FILE"
echo "-- * -- Updated $FILE"