[cmake] refactor: cpmfile, deps prefetch, force system and more

This is a (currently) very WIP PR that basically completely refactors the CPM system for the third time. Dependencies are now managed in a singular json file, where each can be properly prefetched at-will via a script I'll add eventually. For the time being, this is super incomplete and I have to go to class. Once I'm back I'll keep working on it. Signed-off-by: crueter <crueter@eden-emu.dev>
7 months ago · 5d4bf12cd7
5 changed files with 191 additions and 57 deletions
--- a/CMakeModules/CPMUtil.cmake
+++ b/CMakeModules/CPMUtil.cmake
@ -19,10 +19,144 @@ CMAKE_DEPENDENT_OPTION(CPMUTIL_DEFAULT_SYSTEM
 cmake_minimum_required(VERSION 3.22)
 include(CPM)

+set(CPMUTIL_JSON_FILE "${CMAKE_CURRENT_SOURCE_DIR}/cpmfile.json" CACHE STRING "Location of cpmfile.json")
+
+if (EXISTS ${CPMUTIL_JSON_FILE})
+    file(READ ${CPMUTIL_JSON_FILE} CPMFILE_CONTENT)
+else()
+    message(WARNING "[CPMUtil] cpmfile ${CPMUTIL_JSON_FILE} does not exist, AddJsonPackage will be a no-op")
+endif()
+
 function(cpm_utils_message level name message)
    message(${level} "[CPMUtil] ${name}: ${message}")
 endfunction()

+function(array_to_list array length out)
+    math(EXPR range "${length} - 1")
+
+    foreach(IDX RANGE ${range})
+        string(JSON _element GET "${array}" "${IDX}")
+
+        list(APPEND NEW_LIST ${_element})
+    endforeach()
+
+    set("${out}" "${NEW_LIST}" PARENT_SCOPE)
+endfunction()
+
+function(get_json_element object out member default)
+    string(JSON out_type ERROR_VARIABLE err TYPE "${object}" ${member})
+
+    if (err)
+        set("${out}" "${default}" PARENT_SCOPE)
+        return()
+    endif()
+
+    string(JSON outvar GET "${object}" ${member})
+
+    if (out_type STREQUAL "ARRAY")
+        string(JSON _len LENGTH "${object}" ${member})
+        # array_to_list("${outvar}" ${_len} outvar)
+        set("${out}_LENGTH" "${_len}" PARENT_SCOPE)
+    endif()
+
+    set("${out}" "${outvar}" PARENT_SCOPE)
+endfunction()
+
+
+# Kinda cancerous but whatever
+function(AddJsonPackage name)
+
+    # these are overrides that can be generated at runtime, so can be defined separately from the json
+    set(oneValueArgs
+        DOWNLOAD_ONLY
+        SYSTEM_PACKAGE
+        BUNDLED_PACKAGE
+    )
+
+    set(multiValueArgs OPTIONS)
+
+    cmake_parse_arguments(JSON "" "${oneValueArgs}" "${multiValueArgs}"
+                          "${ARGN}")
+
+    if (NOT DEFINED CPMFILE_CONTENT)
+        cpm_utils_message(WARNING ${name} "No cpmfile, AddJsonPackage is a no-op")
+        return()
+    endif()
+
+    string(JSON object ERROR_VARIABLE err GET "${CPMFILE_CONTENT}" "${name}")
+
+    if (err)
+        cpm_utils_message(FATAL_ERROR ${name} "Not found in cpmfile")
+    endif()
+
+    message(STATUS "JSON: ${object}")
+
+    get_json_element("${object}" package package ${name})
+    get_json_element("${object}" hash hash "")
+    get_json_element("${object}" sha sha "")
+    get_json_element("${object}" repo repo "")
+    get_json_element("${object}" bundled bundled "unset")
+    get_json_element("${object}" version version "")
+    get_json_element("${object}" find_args find_args "")
+    get_json_element("${object}" raw_patches patches "")
+
+    # format patchdir
+    if (raw_patches)
+        math(EXPR range "${raw_patches_LENGTH} - 1")
+
+        foreach(IDX RANGE ${range})
+            string(JSON _patch GET "${raw_patches}" "${IDX}")
+
+            set(full_patch "${CMAKE_SOURCE_DIR}/.patch/${name}/${_patch}")
+            if (NOT EXISTS ${full_patch})
+                cpm_utils_message(FATAL_ERROR ${name} "specifies patch ${full_patch} which does not exist")
+            endif()
+
+            list(APPEND patches "${full_patch}")
+        endforeach()
+    endif()
+    # end format patchdir
+
+    # options
+    get_json_element("${object}" raw_options options "")
+
+    if (raw_options)
+        array_to_list("${raw_options}" ${raw_options_LENGTH} options)
+    elseif(JSON_OPTIONS)
+        set(options ${JSON_OPTIONS})
+    endif()
+    # end options
+
+    # system/bundled
+    cpm_utils_message(STATUS ${name} "system: ${system} bundled: ${bundled}")
+    if (bundled STREQUAL "unset" AND DEFINED JSON_BUNDLED_PACKAGE)
+        set(bundled ${JSON_BUNDLED_PACKAGE})
+    else()
+        set(bundled ON)
+    endif()
+
+    AddPackage(
+        NAME "${package}"
+        VERSION "${version}"
+        HASH "${hash}"
+        SHA "${sha}"
+        REPO "${repo}"
+        PATCHES "${patches}"
+        OPTIONS "${options}"
+        FIND_PACKAGE_ARGUMENTS "${find_args}"
+        BUNDLED_PACKAGE "${bundled}"
+    )
+
+    # pass stuff to parent scope
+    set(${package}_ADDED "${${package}_ADDED}"
+        PARENT_SCOPE)
+    set(${package}_SOURCE_DIR "${${package}_SOURCE_DIR}"
+        PARENT_SCOPE)
+    set(${package}_BINARY_DIR "${${package}_BINARY_DIR}"
+        PARENT_SCOPE)
+
+endfunction()
+
 function(AddPackage)
    cpm_set_policies()

@ -43,6 +177,8 @@ function(AddPackage)
          * technically this is unsafe since a hacker can attack that url

        NOTE: hash algo defaults to sha512
+
+        patches are in ${CMAKE_SOURCE_DIR}/.patch/{name}/${patchname}
    #]]
    set(oneValueArgs
        NAME
@ -185,10 +321,9 @@ function(AddPackage)
        set(pkg_hash "${hash_algo}=${pkg_hash_tmp}")
    endif()

+    # TODO(crueter): force system/bundled for specific packages via options e.g httplib_SYSTEM
    if (NOT CPMUTIL_DEFAULT_SYSTEM)
        set(CPM_USE_LOCAL_PACKAGES OFF)
-    elseif (DEFINED PKG_ARGS_SYSTEM_PACKAGE)
-        set(CPM_USE_LOCAL_PACKAGES ${PKG_ARGS_SYSTEM_PACKAGE})
    elseif (DEFINED PKG_ARGS_BUNDLED_PACKAGE)
        if (PKG_ARGS_BUNDLED_PACKAGE)
            set(CPM_USE_LOCAL_PACKAGES OFF)
--- a/cpmfile.json
+++ b/cpmfile.json
@ -0,0 +1,45 @@
+{
+    "mbedtls": {
+        "repo": "Mbed-TLS/mbedtls",
+        "sha": "8c88150ca1",
+        "hash": "769ad1e94c570671071e1f2a5c0f1027e0bf6bcdd1a80ea8ac970f2c86bc45ce4e31aa88d6d8110fc1bed1de81c48bc624df1b38a26f8b340a44e109d784a966",
+        "patches": [
+            "0001-cmake-version.patch"
+        ]
+    },
+    "spirv-headers": {
+        "package": "SPIRV-Headers",
+        "repo": "KhronosGroup/SPIRV-Headers",
+        "sha": "4e209d3d7e",
+        "hash": "f48bbe18341ed55ea0fe280dbbbc0a44bf222278de6e716e143ca1e95ca320b06d4d23d6583fbf8d03e1428f3dac8fa00e5b82ddcd6b425e6236d85af09550a4"
+    },
+    "sirit": {
+        "repo": "eden-emulator/sirit",
+        "sha": "db1f1e8ab5",
+        "hash": "73eb3a042848c63a10656545797e85f40d142009dfb7827384548a385e1e28e1ac72f42b25924ce530d58275f8638554281e884d72f9c7aaf4ed08690a414b05",
+        "options": [
+            "SIRIT_USE_SYSTEM_SPIRV_HEADERS ON"
+        ]
+    },
+    "httplib": {
+        "repo": "yhirose/cpp-httplib",
+        "sha": "a609330e4c",
+        "hash": "dd3fd0572f8367d8549e1319fd98368b3e75801a293b0c3ac9b4adb806473a4506a484b3d389dc5bee5acc460cb90af7a20e5df705a1696b56496b30b9ce7ed2"
+    },
+    "cpp-jwt": {
+        "version": "1.4",
+        "repo": "arun11299/cpp-jwt",
+        "sha": "a54fa08a3b",
+        "hash": "a90f7e594ada0c7e49d5ff9211c71097534e7742a8e44bf0851b0362642a7271d53f5d83d04eeaae2bad17ef3f35e09e6818434d8eaefa038f3d1f7359d0969a",
+        "find_args": "CONFIG",
+        "options": [
+            "CPP_JWT_BUILD_EXAMPLES OFF",
+            "CPP_JWT_BUILD_TESTS OFF",
+            "CPP_JWT_USE_VENDORED_NLOHMANN_JSON OFF"
+        ],
+        "patches": [
+            "0001-no-install.patch",
+            "0002-missing-decl.patch"
+        ]
+    }
+}
--- a/externals/CMakeLists.txt
+++ b/externals/CMakeLists.txt
@ -7,9 +7,6 @@
 # TODO(crueter): A lot of this should be moved to the root.
 # otherwise we have to do weird shenanigans with library linking and stuff

-# cpm
-include(CPMUtil)
-
 # Explicitly declare this option here to propagate to the oaknut CPM call
 option(DYNARMIC_TESTS "Build tests" ${BUILD_TESTING})

@ -70,14 +67,7 @@ endif()
 add_subdirectory(glad)

 # mbedtls
-AddPackage(
-    NAME mbedtls
-    REPO "Mbed-TLS/mbedtls"
-    SHA "8c88150ca1"
-    HASH 769ad1e94c570671071e1f2a5c0f1027e0bf6bcdd1a80ea8ac970f2c86bc45ce4e31aa88d6d8110fc1bed1de81c48bc624df1b38a26f8b340a44e109d784a966
-    PATCHES
-        ${CMAKE_SOURCE_DIR}/.patch/mbedtls/0001-cmake-version.patch
-)
+AddJsonPackage(mbedtls)

 if (mbedtls_ADDED)
    target_include_directories(mbedtls PUBLIC ${mbedtls_SOURCE_DIR}/include)
@ -97,23 +87,13 @@ endif()
 # Sirit
 # TODO(crueter): spirv-tools doesn't work w/ system
 set(SPIRV_WERROR OFF)
-AddPackage(
-    NAME SPIRV-Headers
-    REPO "KhronosGroup/SPIRV-Headers"
-    SHA 4e209d3d7e
-    HASH f48bbe18341ed55ea0fe280dbbbc0a44bf222278de6e716e143ca1e95ca320b06d4d23d6583fbf8d03e1428f3dac8fa00e5b82ddcd6b425e6236d85af09550a4
-)
+AddJsonPackage("spirv-headers")

-AddPackage(
-    NAME sirit
-    REPO "eden-emulator/sirit"
-    SHA db1f1e8ab5
-    HASH 73eb3a042848c63a10656545797e85f40d142009dfb7827384548a385e1e28e1ac72f42b25924ce530d58275f8638554281e884d72f9c7aaf4ed08690a414b05
-    OPTIONS
-        "SIRIT_USE_SYSTEM_SPIRV_HEADERS ON"
-)
+AddJsonPackage("sirit")

-if(MSVC AND USE_CCACHE AND TARGET sirit)
+message(STATUS "Sirit added: ${sirit_ADDED}")
+
+if(MSVC AND USE_CCACHE AND sirit_ADDED)
    get_target_property(_opts sirit COMPILE_OPTIONS)
    list(FILTER _opts EXCLUDE REGEX "/Zi")
    list(APPEND _opts "/Z7")
@ -122,33 +102,12 @@ endif()

 # httplib
 if (ENABLE_WEB_SERVICE OR ENABLE_QT_UPDATE_CHECKER)
-    AddPackage(
-        NAME httplib
-        REPO "yhirose/cpp-httplib"
-        SHA a609330e4c
-        HASH dd3fd0572f8367d8549e1319fd98368b3e75801a293b0c3ac9b4adb806473a4506a484b3d389dc5bee5acc460cb90af7a20e5df705a1696b56496b30b9ce7ed2
-        OPTIONS
-            "HTTPLIB_REQUIRE_OPENSSL ${ENABLE_OPENSSL}"
-    )
+    AddJsonPackage("httplib")
 endif()

 # cpp-jwt
 if (ENABLE_WEB_SERVICE)
-    AddPackage(
-        NAME cpp-jwt
-        VERSION 1.4
-        REPO "arun11299/cpp-jwt"
-        SHA a54fa08a3b
-        HASH a90f7e594ada0c7e49d5ff9211c71097534e7742a8e44bf0851b0362642a7271d53f5d83d04eeaae2bad17ef3f35e09e6818434d8eaefa038f3d1f7359d0969a
-        FIND_PACKAGE_ARGUMENTS "CONFIG"
-        OPTIONS
-            "CPP_JWT_BUILD_EXAMPLES OFF"
-            "CPP_JWT_BUILD_TESTS OFF"
-            "CPP_JWT_USE_VENDORED_NLOHMANN_JSON OFF"
-        PATCHES
-            ${CMAKE_SOURCE_DIR}/.patch/cpp-jwt/0001-no-install.patch
-            ${CMAKE_SOURCE_DIR}/.patch/cpp-jwt/0002-missing-decl.patch
-    )
+    AddJsonPackage("cpp-jwt")
 endif()

 # unordered_dense
--- a/src/dynarmic/externals/CMakeLists.txt
+++ b/src/dynarmic/externals/CMakeLists.txt
@ -1,5 +1,3 @@
-include(CPMUtil)
-
 # Always build externals as static libraries, even when dynarmic is built as shared
 if (BUILD_SHARED_LIBS)
    set(BUILD_SHARED_LIBS OFF)
@ -104,7 +102,7 @@ if ("x86_64" IN_LIST ARCHITECTURE)
        OPTIONS
            "CMAKE_DISABLE_FIND_PACKAGE_Doxygen ON"
        EXCLUDE_FROM_ALL ON
-        SYSTEM_PACKAGE OFF
+        BUNDLED_PACKAGE ON
    )

    AddPackage(
--- a/src/yuzu/externals/CMakeLists.txt
+++ b/src/yuzu/externals/CMakeLists.txt
@ -1,9 +1,6 @@
 # SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project
 # SPDX-License-Identifier: GPL-3.0-or-later

-# cpm
-include(CPMUtil)
-
 # Disable tests/tools in all externals supporting the standard option name
 set(BUILD_TESTING OFF)