diff --git a/CMakeModules/CPMUtil.cmake b/CMakeModules/CPMUtil.cmake index 1c4a4874c6..a5e13e58ed 100644 --- a/CMakeModules/CPMUtil.cmake +++ b/CMakeModules/CPMUtil.cmake @@ -13,6 +13,12 @@ option(CPMUTIL_FORCE_BUNDLED option(CPMUTIL_FORCE_SYSTEM "Force system packages for all CPM dependencies (NOT RECOMMENDED)" OFF) +option(CPMUTIL_DEFAULT_HOST + "Sets the default host when 'git_host' isn't defined" "github.com") + +option(CPMUTIL_FORCE_HOST + "Force host CPMUTIL_DEFAULT_HOST to be used for all CPM dependencies even when 'git_host' is defined" OFF) + cmake_minimum_required(VERSION 3.22) include(CPM) @@ -293,8 +299,8 @@ function(AddPackage) option(${PKG_ARGS_NAME}_FORCE_SYSTEM "Force the system package for ${PKG_ARGS_NAME}") option(${PKG_ARGS_NAME}_FORCE_BUNDLED "Force the bundled package for ${PKG_ARGS_NAME}") - if (NOT DEFINED PKG_ARGS_GIT_HOST) - set(git_host github.com) + if (CPMUTIL_FORCE_HOST OR NOT DEFINED PKG_ARGS_GIT_HOST) + set(git_host ${CPMUTIL_DEFAULT_HOST}) else() set(git_host ${PKG_ARGS_GIT_HOST}) endif() diff --git a/docs/CPMUtil/README.md b/docs/CPMUtil/README.md index 8fe4df5237..aec19411ea 100644 --- a/docs/CPMUtil/README.md +++ b/docs/CPMUtil/README.md @@ -7,6 +7,8 @@ Global Options: - `CPMUTIL_FORCE_SYSTEM` (default `OFF`): Require all CPM dependencies to use system packages. NOT RECOMMENDED! * You may optionally override this (section) - `CPMUTIL_FORCE_BUNDLED` (default `ON` on MSVC and Android, `OFF` elsewhere): Require all CPM dependencies to use bundled packages. +- `CPMUTIL_DEFAULT_HOST`: (default `github.com`): Sets the default `git_host`. +- `CPMUTIL_FORCE_HOST`: (default `OFF`): Forces all CPM packages to use `CPMUTIL_DEFAULT_HOST` instead, even if they have `git_host` defined. You are highly encouraged to read AddPackage first, even if you plan to only interact with CPMUtil via `AddJsonPackage`. diff --git a/tools/README.md b/tools/README.md index 767892e7ac..e45eaef7ab 100644 --- a/tools/README.md +++ b/tools/README.md @@ -32,3 +32,6 @@ It's recommended to run these scritps after almost any Android change, as they a ## Translations - [Translation Scripts](./translations) + +## Mirror +- [Mirroring scripts](./mirror) diff --git a/tools/mirror/README.md b/tools/mirror/README.md new file mode 100644 index 0000000000..183a97e4cf --- /dev/null +++ b/tools/mirror/README.md @@ -0,0 +1,6 @@ +# Mirroring scripts + +Scripts for setting up a Git mirror or performing offline work (i.e no GitHub) when using systems like CPM. + +- `helper.sh`: Mini helper script - made for `cgit` jails but may work for Forgejo as well. +- `common.sh`: Common functions and stuff. diff --git a/tools/mirror/common.sh b/tools/mirror/common.sh new file mode 100644 index 0000000000..0f401082bd --- /dev/null +++ b/tools/mirror/common.sh @@ -0,0 +1,28 @@ +#!/bin/sh -e +# SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +# SPDX-License-Identifier: GPL-3.0-or-later +git_clone_mirror_repo() { + git clone --mirror $1 $(echo $1 | awk -F"/" '{print $4"/"$5".git"}') +} +git_retrieve_file() { + TMPDIR=$1 + BRANCH=$(cd $TMPDIR && git rev-parse --abbrev-ref HEAD) + cd $1 && git show $BRANCH:$2 2>/dev/null | git lfs smudge 2>/dev/null +} +git_retrieve_gitmodules() { + TMP=$1 + git_retrieve_file $TMP '.gitmodules' | awk '{$1=$1};1' \ + | grep "url = " | sed "s,url = ,," \ + | while IFS= read -r line; do + case "$line" in + ../*) + # TODO: maybe one day handle case where its NOT ALL GITHUB - thanks boost + ORGAN=$(echo "$TMP" | awk -F'/' '{print $2}') + echo "$line" | sed "s|../|https://github.com/$ORGAN/|" + ;; + http*) + echo "$line" + ;; + esac + done +} diff --git a/tools/mirror/helper.sh b/tools/mirror/helper.sh new file mode 100755 index 0000000000..dbf8230667 --- /dev/null +++ b/tools/mirror/helper.sh @@ -0,0 +1,49 @@ +#!/bin/sh -e +# SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +# SPDX-License-Identifier: GPL-3.0-or-later +# You must run this at the root of the eden git repo +CPM_DIR=$PWD +NPROC=`nproc` +. ./tools/mirror/common.sh +#read -p "Absolute path of target directory (i.e /srv/git/repos): " TARGET_ROOT_DIR +[ -z "$TARGET_ROOT_DIR" ] && TARGET_ROOT_DIR="/usr/local/jails/containers/classic/srv/git/repos" +echo "Path: " $TARGET_ROOT_DIR +echo "Path will be modified with perms 777" +echo "1 - Do initial clone/world" +echo "2 - Clone submodules (update)" +echo "3 - Force manual remotes update (please use crontab instead)" +read -p "Selection? [none]: " ANSWER +if [ "$ANSWER" = "1" ]; then + echo Cloning to $TARGET_ROOT_DIR... + sudo chmod 777 $TARGET_ROOT_DIR + # stuff to parse the cpmfile json and then spit out full repo path + REPOS=$(find "$CPM_DIR" "$CPM_DIR/src" -maxdepth 3 -type f -name 'cpmfile.json' -not -path 'build' \ + | sort | uniq | xargs jq -s 'reduce .[] as $item ({}; . * $item)' \ + | jq -r 'reduce .[] as $i (""; . + (if $i.git_host == null then "https://github.com" else "https://" + $i.git_host end) + "/" + $i.repo + " ")' \ + | tr ' ' '\n' | xargs -I {} echo {}) + # clone the stuff + cd $TARGET_ROOT_DIR && echo "$REPOS" \ + | xargs -P $NPROC -I {} sh -c ". $CPM_DIR/tools/mirror/common.sh && git_clone_mirror_repo {}" + sudo chmod 755 $TARGET_ROOT_DIR +elif [ "$ANSWER" = "2" ]; then + echo Updating submodules of $TARGET_ROOT_DIR... + sudo chmod 777 $TARGET_ROOT_DIR + cd $TARGET_ROOT_DIR && find . -maxdepth 2 -type d -name '*.git' -print0 \ + | xargs -0 -I {} sh -c ". $CPM_DIR/tools/mirror/common.sh && git_retrieve_gitmodules {}" \ + | while IFS= read -r line; do + NAME=$(echo "$line" | awk -F'/' '{print $5}') + ORG=$(echo "$line" | awk -F'/' '{print $4}') + URL=$(echo "$line") + git clone --mirror $URL $ORG/$NAME || echo "skipped $ORG/$NAME" + done + sudo chmod 755 $TARGET_ROOT_DIR +elif [ "$ANSWER" = "3" ]; then + echo Forcing git updates + sudo chmod 777 $TARGET_ROOT_DIR + cd $TARGET_ROOT_DIR && find . -maxdepth 3 -type d -name '*.git' -print0 \ + | xargs -0 -P $NPROC -I {} sh -c 'cd {} && git remote update && echo {}' + sudo chmod 755 $TARGET_ROOT_DIR +else + echo Aborting +fi +