From 5478ddde8799948280d0958768f2b5f5cea12bf4 Mon Sep 17 00:00:00 2001 From: lizzie Date: Sat, 25 Oct 2025 20:09:49 +0000 Subject: [PATCH] better Signed-off-by: lizzie --- CMakeModules/CPMUtil.cmake | 3 +- tools/README.md | 2 + tools/cpm/common.sh | 2 +- tools/mirror/common.sh | 4 +- tools/mirror/helper.sh | 112 +++++++++++++++++++++++-------------- 5 files changed, 77 insertions(+), 46 deletions(-) diff --git a/CMakeModules/CPMUtil.cmake b/CMakeModules/CPMUtil.cmake index a5e13e58ed..88ca641a0f 100644 --- a/CMakeModules/CPMUtil.cmake +++ b/CMakeModules/CPMUtil.cmake @@ -13,8 +13,7 @@ option(CPMUTIL_FORCE_BUNDLED option(CPMUTIL_FORCE_SYSTEM "Force system packages for all CPM dependencies (NOT RECOMMENDED)" OFF) -option(CPMUTIL_DEFAULT_HOST - "Sets the default host when 'git_host' isn't defined" "github.com") +set(CPMUTIL_DEFAULT_HOST github.com CACHE STRING "Sets the default host when 'git_host' isn't defined") option(CPMUTIL_FORCE_HOST "Force host CPMUTIL_DEFAULT_HOST to be used for all CPM dependencies even when 'git_host' is defined" OFF) diff --git a/tools/README.md b/tools/README.md index e45eaef7ab..51ab12dc2e 100644 --- a/tools/README.md +++ b/tools/README.md @@ -24,6 +24,7 @@ Tools for Eden and other subprojects. - `find-unused-strings.sh`: Find any unused strings in the Android app (XML -> Kotlin). ## Android + It's recommended to run these scritps after almost any Android change, as they are relatively fast and important both for APK bloat and CI. - `unused-strings.sh`: Finds unused strings in `strings.xml` files. @@ -34,4 +35,5 @@ It's recommended to run these scritps after almost any Android change, as they a - [Translation Scripts](./translations) ## Mirror + - [Mirroring scripts](./mirror) diff --git a/tools/cpm/common.sh b/tools/cpm/common.sh index 92a88f0c9b..7f7b8f9f13 100755 --- a/tools/cpm/common.sh +++ b/tools/cpm/common.sh @@ -13,7 +13,7 @@ MAXDEPTH=3 # For your project you'll want to change this to define what dirs you have cpmfiles in # Remember to account for the MAXDEPTH variable! # Adding ./ before each will help to remove duplicates -[ -z "$CPMFILES" ] && CPMFILES=$(find . ./src -maxdepth "$MAXDEPTH" -name cpmfile.json | sort | uniq) +[ -z "$CPMFILES" ] && CPMFILES=$(find . ./src -maxdepth "$MAXDEPTH" -name cpmfile.json -not -path build | sort | uniq) # shellcheck disable=SC2016 [ -z "$PACKAGES" ] && PACKAGES=$(echo "$CPMFILES" | xargs jq -s 'reduce .[] as $item ({}; . * $item)') diff --git a/tools/mirror/common.sh b/tools/mirror/common.sh index 0f401082bd..ebb0783e36 100644 --- a/tools/mirror/common.sh +++ b/tools/mirror/common.sh @@ -2,7 +2,9 @@ # SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project # SPDX-License-Identifier: GPL-3.0-or-later git_clone_mirror_repo() { - git clone --mirror $1 $(echo $1 | awk -F"/" '{print $4"/"$5".git"}') + NAME=$(echo "$1" | cut -d '/' -f 5) + ORG=$(echo "$1" | cut -d '/' -f 4) + git clone --mirror $1 $ORG/$NAME } git_retrieve_file() { TMPDIR=$1 diff --git a/tools/mirror/helper.sh b/tools/mirror/helper.sh index dbf8230667..c325fc7b40 100755 --- a/tools/mirror/helper.sh +++ b/tools/mirror/helper.sh @@ -5,45 +5,73 @@ CPM_DIR=$PWD NPROC=`nproc` . ./tools/mirror/common.sh -#read -p "Absolute path of target directory (i.e /srv/git/repos): " TARGET_ROOT_DIR -[ -z "$TARGET_ROOT_DIR" ] && TARGET_ROOT_DIR="/usr/local/jails/containers/classic/srv/git/repos" -echo "Path: " $TARGET_ROOT_DIR -echo "Path will be modified with perms 777" -echo "1 - Do initial clone/world" -echo "2 - Clone submodules (update)" -echo "3 - Force manual remotes update (please use crontab instead)" -read -p "Selection? [none]: " ANSWER -if [ "$ANSWER" = "1" ]; then - echo Cloning to $TARGET_ROOT_DIR... - sudo chmod 777 $TARGET_ROOT_DIR - # stuff to parse the cpmfile json and then spit out full repo path - REPOS=$(find "$CPM_DIR" "$CPM_DIR/src" -maxdepth 3 -type f -name 'cpmfile.json' -not -path 'build' \ - | sort | uniq | xargs jq -s 'reduce .[] as $item ({}; . * $item)' \ - | jq -r 'reduce .[] as $i (""; . + (if $i.git_host == null then "https://github.com" else "https://" + $i.git_host end) + "/" + $i.repo + " ")' \ - | tr ' ' '\n' | xargs -I {} echo {}) - # clone the stuff - cd $TARGET_ROOT_DIR && echo "$REPOS" \ - | xargs -P $NPROC -I {} sh -c ". $CPM_DIR/tools/mirror/common.sh && git_clone_mirror_repo {}" - sudo chmod 755 $TARGET_ROOT_DIR -elif [ "$ANSWER" = "2" ]; then - echo Updating submodules of $TARGET_ROOT_DIR... - sudo chmod 777 $TARGET_ROOT_DIR - cd $TARGET_ROOT_DIR && find . -maxdepth 2 -type d -name '*.git' -print0 \ - | xargs -0 -I {} sh -c ". $CPM_DIR/tools/mirror/common.sh && git_retrieve_gitmodules {}" \ - | while IFS= read -r line; do - NAME=$(echo "$line" | awk -F'/' '{print $5}') - ORG=$(echo "$line" | awk -F'/' '{print $4}') - URL=$(echo "$line") - git clone --mirror $URL $ORG/$NAME || echo "skipped $ORG/$NAME" - done - sudo chmod 755 $TARGET_ROOT_DIR -elif [ "$ANSWER" = "3" ]; then - echo Forcing git updates - sudo chmod 777 $TARGET_ROOT_DIR - cd $TARGET_ROOT_DIR && find . -maxdepth 3 -type d -name '*.git' -print0 \ - | xargs -0 -P $NPROC -I {} sh -c 'cd {} && git remote update && echo {}' - sudo chmod 755 $TARGET_ROOT_DIR -else - echo Aborting -fi - +. ./tools/cpm/common.sh +die() { + echo "$@" >&2 + exit 1 +} +help() { + cat << EOF +--path Specify the given path (must be the root of your SCM folder) +--initial Initial cloning (no submodules) - fetched from cpmfiles +--clone-submodules Clones submodules, can also be used multiple times to clone + newly referenced submodules +--remote-update Update all remotes (of the repos) - aka. sync with remote +--mirror Perform a mirror clone; URL must be specified before, + if name or organisation are not specified, it is derived from URL +--url Set URL of clone +--org Set organisation folder of clone +--name Set name of clone +EOF +} +[ -z "$SCM_ROOT_DIR" ] && SCM_ROOT_DIR="/usr/local/jails/containers/classic/srv/git/repos" +while true; do + case "$1" in + --path) + shift + SCM_ROOT_DIR=$1 + [ -z "$SCM_ROOT_DIR" ] && "Empty target root dir" + ;; + --initial) + sudo chmod 777 $SCM_ROOT_DIR + # stuff to parse the cpmfile json and then spit out full repo path + REPOS=$(echo "$PACKAGES" \ + | jq -r 'reduce .[] as $i (""; . + (if $i.git_host == null then "https://github.com" else "https://" + $i.git_host end) + "/" + $i.repo + " ")' \ + | tr ' ' '\n' | xargs -I {} echo {}) + # clone the stuff + cd $SCM_ROOT_DIR && echo "$REPOS" \ + | xargs -P $NPROC -I {} sh -c ". $CPM_DIR/tools/mirror/common.sh && git_clone_mirror_repo {}" + sudo chmod 755 $SCM_ROOT_DIR + ;; + --clone-submodules) + sudo chmod 777 $SCM_ROOT_DIR + cd $SCM_ROOT_DIR && find . -maxdepth 2 -type d -name '*.git' -print0 \ + | xargs -0 -I {} sh -c ". $CPM_DIR/tools/mirror/common.sh && git_retrieve_gitmodules {}" \ + | while IFS= read -r url; do + git_clone_mirror_repo $url || echo "skipped $url" + done + sudo chmod 755 $SCM_ROOT_DIR + ;; + --remote-update) + sudo chmod 777 $SCM_ROOT_DIR + cd $SCM_ROOT_DIR && find . -maxdepth 3 -type d -name '*.git' -print0 \ + | xargs -0 -P $NPROC -I {} sh -c 'cd {} && git remote update && echo {}' + sudo chmod 755 $SCM_ROOT_DIR + ;; + --mirror) + sudo chmod 777 $SCM_ROOT_DIR + [ -z "$URL" ] && die "Specify repo --url" + [ -z "$NAME" ] && NAME=$(echo "$URL" | cut -d '/' -f 5) + [ -z "$ORG" ] && ORG=$(echo "$URL" | cut -d '/' -f 4) + cd $SCM_ROOT_DIR && git clone --mirror $URL $ORG/$NAME || echo "skipped $URL" + sudo chmod 755 $SCM_ROOT_DIR + ;; + --url) shift; URL=$1 ;; + --name) shift; NAME=$1;; + --org) shift; ORG=$1;; + --help) help "$@";; + --*) die "Invalid option $1" ;; + "$0" | "") break;; + esac + shift +done