114 lines
2.6 KiB
Plaintext
114 lines
2.6 KiB
Plaintext
|
#!/bin/bash
|
||
|
|
#
|
||
|
|
# Mirror git repository with submodules - recursively
|
||
|
|
#
|
||
|
|
# Usage:
|
||
|
|
# mirror-recursive <main repo url> [<depth=1000>]
|
||
|
|
|
||
|
|
# strict mode
|
||
|
|
set -euo pipefail
|
||
|
|
IFS=$'\n\t'
|
||
|
|
|
||
|
|
# Scratch - temp
|
||
|
|
tmpdir=$(mktemp -d -t mirror-recursive-XXXXXXX)
|
||
|
|
function finish {
|
||
|
|
rm -rf "$tmpdir"
|
||
|
|
}
|
||
|
|
trap finish EXIT
|
||
|
|
|
||
|
|
source $(dirname $(realpath $0))/config
|
||
|
|
source $(dirname $(realpath $0))/gen-mirror-path
|
||
|
|
|
||
|
|
|
||
|
|
function progress(){
|
||
|
|
local progress=${1:-100}
|
||
|
|
echo -n "..$progress%"
|
||
|
|
if [ $progress -eq 100 ]
|
||
|
|
then
|
||
|
|
echo
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
function submoduleDiscovery(){
|
||
|
|
# main parameters
|
||
|
|
local repo=$1
|
||
|
|
local gitdir=$(getRepoPath $repo)
|
||
|
|
# depth (empty or prefixed from main script)
|
||
|
|
local depth=${2:-}
|
||
|
|
# temporary path
|
||
|
|
local tmpname=$(getRepoUniq $repo)
|
||
|
|
tmpname=$tmpdir/$tmpname
|
||
|
|
local tmpCommitList=$tmpname.commits
|
||
|
|
local tmpSubmoduleList=$tmpname.submodules
|
||
|
|
# cache paths
|
||
|
|
local cachePath=$(getRepoCache $repo)
|
||
|
|
local cacheCommits=$cachePath/commits-checked
|
||
|
|
local cacheSubmodules=$cachePath/submodules-checked
|
||
|
|
|
||
|
|
# check, if cache exists
|
||
|
|
[ -d $cachePath ] || mkdir -p $cachePath
|
||
|
|
[ -f $cacheCommits ] || touch $cacheCommits
|
||
|
|
[ -f $cacheSubmodules ] || touch $cacheSubmodules
|
||
|
|
|
||
|
|
# avoid recursion - if commit list exists
|
||
|
|
# there was activity with this run recently
|
||
|
|
if [ ! -f $tmpCommitList ]
|
||
|
|
then
|
||
|
|
|
||
|
|
# cache submodules reuse
|
||
|
|
cat $cacheSubmodules > $tmpSubmoduleList
|
||
|
|
|
||
|
|
echo -n "Discovering submodules of $repo.. "
|
||
|
|
git --git-dir $gitdir log --all $depth --format="%H" | sort > $tmpCommitList
|
||
|
|
|
||
|
|
# check against cache
|
||
|
|
echo -n "cache check.."
|
||
|
|
comm -13 $cacheCommits $tmpCommitList > $tmpname
|
||
|
|
mv $tmpname $tmpCommitList
|
||
|
|
|
||
|
|
local commits=$(wc -l $tmpCommitList | cut -d' ' -f1)
|
||
|
|
echo -n "$commits commits"
|
||
|
|
# this can take long time...
|
||
|
|
local processed=0
|
||
|
|
local nextStamp=$(($(date +"%s") + 3))
|
||
|
|
while read -r line || [[ -n "$line" ]]
|
||
|
|
do
|
||
|
|
submodule-describe $gitdir $line | cut -f 3 >> $tmpSubmoduleList
|
||
|
|
# progress indication
|
||
|
|
processed=$(($processed + 1))
|
||
|
|
if [ $(date +"%s") -gt $nextStamp ]
|
||
|
|
then
|
||
|
|
progress $((100*$processed/$commits))
|
||
|
|
nextStamp=$(($nextStamp + 3))
|
||
|
|
fi
|
||
|
|
done < $tmpCommitList
|
||
|
|
# finish the bar
|
||
|
|
progress
|
||
|
|
|
||
|
|
# archive to cache
|
||
|
|
cat $tmpCommitList $cacheCommits > $tmpname
|
||
|
|
sort $tmpname > $cacheCommits
|
||
|
|
sort $tmpSubmoduleList | uniq > $tmpname
|
||
|
|
cat $tmpname > $cacheSubmodules
|
||
|
|
|
||
|
|
# Recursion++
|
||
|
|
while read -r submodule || [[ -n "$submodule" ]]
|
||
|
|
do
|
||
|
|
mirror-main-repo $submodule
|
||
|
|
submoduleDiscovery $submodule $depth
|
||
|
|
done < $cacheSubmodules
|
||
|
|
fi
|
||
|
|
|
||
|
|
}
|
||
|
|
|
||
|
|
# main repo
|
||
|
|
mainrepo=$1
|
||
|
|
depth=${2:-}
|
||
|
|
|
||
|
|
[ -n "$depth" ] && depth="-$depth"
|
||
|
|
|
||
|
|
# Make first mirror
|
||
|
|
mirror-main-repo $mainrepo
|
||
|
|
submoduleDiscovery $mainrepo $depth
|
||
|
|
|