Working copy of script
This commit is contained in:
commit
73bfa09839
72
cclone
Executable file
72
cclone
Executable file
|
|
@ -0,0 +1,72 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Clone repository from mirror
|
||||
#
|
||||
# Usage:
|
||||
# cclone <main repo url> [ -p <path> ] [ -c <checkout_ref> ]
|
||||
|
||||
# strict mode
|
||||
set -euo pipefail
|
||||
IFS=$'\n\t'
|
||||
|
||||
# include config
|
||||
source $(dirname $(realpath $0))/config
|
||||
source $(dirname $(realpath $0))/gen-mirror-path
|
||||
|
||||
# parse arguments
|
||||
function usage(){
|
||||
echo "Usage: $0 [-p <path>] [-c <checkout_ref>]" 1>&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
repo=$1
|
||||
|
||||
if [[ ! "$repo" =~ ^-.* ]]
|
||||
then
|
||||
# check if repo was mirrored
|
||||
if [ ! -d $(getRepoPath $repo) ]
|
||||
then
|
||||
echo "Specified repo wasn't mirrored yes, please do it so!" 1>&2
|
||||
exit 1
|
||||
fi
|
||||
# kick args +1
|
||||
shift
|
||||
else
|
||||
usage
|
||||
fi
|
||||
|
||||
# other opts
|
||||
while getopts "c:p:" o; do
|
||||
case "${o}" in
|
||||
c)
|
||||
param_c=${OPTARG}
|
||||
;;
|
||||
p)
|
||||
param_p=${OPTARG}
|
||||
;;
|
||||
\?)
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
|
||||
# Run clone
|
||||
cloneurl=$(getRepoPath $repo)
|
||||
clonepath=${param_p:-}
|
||||
|
||||
echo "Cloning $repo"
|
||||
git clone file://$cloneurl $clonepath
|
||||
|
||||
checkout=${param_c:-}
|
||||
if [ -n "$checkout" ]
|
||||
then
|
||||
chdir=${param_p:-$(getRepoUniq $repo)}
|
||||
echo "Checking out $checkout"
|
||||
oldpwd=$(pwd)
|
||||
cd $chdir
|
||||
# -b just to make git less verbose
|
||||
git checkout $checkout -b _tmp_$checkout
|
||||
cd $oldpwd
|
||||
fi
|
||||
|
||||
9
config
Normal file
9
config
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
#!/bin/bash
|
||||
|
||||
cfgMirrorPath=/home/vasek/asdf
|
||||
cfgCachePath=/home/vasek/asdf-cache
|
||||
|
||||
# modify PATH
|
||||
bindir=$(dirname $(realpath $0))
|
||||
PATH=$bindir:$PATH
|
||||
|
||||
42
gen-mirror-path
Executable file
42
gen-mirror-path
Executable file
|
|
@ -0,0 +1,42 @@
|
|||
#!/bin/bash
|
||||
|
||||
source $(dirname $(realpath $0))/config
|
||||
|
||||
|
||||
function getRepoUniq(){
|
||||
base=$(basename $1 .git)
|
||||
rest=$(dirname $1)
|
||||
# .git (working repo)
|
||||
if [ "$base" == ".git" ]
|
||||
then
|
||||
base=$(basename $rest)
|
||||
rest=$(dirname $rest)
|
||||
fi
|
||||
# extract username - or path
|
||||
namespace=$(basename $rest)
|
||||
# solve ssh domain:namespace/repo.git
|
||||
if [[ "$namespace" == *:* ]]
|
||||
then
|
||||
namespace=$(echo $namespace | cut -d':' -f2)
|
||||
fi
|
||||
csum=$(echo $1 | cksum | cut -d' ' -f1)
|
||||
echo ${namespace}_${base}_${csum}
|
||||
}
|
||||
|
||||
function getRepoPath(){
|
||||
uniq=$(getRepoUniq $1)
|
||||
echo $cfgMirrorPath/$uniq.git
|
||||
}
|
||||
|
||||
function getRepoCache(){
|
||||
uniq=$(getRepoUniq $1)
|
||||
echo $cfgCachePath/$uniq
|
||||
}
|
||||
|
||||
# it the script is not sourced
|
||||
if [[ ! "${BASH_SOURCE[0]}" != "${0}" ]]
|
||||
then
|
||||
getRepoPath $1
|
||||
fi
|
||||
|
||||
|
||||
49
mirror-main-repo
Executable file
49
mirror-main-repo
Executable file
|
|
@ -0,0 +1,49 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Just mirror (clone or fetch) specified git repository
|
||||
# - no other mess (eg submodules, just clean mirror)
|
||||
#
|
||||
# Usage:
|
||||
# mirror-main-repo <url>
|
||||
|
||||
|
||||
# Unofficial strict mode
|
||||
set -euo pipefail
|
||||
IFS=$'\n\t'
|
||||
|
||||
source $(dirname $(realpath $0))/gen-mirror-path
|
||||
|
||||
|
||||
function updateOrCreate(){
|
||||
url=$1
|
||||
repodir=$(getRepoPath $url)
|
||||
|
||||
if [ ! -d $repodir ]
|
||||
then
|
||||
echo "Clone of $url"
|
||||
git clone --bare --mirror $url $repodir
|
||||
else
|
||||
cd $repodir
|
||||
echo "Update of $url"
|
||||
git fetch --prune
|
||||
fi
|
||||
|
||||
# tags
|
||||
}
|
||||
|
||||
function getLastCommit(){
|
||||
url=$1
|
||||
repodir=$(getRepoPath $url)
|
||||
if [ -d $repodir ]
|
||||
then
|
||||
cd $repodir
|
||||
git --no-pager log --full-history --all -1 --pretty=format:"%H%n"
|
||||
else
|
||||
echo '-'
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
oldPwd=$(pwd)
|
||||
updateOrCreate $1
|
||||
cd $oldPwd
|
||||
113
mirror-recursive
Executable file
113
mirror-recursive
Executable file
|
|
@ -0,0 +1,113 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Mirror git repository with submodules - recursively
|
||||
#
|
||||
# Usage:
|
||||
# mirror-recursive <main repo url> [<depth=1000>]
|
||||
|
||||
# strict mode
|
||||
set -euo pipefail
|
||||
IFS=$'\n\t'
|
||||
|
||||
# Scratch - temp
|
||||
tmpdir=$(mktemp -d -t mirror-recursive-XXXXXXX)
|
||||
function finish {
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap finish EXIT
|
||||
|
||||
source $(dirname $(realpath $0))/config
|
||||
source $(dirname $(realpath $0))/gen-mirror-path
|
||||
|
||||
|
||||
function progress(){
|
||||
local progress=${1:-100}
|
||||
echo -n "..$progress%"
|
||||
if [ $progress -eq 100 ]
|
||||
then
|
||||
echo
|
||||
fi
|
||||
}
|
||||
|
||||
function submoduleDiscovery(){
|
||||
# main parameters
|
||||
local repo=$1
|
||||
local gitdir=$(getRepoPath $repo)
|
||||
# depth (empty or prefixed from main script)
|
||||
local depth=${2:-}
|
||||
# temporary path
|
||||
local tmpname=$(getRepoUniq $repo)
|
||||
tmpname=$tmpdir/$tmpname
|
||||
local tmpCommitList=$tmpname.commits
|
||||
local tmpSubmoduleList=$tmpname.submodules
|
||||
# cache paths
|
||||
local cachePath=$(getRepoCache $repo)
|
||||
local cacheCommits=$cachePath/commits-checked
|
||||
local cacheSubmodules=$cachePath/submodules-checked
|
||||
|
||||
# check, if cache exists
|
||||
[ -d $cachePath ] || mkdir -p $cachePath
|
||||
[ -f $cacheCommits ] || touch $cacheCommits
|
||||
[ -f $cacheSubmodules ] || touch $cacheSubmodules
|
||||
|
||||
# avoid recursion - if commit list exists
|
||||
# there was activity with this run recently
|
||||
if [ ! -f $tmpCommitList ]
|
||||
then
|
||||
|
||||
# cache submodules reuse
|
||||
cat $cacheSubmodules > $tmpSubmoduleList
|
||||
|
||||
echo -n "Discovering submodules of $repo.. "
|
||||
git --git-dir $gitdir log --all $depth --format="%H" | sort > $tmpCommitList
|
||||
|
||||
# check against cache
|
||||
echo -n "cache check.."
|
||||
comm -13 $cacheCommits $tmpCommitList > $tmpname
|
||||
mv $tmpname $tmpCommitList
|
||||
|
||||
local commits=$(wc -l $tmpCommitList | cut -d' ' -f1)
|
||||
echo -n "$commits commits"
|
||||
# this can take long time...
|
||||
local processed=0
|
||||
local nextStamp=$(($(date +"%s") + 3))
|
||||
while read -r line || [[ -n "$line" ]]
|
||||
do
|
||||
submodule-describe $gitdir $line | cut -f 3 >> $tmpSubmoduleList
|
||||
# progress indication
|
||||
processed=$(($processed + 1))
|
||||
if [ $(date +"%s") -gt $nextStamp ]
|
||||
then
|
||||
progress $((100*$processed/$commits))
|
||||
nextStamp=$(($nextStamp + 3))
|
||||
fi
|
||||
done < $tmpCommitList
|
||||
# finish the bar
|
||||
progress
|
||||
|
||||
# archive to cache
|
||||
cat $tmpCommitList $cacheCommits > $tmpname
|
||||
sort $tmpname > $cacheCommits
|
||||
sort $tmpSubmoduleList | uniq > $tmpname
|
||||
cat $tmpname > $cacheSubmodules
|
||||
|
||||
# Recursion++
|
||||
while read -r submodule || [[ -n "$submodule" ]]
|
||||
do
|
||||
mirror-main-repo $submodule
|
||||
submoduleDiscovery $submodule $depth
|
||||
done < $cacheSubmodules
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
# main repo
|
||||
mainrepo=$1
|
||||
depth=${2:-}
|
||||
|
||||
[ -n "$depth" ] && depth="-$depth"
|
||||
|
||||
# Make first mirror
|
||||
mirror-main-repo $mainrepo
|
||||
submoduleDiscovery $mainrepo $depth
|
||||
|
||||
73
sclone
Executable file
73
sclone
Executable file
|
|
@ -0,0 +1,73 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Clone repository from mirror - recursively with submodules
|
||||
#
|
||||
# Usage:
|
||||
# sclone <main repo url> [ -p <path> ] [ -c <checkout_ref>
|
||||
|
||||
# strict mode
|
||||
set -euo pipefail
|
||||
IFS=$'\n\t'
|
||||
|
||||
# Scratch - temp
|
||||
tmpdir=$(mktemp -d -t mirror-recursive-XXXXXXX)
|
||||
function finish {
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap finish EXIT
|
||||
|
||||
# include config
|
||||
source $(dirname $(realpath $0))/config
|
||||
source $(dirname $(realpath $0))/gen-mirror-path
|
||||
|
||||
# parse arguments
|
||||
function usage(){
|
||||
echo "Usage: $0 [-p <path>] [-c <checkout_ref>]" 1>&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
[ $# -eq 0 ] && usage
|
||||
|
||||
repo=$1
|
||||
|
||||
|
||||
# clone the repo
|
||||
cclone $@ || true
|
||||
# skip url
|
||||
shift
|
||||
|
||||
# parse opts
|
||||
while getopts "c:p:" o; do
|
||||
case "${o}" in
|
||||
c)
|
||||
param_c=${OPTARG}
|
||||
;;
|
||||
p)
|
||||
param_p=${OPTARG}
|
||||
;;
|
||||
\?)
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# change dir and examine the commit + submodules
|
||||
oldpwd=$(pwd)
|
||||
submodules=$tmpdir/submodules
|
||||
cd ${param_p:-$(getRepoUniq $repo)}
|
||||
submodule-describe . > $submodules
|
||||
|
||||
while read -r line || [[ -n "$line" ]]
|
||||
do
|
||||
# read -r retypes \t to ' ' (space)
|
||||
commit=$(echo $line | cut -f1 -d' ')
|
||||
directory=$(echo $line | cut -f2 -d' ')
|
||||
url=$(echo $line | cut -f3 -d' ')
|
||||
# recursion ++
|
||||
sclone $url -p $directory -c $commit
|
||||
done < $submodules
|
||||
|
||||
|
||||
|
||||
cd $oldpwd
|
||||
|
||||
66
submodule-describe
Executable file
66
submodule-describe
Executable file
|
|
@ -0,0 +1,66 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Describe submodules in repository, optionally per commit
|
||||
#
|
||||
# Usage:
|
||||
# submodule-describe <path-to-repo> [<commit>]
|
||||
#
|
||||
# Output:
|
||||
# <hash> <path> <url>
|
||||
# <hash> <path> <url>
|
||||
#
|
||||
# everything separated with tabs
|
||||
|
||||
# Safe mode
|
||||
set -euo pipefail
|
||||
IFS=$'\n\t'
|
||||
|
||||
# get config file for specific commit
|
||||
function getConfigFile() {
|
||||
git --no-pager show $commit:.gitmodules
|
||||
}
|
||||
|
||||
# parse submodule file from file
|
||||
function parseSectionNames(){
|
||||
git config -f $1 --list --name-only | grep '^submodule.' | cut -d. -f2 | sort | uniq
|
||||
}
|
||||
|
||||
# generate line for single submodule
|
||||
function generateDescription(){
|
||||
cfgFile=$1
|
||||
section=$2
|
||||
|
||||
path=$(git config -f $cfgFile --get submodule.$section.path)
|
||||
url=$(git config -f $cfgFile --get submodule.$section.url)
|
||||
hash=$(git ls-tree -l $commit -- $path | cut -d' ' -f3)
|
||||
|
||||
printf "%s\t%s\t%s\n" $hash $path $url
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
# Grab varriables
|
||||
repodir=$1
|
||||
commit=${2:-HEAD}
|
||||
|
||||
# Go to repo directory
|
||||
oldPwd=$(pwd)
|
||||
cd $repodir
|
||||
|
||||
# Are there any submodules registered?
|
||||
test 0 -eq `git ls-tree $commit -- .gitmodules | wc -l` && exit 0
|
||||
|
||||
tmpfile=$(mktemp)
|
||||
|
||||
getConfigFile $repodir $commit > $tmpfile
|
||||
for section in $(parseSectionNames $tmpfile)
|
||||
do
|
||||
generateDescription $tmpfile $section
|
||||
done
|
||||
|
||||
|
||||
# Cleanup
|
||||
rm $tmpfile
|
||||
# Go back home
|
||||
cd $oldPwd
|
||||
Loading…
Reference in New Issue
Block a user