From 73bfa0983938b33e2d445086585bb160cd3a3c1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A1clav=20Val=C3=AD=C4=8Dek=20=28YCNet=29?= Date: Mon, 5 Feb 2018 15:47:39 +0100 Subject: [PATCH] Working copy of script --- cclone | 72 +++++++++++++++++++++++++++++ config | 9 ++++ gen-mirror-path | 42 +++++++++++++++++ mirror-main-repo | 49 ++++++++++++++++++++ mirror-recursive | 113 +++++++++++++++++++++++++++++++++++++++++++++ sclone | 73 +++++++++++++++++++++++++++++ submodule-describe | 66 ++++++++++++++++++++++++++ 7 files changed, 424 insertions(+) create mode 100755 cclone create mode 100644 config create mode 100755 gen-mirror-path create mode 100755 mirror-main-repo create mode 100755 mirror-recursive create mode 100755 sclone create mode 100755 submodule-describe diff --git a/cclone b/cclone new file mode 100755 index 0000000..f68fc5d --- /dev/null +++ b/cclone @@ -0,0 +1,72 @@ +#!/bin/bash +# +# Clone repository from mirror +# +# Usage: +# cclone
[ -p ] [ -c ] + +# strict mode +set -euo pipefail +IFS=$'\n\t' + +# include config +source $(dirname $(realpath $0))/config +source $(dirname $(realpath $0))/gen-mirror-path + +# parse arguments +function usage(){ + echo "Usage: $0 [-p ] [-c ]" 1>&2 + exit 1 +} + +repo=$1 + +if [[ ! "$repo" =~ ^-.* ]] +then + # check if repo was mirrored + if [ ! -d $(getRepoPath $repo) ] + then + echo "Specified repo wasn't mirrored yes, please do it so!" 1>&2 + exit 1 + fi + # kick args +1 + shift +else + usage +fi + +# other opts +while getopts "c:p:" o; do + case "${o}" in + c) + param_c=${OPTARG} + ;; + p) + param_p=${OPTARG} + ;; + \?) + usage + ;; + esac +done + + +# Run clone +cloneurl=$(getRepoPath $repo) +clonepath=${param_p:-} + +echo "Cloning $repo" +git clone file://$cloneurl $clonepath + +checkout=${param_c:-} +if [ -n "$checkout" ] +then + chdir=${param_p:-$(getRepoUniq $repo)} + echo "Checking out $checkout" + oldpwd=$(pwd) + cd $chdir + # -b just to make git less verbose + git checkout $checkout -b _tmp_$checkout + cd $oldpwd +fi + diff --git a/config b/config new file mode 100644 index 0000000..841ae90 --- /dev/null +++ b/config @@ -0,0 +1,9 @@ +#!/bin/bash + +cfgMirrorPath=/home/vasek/asdf +cfgCachePath=/home/vasek/asdf-cache + +# modify PATH +bindir=$(dirname $(realpath $0)) +PATH=$bindir:$PATH + diff --git a/gen-mirror-path b/gen-mirror-path new file mode 100755 index 0000000..7ef1f11 --- /dev/null +++ b/gen-mirror-path @@ -0,0 +1,42 @@ +#!/bin/bash + +source $(dirname $(realpath $0))/config + + +function getRepoUniq(){ + base=$(basename $1 .git) + rest=$(dirname $1) + # .git (working repo) + if [ "$base" == ".git" ] + then + base=$(basename $rest) + rest=$(dirname $rest) + fi + # extract username - or path + namespace=$(basename $rest) + # solve ssh domain:namespace/repo.git + if [[ "$namespace" == *:* ]] + then + namespace=$(echo $namespace | cut -d':' -f2) + fi + csum=$(echo $1 | cksum | cut -d' ' -f1) + echo ${namespace}_${base}_${csum} +} + +function getRepoPath(){ + uniq=$(getRepoUniq $1) + echo $cfgMirrorPath/$uniq.git +} + +function getRepoCache(){ + uniq=$(getRepoUniq $1) + echo $cfgCachePath/$uniq +} + +# it the script is not sourced +if [[ ! "${BASH_SOURCE[0]}" != "${0}" ]] +then + getRepoPath $1 +fi + + diff --git a/mirror-main-repo b/mirror-main-repo new file mode 100755 index 0000000..fb12adc --- /dev/null +++ b/mirror-main-repo @@ -0,0 +1,49 @@ +#!/bin/bash +# +# Just mirror (clone or fetch) specified git repository +# - no other mess (eg submodules, just clean mirror) +# +# Usage: +# mirror-main-repo + + +# Unofficial strict mode +set -euo pipefail +IFS=$'\n\t' + +source $(dirname $(realpath $0))/gen-mirror-path + + +function updateOrCreate(){ + url=$1 + repodir=$(getRepoPath $url) + + if [ ! -d $repodir ] + then + echo "Clone of $url" + git clone --bare --mirror $url $repodir + else + cd $repodir + echo "Update of $url" + git fetch --prune + fi + + # tags +} + +function getLastCommit(){ + url=$1 + repodir=$(getRepoPath $url) + if [ -d $repodir ] + then + cd $repodir + git --no-pager log --full-history --all -1 --pretty=format:"%H%n" + else + echo '-' + fi + +} + +oldPwd=$(pwd) +updateOrCreate $1 +cd $oldPwd diff --git a/mirror-recursive b/mirror-recursive new file mode 100755 index 0000000..372e102 --- /dev/null +++ b/mirror-recursive @@ -0,0 +1,113 @@ +#!/bin/bash +# +# Mirror git repository with submodules - recursively +# +# Usage: +# mirror-recursive
[] + +# strict mode +set -euo pipefail +IFS=$'\n\t' + +# Scratch - temp +tmpdir=$(mktemp -d -t mirror-recursive-XXXXXXX) +function finish { + rm -rf "$tmpdir" +} +trap finish EXIT + +source $(dirname $(realpath $0))/config +source $(dirname $(realpath $0))/gen-mirror-path + + +function progress(){ + local progress=${1:-100} + echo -n "..$progress%" + if [ $progress -eq 100 ] + then + echo + fi +} + +function submoduleDiscovery(){ + # main parameters + local repo=$1 + local gitdir=$(getRepoPath $repo) + # depth (empty or prefixed from main script) + local depth=${2:-} + # temporary path + local tmpname=$(getRepoUniq $repo) + tmpname=$tmpdir/$tmpname + local tmpCommitList=$tmpname.commits + local tmpSubmoduleList=$tmpname.submodules + # cache paths + local cachePath=$(getRepoCache $repo) + local cacheCommits=$cachePath/commits-checked + local cacheSubmodules=$cachePath/submodules-checked + + # check, if cache exists + [ -d $cachePath ] || mkdir -p $cachePath + [ -f $cacheCommits ] || touch $cacheCommits + [ -f $cacheSubmodules ] || touch $cacheSubmodules + + # avoid recursion - if commit list exists + # there was activity with this run recently + if [ ! -f $tmpCommitList ] + then + + # cache submodules reuse + cat $cacheSubmodules > $tmpSubmoduleList + + echo -n "Discovering submodules of $repo.. " + git --git-dir $gitdir log --all $depth --format="%H" | sort > $tmpCommitList + + # check against cache + echo -n "cache check.." + comm -13 $cacheCommits $tmpCommitList > $tmpname + mv $tmpname $tmpCommitList + + local commits=$(wc -l $tmpCommitList | cut -d' ' -f1) + echo -n "$commits commits" + # this can take long time... + local processed=0 + local nextStamp=$(($(date +"%s") + 3)) + while read -r line || [[ -n "$line" ]] + do + submodule-describe $gitdir $line | cut -f 3 >> $tmpSubmoduleList + # progress indication + processed=$(($processed + 1)) + if [ $(date +"%s") -gt $nextStamp ] + then + progress $((100*$processed/$commits)) + nextStamp=$(($nextStamp + 3)) + fi + done < $tmpCommitList + # finish the bar + progress + + # archive to cache + cat $tmpCommitList $cacheCommits > $tmpname + sort $tmpname > $cacheCommits + sort $tmpSubmoduleList | uniq > $tmpname + cat $tmpname > $cacheSubmodules + + # Recursion++ + while read -r submodule || [[ -n "$submodule" ]] + do + mirror-main-repo $submodule + submoduleDiscovery $submodule $depth + done < $cacheSubmodules + fi + +} + +# main repo +mainrepo=$1 +depth=${2:-} + +[ -n "$depth" ] && depth="-$depth" + +# Make first mirror +mirror-main-repo $mainrepo +submoduleDiscovery $mainrepo $depth + diff --git a/sclone b/sclone new file mode 100755 index 0000000..2081316 --- /dev/null +++ b/sclone @@ -0,0 +1,73 @@ +#!/bin/bash +# +# Clone repository from mirror - recursively with submodules +# +# Usage: +# sclone
[ -p ] [ -c + +# strict mode +set -euo pipefail +IFS=$'\n\t' + +# Scratch - temp +tmpdir=$(mktemp -d -t mirror-recursive-XXXXXXX) +function finish { + rm -rf "$tmpdir" +} +trap finish EXIT + +# include config +source $(dirname $(realpath $0))/config +source $(dirname $(realpath $0))/gen-mirror-path + +# parse arguments +function usage(){ + echo "Usage: $0 [-p ] [-c ]" 1>&2 + exit 1 +} + +[ $# -eq 0 ] && usage + +repo=$1 + + +# clone the repo +cclone $@ || true +# skip url +shift + +# parse opts +while getopts "c:p:" o; do + case "${o}" in + c) + param_c=${OPTARG} + ;; + p) + param_p=${OPTARG} + ;; + \?) + usage + ;; + esac +done + +# change dir and examine the commit + submodules +oldpwd=$(pwd) +submodules=$tmpdir/submodules +cd ${param_p:-$(getRepoUniq $repo)} +submodule-describe . > $submodules + +while read -r line || [[ -n "$line" ]] +do + # read -r retypes \t to ' ' (space) + commit=$(echo $line | cut -f1 -d' ') + directory=$(echo $line | cut -f2 -d' ') + url=$(echo $line | cut -f3 -d' ') + # recursion ++ + sclone $url -p $directory -c $commit +done < $submodules + + + +cd $oldpwd + diff --git a/submodule-describe b/submodule-describe new file mode 100755 index 0000000..76a937d --- /dev/null +++ b/submodule-describe @@ -0,0 +1,66 @@ +#!/bin/bash +# +# Describe submodules in repository, optionally per commit +# +# Usage: +# submodule-describe [] +# +# Output: +# +# +# +# everything separated with tabs + +# Safe mode +set -euo pipefail +IFS=$'\n\t' + +# get config file for specific commit +function getConfigFile() { + git --no-pager show $commit:.gitmodules +} + +# parse submodule file from file +function parseSectionNames(){ + git config -f $1 --list --name-only | grep '^submodule.' | cut -d. -f2 | sort | uniq +} + +# generate line for single submodule +function generateDescription(){ + cfgFile=$1 + section=$2 + + path=$(git config -f $cfgFile --get submodule.$section.path) + url=$(git config -f $cfgFile --get submodule.$section.url) + hash=$(git ls-tree -l $commit -- $path | cut -d' ' -f3) + + printf "%s\t%s\t%s\n" $hash $path $url +} + + + + +# Grab varriables +repodir=$1 +commit=${2:-HEAD} + +# Go to repo directory +oldPwd=$(pwd) +cd $repodir + +# Are there any submodules registered? +test 0 -eq `git ls-tree $commit -- .gitmodules | wc -l` && exit 0 + +tmpfile=$(mktemp) + +getConfigFile $repodir $commit > $tmpfile +for section in $(parseSectionNames $tmpfile) +do + generateDescription $tmpfile $section +done + + +# Cleanup +rm $tmpfile +# Go back home +cd $oldPwd