Convert into single docker image, change volume structure etc

Makefile inconsistent!
This commit is contained in:
Václav Valíček (YCNet) 2018-02-23 10:17:37 +01:00
parent 8c4f64cf96
commit 06437ac0b8
No known key found for this signature in database
GPG Key ID: 7CF44871CEA75938
14 changed files with 104 additions and 161 deletions

View File

@ -1,2 +1 @@
creator/* creator/*
launcher/*

View File

@ -3,11 +3,17 @@ FROM alpine:edge
VOLUME /data VOLUME /data
RUN apk add --no-cache git bash openssh RUN apk add --no-cache git bash openssh parallel shadow ; \
useradd -ms /bin/bash executor ; \
chown executor:executor /data ; \
echo "* * * * * /usr/local/bin/cron-command" >> /etc/crontabs/executor ; \
touch /var/run/cloner.pid ; \
chown executor:executor /var/run/cloner.pid ; \
apk del shadow
ADD checker/* src/* /usr/local/bin/ ADD dockerbin/* checker/* src/* /usr/local/bin/
CMD [ "/usr/local/bin/run-checker" ] CMD [ "/usr/local/bin/launch-cron" ]

View File

@ -1,6 +1,7 @@
default: buildall default: buildall
VOLUME ?= cloner-test VOLUME ?= cloner-test
GLOBALVOL ?= cloner-global
TAGOWNER = valicek1 TAGOWNER = valicek1
TAGMAIN = repo-cloner TAGMAIN = repo-cloner
TAGBASE= $(TAGOWNER)/$(TAGMAIN) TAGBASE= $(TAGOWNER)/$(TAGMAIN)
@ -8,12 +9,18 @@ TAGBASE= $(TAGOWNER)/$(TAGMAIN)
buildall: mirror creator launcher buildall: mirror creator launcher
# Mirrorer - root of repo cloner:
mirror: docker build -t $(TAGBASE) .
docker build -t $(TAGBASE)-mirror .
run: cloner
docker run -v $(GLOBALVOL):/data -it --rm $(TAGBASE)
run-once: cloner
docker run -v $(GLOBALVOL):/data -it --rm --user=executor $(TAGBASE) /usr/local/bin/cron-command
run-bash: cloner
docker run -v $(GLOBALVOL):/data -it --rm $(TAGBASE) /bin/bash
run-mirror: mirror
docker run -v $(VOLUME):/data -it --rm $(TAGBASE)-mirror
# Creator - ./creator-image dir # Creator - ./creator-image dir
creator: creator:
@ -22,17 +29,8 @@ creator:
run-creator: creator run-creator: creator
docker run -v /var/run/docker.sock:/var/run/docker.sock -it --rm $(TAGBASE)-creator docker run -v /var/run/docker.sock:/var/run/docker.sock -it --rm $(TAGBASE)-creator
# Launcher - ./launcher-image
launcher:
docker build -t $(TAGBASE)-launcher ./launcher-image
run-launcher: launcher
docker run -e JOBS=4 -v /var/run/docker.sock:/var/run/docker.sock -it --rm $(TAGBASE)-launcher
run-launcher-detached: launcher
docker run -e JOBS=8 -v /var/run/docker.sock:/var/run/docker.sock -itd --rm $(TAGBASE)-launcher
# wizzard # wizzard
# auth dir could be executable (to list)
wizzard: mirror run-creator wizzard: mirror run-creator
# CI Detector # CI Detector

View File

@ -8,7 +8,7 @@ function prepareGitAuth(){
[ -d $confdir ] || mkdir $confdir [ -d $confdir ] || mkdir $confdir
[ -d $confdir/ssh ] || mkdir $confdir/ssh [ -d $confdir/ssh ] || mkdir $confdir/ssh
chmod 0600 $confdir chmod 0700 $confdir
# git configure http authenticator # git configure http authenticator
git config --global credential.helper "store --file=$confdir/git-credentials" git config --global credential.helper "store --file=$confdir/git-credentials"

View File

@ -1,4 +1,3 @@
#!/bin/bash
# library for work with config file # library for work with config file
# check directories if they exist # check directories if they exist

View File

@ -1,6 +1,5 @@
#!/bin/bash #!/bin/bash
# configure # configure
export BASE=/data
export CCLONE_PATH=$BASE/repos export CCLONE_PATH=$BASE/repos
export CCLONE_CACHE=$BASE/cache export CCLONE_CACHE=$BASE/cache
export CONFIG_DIR=$BASE/config export CONFIG_DIR=$BASE/config

View File

@ -35,6 +35,8 @@ fi
submodules=${cloner_submodules:-0} submodules=${cloner_submodules:-0}
depth=${cloner_submodule_depth:-} depth=${cloner_submodule_depth:-}
export HOME=$CCLONE_CACHE
prepareGitAuth $CONFIG_DIR prepareGitAuth $CONFIG_DIR
# without submodule support # without submodule support

30
dockerbin/cron-command Executable file
View File

@ -0,0 +1,30 @@
#!/bin/bash
set -euo pipefail
IFS=$'\n\t'
# check lock
lock=/var/run/cloner.pid
dir_prefix=cloner
max_jobs=${JOBS:-3}
function die(){
echo $@ 1>&2
exit 1
}
# first, check process
if [ -f $lock ]
then
pid=$(cat $lock)
# if it still runs, die not so quietly
[ -n "$pid" ] && [ -d /proc/$pid ] && die "Another process running!"
fi
# else make some mess and setup trap
echo $BASHPID > $lock
find /data -maxdepth 1 -type d -name "${dir_prefix}-*" | \
parallel --lb -j $max_jobs -n 1 run-mirror-update

14
dockerbin/launch-cron Executable file
View File

@ -0,0 +1,14 @@
#!/bin/bash
set -euo pipefail
IFS=$'\n\t'
# make parallel citation shut up
mkdir ~executor/.parallel
touch ~executor/.parallel/will-cite
# repair ownership
find /data \! -user executor -exec chown executor:executor {} \;
# run cron
crond -f

35
dockerbin/run-mirror-update Executable file
View File

@ -0,0 +1,35 @@
#!/bin/bash
set -euo pipefail
IFS=$'\n\t'
function log(){
local title=${raw:-$name}
[ -z "$title" ] || title=" [$title]"
echo "[$(date +"%X")]$title $@"
}
function die(){
log "$@" 1>&2
exit 1
}
scratch=$(mktemp -d -t tmp.XXXXXXXXXX)
function finish {
rm -rf "$scratch"
}
trap finish EXIT
# necessary checks
pathto=${1:-}
[ -n "$pathto" ] || die "No project specified"
raw=$(basename $pathto | sed 's/^cloner-//g')
# is it enabled?
[ -f "$pathto/.enabled" ] || die "$raw not enabled!"
env BASE=$pathto run-checker | while read line; do log "$line"; done

View File

@ -1,11 +0,0 @@
# Alpine base image
FROM alpine:edge
RUN apk add --no-cache bash docker parallel ; \
echo "* * * * * /bin/cron-command" >> /etc/crontabs/root
ADD src/* /bin/
CMD [ "/bin/entrypoint" ]

View File

@ -1,40 +0,0 @@
#!/bin/bash
set -euo pipefail
IFS=$'\n\t'
# trap
lock=/var/run/cloner.pid
volume_prefix=cloner
clone_prefix=cloner-runner
max_jobs=${JOBS:-3}
function die(){
echo $@ 1>&2
exit 1
}
# first, check process
if [ -f $lock ]
then
pid=$(cat $lock)
# if it still runs, die not so quietly
[ -d /proc/$pid ] && die "Another process running!"
fi
# else make some mess and setup trap
echo $BASHPID > $lock
function finish {
rm -rf $lock
}
trap finish EXIT
# declare functions
function listVolumes(){
docker volume ls --filter=name=$volume_prefix- --quiet
}
# run for every volume in parallel
listVolumes | parallel --lb -j $max_jobs -n 1 run-mirror-update

View File

@ -1,11 +0,0 @@
#!/bin/bash
set -euo pipefail
IFS=$'\n\t'
# make parallel citation shut up
mkdir $HOME/.parallel
touch $HOME/.parallel/will-cite
# run cron
crond -f

View File

@ -1,77 +0,0 @@
#!/bin/bash
set -euo pipefail
IFS=$'\n\t'
mirror_image="valicek1/repo-cloner-mirror"
function log(){
local title=${raw:-$name}
[ -z "$title" ] || title=" [$title]"
echo "[$(date +"%X")]$title $@"
}
function die(){
log "$@" 1>&2
exit 1
}
scratch=$(mktemp -d -t tmp.XXXXXXXXXX)
function finish {
rm -rf "$scratch"
}
trap finish EXIT
volume_prefix=cloner
clone_prefix=cloner-runner
# last version of image
last_version=$(docker images $mirror_image --no-trunc --quiet)
name=${1:-}
[ -n "$name" ] || die "No volume specified"
# contains prefix
[[ "$name" == $volume_prefix-* ]] || die "Volume name does not contain prefix!"
raw=$(echo $name | sed "s/^$volume_prefix-//g")
container_name=$clone_prefix-$raw
# does it run?
docker ps --quiet --no-trunc --filter name=^/$container_name$ > $scratch/running
lines=$(wc -l $scratch/running | cut -f1 -d' ')
if ! [ $lines -eq 0 ]
then
log "Another copy of container is running.. Exiting silently"
exit 0
fi
unset lines
# check if volume does exist?
docker volume inspect $name > /dev/null 2>&1 || die "Volume '$name' does not exist - exiting!"
# does it exist?
image_version=$(docker inspect --format='{{.Image}}' $container_name 2>/dev/null || true)
# do I need to recreate image?
if ! [ "x$image_version" = "x$last_version" ]
then
# not empty string - need to delete first
if [ -n "$image_version" ]
then
log "Conflicting container exists, removing.."
docker rm $container_name
fi
# create new container
log "Creating new version of container.."
docker create -v $name:/data --name $container_name $mirror_image
fi
log "Running container of update process..."
docker start -a $container_name | while read line; do log "$line"; done
log "Finished."