From 06437ac0b8751d848f36a226c35753f15bbff7c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A1clav=20Val=C3=AD=C4=8Dek=20=28YCNet=29?= Date: Fri, 23 Feb 2018 10:17:37 +0100 Subject: [PATCH] Convert into single docker image, change volume structure etc Makefile inconsistent! --- .dockerignore | 1 - Dockerfile | 12 +++-- Makefile | 28 +++++----- checker/cloner-lib-auth | 2 +- checker/cloner-lib-cfg | 1 - checker/cloner-lib-general | 1 - checker/run-checker | 2 + dockerbin/cron-command | 30 +++++++++++ dockerbin/launch-cron | 14 +++++ dockerbin/run-mirror-update | 35 +++++++++++++ launcher-image/Dockerfile | 11 ---- launcher-image/src/cron-command | 40 --------------- launcher-image/src/entrypoint | 11 ---- launcher-image/src/run-mirror-update | 77 ---------------------------- 14 files changed, 104 insertions(+), 161 deletions(-) create mode 100755 dockerbin/cron-command create mode 100755 dockerbin/launch-cron create mode 100755 dockerbin/run-mirror-update delete mode 100644 launcher-image/Dockerfile delete mode 100755 launcher-image/src/cron-command delete mode 100755 launcher-image/src/entrypoint delete mode 100755 launcher-image/src/run-mirror-update diff --git a/.dockerignore b/.dockerignore index ad57124..89f47bd 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1 @@ creator/* -launcher/* diff --git a/Dockerfile b/Dockerfile index e40c2cf..8dbe285 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,11 +3,17 @@ FROM alpine:edge VOLUME /data -RUN apk add --no-cache git bash openssh +RUN apk add --no-cache git bash openssh parallel shadow ; \ + useradd -ms /bin/bash executor ; \ + chown executor:executor /data ; \ + echo "* * * * * /usr/local/bin/cron-command" >> /etc/crontabs/executor ; \ + touch /var/run/cloner.pid ; \ + chown executor:executor /var/run/cloner.pid ; \ + apk del shadow -ADD checker/* src/* /usr/local/bin/ +ADD dockerbin/* checker/* src/* /usr/local/bin/ -CMD [ "/usr/local/bin/run-checker" ] +CMD [ "/usr/local/bin/launch-cron" ] diff --git a/Makefile b/Makefile index 65ea437..d08208a 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ default: buildall VOLUME ?= cloner-test +GLOBALVOL ?= cloner-global TAGOWNER = valicek1 TAGMAIN = repo-cloner TAGBASE= $(TAGOWNER)/$(TAGMAIN) @@ -8,12 +9,18 @@ TAGBASE= $(TAGOWNER)/$(TAGMAIN) buildall: mirror creator launcher -# Mirrorer - root of repo -mirror: - docker build -t $(TAGBASE)-mirror . +cloner: + docker build -t $(TAGBASE) . + +run: cloner + docker run -v $(GLOBALVOL):/data -it --rm $(TAGBASE) + +run-once: cloner + docker run -v $(GLOBALVOL):/data -it --rm --user=executor $(TAGBASE) /usr/local/bin/cron-command + +run-bash: cloner + docker run -v $(GLOBALVOL):/data -it --rm $(TAGBASE) /bin/bash -run-mirror: mirror - docker run -v $(VOLUME):/data -it --rm $(TAGBASE)-mirror # Creator - ./creator-image dir creator: @@ -22,17 +29,8 @@ creator: run-creator: creator docker run -v /var/run/docker.sock:/var/run/docker.sock -it --rm $(TAGBASE)-creator -# Launcher - ./launcher-image -launcher: - docker build -t $(TAGBASE)-launcher ./launcher-image - -run-launcher: launcher - docker run -e JOBS=4 -v /var/run/docker.sock:/var/run/docker.sock -it --rm $(TAGBASE)-launcher - -run-launcher-detached: launcher - docker run -e JOBS=8 -v /var/run/docker.sock:/var/run/docker.sock -itd --rm $(TAGBASE)-launcher - # wizzard +# auth dir could be executable (to list) wizzard: mirror run-creator # CI Detector diff --git a/checker/cloner-lib-auth b/checker/cloner-lib-auth index 0c021ce..b7d32a3 100644 --- a/checker/cloner-lib-auth +++ b/checker/cloner-lib-auth @@ -8,7 +8,7 @@ function prepareGitAuth(){ [ -d $confdir ] || mkdir $confdir [ -d $confdir/ssh ] || mkdir $confdir/ssh - chmod 0600 $confdir + chmod 0700 $confdir # git configure http authenticator git config --global credential.helper "store --file=$confdir/git-credentials" diff --git a/checker/cloner-lib-cfg b/checker/cloner-lib-cfg index ad6e3f3..6e663af 100644 --- a/checker/cloner-lib-cfg +++ b/checker/cloner-lib-cfg @@ -1,4 +1,3 @@ -#!/bin/bash # library for work with config file # check directories if they exist diff --git a/checker/cloner-lib-general b/checker/cloner-lib-general index 9bcf2f1..84e5ea0 100644 --- a/checker/cloner-lib-general +++ b/checker/cloner-lib-general @@ -1,6 +1,5 @@ #!/bin/bash # configure -export BASE=/data export CCLONE_PATH=$BASE/repos export CCLONE_CACHE=$BASE/cache export CONFIG_DIR=$BASE/config diff --git a/checker/run-checker b/checker/run-checker index 733df57..f4ba498 100755 --- a/checker/run-checker +++ b/checker/run-checker @@ -35,6 +35,8 @@ fi submodules=${cloner_submodules:-0} depth=${cloner_submodule_depth:-} +export HOME=$CCLONE_CACHE + prepareGitAuth $CONFIG_DIR # without submodule support diff --git a/dockerbin/cron-command b/dockerbin/cron-command new file mode 100755 index 0000000..4bd49b7 --- /dev/null +++ b/dockerbin/cron-command @@ -0,0 +1,30 @@ +#!/bin/bash +set -euo pipefail +IFS=$'\n\t' + +# check lock +lock=/var/run/cloner.pid + +dir_prefix=cloner + +max_jobs=${JOBS:-3} + +function die(){ + echo $@ 1>&2 + exit 1 +} + +# first, check process +if [ -f $lock ] +then + pid=$(cat $lock) + # if it still runs, die not so quietly + [ -n "$pid" ] && [ -d /proc/$pid ] && die "Another process running!" +fi +# else make some mess and setup trap +echo $BASHPID > $lock + +find /data -maxdepth 1 -type d -name "${dir_prefix}-*" | \ + parallel --lb -j $max_jobs -n 1 run-mirror-update + + diff --git a/dockerbin/launch-cron b/dockerbin/launch-cron new file mode 100755 index 0000000..174168b --- /dev/null +++ b/dockerbin/launch-cron @@ -0,0 +1,14 @@ +#!/bin/bash +set -euo pipefail +IFS=$'\n\t' + +# make parallel citation shut up +mkdir ~executor/.parallel +touch ~executor/.parallel/will-cite + +# repair ownership +find /data \! -user executor -exec chown executor:executor {} \; + +# run cron +crond -f + diff --git a/dockerbin/run-mirror-update b/dockerbin/run-mirror-update new file mode 100755 index 0000000..f45d4da --- /dev/null +++ b/dockerbin/run-mirror-update @@ -0,0 +1,35 @@ +#!/bin/bash +set -euo pipefail +IFS=$'\n\t' + +function log(){ + local title=${raw:-$name} + [ -z "$title" ] || title=" [$title]" + echo "[$(date +"%X")]$title $@" +} + +function die(){ + log "$@" 1>&2 + exit 1 +} + + +scratch=$(mktemp -d -t tmp.XXXXXXXXXX) +function finish { + rm -rf "$scratch" +} +trap finish EXIT + + +# necessary checks +pathto=${1:-} +[ -n "$pathto" ] || die "No project specified" + +raw=$(basename $pathto | sed 's/^cloner-//g') + +# is it enabled? +[ -f "$pathto/.enabled" ] || die "$raw not enabled!" + + +env BASE=$pathto run-checker | while read line; do log "$line"; done + diff --git a/launcher-image/Dockerfile b/launcher-image/Dockerfile deleted file mode 100644 index 20277f6..0000000 --- a/launcher-image/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -# Alpine base image -FROM alpine:edge - -RUN apk add --no-cache bash docker parallel ; \ - echo "* * * * * /bin/cron-command" >> /etc/crontabs/root - -ADD src/* /bin/ - -CMD [ "/bin/entrypoint" ] - - diff --git a/launcher-image/src/cron-command b/launcher-image/src/cron-command deleted file mode 100755 index 4f0941a..0000000 --- a/launcher-image/src/cron-command +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -set -euo pipefail -IFS=$'\n\t' - -# trap -lock=/var/run/cloner.pid - -volume_prefix=cloner -clone_prefix=cloner-runner - -max_jobs=${JOBS:-3} - -function die(){ - echo $@ 1>&2 - exit 1 -} - -# first, check process -if [ -f $lock ] -then - pid=$(cat $lock) - # if it still runs, die not so quietly - [ -d /proc/$pid ] && die "Another process running!" -fi -# else make some mess and setup trap -echo $BASHPID > $lock -function finish { - rm -rf $lock -} -trap finish EXIT - - -# declare functions -function listVolumes(){ - docker volume ls --filter=name=$volume_prefix- --quiet -} - -# run for every volume in parallel -listVolumes | parallel --lb -j $max_jobs -n 1 run-mirror-update - diff --git a/launcher-image/src/entrypoint b/launcher-image/src/entrypoint deleted file mode 100755 index 39e8d52..0000000 --- a/launcher-image/src/entrypoint +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -set -euo pipefail -IFS=$'\n\t' - -# make parallel citation shut up -mkdir $HOME/.parallel -touch $HOME/.parallel/will-cite - -# run cron -crond -f - diff --git a/launcher-image/src/run-mirror-update b/launcher-image/src/run-mirror-update deleted file mode 100755 index 47e1178..0000000 --- a/launcher-image/src/run-mirror-update +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash -set -euo pipefail -IFS=$'\n\t' - -mirror_image="valicek1/repo-cloner-mirror" - - -function log(){ - local title=${raw:-$name} - [ -z "$title" ] || title=" [$title]" - echo "[$(date +"%X")]$title $@" -} - -function die(){ - log "$@" 1>&2 - exit 1 -} - - -scratch=$(mktemp -d -t tmp.XXXXXXXXXX) -function finish { - rm -rf "$scratch" -} -trap finish EXIT - -volume_prefix=cloner -clone_prefix=cloner-runner - -# last version of image -last_version=$(docker images $mirror_image --no-trunc --quiet) - -name=${1:-} -[ -n "$name" ] || die "No volume specified" - -# contains prefix -[[ "$name" == $volume_prefix-* ]] || die "Volume name does not contain prefix!" - -raw=$(echo $name | sed "s/^$volume_prefix-//g") - -container_name=$clone_prefix-$raw - -# does it run? -docker ps --quiet --no-trunc --filter name=^/$container_name$ > $scratch/running -lines=$(wc -l $scratch/running | cut -f1 -d' ') - -if ! [ $lines -eq 0 ] -then - log "Another copy of container is running.. Exiting silently" - exit 0 -fi -unset lines - -# check if volume does exist? -docker volume inspect $name > /dev/null 2>&1 || die "Volume '$name' does not exist - exiting!" - -# does it exist? -image_version=$(docker inspect --format='{{.Image}}' $container_name 2>/dev/null || true) - -# do I need to recreate image? -if ! [ "x$image_version" = "x$last_version" ] -then - # not empty string - need to delete first - if [ -n "$image_version" ] - then - log "Conflicting container exists, removing.." - docker rm $container_name - fi - # create new container - log "Creating new version of container.." - docker create -v $name:/data --name $container_name $mirror_image -fi - - -log "Running container of update process..." -docker start -a $container_name | while read line; do log "$line"; done -log "Finished." -