Convert into single docker image, change volume structure etc
Makefile inconsistent!
This commit is contained in:
parent
8c4f64cf96
commit
06437ac0b8
|
@ -1,2 +1 @@
|
||||||
creator/*
|
creator/*
|
||||||
launcher/*
|
|
||||||
|
|
12
Dockerfile
12
Dockerfile
|
@ -3,11 +3,17 @@ FROM alpine:edge
|
||||||
|
|
||||||
VOLUME /data
|
VOLUME /data
|
||||||
|
|
||||||
RUN apk add --no-cache git bash openssh
|
RUN apk add --no-cache git bash openssh parallel shadow ; \
|
||||||
|
useradd -ms /bin/bash executor ; \
|
||||||
|
chown executor:executor /data ; \
|
||||||
|
echo "* * * * * /usr/local/bin/cron-command" >> /etc/crontabs/executor ; \
|
||||||
|
touch /var/run/cloner.pid ; \
|
||||||
|
chown executor:executor /var/run/cloner.pid ; \
|
||||||
|
apk del shadow
|
||||||
|
|
||||||
ADD checker/* src/* /usr/local/bin/
|
ADD dockerbin/* checker/* src/* /usr/local/bin/
|
||||||
|
|
||||||
|
|
||||||
CMD [ "/usr/local/bin/run-checker" ]
|
CMD [ "/usr/local/bin/launch-cron" ]
|
||||||
|
|
||||||
|
|
||||||
|
|
28
Makefile
28
Makefile
|
@ -1,6 +1,7 @@
|
||||||
default: buildall
|
default: buildall
|
||||||
|
|
||||||
VOLUME ?= cloner-test
|
VOLUME ?= cloner-test
|
||||||
|
GLOBALVOL ?= cloner-global
|
||||||
TAGOWNER = valicek1
|
TAGOWNER = valicek1
|
||||||
TAGMAIN = repo-cloner
|
TAGMAIN = repo-cloner
|
||||||
TAGBASE= $(TAGOWNER)/$(TAGMAIN)
|
TAGBASE= $(TAGOWNER)/$(TAGMAIN)
|
||||||
|
@ -8,12 +9,18 @@ TAGBASE= $(TAGOWNER)/$(TAGMAIN)
|
||||||
|
|
||||||
buildall: mirror creator launcher
|
buildall: mirror creator launcher
|
||||||
|
|
||||||
# Mirrorer - root of repo
|
cloner:
|
||||||
mirror:
|
docker build -t $(TAGBASE) .
|
||||||
docker build -t $(TAGBASE)-mirror .
|
|
||||||
|
run: cloner
|
||||||
|
docker run -v $(GLOBALVOL):/data -it --rm $(TAGBASE)
|
||||||
|
|
||||||
|
run-once: cloner
|
||||||
|
docker run -v $(GLOBALVOL):/data -it --rm --user=executor $(TAGBASE) /usr/local/bin/cron-command
|
||||||
|
|
||||||
|
run-bash: cloner
|
||||||
|
docker run -v $(GLOBALVOL):/data -it --rm $(TAGBASE) /bin/bash
|
||||||
|
|
||||||
run-mirror: mirror
|
|
||||||
docker run -v $(VOLUME):/data -it --rm $(TAGBASE)-mirror
|
|
||||||
|
|
||||||
# Creator - ./creator-image dir
|
# Creator - ./creator-image dir
|
||||||
creator:
|
creator:
|
||||||
|
@ -22,17 +29,8 @@ creator:
|
||||||
run-creator: creator
|
run-creator: creator
|
||||||
docker run -v /var/run/docker.sock:/var/run/docker.sock -it --rm $(TAGBASE)-creator
|
docker run -v /var/run/docker.sock:/var/run/docker.sock -it --rm $(TAGBASE)-creator
|
||||||
|
|
||||||
# Launcher - ./launcher-image
|
|
||||||
launcher:
|
|
||||||
docker build -t $(TAGBASE)-launcher ./launcher-image
|
|
||||||
|
|
||||||
run-launcher: launcher
|
|
||||||
docker run -e JOBS=4 -v /var/run/docker.sock:/var/run/docker.sock -it --rm $(TAGBASE)-launcher
|
|
||||||
|
|
||||||
run-launcher-detached: launcher
|
|
||||||
docker run -e JOBS=8 -v /var/run/docker.sock:/var/run/docker.sock -itd --rm $(TAGBASE)-launcher
|
|
||||||
|
|
||||||
# wizzard
|
# wizzard
|
||||||
|
# auth dir could be executable (to list)
|
||||||
wizzard: mirror run-creator
|
wizzard: mirror run-creator
|
||||||
|
|
||||||
# CI Detector
|
# CI Detector
|
||||||
|
|
|
@ -8,7 +8,7 @@ function prepareGitAuth(){
|
||||||
[ -d $confdir ] || mkdir $confdir
|
[ -d $confdir ] || mkdir $confdir
|
||||||
[ -d $confdir/ssh ] || mkdir $confdir/ssh
|
[ -d $confdir/ssh ] || mkdir $confdir/ssh
|
||||||
|
|
||||||
chmod 0600 $confdir
|
chmod 0700 $confdir
|
||||||
|
|
||||||
# git configure http authenticator
|
# git configure http authenticator
|
||||||
git config --global credential.helper "store --file=$confdir/git-credentials"
|
git config --global credential.helper "store --file=$confdir/git-credentials"
|
||||||
|
|
|
@ -1,4 +1,3 @@
|
||||||
#!/bin/bash
|
|
||||||
# library for work with config file
|
# library for work with config file
|
||||||
|
|
||||||
# check directories if they exist
|
# check directories if they exist
|
||||||
|
|
|
@ -1,6 +1,5 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
# configure
|
# configure
|
||||||
export BASE=/data
|
|
||||||
export CCLONE_PATH=$BASE/repos
|
export CCLONE_PATH=$BASE/repos
|
||||||
export CCLONE_CACHE=$BASE/cache
|
export CCLONE_CACHE=$BASE/cache
|
||||||
export CONFIG_DIR=$BASE/config
|
export CONFIG_DIR=$BASE/config
|
||||||
|
|
|
@ -35,6 +35,8 @@ fi
|
||||||
submodules=${cloner_submodules:-0}
|
submodules=${cloner_submodules:-0}
|
||||||
depth=${cloner_submodule_depth:-}
|
depth=${cloner_submodule_depth:-}
|
||||||
|
|
||||||
|
export HOME=$CCLONE_CACHE
|
||||||
|
|
||||||
prepareGitAuth $CONFIG_DIR
|
prepareGitAuth $CONFIG_DIR
|
||||||
|
|
||||||
# without submodule support
|
# without submodule support
|
||||||
|
|
30
dockerbin/cron-command
Executable file
30
dockerbin/cron-command
Executable file
|
@ -0,0 +1,30 @@
|
||||||
|
#!/bin/bash
|
||||||
|
set -euo pipefail
|
||||||
|
IFS=$'\n\t'
|
||||||
|
|
||||||
|
# check lock
|
||||||
|
lock=/var/run/cloner.pid
|
||||||
|
|
||||||
|
dir_prefix=cloner
|
||||||
|
|
||||||
|
max_jobs=${JOBS:-3}
|
||||||
|
|
||||||
|
function die(){
|
||||||
|
echo $@ 1>&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
# first, check process
|
||||||
|
if [ -f $lock ]
|
||||||
|
then
|
||||||
|
pid=$(cat $lock)
|
||||||
|
# if it still runs, die not so quietly
|
||||||
|
[ -n "$pid" ] && [ -d /proc/$pid ] && die "Another process running!"
|
||||||
|
fi
|
||||||
|
# else make some mess and setup trap
|
||||||
|
echo $BASHPID > $lock
|
||||||
|
|
||||||
|
find /data -maxdepth 1 -type d -name "${dir_prefix}-*" | \
|
||||||
|
parallel --lb -j $max_jobs -n 1 run-mirror-update
|
||||||
|
|
||||||
|
|
14
dockerbin/launch-cron
Executable file
14
dockerbin/launch-cron
Executable file
|
@ -0,0 +1,14 @@
|
||||||
|
#!/bin/bash
|
||||||
|
set -euo pipefail
|
||||||
|
IFS=$'\n\t'
|
||||||
|
|
||||||
|
# make parallel citation shut up
|
||||||
|
mkdir ~executor/.parallel
|
||||||
|
touch ~executor/.parallel/will-cite
|
||||||
|
|
||||||
|
# repair ownership
|
||||||
|
find /data \! -user executor -exec chown executor:executor {} \;
|
||||||
|
|
||||||
|
# run cron
|
||||||
|
crond -f
|
||||||
|
|
35
dockerbin/run-mirror-update
Executable file
35
dockerbin/run-mirror-update
Executable file
|
@ -0,0 +1,35 @@
|
||||||
|
#!/bin/bash
|
||||||
|
set -euo pipefail
|
||||||
|
IFS=$'\n\t'
|
||||||
|
|
||||||
|
function log(){
|
||||||
|
local title=${raw:-$name}
|
||||||
|
[ -z "$title" ] || title=" [$title]"
|
||||||
|
echo "[$(date +"%X")]$title $@"
|
||||||
|
}
|
||||||
|
|
||||||
|
function die(){
|
||||||
|
log "$@" 1>&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
scratch=$(mktemp -d -t tmp.XXXXXXXXXX)
|
||||||
|
function finish {
|
||||||
|
rm -rf "$scratch"
|
||||||
|
}
|
||||||
|
trap finish EXIT
|
||||||
|
|
||||||
|
|
||||||
|
# necessary checks
|
||||||
|
pathto=${1:-}
|
||||||
|
[ -n "$pathto" ] || die "No project specified"
|
||||||
|
|
||||||
|
raw=$(basename $pathto | sed 's/^cloner-//g')
|
||||||
|
|
||||||
|
# is it enabled?
|
||||||
|
[ -f "$pathto/.enabled" ] || die "$raw not enabled!"
|
||||||
|
|
||||||
|
|
||||||
|
env BASE=$pathto run-checker | while read line; do log "$line"; done
|
||||||
|
|
|
@ -1,11 +0,0 @@
|
||||||
# Alpine base image
|
|
||||||
FROM alpine:edge
|
|
||||||
|
|
||||||
RUN apk add --no-cache bash docker parallel ; \
|
|
||||||
echo "* * * * * /bin/cron-command" >> /etc/crontabs/root
|
|
||||||
|
|
||||||
ADD src/* /bin/
|
|
||||||
|
|
||||||
CMD [ "/bin/entrypoint" ]
|
|
||||||
|
|
||||||
|
|
|
@ -1,40 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
set -euo pipefail
|
|
||||||
IFS=$'\n\t'
|
|
||||||
|
|
||||||
# trap
|
|
||||||
lock=/var/run/cloner.pid
|
|
||||||
|
|
||||||
volume_prefix=cloner
|
|
||||||
clone_prefix=cloner-runner
|
|
||||||
|
|
||||||
max_jobs=${JOBS:-3}
|
|
||||||
|
|
||||||
function die(){
|
|
||||||
echo $@ 1>&2
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
# first, check process
|
|
||||||
if [ -f $lock ]
|
|
||||||
then
|
|
||||||
pid=$(cat $lock)
|
|
||||||
# if it still runs, die not so quietly
|
|
||||||
[ -d /proc/$pid ] && die "Another process running!"
|
|
||||||
fi
|
|
||||||
# else make some mess and setup trap
|
|
||||||
echo $BASHPID > $lock
|
|
||||||
function finish {
|
|
||||||
rm -rf $lock
|
|
||||||
}
|
|
||||||
trap finish EXIT
|
|
||||||
|
|
||||||
|
|
||||||
# declare functions
|
|
||||||
function listVolumes(){
|
|
||||||
docker volume ls --filter=name=$volume_prefix- --quiet
|
|
||||||
}
|
|
||||||
|
|
||||||
# run for every volume in parallel
|
|
||||||
listVolumes | parallel --lb -j $max_jobs -n 1 run-mirror-update
|
|
||||||
|
|
|
@ -1,11 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
set -euo pipefail
|
|
||||||
IFS=$'\n\t'
|
|
||||||
|
|
||||||
# make parallel citation shut up
|
|
||||||
mkdir $HOME/.parallel
|
|
||||||
touch $HOME/.parallel/will-cite
|
|
||||||
|
|
||||||
# run cron
|
|
||||||
crond -f
|
|
||||||
|
|
|
@ -1,77 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
set -euo pipefail
|
|
||||||
IFS=$'\n\t'
|
|
||||||
|
|
||||||
mirror_image="valicek1/repo-cloner-mirror"
|
|
||||||
|
|
||||||
|
|
||||||
function log(){
|
|
||||||
local title=${raw:-$name}
|
|
||||||
[ -z "$title" ] || title=" [$title]"
|
|
||||||
echo "[$(date +"%X")]$title $@"
|
|
||||||
}
|
|
||||||
|
|
||||||
function die(){
|
|
||||||
log "$@" 1>&2
|
|
||||||
exit 1
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
scratch=$(mktemp -d -t tmp.XXXXXXXXXX)
|
|
||||||
function finish {
|
|
||||||
rm -rf "$scratch"
|
|
||||||
}
|
|
||||||
trap finish EXIT
|
|
||||||
|
|
||||||
volume_prefix=cloner
|
|
||||||
clone_prefix=cloner-runner
|
|
||||||
|
|
||||||
# last version of image
|
|
||||||
last_version=$(docker images $mirror_image --no-trunc --quiet)
|
|
||||||
|
|
||||||
name=${1:-}
|
|
||||||
[ -n "$name" ] || die "No volume specified"
|
|
||||||
|
|
||||||
# contains prefix
|
|
||||||
[[ "$name" == $volume_prefix-* ]] || die "Volume name does not contain prefix!"
|
|
||||||
|
|
||||||
raw=$(echo $name | sed "s/^$volume_prefix-//g")
|
|
||||||
|
|
||||||
container_name=$clone_prefix-$raw
|
|
||||||
|
|
||||||
# does it run?
|
|
||||||
docker ps --quiet --no-trunc --filter name=^/$container_name$ > $scratch/running
|
|
||||||
lines=$(wc -l $scratch/running | cut -f1 -d' ')
|
|
||||||
|
|
||||||
if ! [ $lines -eq 0 ]
|
|
||||||
then
|
|
||||||
log "Another copy of container is running.. Exiting silently"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
unset lines
|
|
||||||
|
|
||||||
# check if volume does exist?
|
|
||||||
docker volume inspect $name > /dev/null 2>&1 || die "Volume '$name' does not exist - exiting!"
|
|
||||||
|
|
||||||
# does it exist?
|
|
||||||
image_version=$(docker inspect --format='{{.Image}}' $container_name 2>/dev/null || true)
|
|
||||||
|
|
||||||
# do I need to recreate image?
|
|
||||||
if ! [ "x$image_version" = "x$last_version" ]
|
|
||||||
then
|
|
||||||
# not empty string - need to delete first
|
|
||||||
if [ -n "$image_version" ]
|
|
||||||
then
|
|
||||||
log "Conflicting container exists, removing.."
|
|
||||||
docker rm $container_name
|
|
||||||
fi
|
|
||||||
# create new container
|
|
||||||
log "Creating new version of container.."
|
|
||||||
docker create -v $name:/data --name $container_name $mirror_image
|
|
||||||
fi
|
|
||||||
|
|
||||||
|
|
||||||
log "Running container of update process..."
|
|
||||||
docker start -a $container_name | while read line; do log "$line"; done
|
|
||||||
log "Finished."
|
|
||||||
|
|
Loading…
Reference in New Issue
Block a user