New behavior, new tests, added wizzard and processor

Signed-off-by: Václav Valíček <valicek1994@gmail.com>

old/dockerbin/wizzard

@@ -1,70 +1,5 @@
 #!/bin/bash
 
-function checkProjectName(){
-	# check, if volume does not exist yet
-	name=$1
-	# should not be empty
-	[ -n "$read_project_name" ] || die "Empty project name is not allowed"
-	if [ -d /data/$dir_prefix-$name ] 
-	then
-		die "Target volume for project '$name' exists - please try again!"
-	fi
-}
-
-function generateSSHKey(){
-	# generates ssh key with $1 path and $2 description
-	local keyfile=$1/identity
-	local description=$2
-	echo "Creating SSH deployment key.."
-	ssh-keygen -f $keyfile -t ed25519 -C "$description" -N ""
-	echo 
-	echo "Public key is:"
-	echo "-----------------------------------------------------"
-	cat $keyfile.pub
-	echo "-----------------------------------------------------"
-	echo -n "Please make sure that key is set up at your git hosting and press enter.."
-	read
-}
-
-
-function reuseSSHKey(){
-	# pastes ssh key to file $1 with vim
-	local keyfile=$1
-	local scratch=$(mktemp)
-	echo "# Please paste private ssh key here and save this file" > $scratch
-	vim $scratch
-	sed -e 's/#.*$//' $scratch > $keyfile
-	rm $scratch
-	echo "Checking key..."
-	chmod 0700 $keyfile
-	ssh-keygen -y -f $keyfile -P ""	|| true # will fail in the end, so script will continue and clean up the mess
-}
-
-# use ssh config?
-echo -n "Would you like to use SSH auth? ([C]reate new key/[U]se existing key/[N]o) [C/u/n]: "
-read read_ssh
-[ -n "$read_ssh" ] || read_ssh=C
-[[ "$read_ssh" =~ ^[CcUuNn]$ ]] || die "Invalid SSH key option, script is exiting now.."
-
-# ssh resolutions? 
-# create dir if needed
-[[ "$read_ssh" =~ ^[nN]$ ]] || mkdir -p $root/config/auth/ssh
-
-# generate new key
-if [[ "$read_ssh" =~ ^[Cc]$ ]] 
-then
-	# create key
-	generateSSHKey $root/config/auth/ssh "cloner-deploy-key-$read_project_name"
-fi
-
-# use existing key
-if [[ "$read_ssh" =~ ^[Uu]$ ]] 
-then
-	# load key
-	reuseSSHKey $root/config/auth/ssh/identity
-fi
-
-
 echo "First run - initialization of repos..."
 if ! env BASE=$root run-checker
 then 
@@ -76,7 +11,7 @@ then
 		rm -Rf $root
 	fi
 else
-	createDetectorConfig $root/config/detector.cfg
+
 	echo "Setup has finished!"
 	touch $root/.enabled
 fi

repo_cloner/initialization_wizzard.py

@@ -6,12 +6,15 @@ import os
 import pyinputplus as pyip
 from typing import Optional, Callable
 from pathlib import Path
+import subprocess
+from repo_cloner.process_repository_dir import clone_or_fetch
 
 # base dir
 base_dir: Optional[Path] = None
 cloner_prefix: str = "cloner-"
 
 data: dict = {}
+initialized_folder = None
 
 
 # determine starting user and devote UID/GID to unprivileged user - safety first :)
@@ -122,11 +125,12 @@ def input_default_str(query: str, default_value: str, validation: Optional[Calla
 
 def input_default_int(query: str, default_value: int, validation: Optional[Callable[[int], None]] = None) -> int:
 	log.debug(f"Input query {query} with default {default_value}")
+	ret = default_value
 	while True:
 		new_query = f"{query} [{default_value}] "
 
 		ret = pyip.inputInt(new_query, blank = True, strip = True, min = 0)
-		if not ret:
+		if not ret and not ret == 0:
 			log.debug(f"Empty query answer => using previous/default value")
 			ret = default_value
 		try:
@@ -160,11 +164,14 @@ def input_default_bool(query: str, default_value: bool) -> bool:
 def query_repo_info() -> bool:
 	log.debug(f"Querying base info")
 
+	if not initialized_folder:
 		# project name
 		data["cloner_project_name"] = input_default_str(
 			"Enter project name:",
 			data["cloner_project_name"],
 			check_project_name)
+	else:
+		log.warning(f"Project folder was initialized -> unable to change")
 
 	# url
 	data["cloner_repo_url"] = input_default_str("Enter project url:", data["cloner_repo_url"], check_url)
@@ -183,6 +190,8 @@ def query_repo_info() -> bool:
 
 	data["detector"] = input_default_bool("Do you want to enable CI support? (detector) [y/n]", data["detector"])
 
+	data["detector_init"] = input_default_bool("Do you want to init CI detector caches? [y/n]", data["detector_init"])
+
 
 def query_repo_info_recursive():
 	while True:
@@ -195,6 +204,75 @@ def query_repo_info_recursive():
 			break
 
 
+def create_ssh_key(ssh_dir: Path):
+	log.info(f"Creating SSH key...")
+
+	keyfile = ssh_dir.joinpath("identity")
+	subprocess.run([
+		"/usr/bin/ssh-keygen",
+		"-f", keyfile.as_posix(),
+		"-t", "ed25519",
+		"-C", f"cloner-deploy-key-{data['cloner_project_name']}"
+		, "-N", ""
+	])
+	public = ssh_dir.joinpath("identity.pub").read_text().strip()
+	print("Public key is:")
+	print("-----------------------------------------------------")
+	print(public)
+	print("-----------------------------------------------------")
+	print("Please make sure that key is set up at your git hosting and press enter..")
+	pyip.inputStr("", blank = True)
+
+
+def reuse_ssh_key(ssh_dir: Path):
+	log.info("Reusing any ssh key")
+	key_contents = ""
+	print("Paste password-less private key here. End with empty line:")
+	while True:
+		ret = pyip.inputStr("", blank = True)
+		if not len(ret):
+			break
+		key_contents += f"{ret}\n"
+	# public keyfile
+	keyfile_pub = ssh_dir.joinpath("identity.pub")
+	if keyfile_pub.exists():
+		log.debug(f"Removing old public key")
+		keyfile_pub.unlink()
+	# private keyfile
+	keyfile = ssh_dir.joinpath("identity")
+	log.info(f"Writing key")
+	keyfile.write_text(key_contents)
+	keyfile.chmod(0o600)
+	log.info(f"Checking supplied key for validity...")
+	rc = subprocess.run(["/usr/bin/ssh-keygen", "-y", "-f", keyfile.as_posix(), "-P", ""], stdout = subprocess.PIPE)
+	if not rc.returncode == 0:
+		log.critical(f"Supplied key is invalid.")
+		if input_default_bool("Do you want to try again? [y/n]", True):
+			reuse_ssh_key(ssh_dir)
+	else:
+		print("Public key is:")
+		print("-----------------------------------------------------")
+		public = rc.stdout.decode().strip() + f" cloner-deploy-key-{data['cloner_project_name']}"
+		print(public)
+		print("-----------------------------------------------------")
+		log.info(f"Writing new public key...")
+		keyfile_pub.write_text(public)
+
+
+def solve_authorization(conf_dir: Path):
+	decision: str = pyip.inputMenu(
+		["Create ssh key", "Reuse ssh key", "Keep unchanged / unauthorized / usernames access only"],
+		numbered = True)
+	log.info(f"Preparing auth file..")
+	ssh_dir = conf_dir.joinpath("auth", "ssh")
+	if not ssh_dir.exists():
+		ssh_dir.mkdir(mode = 0o700, parents = True)
+	if decision.startswith("Create"):
+		create_ssh_key(ssh_dir)
+	elif decision.startswith("Reuse"):
+		reuse_ssh_key(ssh_dir)
+
+
 def main() -> int:
 	check_privileges()
 	parse_args()
@@ -211,18 +289,61 @@ def main() -> int:
 	data["cloner_submodules"] = False
 	data["cloner_submodule_depth"] = 0
 	data["detector"] = True
+	data["detector_init"] = True
 
+	edit_config = True
+	# endless loop until finished or canceled
+	while True:
+		# query params from user?
+		if edit_config:
 			query_repo_info_recursive()
 
+		# create dir
 		project_path = base_dir.joinpath(f"{cloner_prefix}{data['cloner_project_name']}")
-	project_path.mkdir()
 
-	project_path.joinpath("repos").mkdir()
+		global initialized_folder
-	project_path.joinpath("cache").mkdir()
+		initialized_folder = True
+
+		project_path.mkdir(exist_ok = True)
+		project_path.joinpath("repos").mkdir(exist_ok = True)
+		project_path.joinpath("cache").mkdir(exist_ok = True)
 
 		config_dir = project_path.joinpath("config")
-	config_dir.mkdir()
+		config_dir.mkdir(exist_ok = True)
 
 		gen_config_file(config_dir, **data)
 
+		if edit_config:
+			solve_authorization(config_dir)
+
+		# try initializing repos
+		x = clone_or_fetch(project_path.as_posix(), clone_init = True, detector_init = data["detector_init"])
+
+		if x == 0:
 			return 0
+
+		# determine if to continue
+		log.critical(f"Something has failed. Please see log above and decide what to do")
+		menu_config = "Edit config and start over"
+		menu_reclone = "Try cloning again, including base repo"
+		menu_quit_clean = "Quit & clean"
+		decision: str = pyip.inputMenu(
+			[menu_config, menu_reclone, menu_quit_clean],
+			numbered = True)
+
+		# other stuff
+		if decision == menu_quit_clean:
+			log.warning(f"Removing old stuff and quitting")
+			subprocess.run(["/usr/bin/rm", "-Rvf", project_path.as_posix()])
+			return 0
+
+		if decision == menu_config:
+			edit_config = True
+		else:
+			edit_config = False
+
+		# remove caches & repos
+		log.info(f"Cleaning unwanted stuff")
+		subprocess.run(["/usr/bin/rm", "-Rvf", project_path.joinpath("repos").as_posix()])
+		subprocess.run(["/usr/bin/rm", "-Rvf", project_path.joinpath("cache").as_posix()])
+		log.info(f"Starting over.. ")

repo_cloner/lib/cloner.py

@@ -66,7 +66,6 @@ class Cloner:
 		self._repo = RepoTool(repo_path)
 		return self.__opened
 
-
 	@property
 	def __opened(self) -> bool:
 		if not self._repo:
@@ -221,3 +220,7 @@ class Cloner:
 		if detector.check_fingerprint():
 			log.debug(f"Starting detector discovery")
 			detector.run(callback)
+
+	def detector_init(self):
+		detector = Detector(Path(self.main_repo_path), Path(self._dirs.cache_dir), self._config.cloner_project_name)
+		detector.initialize_caches()

repo_cloner/lib/cloner_config.py

@@ -97,8 +97,8 @@ class ClonerConfigParser:
 				self.__invalid_lines.append((line, None))
 				continue
 			eq = line.find("=")
-			key: str = line[0:eq]
+			key: str = line[0:eq].strip()
-			val: str = line[eq + 1:]
+			val: str = line[eq + 1:].strip()
 			l.debug(f"Found config pair: {key} => '{val}'")
 
 			try:
@@ -118,4 +118,3 @@ class ClonerConfigParser:
 	@property
 	def invalid_lines(self):
 		return self.__invalid_lines
-

repo_cloner/lib/cred_helper.py

@@ -48,23 +48,22 @@ def config_try_override(config_writer: GitConfigParser, section: str, option: st
 	config_writer.set(section, option, value)
 
 
-def prepare_git_auth(repo: str, config_dir):
+def prepare_git_auth(config_dir: str):
-	log.debug(f"CFG: Opening repo {repo}")
+	# create mockup repo
-	repo = Repo(repo)
+	git_config = Path(config_dir).joinpath("git")
-	path: str = repo._get_config_path("user")
+	config_file = git_config.joinpath("config")
+	path: str = config_file.as_posix()
+
 	log.debug(f"CFG config path: {path}")
-	path = os.path.dirname(path)
+	if not git_config.is_dir():
-	log.debug(f"CFG parent path: {path}")
-	if not os.path.isdir(path):
 		log.debug(f"CFG Creating config dir")
-		os.mkdir(path)
+		git_config.mkdir()
 
-	cred_store: str = os.path.join(path, "git-credentials")
+	cred_store: str = os.path.join(config_dir, "auth", "git-credentials")
-	ssh_identity: str = os.path.join(config_dir, "ssh", "identity")
+	ssh_identity: str = os.path.join(config_dir, "auth", "ssh", "identity")
 
-	with repo.config_writer("user") as cfgw:
+	with GitConfigParser(path, read_only = False) as cfgw:
 		# github personal token
-		# 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
 		if token:
 			config_try_override(
 				cfgw,

repo_cloner/process_repository_dir.py

@@ -31,18 +31,9 @@ def detector_executor(commit: DetectedCommit):
 	subprocess.run(arg_list)
 
 
-def main() -> int:
+def clone_or_fetch(base_dir: str, clone_init: bool = False, detector_init: bool = False):
-	# parse input arguments
+	log.info(f"Started processing git group in folder: {base_dir}")
-	parser = argparse.ArgumentParser(description = "repo-cloner entering script")
+	dirs = RepoDirStructure(base_dir)
-	parser.add_argument('--base-dir', help = 'path to directory containing whole cloner structure', required = True,
-						default = None, type = str)
-	parser.add_argument('--debug', '-d', help = "enable debug output", action = 'store_true')
-	args = parser.parse_args()
-	if args.debug:
-		log.setLevel(logging.DEBUG)
-
-	log.info(f"Started processing git group in folder: {args.base_dir}")
-	dirs = RepoDirStructure(args.base_dir)
 	log.debug(f"Patching XDG_CONFIG_HOME to mock up git config")
 	os.environ['XDG_CONFIG_HOME'] = dirs.conf_dir
 
@@ -73,7 +64,17 @@ def main() -> int:
 		log.warning("Config directive cloner_project_name should not be omitted!")
 
 	cloner = Cloner(dirs)
-	prepare_git_auth(cloner.main_repo_path, dirs.conf_dir)
+	prepare_git_auth(dirs.conf_dir)
+
+	if clone_init:
+		log.info(f"Initial cloning of repositories")
+		if not cloner.clone():
+			return 1
+		if detector_init:
+			cloner.detector_init()
+		return 0
+
+	# regular run
 	if not cloner.sync():
 		log.warning(f"Repo sync did not succeed")
 
@@ -83,5 +84,18 @@ def main() -> int:
 	return 0
 
 
+def main() -> int:
+	# parse input arguments
+	parser = argparse.ArgumentParser(description = "repo-cloner entering script")
+	parser.add_argument('--base-dir', help = 'path to directory containing whole cloner structure', required = True,
+						default = None, type = str)
+	parser.add_argument('--debug', '-d', help = "enable debug output", action = 'store_true')
+	args = parser.parse_args()
+	if args.debug:
+		log.setLevel(logging.DEBUG)
+
+	return clone_or_fetch(args.base_dir)
+
+
 if __name__ == "__main__":
 	exit(main())

tests/lib/test_cloner.py

@@ -523,3 +523,17 @@ def test_detector_run(cloner_dir_with_config):
 		cl.detector_run(callback_test)
 		assert repo_cloner.lib.cloner.Detector.run.called
 		assert called
+
+
+def test_detector_init(cloner_dir_with_config):
+	ds = MockDirStruct(cloner_dir_with_config)
+	ds.config.cloner_repo_url = "https://mock"
+
+	mocks = {
+		'initialize_caches': MagicMock(),
+	}
+
+	with patch.multiple('repo_cloner.lib.cloner.Detector', **mocks):
+		cl = Cloner(ds)
+		cl.detector_init()
+		assert repo_cloner.lib.cloner.Detector.initialize_caches.called

tests/lib/test_cred_helper.py

@@ -62,33 +62,33 @@ def test_config_try_override(tmp_path, path_repo_base, monkeypatch):
 	assert x == "[user]\n	name = Loser\n"
 
 
-def test_prepare_git_auth(tmp_path, path_repo_base, monkeypatch):
+def test_prepare_git_auth(tmp_path, monkeypatch):
 	tmp_path.joinpath("git").mkdir()
 	cred_helper.token = None
 
 	with monkeypatch.context() as mp:
 		mp.setenv("XDG_CONFIG_HOME", tmp_path.as_posix())
 
-		prepare_git_auth(path_repo_base.as_posix(), tmp_path.as_posix())
+		prepare_git_auth(tmp_path.as_posix())
 
 	x = tmp_path.joinpath("git", "config")
 	assert x.exists()
 	config = x.read_text()
 	assert config == \
 		   f"[credential]\n" \
-		   f"	helper = store --file={tmp_path.as_posix()}/git/git-credentials\n" \
+		   f"	helper = store --file={tmp_path.as_posix()}/auth/git-credentials\n" \
 		   "[core]\n" \
-		   f"	sshcommand = ssh -i {tmp_path.as_posix()}/ssh/identity -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -q\n"
+		   f"	sshcommand = ssh -i {tmp_path.as_posix()}/auth/ssh/identity -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -q\n"
 
 
-def test_prepare_git_auth_token(tmp_path, path_repo_base, monkeypatch):
+def test_prepare_git_auth_token(tmp_path, monkeypatch):
 	# tmp_path.joinpath("git").mkdir()
 
 	with monkeypatch.context() as mp:
 		mp.setenv("XDG_CONFIG_HOME", tmp_path.as_posix())
 		cred_helper.token = "token123"
 
-		prepare_git_auth(path_repo_base.as_posix(), tmp_path.as_posix())
+		prepare_git_auth(tmp_path.as_posix())
 
 	x = tmp_path.joinpath("git", "config")
 	assert x.exists()
@@ -97,6 +97,6 @@ def test_prepare_git_auth_token(tmp_path, path_repo_base, monkeypatch):
 		   "[url \"https://token123:x-oauth-basic@github.com/\"]\n" \
 		   "	insteadOf = https://github.com/\n" \
 		   f"[credential]\n" \
-		   f"	helper = store --file={tmp_path.as_posix()}/git/git-credentials\n" \
+		   f"	helper = store --file={tmp_path.as_posix()}/auth/git-credentials\n" \
 		   "[core]\n" \
-		   f"	sshcommand = ssh -i {tmp_path.as_posix()}/ssh/identity -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -q\n"
+		   f"	sshcommand = ssh -i {tmp_path.as_posix()}/auth/ssh/identity -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -q\n"