New tests, Cloner.sync() without tests, but recursive

Signed-off-by: Václav Valíček <valicek1994@gmail.com>

repo_cloner/lib/__init__.py

@@ -0,0 +1,9 @@
+from .checksum import gen_repo_hashed_name
+from .cloner_config import ClonerConfig, ClonerConfigParser
+from .config_file_not_found_error import ConfigFileNotFoundError
+from .default_cloner_config import DefaultClonerConfig
+from .dir_not_found_error import DirNotFoundError
+from .disk_stored_list import DiskStoredList
+from .repo_tool import RepoTool
+from .repo_dir_structure import RepoDirStructure
+from .cloner import Cloner

repo_cloner/lib/cloner.py

@@ -1,8 +1,7 @@
-from repo_cloner.lib.cloner_config import ClonerConfig
-from repo_cloner.lib.repo_dir_structure import RepoDirStructure
-from repo_cloner.lib.dir_not_found_error import DirNotFoundError
-from repo_cloner.lib.repo_tool import RepoTool
-from repo_cloner.lib.checksum import gen_repo_hashed_name
+from repo_cloner.lib import gen_repo_hashed_name
+from repo_cloner.lib import DirNotFoundError
+from repo_cloner.lib import ClonerConfig, DiskStoredList, RepoDirStructure, RepoTool
+
 from pathlib import Path
 from typing import Optional
 from time import time
@@ -40,7 +39,6 @@ class Cloner:
 			log.info("Submodule cache dir does not exist! -> creating")
 			Path(self.__submodule_cache).mkdir(parents = True)
 
-
 	def check_interval(self):
 		log.debug(f"Checking interval for {self._config.cloner_project_name}")
 		# get interval
@@ -82,12 +80,97 @@ class Cloner:
 	def __main_repo_path(self) -> str:
 		return self._repo_path_by_url(self._config.cloner_repo_url)
 
+	@classmethod
+	def check_submodules_repo(cls, repo_tool: RepoTool, cache_file: str, submodule_list: DiskStoredList,
+							  scan_depth: Optional[int]):
+		base = os.path.basename(repo_tool.path)
+		log.debug(f"Loading submodule cache for repo {base}")
+		repo_commits = DiskStoredList(cache_file)
+		log.debug(f"Loaded {len(repo_commits)} commits")
+
+		# list fetched repo
+		log.debug(f"Scanning repo {base} for new submodules")
+		new_commits = repo_tool.list_commits(scan_depth)
+
+		# discover new submodules in new commits
+		for commit in new_commits:
+			log.debug(f"Discovering submodules in {commit.hexsha}")
+			if commit.hexsha in repo_commits:
+				log.debug(f"Cached commit... Okay")
+				continue
+			discovered = repo_tool.list_submodules(commit)
+			if discovered:
+				for submodule in discovered:
+					if submodule not in submodule_list:
+						log.warning(f"Found new submodule: {submodule}")
+						submodule_list.append(submodule)
+			repo_commits.append(commit.hexsha)
+		return submodule_list
+
 	def sync(self) -> bool:
+
 		if not self.__opened:
 			self._repo = RepoTool(self.__main_repo_path)
 			if not self._repo.initialized:
 				return False
-		return self._repo.fetch()
+		# determine recursive behavior
+		if not self._config.cloner_submodules:
+			return self._repo.fetch()
+
+		# recursive now
+		if not self._repo.fetch():
+			log.critical(f"Repo fetch failed for {self._config.cloner_project_name}")
+			return False
+
+		log.debug(f"Loading submodules.cache")
+		submodules = DiskStoredList(os.path.join(self.__submodule_cache, "submodules.cache"))
+		log.debug(f"Loaded submodules.cache - {len(submodules)} items")
+		path = gen_repo_hashed_name(self._config.cloner_repo_url)
+		log.debug(f"Main repo hashed name to load: {path}")
+
+		# recursion limit?
+		scan_depth = self._config.cloner_submodule_depth
+		log.debug(f"Scan depth is {scan_depth} commits")
+		if scan_depth == 0:
+			log.debug(f"Repository scan depth is not limited! -> setting scan_depth to none")
+			scan_depth = None
+
+		submodules = Cloner.check_submodules_repo(
+			self._repo, os.path.join(self.__submodule_cache, path), submodules, scan_depth)
+
+		everything_succeed: bool = True
+		everything_checked: bool = False
+		fetched_repos = set()
+		while not everything_checked:
+			# recursively scan and clone repositories
+			everything_checked = True
+			# for every url in list
+			# list() is needed - Runtime Error for set() changed during iteration
+			for url in list(submodules):
+				if url not in fetched_repos:
+					everything_checked = False
+					# generate new path
+					directory = os.path.dirname(self.__main_repo_path)
+					submodule_cloner = RepoTool(os.path.join(directory, gen_repo_hashed_name(url)))
+					# clone or checkout?
+					if not submodule_cloner.initialized:
+						log.info(f"New uninitialized submodule found: {url}. Cloning...")
+						checked: bool = submodule_cloner.clone(url)
+					else:
+						checked: bool = submodule_cloner.fetch()
+
+					# mark cloned even if failed afterwards - while loop stuck solution
+					fetched_repos.add(url)
+					if not checked:
+						log.critical(f"Clone/fetch of submodule: {url} failed")
+						everything_succeed = False
+						continue
+
+					submodules = Cloner.check_submodules_repo(
+						submodule_cloner,
+						os.path.join(self.__submodule_cache, gen_repo_hashed_name(url)),
+						submodules, scan_depth)
+		return everything_succeed
 
 	def perform_check(self):
 		log.info(f"Started check for {self._config.cloner_project_name}, url: {self._config.cloner_repo_url}")

repo_cloner/lib/repo_tool.py

@@ -184,7 +184,7 @@ class RepoTool:
 
 	@__check_initialized
 	def fetch(self) -> bool:
-		log.info("Fetching repo state")
+		log.info("Fetching repo :)")
 		if not len(self._repo.remotes):
 			log.warning(f"Repo: {self._path} does not contain any remotes!")
 			return False
@@ -214,6 +214,10 @@ class RepoTool:
 		log.debug(f"Repo fingerprint is {x}")
 		return x
 
+	@__check_initialized
+	def list_commits(self, max_depth: Optional[int] = None):
+		return self._repo.iter_commits(all = True, max_count = max_depth)
+
 	@__check_initialized
 	def list_submodules(self, commit: str = "HEAD") -> Union[list, bool]:
 		commit = self._repo.commit(commit)
@@ -241,7 +245,7 @@ class RepoTool:
 	def list_submodules_history(self, limit_of_commits: Optional[int] = None) -> Union[list, bool]:
 
 		log.info(f"Listing repository submodule history")
-		iterator = self._repo.iter_commits(all = True, max_count = limit_of_commits)
+		iterator = self.list_commits(limit_of_commits)
 		submodules = set()
 		counter: int = 0
 		last_status = time.time()