Cloner: clone() recursion support, TODO: recursion tests

Signed-off-by: Václav Valíček <valicek1994@gmail.com>

.idea/vcs.xml

@@ -3,7 +3,28 @@
   <component name="VcsDirectoryMappings">
     <mapping directory="$PROJECT_DIR$" vcs="Git" />
     <mapping directory="$PROJECT_DIR$/tests/_support_data/test-repo-base" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/tests/_support_data/test-repo-changed-branches" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/tests/_support_data/test-repo-different-tags" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/tests/_support_data/test-repo-new-commits" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/tests/_support_data/test-repo-reduced" vcs="Git" />
     <mapping directory="$PROJECT_DIR$/tests/_support_data/test-repo-striped" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/tests/_support_data/test-repo-submodules" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/tests/_support_data/test-repo-submodules-long" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/tests/_support_data/test-repo-submodules-multilevel" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/tests/_support_data/test-submodules-adhoc/submodule-failed" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/tests/_support_data/test-submodules-adhoc/submodule-failed-cfg" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/tests/_support_data/test-submodules-adhoc/submodule-failed/submodule-failed-cfg" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/tests/_support_data/test-submodules-adhoc/submodule-failed/test-repo-base" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/tests/_support_data/test-submodules-adhoc/submodules-level-two" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/tests/_support_data/test-submodules-adhoc/submodules-level-two/test-repo-base" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/tests/_support_data/test-submodules-adhoc/submodules-level-two/test-repo-reduced" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/tests/_support_data/test-submodules-adhoc/submodules-root" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/tests/_support_data/test-submodules-adhoc/submodules-root/submodule-failed" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/tests/_support_data/test-submodules-adhoc/submodules-root/submodule-failed-cfg" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/tests/_support_data/test-submodules-adhoc/submodules-root/submodules-level-two" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/tests/_support_data/test-submodules-adhoc/submodules-root/test-repo-base" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/tests/_support_data/test-submodules-adhoc/submodules-root/test-repo-different-tags" vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/tests/_support_data/test-submodules-adhoc/submodules-root/test-repo-new-commits" vcs="Git" />
     <mapping directory="$PROJECT_DIR$/tests/_support_data/tool/non-bare-init.git" vcs="Git" />
   </component>
 </project>

repo_cloner/lib/cloner.py

@@ -4,6 +4,7 @@ from repo_cloner.lib.dir_not_found_error import DirNotFoundError
 from repo_cloner.lib.repo_tool import RepoTool
 from repo_cloner.lib.checksum import gen_repo_hashed_name
 from pathlib import Path
+from typing import Optional
 from time import time
 import os
 import logging
@@ -15,6 +16,7 @@ class Cloner:
 	_dirs: RepoDirStructure = None
 	_config: ClonerConfig = None
 	_interval_file: str = "last-check-time"
+	__submodule_cache: str = None
 	_repo: RepoTool = None
 	_repo_url: str = ""
 
@@ -32,6 +34,12 @@ class Cloner:
 			log.info(f"Cache dir for project {self._config.cloner_project_name} not found -> creating")
 			Path(self._dirs.cache_dir).mkdir()
 			log.debug(f"Cache dir created")
+		# submodule cache
+		self.__submodule_cache = os.path.join(self._dirs.cache_dir, "submodules")
+		if not os.path.exists(self.__submodule_cache):
+			log.info("Submodule cache dir does not exist! -> creating")
+			Path(self.__submodule_cache).mkdir(parents = True)
+
 
 	def check_interval(self):
 		log.debug(f"Checking interval for {self._config.cloner_project_name}")
@@ -87,11 +95,26 @@ class Cloner:
 			self.sync()
 		log.info(f"Check finished")
 
-	def clone_from_url(self, url: str) -> bool:
+	def clone(self, url: Optional[str] = None) -> bool:
+		# optional parameters - othervise use config
+		if not url:
+			url = self._config.cloner_repo_url
+		# generate path
 		path = self._repo_path_by_url(url)
 		self._repo_url = url
 		self._repo = RepoTool(path)
+		# uninitialized repo
 		if self._repo.initialized:
 			log.critical(f"Repo path {path} is initialized... Refusing clone!")
 			return False
+
+		# recursive or standard?
+		if not self._config.cloner_submodules:
 			return self._repo.clone(url)
+		else:
+			scan_depth_limit = self._config.cloner_submodule_depth
+			# handle dept limit for submodule discovery
+			if scan_depth_limit == 0:
+				scan_depth_limit = None
+			# another levels are handled internally as non-recursive clones and discovers by repo-tool
+			return self._repo.clone_recursive(url, self.__submodule_cache, scan_depth = scan_depth_limit)

tests/lib/test_cloner.py

@@ -17,6 +17,7 @@ class MockConfig:
 		self.cloner_repo_url = ""
 		self.cloner_project_name = "Mocked Project"
 		self.cloner_interval = 0
+		self.cloner_submodules = False
 
 
 class MockDirStruct:
@@ -239,7 +240,7 @@ def test_clone_from_url(tmp_path, path_repo_base):
 	mock = MockDirStruct(tmp_path)
 	mock.config.cloner_repo_url = "invalid"
 	c = Cloner(mock)
-	assert c.clone_from_url(path_repo_base.as_uri())
+	assert c.clone(path_repo_base.as_uri())
 	assert "e0c7e2a72579e24657c05e875201011d2b48bf94" == c._repo._repo.head.commit.hexsha
 
 
@@ -250,6 +251,6 @@ def test_clone_from_url_initialized(tmp_path, path_repo_base, caplog):
 	mock.config.cloner_repo_url = path_repo_base.as_uri()
 	git.Repo().init(path)
 	c = Cloner(mock)
-	assert not c.clone_from_url(path_repo_base.as_uri())
+	assert not c.clone(path_repo_base.as_uri())
 	assert caplog.records[0].levelname == "CRITICAL"
 	assert caplog.records[0].message == f"Repo path {path} is initialized... Refusing clone!"