kamikaze/repo-cloner
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Cloner: sync - check fingerprint before costy discovery

Browse Source 
Signed-off-by: Václav Valíček <valicek1994@gmail.com>
This commit is contained in:
Václav Valíček 2022-08-04 09:50:42 +02:00
parent 5ca24960c3
commit d74c67d4f6
Signed by: valicek
GPG Key ID: FF05BDCA0C73BB31
2 changed files with 48 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
repo_cloner/lib/cloner.py
View File

@ -1,9 +1,9 @@
from repo_cloner.lib import gen_repo_hashed_name
from repo_cloner.lib import DirNotFoundError
from repo_cloner.lib import ClonerConfig, DiskStoredList, RepoDirStructure, RepoTool
from repo_cloner.lib import ClonerConfig, DiskStoredList, RepoDirStructure, RepoTool, DetectedCommit, Detector
from pathlib import Path
from typing import Optional
from typing import Optional, Callable
from time import time
import os
import logging
@ -119,11 +119,17 @@ class Cloner:
if not self._config.cloner_submodules:
return self._repo.fetch()
fingerprint = self._repo.repo_fingerprint
# recursive now
if not self._repo.fetch():
log.critical(f"Repo fetch failed for {self._config.cloner_project_name}")
return False
if fingerprint == self._repo.repo_fingerprint:
log.info(f"Repo fingerpring unchanged - submodule discovery skipped")
return True
log.debug(f"Loading submodules.cache")
submodules = DiskStoredList(os.path.join(self.__submodule_cache, "submodules.cache"))
log.debug(f"Loaded submodules.cache - {len(submodules)} items")
@ -207,3 +213,7 @@ class Cloner:
@property
def detector_enabled(self) -> bool:
return os.path.exists(os.path.join(self._dirs.conf_dir, self.__detector_cfg))
def run_detector(self, callback: Callable[[DetectedCommit], None]):
detector = Detector(self.__main_repo_path, self._dirs.cache_dir, self._config.cloner_project_name)
detector.run(callback)

43
tests/lib/test_cloner.py
View File

@ -306,9 +306,10 @@ def test_submodules_sync_succeed(cloner_dir_with_config, caplog):
# mock almost everything
mocks = {
'initialized': PropertyMock(side_effect = [True, False, True, True, True]),
'fetch': MagicMock(return_value = True),
'clone': MagicMock(return_value = True),
'initialized': PropertyMock(side_effect = [True, False, True, True, True]),
'fetch': MagicMock(return_value = True),
'clone': MagicMock(return_value = True),
'repo_fingerprint': PropertyMock(side_effect = ["fp1", "fp2", "fp3"]),
}
with patch.multiple("repo_cloner.lib.cloner.RepoTool", **mocks):
@ -320,6 +321,33 @@ def test_submodules_sync_succeed(cloner_dir_with_config, caplog):
assert repo_cloner.lib.cloner.RepoTool.clone.call_count == 1
def test_submodules_sync_unchanged_main_repo(cloner_dir_with_config, caplog):
caplog.set_level(0)
cloner_dir_with_config.joinpath("cache", "submodules").mkdir(parents = True)
cloner_dir_with_config.joinpath("cache", "submodules", "submodules.cache") \
.write_text("https://git.hosting:previous/submodule.git")
ds = MockDirStruct(cloner_dir_with_config)
ds.config.cloner_repo_url = "https://git.hosting:namespace/whatever.git"
ds.config.cloner_submodules = True
# mock almost everything
mocks = {
'initialized': PropertyMock(side_effect = [True, False, True, True, True]),
'fetch': MagicMock(return_value = True),
'clone': MagicMock(return_value = True),
# fingerprint is same for both queries, so nothing is fetched/synced afterwards
'repo_fingerprint': PropertyMock(return_value = "fp1"),
}
with patch.multiple("repo_cloner.lib.cloner.RepoTool", **mocks):
Cloner.check_submodules_repo = my_check_submodules_repo
cl = Cloner(ds)
assert cl.sync()
assert repo_cloner.lib.cloner.RepoTool.fetch.call_count == 1
assert repo_cloner.lib.cloner.RepoTool.clone.call_count == 0
def test_submodules_sync_one_fail(cloner_dir_with_config, caplog):
cloner_dir_with_config.joinpath("cache", "submodules").mkdir(parents = True)
cloner_dir_with_config.joinpath("cache", "submodules", "submodules.cache") \
@ -332,9 +360,10 @@ def test_submodules_sync_one_fail(cloner_dir_with_config, caplog):
# mock almost everything
mocks = {
'initialized': PropertyMock(return_value = True),
'fetch': MagicMock(side_effect = [True, True, True, False, True]),
'clone': MagicMock(return_value = True),
'initialized': PropertyMock(return_value = True),
'fetch': MagicMock(side_effect = [True, True, True, False, True]),
'clone': MagicMock(return_value = True),
'repo_fingerprint': PropertyMock(side_effect = ["fp1", "fp2", "fp3"]),
}
with patch.multiple("repo_cloner.lib.cloner.RepoTool", **mocks):
@ -458,6 +487,7 @@ def test_clone_recursive(tmp_path, path_repo_base, caplog):
patch_clone_recursive.assert_called_with(
'https://repo', tmp_path.joinpath('cache', 'submodules').as_posix(), scan_depth = None)
def test_detector_enabled(cloner_dir_with_config):
ds = MockDirStruct(cloner_dir_with_config)
ds.config.cloner_repo_url = "http://mock"
@ -465,4 +495,3 @@ def test_detector_enabled(cloner_dir_with_config):
assert not cl.detector_enabled
Path(cloner_dir_with_config).joinpath("config", "detector.cfg").touch()
assert cl.detector_enabled