import git import pytest import repo_cloner.lib.cloner from repo_cloner.lib import gen_repo_hashed_name, DirNotFoundError, Cloner from cloner_test_fixtures import * from pathlib import Path import logging from unittest.mock import patch, PropertyMock, MagicMock, call some_commits = [ "d766d9c302463257695a4d53d857a2cecd024414", "a2e308bb9c5e59e7a0c319dca0adf3966f3f3a60", "36021088abb6ebcb2202897f3f27b26b21c25068", "e811211f0895e9e792f569b1b38f8452e0efe42f", "6854c793fd2cbd848650a94b9fed4924e5e428b2", "2f08263b5aff30ce776bb0350503be69d45647da", "fc53c9d8bc9e01c63632376df51abb67ca344a89", "c95a862af3d13623da3d7a6b73ff79d68ff2d7ad", "b21c1e9e329bcdef1980462cf62c09a9e807dafd", "dec7ff27d658412f0088c4c3220990f1f1dc98f1", "ad66f820c9dfa06c3767f81cb42b1398fa61ff05", "b9af12b9f80b372c645b5f64c5a8f8b6a148ccfc", "1fda6025ed303ecf03b9afa7cce882aaddc058f9", "d33585f0a72d9de6c481891787e4d956b769ccb5", "7f685afca71731cb9276f937fc8cceca6b47b1f2", "af112317aa2459e88db9cb10533ea6f29818afee", "ddc24a34983ff43b61e1bdf27b7ec2357932e7e0", "7ccca22ba367433eedb226310373fd250a2af725", "e4843c6d2391d0b45beea81e0dd34078484b474f", "fe942d46c33fee0e68d2e0ed64dfa6515dbf89c4", "1d80638229e0a7ef62b3b9cdab9ad9f63acf6d38", "f968ea00a3007034ddf196285d7b60fec5e7fcf0", "490859dbebe02af6bd643a89b6897961674f90fd", "066de679c906a78f7f817ee9a48d3021fd8b6b7c", "2a8277687fb6dee742e6e0193ea95fcd2264fbc4", "8808c649a8b0a5279a2d9a46b730254cad649205", "ad5db175fab144a8851b712195733c4a9c00d699", "36d45a4c1e5ad8c47a14881c5427045c3de095d0", "6b0b9affedb3e3daa4df00cd54678d78ea1c1d94", "aa5056610ff57f73bae9633a985c6a8e41f3bc23", ] def mock_time() -> float: return 1658702099.4258687 class MockConfig: def __init__(self): self.cloner_repo_url = "" self.cloner_project_name = "Mocked Project" self.cloner_interval = 0 self.cloner_submodules = False self.cloner_submodule_depth = 100 class MockDirStruct: raise_cache_exists = False def __init__(self, tmp: Path): self.config = MockConfig() self.cache_dir = tmp.joinpath("cache") self.conf_dir = tmp.joinpath("config") self.repos_dir = tmp.joinpath("repos") self.raise_cache_exists = False @property def cache_dir_exists(self): if self.raise_cache_exists: raise DirNotFoundError("mock_dir") return True class MockRepoTool: path = None initialized = False def __init__(self, path: str): self.path = path self.initialized = False def test_init_invalid_config(tmp_path, caplog): caplog.set_level(logging.INFO) mock = MockDirStruct(tmp_path) with pytest.raises(KeyError) as e: Cloner(mock) assert "KeyError: 'cloner_repo_url not defined in config!'" == e.exconly() assert "CRITICAL root:cloner.py:" in caplog.text assert "Undefined repo cloner URL in config!\n" in caplog.text # valid config caplog.clear() # set mocks mock.config.cloner_repo_url = "git@mocked:/path" cache_dir = tmp_path.joinpath("cache") # validate dir does not exist assert not cache_dir.exists() # set mock to raise error mock.raise_cache_exists = True # create new cloner class Cloner(mock) # dir should exist now assert cache_dir.exists() assert "Cache dir for project Mocked Project not found -> creating" in caplog.text def test_init_create_dirs(cloner_dir_with_config): dirs = MockDirStruct(cloner_dir_with_config) dirs.config.cloner_repo_url = "https://repo" Cloner(dirs) assert cloner_dir_with_config.joinpath("cache", "submodules").exists() def test_check_interval(cloner_dir_struct: Path, monkeypatch): mock = MockDirStruct(cloner_dir_struct) mock.config.cloner_repo_url = "git@mocked:/path" timestamp_file = cloner_dir_struct.joinpath("cache", "last-check-time") with monkeypatch.context() as m: m.setattr("repo_cloner.lib.cloner.time", mock_time) cloner = Cloner(mock) # no timestamp file exists -> timer has to run assert cloner.check_interval() # timestamp file in future -> no run timestamp_file.touch() timestamp_file.write_text(str(int(mock_time() + 11))) assert not cloner.check_interval() # timestamp in history, but run everytime is set >> RUN timestamp_file.write_text(str(int(mock_time() - 1))) assert cloner.check_interval() # second ago, but with 5 minute interval mock.config.cloner_interval = 5 assert not cloner.check_interval() # 6 minutes ago, with 5 min interval -> run! timestamp_file.write_text(str(int(mock_time() - 360))) assert cloner.check_interval() # corrupted file timestamp_file.write_text("1234WTF") assert cloner.check_interval() def test__opened(cloner_dir_with_config, monkeypatch): ds = MockDirStruct(cloner_dir_with_config) ds.config.cloner_repo_url = cloner_dir_with_config.as_uri() c = Cloner(ds) # no repo defined assert not c._Cloner__opened # with mocked repo monkeypatch.setattr(c, "_repo", MockRepoTool(cloner_dir_with_config.as_posix())) c._repo.initialized = True assert c._Cloner__opened c._repo.initialized = False assert not c._Cloner__opened def test_repo_path_by_url(cloner_dir_struct: Path, path_repo_base: Path): ds = MockDirStruct(cloner_dir_struct) ds.config.cloner_repo_url = path_repo_base.as_uri() c = Cloner(ds) x = c._repo_path_by_url("git@server:namespace/repo.git") assert x == ds.repos_dir.joinpath("namespace_repo_3375634822.git").as_posix() def test_main_repo_path(cloner_dir_struct: Path, path_repo_base: Path): ds = MockDirStruct(cloner_dir_struct) ds.config.cloner_repo_url = path_repo_base.as_uri() hashed = gen_repo_hashed_name(path_repo_base.as_uri()) c = Cloner(ds) x = c.main_repo_path assert x == ds.repos_dir.joinpath(hashed).as_posix() def test_check_submodules_repo(tmp_path): from collections import namedtuple from repo_cloner.lib import DiskStoredList def submodule_scanner_mock(commit = None): if commit.hexsha in ["f968ea00a3007034ddf196285d7b60fec5e7fcf0", "36d45a4c1e5ad8c47a14881c5427045c3de095d0"]: return ["https://git.hosting:namespace/repo2.git"] if commit.hexsha in ["6b0b9affedb3e3daa4df00cd54678d78ea1c1d94", "aa5056610ff57f73bae9633a985c6a8e41f3bc23"]: return ["https://git.hosting:namespace/repo2.git", "https://git.hosting:namespace/repo3.git"] return False repo_path = tmp_path.joinpath("repo.git").as_posix() cache_path = tmp_path.joinpath("cache.list") submodule_cache = tmp_path.joinpath("submodules").as_posix() submodules = DiskStoredList(submodule_cache) submodules.append("https://git.hosting:namespace/repo1.git") Commit = namedtuple("Commit", ["hexsha"]) # 30 commits mocked_commits = [Commit(commit) for commit in some_commits] # write cache file with collected commits with open(cache_path.as_posix(), "w") as f: f.writelines([f"{commit}\n" for commit in some_commits[0:20]]) mocks = { 'initialized': PropertyMock(return_value = True), "path": PropertyMock(return_value = repo_path), "list_commits": MagicMock(return_value = mocked_commits), "list_submodules": MagicMock(side_effect = submodule_scanner_mock), } # prepare mocks for RepoTool with patch.multiple("repo_cloner.lib.repo_tool.RepoTool", **mocks): import repo_cloner.lib.repo_tool rt = repo_cloner.lib.repo_tool.RepoTool(tmp_path.as_posix()) Cloner.check_submodules_repo(rt, cache_path.as_posix(), submodules, 100) assert rt.list_commits.call_args == call(100) assert rt.list_submodules.call_count == 10 returned_submodules = [x for x in submodules] assert returned_submodules == [ 'https://git.hosting:namespace/repo1.git', 'https://git.hosting:namespace/repo2.git', 'https://git.hosting:namespace/repo3.git' ] def test_sync(cloner_dir_struct, tmp_path, monkeypatch): called: bool = False def fetch(): nonlocal called called = True return True class PatchCloner(Cloner): open = False @property def __opened(self) -> bool: return self.open ds = MockDirStruct(cloner_dir_struct) ds.config.cloner_repo_url = "file://wut!" repo = tmp_path.joinpath("repo.git") c = PatchCloner(ds) # invalid repo, closed c.open = False assert not c.sync() assert not called # mocked valid repo, opened c._repo = MockRepoTool(repo.as_posix()) c._repo.initialized = True c._repo.fetch = fetch assert c.sync() assert called called = False # mocked repo, invalid, closed c._repo.initialized = False assert not called assert not c.sync() def test_sync_submodules_failed(cloner_dir_with_config, caplog): ds = MockDirStruct(cloner_dir_with_config) # mock config for submodules ds.config.cloner_submodules = True ds.config.cloner_repo_url = "asdhlk" # mock repo_tool.fetch to return false mocks = { 'initialized': PropertyMock(return_value = True), 'fetch': MagicMock(return_value = False) } with patch.multiple("repo_cloner.lib.cloner.RepoTool", **mocks) as mock: cl = Cloner(ds) assert not cl.sync() assert caplog.records[1].levelname == "CRITICAL" assert caplog.records[1].message == "Repo fetch failed for Mocked Project" def my_check_submodules_repo( repo_tool, cache_file: str, submodule_list: list, scan_depth = None): if "namespace_whatever_" in cache_file: submodule_list.append("https://git.hosting:namespace/submodule1.git") submodule_list.append("https://git.hosting:namespace/submodule2.git") if "/namespace_submodule1_" in cache_file: submodule_list.append("https://git.hosting:namespace/submodule3.git") return submodule_list def test_submodules_sync_succeed(cloner_dir_with_config, caplog): cloner_dir_with_config.joinpath("cache", "submodules").mkdir(parents = True) cloner_dir_with_config.joinpath("cache", "submodules", "submodules.cache") \ .write_text("https://git.hosting:previous/submodule.git") ds = MockDirStruct(cloner_dir_with_config) ds.config.cloner_repo_url = "https://git.hosting:namespace/whatever.git" ds.config.cloner_submodules = True # mock almost everything mocks = { 'initialized': PropertyMock(side_effect = [True, False, True, True, True]), 'fetch': MagicMock(return_value = True), 'clone': MagicMock(return_value = True), 'repo_fingerprint': PropertyMock(side_effect = ["fp1", "fp2", "fp3"]), } with patch.multiple("repo_cloner.lib.cloner.RepoTool", **mocks): Cloner.check_submodules_repo = my_check_submodules_repo cl = Cloner(ds) assert cl.sync() assert repo_cloner.lib.cloner.RepoTool.fetch.call_count == 4 assert repo_cloner.lib.cloner.RepoTool.clone.call_count == 1 def test_submodules_sync_unchanged_main_repo(cloner_dir_with_config, caplog): caplog.set_level(0) cloner_dir_with_config.joinpath("cache", "submodules").mkdir(parents = True) cloner_dir_with_config.joinpath("cache", "submodules", "submodules.cache") \ .write_text("https://git.hosting:previous/submodule.git") ds = MockDirStruct(cloner_dir_with_config) ds.config.cloner_repo_url = "https://git.hosting:namespace/whatever.git" ds.config.cloner_submodules = True # mock almost everything mocks = { 'initialized': PropertyMock(side_effect = [True, False, True, True, True]), 'fetch': MagicMock(return_value = True), 'clone': MagicMock(return_value = True), # fingerprint is same for both queries, so nothing is fetched/synced afterwards 'repo_fingerprint': PropertyMock(return_value = "fp1"), } with patch.multiple("repo_cloner.lib.cloner.RepoTool", **mocks): Cloner.check_submodules_repo = my_check_submodules_repo cl = Cloner(ds) assert cl.sync() assert repo_cloner.lib.cloner.RepoTool.fetch.call_count == 1 assert repo_cloner.lib.cloner.RepoTool.clone.call_count == 0 def test_submodules_sync_one_fail(cloner_dir_with_config, caplog): cloner_dir_with_config.joinpath("cache", "submodules").mkdir(parents = True) cloner_dir_with_config.joinpath("cache", "submodules", "submodules.cache") \ .write_text("https://git.hosting:previous/submodule.git") ds = MockDirStruct(cloner_dir_with_config) ds.config.cloner_repo_url = "https://git.hosting:namespace/whatever.git" ds.config.cloner_submodules = True # just to make coverage nicer ds.config.cloner_submodule_depth = 0 # mock almost everything mocks = { 'initialized': PropertyMock(return_value = True), 'fetch': MagicMock(side_effect = [True, True, True, False, True]), 'clone': MagicMock(return_value = True), 'repo_fingerprint': PropertyMock(side_effect = ["fp1", "fp2", "fp3"]), } with patch.multiple("repo_cloner.lib.cloner.RepoTool", **mocks): Cloner.check_submodules_repo = my_check_submodules_repo cl = Cloner(ds) assert not cl.sync() assert repo_cloner.lib.cloner.RepoTool.fetch.call_count == 5 assert repo_cloner.lib.cloner.RepoTool.clone.call_count == 0 assert "Clone/fetch of submodule: https://git.hosting:namespace/submodule2.git failed" in caplog.text def test_perform_check(cloner_dir_with_config, monkeypatch, caplog): call_counter: int = 0 def ret_true(): return True def ret_false(): return False def call_counter_inc(): nonlocal call_counter call_counter += 1 # set logger caplog.set_level(logging.INFO) mock = MockDirStruct(cloner_dir_with_config) mock.config.cloner_repo_url = "git://repo" cloner = Cloner(mock) # timer did not pass monkeypatch.setattr(cloner, "check_interval", ret_false) monkeypatch.setattr(cloner, "sync", call_counter_inc) assert call_counter == 0 cloner.perform_check() assert call_counter == 0 # timer passed monkeypatch.setattr(cloner, "check_interval", ret_true) cloner.perform_check() assert call_counter == 1 # right logs assert 2 == sum( r.message == "Started check for Mocked Project, url: git://repo" and r.levelname == "INFO" for r in caplog.records ) assert 2 == sum( r.message == "Check finished" and r.levelname == "INFO" for r in caplog.records ) def test_open_uninitialized(cloner_dir_with_config, path_repo_base): url = path_repo_base.as_uri() mock = MockDirStruct(cloner_dir_with_config) mock.config.cloner_repo_url = path_repo_base.as_uri() c = Cloner(mock) assert not c.open(mock.config.cloner_repo_url) assert c._repo_url == path_repo_base.as_uri() def test_open_initialized(cloner_dir_with_config, path_repo_base, caplog): mock = MockDirStruct(cloner_dir_with_config) hashed = gen_repo_hashed_name(path_repo_base.as_uri()) path = cloner_dir_with_config.joinpath("repos", hashed).as_posix() mock.config.cloner_repo_url = path_repo_base.as_uri() r = git.Repo().clone_from(path_repo_base.as_uri(), path, bare = True) commit = r.head.commit.hexsha assert r.bare c = Cloner(mock) assert c.open(path_repo_base.as_uri()) assert c._repo._repo.head.commit.hexsha == commit def test_clone(tmp_path, path_repo_base): mock = MockDirStruct(tmp_path) mock.config.cloner_repo_url = "invalid" c = Cloner(mock) assert c.clone(path_repo_base.as_uri()) assert "e0c7e2a72579e24657c05e875201011d2b48bf94" == c._repo._repo.head.commit.hexsha def test_clone_initialized(tmp_path, path_repo_base, caplog): mock = MockDirStruct(tmp_path) hashed = gen_repo_hashed_name(path_repo_base.as_uri()) path = tmp_path.joinpath("repos", hashed).as_posix() mock.config.cloner_repo_url = path_repo_base.as_uri() git.Repo().init(path) c = Cloner(mock) assert not c.clone(path_repo_base.as_uri()) assert caplog.records[0].levelname == "CRITICAL" assert caplog.records[0].message == f"Repo path {path} is initialized... Refusing clone!" def test_clone_recursive(tmp_path, path_repo_base, caplog): mock = MockDirStruct(tmp_path) mock.config.cloner_repo_url = "https://repo" mock.config.cloner_submodules = True Path(mock.repos_dir).mkdir() with patch( "repo_cloner.lib.cloner.RepoTool.initialized", new_callable = PropertyMock, return_value = False) as patch_initialized, \ patch( "repo_cloner.lib.cloner.RepoTool.clone_recursive", return_value = True) as patch_clone_recursive: cloner = Cloner(mock) assert cloner.clone() assert patch_initialized.call_count == 1 assert patch_clone_recursive.call_count == 1 patch_clone_recursive.assert_called_with( 'https://repo', tmp_path.joinpath('cache', 'submodules').as_posix(), scan_depth = 100) mock.config.cloner_submodule_depth = 0 assert cloner.clone() patch_clone_recursive.assert_called_with( 'https://repo', tmp_path.joinpath('cache', 'submodules').as_posix(), scan_depth = None) def test_detector_enabled(cloner_dir_with_config): ds = MockDirStruct(cloner_dir_with_config) ds.config.cloner_repo_url = "http://mock" cl = Cloner(ds) assert not cl.detector_enabled Path(cloner_dir_with_config).joinpath("config", "detector.cfg").touch() assert cl.detector_enabled def test_detector_run(cloner_dir_with_config): called = False def callback_test(commit): nonlocal called called = True def run_mock(callback): callback("Anything") ds = MockDirStruct(cloner_dir_with_config) ds.config.cloner_repo_url = "https://mock" mocks = { 'check_fingerprint': MagicMock(side_effect = [False, True]), 'run': MagicMock(side_effect = run_mock), } with patch.multiple('repo_cloner.lib.cloner.Detector', **mocks): cl = Cloner(ds) cl.detector_run(callback_test) assert not repo_cloner.lib.cloner.Detector.run.called assert not called cl.detector_run(callback_test) assert repo_cloner.lib.cloner.Detector.run.called assert called def test_detector_init(cloner_dir_with_config): ds = MockDirStruct(cloner_dir_with_config) ds.config.cloner_repo_url = "https://mock" mocks = { 'initialize_caches': MagicMock(), } with patch.multiple('repo_cloner.lib.cloner.Detector', **mocks): cl = Cloner(ds) cl.detector_init() assert repo_cloner.lib.cloner.Detector.initialize_caches.called