From b1b0554e60fc5f0feb542bf54d1cadbc1d0418d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A1clav=20Val=C3=AD=C4=8Dek?= Date: Thu, 28 Jul 2022 21:03:20 +0200 Subject: [PATCH] Cloner begining MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Václav Valíček --- repo_cloner/lib/cloner.py | 97 ++++++++++++ tests/lib/cloner_test_fixtures.py | 7 +- tests/lib/test_cloner.py | 251 ++++++++++++++++++++++++++++++ 3 files changed, 352 insertions(+), 3 deletions(-) create mode 100644 repo_cloner/lib/cloner.py create mode 100644 tests/lib/test_cloner.py diff --git a/repo_cloner/lib/cloner.py b/repo_cloner/lib/cloner.py new file mode 100644 index 0000000..3c7c02d --- /dev/null +++ b/repo_cloner/lib/cloner.py @@ -0,0 +1,97 @@ +from repo_cloner.lib.cloner_config import ClonerConfig +from repo_cloner.lib.repo_dir_structure import RepoDirStructure +from repo_cloner.lib.dir_not_found_error import DirNotFoundError +from repo_cloner.lib.repo_tool import RepoTool +from repo_cloner.lib.checksum import gen_repo_hashed_name +from pathlib import Path +from time import time +import os +import logging + +log = logging.getLogger("rc.cloner") + + +class Cloner: + _dirs: RepoDirStructure = None + _config: ClonerConfig = None + _interval_file: str = "last-check-time" + _repo: RepoTool = None + _repo_url: str = "" + + def __init__(self, dir_structure: RepoDirStructure): + self._dirs = dir_structure + self._config = self._dirs.config + if len(self._config.cloner_repo_url) == 0: + logging.critical(f"Undefined repo cloner URL in config!") + raise KeyError(f"cloner_repo_url not defined in config!") + + # create cache dir, if missing + try: + assert self._dirs.cache_dir_exists + except DirNotFoundError: + log.info(f"Cache dir for project {self._config.cloner_project_name} not found -> creating") + Path(self._dirs.cache_dir).mkdir() + log.debug(f"Cache dir created") + + def check_interval(self): + log.debug(f"Checking interval for {self._config.cloner_project_name}") + # get interval + interval = self._config.cloner_interval + # interval file? + interval_file: Path = Path(self._dirs.cache_dir).joinpath(self._interval_file) + log.debug(f"Interval file: {interval_file}") + file_stamp: int = 0 + if interval_file.exists(): + str_val = interval_file.read_text() + try: + file_stamp = int(str_val) + except ValueError: + log.warning(f"Interval file file is corrupted, keeping value as nothing happened") + # check time + if time() > file_stamp + interval * 60: + return True + return False + + def open(self, url: str) -> bool: + log.debug(f"Opening repo with url: {url}") + repo_path = self._repo_path_by_url(url) + self._repo_url = url + self._repo = RepoTool(repo_path) + return self.__opened + + @property + def __opened(self) -> bool: + if not self._repo: + return False + return self._repo.initialized + + def _repo_path_by_url(self, url: str) -> str: + hashed_name: str = gen_repo_hashed_name(url) + log.debug(f"Repo hashed name for {url} is {hashed_name}") + return os.path.join(self._dirs.repos_dir, hashed_name) + + @property + def __main_repo_path(self) -> str: + return self._repo_path_by_url(self._config.cloner_repo_url) + + def sync(self) -> bool: + if not self.__opened: + self._repo = RepoTool(self.__main_repo_path) + if not self._repo.initialized: + return False + return self._repo.fetch() + + def perform_check(self): + log.info(f"Started check for {self._config.cloner_project_name}, url: {self._config.cloner_repo_url}") + if self.check_interval(): + self.sync() + log.info(f"Check finished") + + def clone_from_url(self, url: str) -> bool: + path = self._repo_path_by_url(url) + self._repo_url = url + self._repo = RepoTool(path) + if self._repo.initialized: + log.critical(f"Repo path {path} is initialized... Refusing clone!") + return False + return self._repo.clone(url) diff --git a/tests/lib/cloner_test_fixtures.py b/tests/lib/cloner_test_fixtures.py index 42fb8e2..c7ca9de 100644 --- a/tests/lib/cloner_test_fixtures.py +++ b/tests/lib/cloner_test_fixtures.py @@ -17,11 +17,12 @@ def cloner_dir_struct(tmp_path: Path) -> Path: @pytest.fixture -def cloner_dir_with_config(cloner_dir_struct: Path) -> Path: +def cloner_dir_with_config(cloner_dir_struct: Path, support_data_path) -> Path: + repo_source = support_data_path.joinpath("test-repo-base").as_uri() cfg_file = cloner_dir_struct.joinpath("config", "cloner.cfg") cfg_file.touch() - cfg_file.write_text("# cloner.cfg" - "cloner_repo_url=https://git.sw3.cz/kamikaze/test-repo-base.git" + cfg_file.write_text(f"# cloner.cfg" + f"cloner_repo_url={repo_source}" "cloner_project_name=test-repo" ) return cloner_dir_struct diff --git a/tests/lib/test_cloner.py b/tests/lib/test_cloner.py new file mode 100644 index 0000000..f34ab8f --- /dev/null +++ b/tests/lib/test_cloner.py @@ -0,0 +1,251 @@ +import git +import pytest +import repo_cloner.lib.dir_not_found_error +from cloner_test_fixtures import * +from repo_cloner.lib.cloner import Cloner +from pathlib import Path +import logging + + +def mock_time() -> float: + return 1658702099.4258687 + + +class MockConfig: + def __init__(self): + self.cloner_repo_url = "" + self.cloner_project_name = "Mocked Project" + self.cloner_interval = 0 + + +class MockDirStruct: + raise_cache_exists = False + + def __init__(self, tmp: Path): + self.config = MockConfig() + self.cache_dir = tmp.joinpath("cache") + self.config_dir = tmp.joinpath("config") + self.repos_dir = tmp.joinpath("repos") + self.raise_cache_exists = False + + @property + def cache_dir_exists(self): + if self.raise_cache_exists: + raise repo_cloner.lib.dir_not_found_error.DirNotFoundError("mock_dir") + return True + + +class MockRepoTool: + path = None + initialized = False + + def __init__(self, path: str): + self.path = path + self.initialized = False + + +def test_init_invalid_config(tmp_path, caplog): + caplog.set_level(logging.INFO) + mock = MockDirStruct(tmp_path) + with pytest.raises(KeyError) as e: + Cloner(mock) + assert "KeyError: 'cloner_repo_url not defined in config!'" == e.exconly() + assert "CRITICAL root:cloner.py:" in caplog.text + assert "Undefined repo cloner URL in config!\n" in caplog.text + + # valid config + caplog.clear() + # set mocks + mock.config.cloner_repo_url = "git@mocked:/path" + cache_dir = tmp_path.joinpath("cache") + # validate dir does not exist + assert not cache_dir.exists() + # set mock to raise error + mock.raise_cache_exists = True + # create new cloner class + Cloner(mock) + # dir should exist now + assert cache_dir.exists() + assert "Cache dir for project Mocked Project not found -> creating" in caplog.text + + +def test_check_interval(cloner_dir_struct: Path, monkeypatch): + mock = MockDirStruct(cloner_dir_struct) + mock.config.cloner_repo_url = "git@mocked:/path" + timestamp_file = cloner_dir_struct.joinpath("cache", "last-check-time") + + with monkeypatch.context() as m: + m.setattr("repo_cloner.lib.cloner.time", mock_time) + cloner = Cloner(mock) + # no timestamp file exists -> timer has to run + assert cloner.check_interval() + + # timestamp file in future -> no run + timestamp_file.touch() + timestamp_file.write_text(str(int(mock_time() + 1))) + assert not cloner.check_interval() + + # timestamp in history, but run everytime is set >> RUN + timestamp_file.write_text(str(int(mock_time() - 1))) + assert cloner.check_interval() + + # second ago, but with 5 minute interval + mock.config.cloner_interval = 5 + assert not cloner.check_interval() + + # 6 minutes ago, with 5 min interval -> run! + timestamp_file.write_text(str(int(mock_time() - 360))) + assert cloner.check_interval() + + # corrupted file + timestamp_file.write_text("1234WTF") + assert cloner.check_interval() + + +def test__opened(cloner_dir_with_config, monkeypatch): + ds = MockDirStruct(cloner_dir_with_config) + ds.config.cloner_repo_url = cloner_dir_with_config.as_uri() + c = Cloner(ds) + # no repo defined + assert not c._Cloner__opened + # with mocked repo + monkeypatch.setattr(c, "_repo", MockRepoTool(cloner_dir_with_config.as_posix())) + c._repo.initialized = True + assert c._Cloner__opened + c._repo.initialized = False + assert not c._Cloner__opened + + +def test_repo_path_by_url(cloner_dir_struct: Path, path_repo_base: Path): + ds = MockDirStruct(cloner_dir_struct) + ds.config.cloner_repo_url = path_repo_base.as_uri() + c = Cloner(ds) + x = c._repo_path_by_url("git@server:namespace/repo.git") + assert x == ds.repos_dir.joinpath("namespace_repo_3375634822.git").as_posix() + + +def test__main_repo_path(cloner_dir_struct: Path, path_repo_base: Path): + ds = MockDirStruct(cloner_dir_struct) + ds.config.cloner_repo_url = path_repo_base.as_uri() + c = Cloner(ds) + x = c._Cloner__main_repo_path + assert x == ds.repos_dir.joinpath("_support_data_test-repo-base_402961715.git").as_posix() + + +def test_sync(cloner_dir_struct, tmp_path, monkeypatch): + called: bool = False + + def fetch(): + nonlocal called + called = True + return True + + class PatchCloner(Cloner): + open = False + + @property + def __opened(self) -> bool: + return self.open + + ds = MockDirStruct(cloner_dir_struct) + ds.config.cloner_repo_url = "file://wut!" + repo = tmp_path.joinpath("repo.git") + c = PatchCloner(ds) + # invalid repo, closed + c.open = False + assert not c.sync() + assert not called + # mocked valid repo, opened + c._repo = MockRepoTool(repo.as_posix()) + c._repo.initialized = True + c._repo.fetch = fetch + assert c.sync() + assert called + + called = False + # mocked repo, invalid, closed + c._repo.initialized = False + assert not called + assert not c.sync() + + +def test_perform_check(cloner_dir_with_config, monkeypatch, caplog): + call_counter: int = 0 + + def ret_true(): + return True + + def ret_false(): + return False + + def call_counter_inc(): + nonlocal call_counter + call_counter += 1 + + # set logger + caplog.set_level(logging.INFO) + + mock = MockDirStruct(cloner_dir_with_config) + mock.config.cloner_repo_url = "git://repo" + cloner = Cloner(mock) + # timer did not pass + monkeypatch.setattr(cloner, "check_interval", ret_false) + monkeypatch.setattr(cloner, "sync", call_counter_inc) + assert call_counter == 0 + cloner.perform_check() + assert call_counter == 0 + # timer passed + monkeypatch.setattr(cloner, "check_interval", ret_true) + cloner.perform_check() + assert call_counter == 1 + # right logs + assert 2 == sum( + r.message == "Started check for Mocked Project, url: git://repo" + and r.levelname == "INFO" + for r in caplog.records + ) + assert 2 == sum( + r.message == "Check finished" + and r.levelname == "INFO" + for r in caplog.records + ) + + +def test_open_uninitialized(cloner_dir_with_config, path_repo_base): + url = path_repo_base.as_uri() + mock = MockDirStruct(cloner_dir_with_config) + mock.config.cloner_repo_url = path_repo_base.as_uri() + c = Cloner(mock) + assert not c.open(mock.config.cloner_repo_url) + assert c._repo_url == path_repo_base.as_uri() + + +def test_open_initialized(cloner_dir_with_config, path_repo_base, caplog): + mock = MockDirStruct(cloner_dir_with_config) + path = cloner_dir_with_config.joinpath("repos", "_support_data_test-repo-base_402961715.git").as_posix() + mock.config.cloner_repo_url = path_repo_base.as_uri() + r = git.Repo().clone_from(path_repo_base.as_uri(), path, bare = True) + commit = r.head.commit.hexsha + assert r.bare + c = Cloner(mock) + assert c.open(path_repo_base.as_uri()) + assert c._repo._repo.head.commit.hexsha == commit + + +def test_clone_from_url(tmp_path, path_repo_base): + mock = MockDirStruct(tmp_path) + mock.config.cloner_repo_url = "invalid" + c = Cloner(mock) + assert c.clone_from_url(path_repo_base.as_uri()) + assert "e0c7e2a72579e24657c05e875201011d2b48bf94" == c._repo._repo.head.commit.hexsha + + +def test_clone_from_url_initialized(tmp_path, path_repo_base, caplog): + mock = MockDirStruct(tmp_path) + path = tmp_path.joinpath("repos", "_support_data_test-repo-base_402961715.git").as_posix() + mock.config.cloner_repo_url = path_repo_base.as_uri() + r = git.Repo().init(path) + c = Cloner(mock) + assert not c.clone_from_url(path_repo_base.as_uri()) + assert caplog.records[0].levelname == "CRITICAL" + assert caplog.records[0].message == f"Repo path {path} is initialized... Refusing clone!"