From c196e33b4b2caee0303ddc031d6f0edafc417312 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A1clav=20Val=C3=AD=C4=8Dek?= Date: Tue, 26 Jul 2022 22:04:51 +0200 Subject: [PATCH] Repo clone test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Václav Valíček --- .idea/vcs.xml | 1 + repo_cloner/lib/repo_tool.py | 133 ++++++++++++++++++++++++++++++ tests/_support_data/gen-data.sh | 1 + tests/lib/cloner_test_fixtures.py | 17 +++- tests/lib/test_repo_tool.py | 107 ++++++++++++++++++++++++ 5 files changed, 258 insertions(+), 1 deletion(-) create mode 100644 repo_cloner/lib/repo_tool.py create mode 100644 tests/lib/test_repo_tool.py diff --git a/.idea/vcs.xml b/.idea/vcs.xml index e3e423e..fc02c78 100644 --- a/.idea/vcs.xml +++ b/.idea/vcs.xml @@ -3,5 +3,6 @@ + \ No newline at end of file diff --git a/repo_cloner/lib/repo_tool.py b/repo_cloner/lib/repo_tool.py new file mode 100644 index 0000000..835776d --- /dev/null +++ b/repo_cloner/lib/repo_tool.py @@ -0,0 +1,133 @@ +from git import Repo +from git.exc import NoSuchPathError, InvalidGitRepositoryError +from git import RemoteProgress +import logging +import time + +log = logging.getLogger("rc.repo") + + +class GitRemoteProgress(RemoteProgress): + OP_CODES = [ + "BEGIN", + "CHECKING_OUT", + "COMPRESSING", + "COUNTING", + "END", + "FINDING_SOURCES", + "RECEIVING", + "RESOLVING", + "WRITING", + ] + OP_CODE_MAP = { + getattr(RemoteProgress, _op_code): _op_code for _op_code in OP_CODES + } + + last_step_time = time.time() + time_thr = 0.5 + + cur_task: str = "" + cur_task_max: int = 0 + + def __init__(self) -> None: + super().__init__() + self.last_step_time = time.time() - self.time_thr + self.cur_task_max = 0 + self.cur_task = "" + + def __del__(self) -> None: + self.finish() + + @classmethod + def get_curr_op(cls, op_code: int) -> str: + """Get OP name from OP code.""" + # Remove BEGIN- and END-flag and get op name + op_code_masked = op_code & cls.OP_MASK + return cls.OP_CODE_MAP.get(op_code_masked, "?").title() + + def finish(self): + log.info(f"GIT {self.cur_task}: 100.00% ({self.cur_task_max})") + + def update( + self, + op_code: int, + cur_count: str | float, + max_count: str | float | None = None, + message: str | None = "", + ) -> None: + # Do i need to update? + # -> begin : YES + # -> end : YES + # -> timer: YES + + # so check timer + if (self.last_step_time + self.time_thr) > time.time(): + # timer not passed yet + if not ((op_code & self.BEGIN) or (op_code & self.BEGIN)): + # skip -> no begin or end + return + # update timer + self.last_step_time = time.time() + + # Start new bar on each BEGIN-flag + if op_code & self.BEGIN: + self.cur_task = self.get_curr_op(op_code).upper() + try: + self.cur_task_max = int(max_count) + except ValueError: + self.cur_task_max = 100 + + log.info(f"GIT {self.cur_task} started") + + percent = round(100 * (cur_count / self.cur_task_max), 2) + + # End progress monitoring on each END-flag + if op_code & self.END: + # logger.info("Done: %s", self.curr_op) + percent = 100 + + log.info(f"GIT {self.cur_task}: {percent}% ({cur_count}; {message})") + + +class RepoTool: + _repo: Repo = None + _initialized: bool = False + _bare: bool = False + _path: str = "" + + def __init__(self, path: str): + log.info(f"Initializing repository at {path}") + self._path = str(path) + try: + self._repo = Repo(path, expand_vars = False) + self._initialized = True + self._bare = self._repo.bare + + except (NoSuchPathError, InvalidGitRepositoryError) as e: + log.warning(f"Init failed: {str(e)}, continuing with uninitialized repo") + self._initialized = False + self._bare = False + + @property + def initialized(self) -> bool: + return self._initialized + + @property + def bare(self) -> bool: + return self._bare + + @property + def path(self) -> str: + return self._path + + def clone(self, url: str) -> bool: + if self._initialized: + log.warning(f"Trying to clone to initialized repository!") + return False + + log.info(f"Cloning repository from url: {url}") + self._repo = Repo.clone_from(url, to_path = self._path, progress = GitRemoteProgress(), bare = True) + self._initialized = True + self._bare = self._repo.bare + + return True diff --git a/tests/_support_data/gen-data.sh b/tests/_support_data/gen-data.sh index 396721e..f2c5189 100755 --- a/tests/_support_data/gen-data.sh +++ b/tests/_support_data/gen-data.sh @@ -13,4 +13,5 @@ then echo "Initializing tool_repos" mkdir -p tool_repos/uninitialized.git git init --bare tool_repos/initialized.git +git init tool_repos/non-bare-init diff --git a/tests/lib/cloner_test_fixtures.py b/tests/lib/cloner_test_fixtures.py index 7a2aeb9..3ed8631 100644 --- a/tests/lib/cloner_test_fixtures.py +++ b/tests/lib/cloner_test_fixtures.py @@ -1,6 +1,13 @@ import pytest from pathlib import Path + +@pytest.fixture +def support_data_path() -> Path: + path = Path(__file__).parent.parent.joinpath("_support_data") + return path + + @pytest.fixture def cloner_dir_struct(tmp_path: Path) -> Path: tmp_path.joinpath("config").mkdir() @@ -9,4 +16,12 @@ def cloner_dir_struct(tmp_path: Path) -> Path: return tmp_path - +@pytest.fixture +def cloner_dir_with_config(cloner_dir_struct: Path) -> Path: + cfg_file = cloner_dir_struct.joinpath("config", "cloner.cfg") + cfg_file.touch() + cfg_file.write_text("# cloner.cfg" + "cloner_repo_url=https://git.sw3.cz/kamikaze/test-repo-base.git" + "cloner_project_name=test-repo" + ) + return cloner_dir_struct diff --git a/tests/lib/test_repo_tool.py b/tests/lib/test_repo_tool.py new file mode 100644 index 0000000..0438f3b --- /dev/null +++ b/tests/lib/test_repo_tool.py @@ -0,0 +1,107 @@ +import logging +import pytest +from cloner_test_fixtures import support_data_path +from pathlib import Path +from repo_cloner.lib.repo_tool import RepoTool + + +def test_init(support_data_path: Path): + test_repos_path = support_data_path.joinpath("tool_repos") + # test on non-existent dir + rt = RepoTool(test_repos_path.joinpath("nonexistent.git").as_posix()) + assert not rt.initialized + + # uninitialized, but existing repo + rt = RepoTool(test_repos_path.joinpath("uninitialized.git").as_posix()) + assert not rt.initialized + assert not rt.bare + + # initialized, existing repo + rt = RepoTool(test_repos_path.joinpath("initialized.git").as_posix()) + assert rt.initialized + assert rt.bare + + rt = RepoTool(test_repos_path.joinpath("non-bare-init").as_posix()) + assert rt.initialized + assert not rt.bare + + +def test_initialized(tmp_path, monkeypatch): + rt = RepoTool(tmp_path.as_posix()) + monkeypatch.setattr(rt, "_initialized", False) + assert not rt.initialized + monkeypatch.setattr(rt, "_initialized", True) + assert rt.initialized + + +def test_bare(tmp_path, monkeypatch): + rt = RepoTool(tmp_path.as_posix()) + monkeypatch.setattr(rt, "_bare", False) + assert not rt.bare + monkeypatch.setattr(rt, "_bare", True) + assert rt.bare + + +def test_path(tmp_path, monkeypatch): + rt = RepoTool(tmp_path) + assert tmp_path.as_posix() == rt.path + monkeypatch.setattr(rt, "_path", "/tmp") + assert "/tmp" == rt.path + + +def test_clone_initialized_repo(tmp_path, caplog, support_data_path): + from git import Repo + # initialize repo + Repo().init(tmp_path, bare = True) + rt = RepoTool(tmp_path.as_posix()) + # check it + assert rt.initialized + assert rt.bare + + # try clone + test_repo = support_data_path.joinpath("test-repo-base").as_uri() + assert not rt.clone(test_repo) + + r = caplog.records[0] + assert r.levelname == "WARNING" + assert r.message == "Trying to clone to initialized repository!" + + +def test_clone_okay(tmp_path, caplog, support_data_path): + rt = RepoTool(tmp_path.as_posix()) + assert not rt.initialized + + caplog.set_level(logging.INFO) + + # try clone + test_repo = support_data_path.joinpath("test-repo-base").as_uri() + assert rt.clone(test_repo) + + # warning about uninit repo + assert caplog.records[0].levelname == "WARNING" + # info cloning + assert caplog.records[1].levelname == "INFO" + assert "Cloning repository from url: file:///" in caplog.records[1].message + + # progress states + counting_cnt = 0 + compressing_cnt = 0 + receiving_cnt = 0 + resolving_cnt = 0 + + for x in range(1, len(caplog.records)): + rec = caplog.records[x] + assert rec.levelname == "INFO" + if "GIT COUNTING" in rec.message: + counting_cnt += 1 + if "GIT COMPRESSING" in rec.message: + compressing_cnt += 1 + if "GIT RECEIVING" in rec.message: + receiving_cnt += 1 + if "GIT RESOLVING" in rec.message: + resolving_cnt += 1 + + assert counting_cnt >= 2 + assert compressing_cnt >= 2 + assert receiving_cnt >= 2 + assert resolving_cnt >= 2