From 8e92bc70a797439a31c3bad27d353934e00e0c61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=A1clav=20Val=C3=AD=C4=8Dek?= Date: Sun, 24 Jul 2022 21:14:05 +0200 Subject: [PATCH] Add config parser functionality MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Václav Valíček --- repo_cloner/lib/cloner_config.py | 83 +++++++++++++++++++++++++- repo_cloner/lib/repo_dir_structure.py | 15 +++++ repo_cloner/process_repository_dir.py | 31 +++++++--- tests/_support_data/config/empty.cfg | 1 + tests/_support_data/config/invalid.cfg | 20 +++++++ tests/_support_data/config/valid.cfg | 18 ++++++ tests/lib/test_cloner_config.py | 14 ++++- tests/lib/test_cloner_config_parser.py | 72 ++++++++++++++++++++++ tests/lib/test_repo_dir_structure.py | 22 +++++++ 9 files changed, 265 insertions(+), 11 deletions(-) create mode 100644 tests/_support_data/config/empty.cfg create mode 100644 tests/_support_data/config/invalid.cfg create mode 100644 tests/_support_data/config/valid.cfg create mode 100644 tests/lib/test_cloner_config_parser.py diff --git a/repo_cloner/lib/cloner_config.py b/repo_cloner/lib/cloner_config.py index 1f4e194..325e49a 100644 --- a/repo_cloner/lib/cloner_config.py +++ b/repo_cloner/lib/cloner_config.py @@ -1,4 +1,10 @@ +import os + from repo_cloner.lib.default_cloner_config import DefaultClonerConfig +from repo_cloner.lib.config_file_not_found_error import ConfigFileNotFoundError +import logging + +l = logging.getLogger("rc.cfg") class ClonerConfig(DefaultClonerConfig): @@ -37,6 +43,79 @@ class ClonerConfig(DefaultClonerConfig): def _set_property(self, key: str, value): if not self.has_property(key): raise KeyError(f"{key} is not recognized config option") - if not isinstance(value, type(self.__getattribute__(key))): - raise ValueError(f"Invalid value for key {key}: type is {type(value)}") + + requested_type = type(self.__getattribute__(key)); + if not isinstance(value, requested_type): + l.debug(f"Trying conversion of {key} from {type(value)} to {requested_type}") + try: + if requested_type == bool: + value = True if value == "1" else False + else: + value = requested_type(value) + l.debug(f"Conversion result: {type(value)}: {value}") + except Exception as e: + l.critical(f"Conversion failed! ({str(e)}") + raise ValueError( + f"Invalid value for key {key}: type is {type(value)} instead of {requested_type}, conversion failed") self.__values[key] = value + + +class ClonerConfigParser: + __file: str = "" + __invalid_lines: list = [] + + def __init__(self, config_file: str): + self.__file = config_file + self.__config = ClonerConfig() + self.__invalid_lines = [] + l.info(f"Parsing cloner config file: {self.__file}") + + if not os.path.exists(self.__file): + l.critical(f"Config file: {self.__file} does not exist!") + raise ConfigFileNotFoundError(self.__file) + + # parse config file + raw_lines = [] + with open(self.__file, "r") as f: + raw_lines = f.readlines() + l.debug(f"Readen {len(raw_lines)} lines") + lines = [] + + # strip lines, remove comments + line: str = "" + for line in raw_lines: + if not line.startswith('#'): + line = line.strip() + # empty line? skip + if len(line) > 0: + lines.append(line) + l.debug(f"Found {len(lines)} config lines") + + for line in lines: + if not "=" in line: + l.warning(f"Line '{line}' has invalid format!") + self.__invalid_lines.append((line, None)) + continue + eq = line.find("=") + key: str = line[0:eq] + val: str = line[eq + 1:] + l.debug(f"Found config pair: {key} => '{val}'") + + try: + self.__config._set_property(key, val) + except BaseException as e: + self.__invalid_lines.append((key, val)) + l.critical(str(e)) + + @property + def config(self) -> ClonerConfig: + return self.__config + + @property + def invalid(self) -> bool: + return len(self.__invalid_lines) > 0 + + @property + def invalid_lines(self): + return self.__invalid_lines + diff --git a/repo_cloner/lib/repo_dir_structure.py b/repo_cloner/lib/repo_dir_structure.py index 7286aff..805795f 100644 --- a/repo_cloner/lib/repo_dir_structure.py +++ b/repo_cloner/lib/repo_dir_structure.py @@ -1,6 +1,7 @@ import os.path from repo_cloner.lib.dir_not_found_error import DirNotFoundError from repo_cloner.lib.config_file_not_found_error import ConfigFileNotFoundError +from repo_cloner.lib.cloner_config import ClonerConfig, ClonerConfigParser class RepoDirStructure(): @@ -8,12 +9,14 @@ class RepoDirStructure(): _conf_dir: str = "" _cache_dir: str = "" _repos_dir: str = "" + _config = None def __init__(self, base_dir: str): self._base_dir = base_dir self._conf_dir = os.path.join(self._base_dir, "config") self._cache_dir = os.path.join(self._base_dir, "cache") self._repos_dir = os.path.join(self._base_dir, "repos") + self._config = None @property def __config_filename(self) -> str: @@ -75,3 +78,15 @@ class RepoDirStructure(): if not os.path.exists(self.__config_filename): raise ConfigFileNotFoundError(self.__config_filename) return True + + @property + def config_file(self): + return self.__config_filename + + @property + def config(self) -> ClonerConfig: + self.has_config + if not self._config: + parser = ClonerConfigParser(self.__config_filename) + self._config = parser.config + return self._config diff --git a/repo_cloner/process_repository_dir.py b/repo_cloner/process_repository_dir.py index f772fde..f50d00b 100755 --- a/repo_cloner/process_repository_dir.py +++ b/repo_cloner/process_repository_dir.py @@ -1,24 +1,37 @@ #!/usr/bin/env python3 import argparse +import logging import os +import logging as l + +#l.basicConfig(level = 0) + +# create console handler with a higher log level +console_logger = l.StreamHandler() +console_formatter = l.Formatter( + "%(asctime)-15s :: [%(levelname)8s] :: %(name)-15s :: %(message)s (%(filename)s:%(lineno)s)", + "%Y-%m-%d %H:%M:%S") +# setup logger +console_logger.setFormatter(console_formatter) +log = l.getLogger("rc") +log.addHandler(console_logger) +log.setLevel(logging.DEBUG) + from repo_cloner.lib.repo_dir_structure import RepoDirStructure from git.config import GitConfigParser -import logging as l - -l.basicConfig(level = 0) def config_try_override(config_writer: GitConfigParser, section: str, option: str, value: str): if not section in config_writer.sections(): - l.debug(f"CFG Creating section: {section}") + log.debug(f"CFG Creating section: {section}") config_writer.add_section(section) if not config_writer.has_option(section, option): - l.debug(f"CFG Creating option: {option}") + log.debug(f"CFG Creating option: {option}") config_writer.add_value(section, option, "") - l.debug(f"Setting {section}.{option} = {value}") + log.debug(f"Setting {section}.{option} = {value}") config_writer.set(section, option, value) @@ -48,9 +61,11 @@ def main(): with r.config_writer("user") as cfgw: config_try_override(cfgw, "credential", "helper", f"store --file={cred_store}") config_try_override(cfgw, "core", "sshcommand", - f"ssh -i {ssh_identity} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -q") + f"ssh -i {ssh_identity} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=yes -q") + + from repo_cloner.lib.cloner_config import ClonerConfigParser + ClonerConfigParser(os.path.join(dirs.conf_dir, "cloner.cfg")) if __name__ == "__main__": - main() diff --git a/tests/_support_data/config/empty.cfg b/tests/_support_data/config/empty.cfg new file mode 100644 index 0000000..7f2d23d --- /dev/null +++ b/tests/_support_data/config/empty.cfg @@ -0,0 +1 @@ +# empty file with no directives \ No newline at end of file diff --git a/tests/_support_data/config/invalid.cfg b/tests/_support_data/config/invalid.cfg new file mode 100644 index 0000000..3a7acbf --- /dev/null +++ b/tests/_support_data/config/invalid.cfg @@ -0,0 +1,20 @@ +# cloner.cfg +# main config +# created at 2018-03-23 20:24:57 +invalid_option1 +invalid_option=2 + +# main url - url of main repo - just to clone +cloner_repo_url=git@git.somerandomserver.test:/path + +# project name (names of volumes are derrived from this +cloner_project_name=some-random-project + +# cloner interval (in minutes, default=0 - run always) +cloner_interval=666 + +# do you need submodules support? (1/0) +cloner_submodules=1 + +# max depth of submodule scan (default = unlimited, uncomment to use) +cloner_submodule_depth=500000 \ No newline at end of file diff --git a/tests/_support_data/config/valid.cfg b/tests/_support_data/config/valid.cfg new file mode 100644 index 0000000..d064d45 --- /dev/null +++ b/tests/_support_data/config/valid.cfg @@ -0,0 +1,18 @@ +# cloner.cfg +# main config +# created at 2018-03-23 20:24:57 + +# main url - url of main repo - just to clone +cloner_repo_url=git@git.somerandomserver.test:/path + +# project name (names of volumes are derrived from this +cloner_project_name=some-random-project + +# cloner interval (in minutes, default=0 - run always) +cloner_interval=666 + +# do you need submodules support? (1/0) +cloner_submodules=1 + +# max depth of submodule scan (default = unlimited, uncomment to use) +cloner_submodule_depth=500000 \ No newline at end of file diff --git a/tests/lib/test_cloner_config.py b/tests/lib/test_cloner_config.py index cfeb1d2..b5faa6a 100644 --- a/tests/lib/test_cloner_config.py +++ b/tests/lib/test_cloner_config.py @@ -66,7 +66,7 @@ def test_set_property(): # invalid type with pytest.raises(ValueError) as exc: x._set_property("cloner_interval", "č") - assert exc.exconly() == "ValueError: Invalid value for key cloner_interval: type is " + assert exc.exconly() == "ValueError: Invalid value for key cloner_interval: type is instead of , conversion failed" x._set_property("cloner_project_name", "č") @@ -74,3 +74,15 @@ def test_set_property(): with pytest.raises(KeyError) as exc: x._set_property("nonexistent_key", 888) assert exc.exconly() == "KeyError: 'nonexistent_key is not recognized config option'" + + # boolean conversion + x._set_property("cloner_submodules", "1") + assert x.cloner_submodules == True + + x._set_property("cloner_submodules", "0") + assert x.cloner_submodules == False + + # integer conversion + x._set_property("cloner_interval", 60) + assert x.cloner_interval == 60 + diff --git a/tests/lib/test_cloner_config_parser.py b/tests/lib/test_cloner_config_parser.py new file mode 100644 index 0000000..9d401e0 --- /dev/null +++ b/tests/lib/test_cloner_config_parser.py @@ -0,0 +1,72 @@ +from repo_cloner.lib.cloner_config import ClonerConfigParser +import pytest +from repo_cloner.lib.config_file_not_found_error import ConfigFileNotFoundError +from pathlib import Path + + +@pytest.fixture +def config_path() -> Path: + path = Path(__file__).parent.parent + return path.joinpath("_support_data", "config") + + +def test_no_config_file(): + with pytest.raises(ConfigFileNotFoundError) as e: + x = ClonerConfigParser("/tmp/path/that/does/not/exist") + assert e.__str__() == "" + + +def test_valid_config(config_path: Path): + cfg = config_path.joinpath("valid.cfg") + x = ClonerConfigParser(cfg) + assert x.invalid == False + assert x.invalid_lines == [] + c = x.config + assert c.cloner_project_name == "some-random-project" + assert c.cloner_repo_url == "git@git.somerandomserver.test:/path" + assert c.cloner_interval == 666 + assert c.cloner_submodules == True + assert c.cloner_submodule_depth == 500000 + + +def test_empty_config(config_path: Path): + cfg = config_path.joinpath("empty.cfg") + x = ClonerConfigParser(cfg) + assert x.invalid == False + assert x.invalid_lines == [] + c = x.config + assert c.cloner_project_name == "" + assert c.cloner_repo_url == "" + assert c.cloner_interval == 5 + assert c.cloner_submodules == True + assert c.cloner_submodule_depth == 50000 + + +def test_invalid_config(config_path: Path, caplog): + cfg = config_path.joinpath("invalid.cfg") + with caplog.at_level(30): + x = ClonerConfigParser(cfg) + + # two invalid lines + assert x.invalid == True + assert x.invalid_lines == [ + ('invalid_option1', None), + ('invalid_option', "2"), + ] + # test logging + assert any( + (r.levelname == "CRITICAL" and r.message == "'invalid_option is not recognized config option'") + for r in caplog.records + ) + + assert any( + (r.levelname == "WARNING" and r.message == "Line 'invalid_option1' has invalid format!") + for r in caplog.records + ) + + c = x.config + assert c.cloner_project_name == "some-random-project" + assert c.cloner_repo_url == "git@git.somerandomserver.test:/path" + assert c.cloner_interval == 666 + assert c.cloner_submodules == True + assert c.cloner_submodule_depth == 500000 diff --git a/tests/lib/test_repo_dir_structure.py b/tests/lib/test_repo_dir_structure.py index 69fde80..e747349 100644 --- a/tests/lib/test_repo_dir_structure.py +++ b/tests/lib/test_repo_dir_structure.py @@ -102,3 +102,25 @@ def test_has_config(tmp_path: PosixPath): tmp_path.joinpath("config", "cloner.cfg").touch() assert X.has_config + + +def test_config_file(tmp_path: PosixPath): + X = RepoDirStructure(tmp_path) + assert X.config_file == tmp_path.joinpath("config", "cloner.cfg").__str__() + + +def test_get_config(tmp_path: PosixPath): + tmp_path.joinpath("config").mkdir() + tmp_path.joinpath("cache").mkdir() + tmp_path.joinpath("repos").mkdir() + + X = RepoDirStructure(tmp_path) + # no file provided + with pytest.raises(ConfigFileNotFoundError) as excinfo: + X.config + + # create config file + tmp_path.joinpath("config", "cloner.cfg").touch() + X = RepoDirStructure(tmp_path) + assert X.has_config == True + assert 5 == X.config.cloner_interval