kamikaze/repo-cloner
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Update, repo tool cloner: recursive clones

Browse Source 
Signed-off-by: Václav Valíček <valicek1994@gmail.com>
This commit is contained in:
Václav Valíček
2022-07-29 17:03:15 +02:00
parent 8a150c63c5
commit 6463a6bb95
6 changed files with 142 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
repo_cloner/lib/repo_tool.py
View File

@@ -1,3 +1,5 @@
import os.path
import git
from git import Repo
from git.exc import NoSuchPathError, InvalidGitRepositoryError
@@ -5,6 +7,7 @@ from git import RemoteProgress
import logging
import time
from typing import Union, Optional
from repo_cloner.lib.checksum import gen_repo_hashed_name
log = logging.getLogger("rc.repo")
@@ -64,7 +67,7 @@ class GitRemoteProgress(RemoteProgress):
# so check timer
if (self.last_step_time + self.time_thr) > time.time():
# timer not passed yet
# timer not passed yet repo.head.reset(commit = 'origin/master', index = True, working_tree = True)
if not ((op_code & self.BEGIN) or (op_code & self.BEGIN)):
# skip -> no begin or end
return
@@ -97,10 +100,16 @@ class RepoTool:
_bare: bool = False
_path: str = ""
_last_fetch_data = []
_recursive_discovery_urls: set = set()
_recursive_discovery_cloned: set = set()
_submodule_discovery_history: list = []
def __init__(self, path: str):
log.info(f"Initializing repository at {path}")
self._path = str(path)
self._recursive_discovery_cloned = set()
self._recursive_discovery_urls = set()
self._submodule_discovery_history: list = []
try:
self._repo = Repo(path, expand_vars = False)
self._initialized = True
@@ -123,6 +132,14 @@ class RepoTool:
def path(self) -> str:
return self._path
@property
def cloned_submodules_url_list(self) -> list:
return list(self._recursive_discovery_cloned)
@property
def discovered_submodules_commits(self) -> list:
return self._submodule_discovery_history
def __check_initialized(self):
def inner(*args):
fake_self: RepoTool = args[0]
@@ -183,7 +200,8 @@ class RepoTool:
log.debug(f"Repo fingerprint is {x}")
return x
def list_submodules(self, commit: str = "HEAD") -> list:
@__check_initialized
def list_submodules(self, commit: str = "HEAD") -> Union[list, bool]:
commit = self._repo.commit(commit)
submodules = []
@@ -198,9 +216,13 @@ class RepoTool:
if parser.has_option(section, "url"):
submodules.append(parser.get_value(section, "url"))
if commit.hexsha not in self._submodule_discovery_history:
self._submodule_discovery_history.append(commit.hexsha)
return submodules
def list_submodules_history(self, limit_of_commits: Optional[int] = None):
@__check_initialized
def list_submodules_history(self, limit_of_commits: Optional[int] = None) -> Union[list, bool]:
log.info(f"Listing repository submodule history")
iterator = self._repo.iter_commits(all = True, max_count = limit_of_commits)
@@ -216,3 +238,55 @@ class RepoTool:
log.info(f"Submodule discovery: {counter} commits finished, {len(submodules)} discovered")
last_status = time.time()
return list(submodules)
def clone_recursive(
self,
main_url: str,
scan_cache_dir: Optional[str] == None,
scan_depth: Optional[int] = None
) -> bool:
log.info(f"Started recursive clone of {main_url} with recursive discovery limited to {scan_depth} commits")
# clone main repo
if not self.clone(main_url):
log.critical(f"Clone of main repository failed!")
return False
# discover submodules for repository
submodules = self.list_submodules_history(scan_depth)
if submodules:
for submodule in submodules:
self._recursive_discovery_urls.add(submodule)
everything_succeed: bool = True
everything_cloned: bool = False
while not everything_cloned:
# recursively scan and clone repositories
everything_cloned = True
# for every url in list
# list() is needed - Runtime Error for set() changed during iteration
for url in list(self._recursive_discovery_urls):
if url not in self._recursive_discovery_cloned:
everything_cloned = False
# generate new path
directory = os.path.dirname(self.path)
submodule_cloner = RepoTool(os.path.join(directory, gen_repo_hashed_name(url)))
# clone
cloned: bool = submodule_cloner.clone(url)
# mark cloned even if failed afterwards - while loop stuck solution
self._recursive_discovery_cloned.add(url)
if not cloned:
log.critical(f"Clone of submodule: {url} failed")
everything_succeed = False
continue
# scan for submodules
submodules = submodule_cloner.list_submodules_history(scan_depth)
if type(submodules) == bool and not submodules:
log.critical(f"Submodule discovery for {url} failed!")
everything_succeed = False
continue
for submodule in submodules:
self._recursive_discovery_urls.add(submodule)
return everything_succeed

33
repo_cloner/process_repository_dir.py
View File

@@ -6,9 +6,6 @@ import os
import logging as l
# l.basicConfig(level = 0)
# create console handler with a higher log level
console_logger = l.StreamHandler()
console_formatter = l.Formatter(
"%(asctime)-15s :: [%(levelname)8s] :: %(name)-15s :: %(message)s (%(filename)s:%(lineno)s)",
@@ -20,7 +17,11 @@ log.addHandler(console_logger)
log.setLevel(logging.DEBUG)
from repo_cloner.lib.repo_dir_structure import RepoDirStructure
from repo_cloner.lib.cloner import Cloner
from repo_cloner.lib.repo_tool import RepoTool
from git.config import GitConfigParser
from git.repo import Repo
from typing import Union
def config_try_override(config_writer: GitConfigParser, section: str, option: str, value: str):
@@ -36,6 +37,10 @@ def config_try_override(config_writer: GitConfigParser, section: str, option: st
def main() -> int:
def update(op_code: int, cur_count: Union[str, float], max_count: Union[str, float, None] = None,
message: str = ''):
log.debug(f"op: {op_code}; cur: {cur_count}/{max_count}; mess: {message}")
# parse input arguments
parser = argparse.ArgumentParser(description = "repo-cloner entering script")
parser.add_argument('--base-dir', help = 'path to directory containing whole cloner structure', required = True,
@@ -72,11 +77,29 @@ def main() -> int:
if len(config.cloner_project_name) == 0:
log.warning("Config directive cloner_project_name should not be omitted!")
# cloner = Cloner(dirs)
# cloner.check_interval()
import subprocess
subprocess.run(["/usr/bin/rm", "-Rf", "/tmp/test/repos"])
subprocess.run(["/usr/bin/mkdir", "/tmp/test/repos"])
rt = RepoTool("/tmp/test/repos/main.git")
# rt.clone("https://github.com/u-boot/u-boot.git")
x = rt.clone_recursive("file:///home/vasek/dev/repo-cloner/tests/_support_data/test-repo-submodules-multilevel")
print(x)
# url = ""
# for x in rt._repo.remote("origin").urls:
# url = x
# url = url.replace("test-repo-base", "test-repo-reduced")
# url = url.replace("test-repo-base", "test-repo-changed-branches")
# rt._repo.remote("origin").set_url(url)
return 0
from git import Repo
r = Repo("/home/vasek/dev/repo-cloner")
# from git import Repo
r = Repo("file:///home/vasek/dev/repo-cloner")
path: str = r._get_config_path("user")
print(path)
path = os.path.dirname(path)