From 51a8fb66db69027736ef7ef3ff0e3ad076759968 Mon Sep 17 00:00:00 2001 From: Vadim Troshchinskiy Date: Mon, 16 Dec 2024 13:09:43 +0100 Subject: [PATCH] Improve repository initialization Improve performance, better progress display --- gitlib/gitlib.py | 86 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 71 insertions(+), 15 deletions(-) diff --git a/gitlib/gitlib.py b/gitlib/gitlib.py index b1be526..0e24859 100755 --- a/gitlib/gitlib.py +++ b/gitlib/gitlib.py @@ -1209,6 +1209,8 @@ class OpengnsysGitLibrary: repo.config_writer().add_value("user", "name", "OpenGnsys").release() repo.config_writer().add_value("user", "email", "OpenGnsys@opengnsys.com").release() repo.config_writer().add_value("core", "filemode", "false").release() + repo.config_writer().add_value("push", "autoSetupRemote", "true").release() + repo.config_writer().add_value("maintenance", "autoDetach", "false").release() def initRepo(self, device, repo_name): """ @@ -1231,12 +1233,16 @@ class OpengnsysGitLibrary: - The method sets up a remote origin and pushes the initial commit. """ + if not self.check_remote_exists(repo_name): + self.logger.error("Specified repository can't be used, aborting.") + return + path = self.fs.ensure_mounted(device) self.logger.info("Initializing repository: %s", path) git_dir = os.path.join(path, ".git") real_git_dir = os.path.join(self.cache_dir, f"git-{repo_name}") - + repo_url = self._getOgRepository(repo_name) if os.path.exists(real_git_dir): self.logger.debug(f"Removing existing repository {real_git_dir}") @@ -1267,11 +1273,35 @@ class OpengnsysGitLibrary: os.symlink(real_git_dir, git_dir) - repo = git.Repo.init(path) - self._configure_repo(repo) - self._write_ignore_list(path) + with git.Repo.init(path) as repo: + # On NTFS, we have to unmount the filesystem to do secaudit. + # Gitpython objects existing at that time may mean a dangling git process that prevents + # the required unmounting. + # + # So we make sure we destroy gitpython after this initial stage, to recreate it + # right after _create_metadata. + self._configure_repo(repo) + self._write_ignore_list(path) + + + # Adding the gitignore and doing the manual --force saves us an expensive fetch if + # the repo already had data in it, and allows us to use the gitpython functions with + # progress reports for doing the full push later. + origin = repo.create_remote("origin", repo_url) + repo.index.add(f"{path}/.gitignore") + repo.index.commit("Initial commit") + repo.git.push("--force") # Obliterate whatever might have been there + + self.logger.debug("Fetching origin") + origin.fetch(progress=OgProgressPrinter(self.logger)) + + repo.heads.master.set_tracking_branch(origin.refs.master) + + metadata_ret = self._create_metadata(path, initial_creation=True) + repo = git.Repo(path) + self.logger.debug(f"Building list of files to add from path {path}") add_files = [] @@ -1337,21 +1367,17 @@ class OpengnsysGitLibrary: self.fs.unload_ntfs() - repo_url = self._getOgRepository(repo_name) - self.logger.debug(f"Creating remote origin: {repo_url}") - if "origin" in repo.remotes: - repo.delete_remote("origin") - origin = repo.create_remote("origin", repo_url) - self.logger.debug("Fetching origin") - origin.fetch() # repo.create_head - # repo.heads.master.set_tracking_branch(origin.refs.master) + + # repo.heads.master.set_tracking_branch(origin.refs.master) self.logger.info("Uploading to ogrepository") - repo.git.push("--set-upstream", "origin", repo.head.ref, "--force") + origin.push(progress=OgProgressPrinter(self.logger)) + + #repo.git.push("--set-upstream", "origin", repo.head.ref, "--force") def cloneRepo(self, repo_name, destination, boot_device): """ @@ -1481,7 +1507,15 @@ class OpengnsysGitLibrary: repo = git.Repo(path) self.logger.info("Uploading to ogrepository") - repo.git.push("--set-upstream", "origin", repo.head.ref, "--force") # force = True) + if not "origin" in repo.remotes: + self.logger.critical("'origin' remote not found!") + return + + origin = repo.remotes["origin"] + repo.heads.master.set_tracking_branch(origin.refs.master) + origin.push(progress=OgProgressPrinter(self.logger)) + + #repo.git.push("--set-upstream", "origin", repo.head.ref, "--force") # force = True) def fetch(self, path = None, device = None): @@ -1519,6 +1553,21 @@ class OpengnsysGitLibrary: # Restaurar cosas modificadas para git self._restore_metadata(path, destructive_only=True) + def check_remote_exists(self, repo_name): + repo_url = self._getOgRepository(repo_name) + + self.logger.info("Checking whether %s exists and is accessible", repo_url) + + ret = subprocess.run(["/usr/bin/git", "ls-remote", repo_url], encoding='utf-8', capture_output=True, check=False) + if ret.returncode == 0: + return True + else: + self.logger.warning("Remote can't be accessed, git said: %s", ret.stderr) + return False + + + + if __name__ == '__main__': @@ -1577,6 +1626,7 @@ if __name__ == '__main__': parser.add_argument("--pull", type=str, metavar='DEV', help="Bajar cambios de ogrepository") parser.add_argument("--fetch", type=str, metavar='DEV', help="Fetch changes from ogrepository") parser.add_argument("--efi-config", type=str, metavar="NAME", help="Name of the custom EFI configuration to deploy") + parser.add_argument("--verify-repo", action='store_true', help="Verify whether the indicated repository exists and can be used") parser.add_argument("--ntfs-type", type=str, metavar="FS", help="Tipo de NTFS, 'kernel' o 'fuse'") @@ -1644,7 +1694,13 @@ if __name__ == '__main__': og_git.fetch(device = args.fetch) elif args.pull: with OperationTimer(og_git, "git pull"): - og_git.pullRepo(args.pull) + og_git.pull(device = args.pull) + elif args.verify_repo: + if og_git.check_remote_exists(args.repo): + print("Remote checks OK") + else: + print("Check failed") + elif args.test_create_metadata: og_git._create_metadata(args.test_create_metadata, initial_creation=False) # pylint: disable=protected-access elif args.test_restore_metadata: