From d4a6e097719f329dfc6927031bc8802661b4b761 Mon Sep 17 00:00:00 2001 From: Vadim Troshchinskiy Date: Wed, 3 Sep 2025 13:30:15 +0200 Subject: [PATCH 1/4] refs #2705: Improve dealing with blocked unmount by our own script --- ogclient/lib/python3/GitLib/filesystem.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/ogclient/lib/python3/GitLib/filesystem.py b/ogclient/lib/python3/GitLib/filesystem.py index b724f46..eaee785 100644 --- a/ogclient/lib/python3/GitLib/filesystem.py +++ b/ogclient/lib/python3/GitLib/filesystem.py @@ -89,6 +89,8 @@ class FilesystemLibrary: if not pid_dir.isdigit(): continue # Not a pid directory + pid_num = int(pid_dir) + pid_dir_path = os.path.join(proc_path, pid_dir) fd_path = os.path.join(pid_dir_path, "fd") @@ -100,11 +102,11 @@ class FilesystemLibrary: if command_name.startswith(path): - self.logger.debug("PID %s (%s) is running from within %s: %s", pid_dir, command_name, path, command_name) - pids_using_path[pid_dir] = { "cmd" : command_name, "args" : cmdline} + self.logger.debug("PID %i (%s) is running from within %s: %s", pid_num, command_name, path, command_name) + pids_using_path[pid_num] = { "cmd" : command_name, "args" : cmdline} elif working_dir.startswith(path): - self.logger.debug("PID %s (%s) is has a working directory within %s: %s", pid_dir, command_name, path, working_dir) - pids_using_path[pid_dir] = { "cmd" : command_name, "args" : cmdline} + self.logger.debug("PID %i (%s) is has a working directory within %s: %s", pid_num, command_name, path, working_dir) + pids_using_path[pid_num] = { "cmd" : command_name, "args" : cmdline} else: try: for fd_file in os.listdir(fd_path): @@ -112,8 +114,8 @@ class FilesystemLibrary: target = self._read_link(fd_file_full_path) if target.startswith(path): - self.logger.debug("PID %s (%s) is has an open file within %s: %s", pid_dir, command_name, path, target) - pids_using_path[pid_dir] = { "cmd" : command_name, "args" : cmdline} + self.logger.debug("PID %i (%s) is has an open file within %s: %s", pid_num, command_name, path, target) + pids_using_path[pid_num] = { "cmd" : command_name, "args" : cmdline} except IOError as ioerr: self.logger.warning("Exception during listdir: %s", ioerr) @@ -140,6 +142,11 @@ class FilesystemLibrary: lsof_data = self.lsof(path) for pid, pid_data in lsof_data.items(): + if pid == os.getpid(): + self.logger.error("We're trying to terminate our own process!") + self.logger.error("This script is probably being run from within a mounted filesystem we need unmounted.") + self.logger.error("Our working directory is %s", os.getcwd()) + self.logger.error("We need %s to be unmounted", path) try: if use_sigkill: -- 2.40.1 From 090f6d3cf20f13803bc7499d79b72611a07fd3ad Mon Sep 17 00:00:00 2001 From: Vadim Troshchinskiy Date: Wed, 3 Sep 2025 13:35:32 +0200 Subject: [PATCH 2/4] refs #2705: fix commit message in image creation --- ogclient/interfaceAdm/CrearImagenGit | 8 ++++---- ogclient/interfaceAdm/ModificarImagenGit | 2 +- ogclient/lib/python3/GitLib/__init__.py | 8 ++++++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/ogclient/interfaceAdm/CrearImagenGit b/ogclient/interfaceAdm/CrearImagenGit index 87c66ee..585abc7 100755 --- a/ogclient/interfaceAdm/CrearImagenGit +++ b/ogclient/interfaceAdm/CrearImagenGit @@ -35,7 +35,7 @@ class OgLogger(logging.StreamHandler): -def create_image(disk_num, partition_num, repo, image_name): +def create_image(disk_num, partition_num, repo, image_name, commit_message): ntfs_impl = NTFSImplementation.NTFS3G og_git = OpengnsysGitLibrary(ntfs_implementation = ntfs_impl) @@ -45,7 +45,7 @@ def create_image(disk_num, partition_num, repo, image_name): if device is None: sys.exit(SystemLib.ogRaiseError([], ogGlobals.OG_ERR_FORMAT, f"Failed to translate disk {disk_num} partition {partition_num} to a device")) - if og_git.initRepo(device, image_name): + if og_git.initRepo(device, image_name, message = commit_message): return 0 else: return 1 @@ -73,7 +73,7 @@ def main(): parser.add_argument("--repository", type=str, metavar="REPO", required=True, help="Address of the Git repository to clone") parser.add_argument("--image-name", type=str, metavar="REPO", required=True, help="Name of the new image at the repository") parser.add_argument("--tag", type=str, metavar="TAG", required=False, help="Tag to automatically create") - parser.add_argument("--message", type=str, metavar="MSG", required=False, help="Commit message") + parser.add_argument("--message", type=str, metavar="MSG", required=False, default="", help="Commit message") parser.add_help = True @@ -106,7 +106,7 @@ def main(): # repo = repositorio, oggit@server.com:/oggit # image = nombre repo - retval = create_image(args.disk, args.partition, args.repository, args.image_name) + retval = create_image(args.disk, args.partition, args.repository, args.image_name, args.message) diff --git a/ogclient/interfaceAdm/ModificarImagenGit b/ogclient/interfaceAdm/ModificarImagenGit index 4e99a03..b517099 100755 --- a/ogclient/interfaceAdm/ModificarImagenGit +++ b/ogclient/interfaceAdm/ModificarImagenGit @@ -110,7 +110,7 @@ def main(): parser.add_argument("--branch", type=str, metavar="BRANCH", required=False, help="Branch to automatically create") parser.add_argument("--options", type=str, metavar="OPTS", required=False, help="Options to branch creation (forcepush)") - parser.add_argument("--message", type=str, metavar="MSG", required=False, help="Commit message") + parser.add_argument("--message", type=str, metavar="MSG", required=False, default="", help="Commit message") parser.add_help = True diff --git a/ogclient/lib/python3/GitLib/__init__.py b/ogclient/lib/python3/GitLib/__init__.py index c86afc1..01d371a 100755 --- a/ogclient/lib/python3/GitLib/__init__.py +++ b/ogclient/lib/python3/GitLib/__init__.py @@ -1254,7 +1254,7 @@ class OpengnsysGitLibrary: repo.config_writer().add_value("push", "followTags", "true").release() - def initRepo(self, device, repo_name): + def initRepo(self, device, repo_name, message = None): """ Initialize a Git repository on a specified device. @@ -1396,7 +1396,11 @@ class OpengnsysGitLibrary: self.logger.info("Committing") - repo.index.commit("Initialize repository") + + if message: + repo.index.commit(message) + else: + repo.index.commit("Initialize repository") # Restaurar cosas modificadas para git self._restore_metadata(path, destructive_only=True, set_device_uuids=False) -- 2.40.1 From 17e8788f5f0eb3798cd03c6e8b4d7393a2e5e9a6 Mon Sep 17 00:00:00 2001 From: Vadim Troshchinskiy Date: Wed, 3 Sep 2025 15:58:49 +0200 Subject: [PATCH 3/4] refs #2705: fix repository argument --- ogclient/interfaceAdm/CrearImagenGit | 1 + ogclient/interfaceAdm/ModificarImagenGit | 1 + ogclient/interfaceAdm/RestaurarImagenGit | 3 +++ 3 files changed, 5 insertions(+) diff --git a/ogclient/interfaceAdm/CrearImagenGit b/ogclient/interfaceAdm/CrearImagenGit index 585abc7..a1b54de 100755 --- a/ogclient/interfaceAdm/CrearImagenGit +++ b/ogclient/interfaceAdm/CrearImagenGit @@ -40,6 +40,7 @@ def create_image(disk_num, partition_num, repo, image_name, commit_message): ntfs_impl = NTFSImplementation.NTFS3G og_git = OpengnsysGitLibrary(ntfs_implementation = ntfs_impl) og_git.progress_callback = OgProgressPrinterWeb() + og_git.repo_server = repo device = DiskLib.ogDiskToDev(disk_num, partition_num) if device is None: diff --git a/ogclient/interfaceAdm/ModificarImagenGit b/ogclient/interfaceAdm/ModificarImagenGit index b517099..ab5861c 100755 --- a/ogclient/interfaceAdm/ModificarImagenGit +++ b/ogclient/interfaceAdm/ModificarImagenGit @@ -72,6 +72,7 @@ def commit_image(disk_num, partition_num, repository, branch, options, msg): ntfs_impl = NTFSImplementation.NTFS3G og_git = OpengnsysGitLibrary(ntfs_implementation = ntfs_impl) og_git.progress_callback = OgProgressPrinterWeb() + og_git.repo_server = repository device = DiskLib.ogDiskToDev(disk_num, partition_num) if device is None: diff --git a/ogclient/interfaceAdm/RestaurarImagenGit b/ogclient/interfaceAdm/RestaurarImagenGit index 0fd79f2..348e1cc 100755 --- a/ogclient/interfaceAdm/RestaurarImagenGit +++ b/ogclient/interfaceAdm/RestaurarImagenGit @@ -94,6 +94,9 @@ if __name__ == "__main__": if device is None: sys.exit(SystemLib.ogRaiseError([], ogGlobals.OG_ERR_FORMAT, f"Failed to translate disk {args.disk} partition {args.partition} to a device")) + if args.repository: + og_git.repo_server = args.repository + og_git.cloneRepo(args.image_name, destination = device, boot_device = device, ref = args.commit, branch = args.branch) logger.info("RestaurarImagenGit Finished.") -- 2.40.1 From 54581e36a80b10c06c87bd7092059d52f46e76f2 Mon Sep 17 00:00:00 2001 From: Vadim Troshchinskiy Date: Wed, 3 Sep 2025 15:59:33 +0200 Subject: [PATCH 4/4] refs #2703: mark and detect broken git clones --- ogclient/lib/python3/GitLib/__init__.py | 66 ++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/ogclient/lib/python3/GitLib/__init__.py b/ogclient/lib/python3/GitLib/__init__.py index 01d371a..ee78d2a 100755 --- a/ogclient/lib/python3/GitLib/__init__.py +++ b/ogclient/lib/python3/GitLib/__init__.py @@ -240,6 +240,23 @@ class RequirementException(Exception): super().__init__(message) self.message = message +class OgGitInternalException(Exception): + """Excepción que indica que hay un problema interno + + Attributes: + message (str): Mensaje de error mostrado al usuario + """ + + def __init__(self, message): + """Inicializar RequirementException. + + Args: + message (str): Mensaje de error mostrado al usuario + """ + super().__init__(message) + self.message = message + + class OpengnsysGitLibrary: """OpenGnsys Git Library""" @@ -697,10 +714,14 @@ class OpengnsysGitLibrary: git_dir = os.path.normpath(os.path.join(path, ".git")) meta_dir = os.path.join(path, ".opengnsys-metadata") + meta_lock = os.path.join(meta_dir, "create_metadata.lock") if not os.path.exists(meta_dir): os.mkdir(meta_dir, mode=0o700) + + Path(meta_lock).touch() + # https://jsonlines.org/ metadata_file = open(os.path.join(meta_dir, "metadata.json.new"), "w", encoding='utf-8') @@ -990,6 +1011,8 @@ class OpengnsysGitLibrary: self._ntfs_secaudit(audit) + Path(meta_lock).unlink() + self.logger.debug("Metadata updated") return return_data @@ -1014,11 +1037,14 @@ class OpengnsysGitLibrary: self.logger.info(f"Restoring metadata in {path}") meta_dir = os.path.join(path, ".opengnsys-metadata") + meta_lock = os.path.join(meta_dir, "restore_metadata.lock") + if not os.path.exists(meta_dir): self.logger.error(f"Metadata directory not found: {meta_dir}") return + Path(meta_lock).touch() if set_device_uuids: # Windows boot manager uses partition UUIDs in at least some cases. One option to make booting work @@ -1235,7 +1261,7 @@ class OpengnsysGitLibrary: - + Path(meta_lock).unlink() self.logger.info("Metadata restoration completed.") def _configure_repo(self, repo): @@ -1549,6 +1575,9 @@ class OpengnsysGitLibrary: if path is None: path = self.fs.ensure_mounted(device) + if self.is_broken(path = path, device = device): + raise OgGitInternalException("Cloned repository is in a broken state, see log.") + self.logger.info("Committing changes from %s to repository", path) repo = git.Repo(path) @@ -1692,7 +1721,42 @@ class OpengnsysGitLibrary: self.logger.warning("Remote can't be accessed, git said: %s", ret.stderr) return False + def is_broken(self, path = None, device = None): + """ + Checks whether a given checked out Git filesystem is in a broken state. + This mainly checks for the presence of lock files that indicate some process didn't run to completion. + + Args: + path (_type_, optional): Path to check. + device (_type_, optional): Device to check. + + Returns: + _Bool_: Whether the check passes + """ + if path is None: + path = self.fs.ensure_mounted(device) + + self.logger.info("Checking whether %s is broken", path) + + meta_dir = os.path.join(path, ".opengnsys-metadata") + if not os.path.exists(meta_dir): + self.logger.error("Metadata directory is missing") + return False + + restore_meta_lock = os.path.join(meta_dir, "restore_metadata.lock") + create_meta_lock = os.path.join(meta_dir, "restore_metadata.lock") + + if os.path.exists(restore_meta_lock): + self.logger.error("Metadata restoration didn't finish") + return False + + if os.path.exists(create_meta_lock): + self.logger.error("Metadata creation didn't finish") + return False + + self.logger.info("Check ok") + return True -- 2.40.1