From 7315ebe2ffd4210bc4f17c75d806f5288ec52214 Mon Sep 17 00:00:00 2001 From: Vadim Troshchinskiy Date: Fri, 11 Jul 2025 12:10:29 +0200 Subject: [PATCH 1/2] refs #2230: Fix logging --- ogclient/interfaceAdm/CrearImagenGit.py | 26 ++++++++++++++++++ ogclient/interfaceAdm/ModificarImagenGit.py | 29 +++++++++++++++++++-- ogclient/interfaceAdm/RestaurarImagenGit.py | 25 ++++++++++++++++++ ogclient/lib/python3/GitLib/__init__.py | 2 +- 4 files changed, 79 insertions(+), 3 deletions(-) diff --git a/ogclient/interfaceAdm/CrearImagenGit.py b/ogclient/interfaceAdm/CrearImagenGit.py index 6a58015..946c274 100755 --- a/ogclient/interfaceAdm/CrearImagenGit.py +++ b/ogclient/interfaceAdm/CrearImagenGit.py @@ -14,6 +14,27 @@ import NetLib from GitLib import OpengnsysGitLibrary, NTFSImplementation, OgProgressPrinterWeb +class OgLogger(logging.StreamHandler): + def emit(self, record): + log_types = ["command"] + log_level = "info" + + match(record.levelno): + case logging.DEBUG: + log_level = None + case logging.WARNING: + log_level = "warning" + case logging.INFO: + log_level = "info" + case logging.ERROR: + log_level = "error" + case logging.CRITICAL: + SystemLib.ogRaiseError() + + SystemLib.ogEcho(log_types, log_level, record.getMessage()) + + + def create_image(disk_num, partition_num, repo, image_name, tagName): ntfs_impl = NTFSImplementation.NTFS3G @@ -56,6 +77,9 @@ def main(): ip_address = NetLib.ogGetIpAddress() logFilePath = f"{opengnsys_log_dir}/{ip_address}.log" + ogLog = OgLogger() + ogLog.setLevel(logging.DEBUG) + fileLog = logging.FileHandler(logFilePath) fileLog.setLevel(logging.DEBUG) @@ -66,6 +90,8 @@ def main(): logger = logging.getLogger(__package__) logger.setLevel(logging.DEBUG) logger.addHandler(fileLog) + logger.addHandler(ogLog) + logger.info("Starting CrearImagenGit") diff --git a/ogclient/interfaceAdm/ModificarImagenGit.py b/ogclient/interfaceAdm/ModificarImagenGit.py index e0a58b5..727d410 100755 --- a/ogclient/interfaceAdm/ModificarImagenGit.py +++ b/ogclient/interfaceAdm/ModificarImagenGit.py @@ -46,6 +46,28 @@ import NetLib from GitLib import OpengnsysGitLibrary, NTFSImplementation, OgProgressPrinterWeb +class OgLogger(logging.StreamHandler): + def emit(self, record): + log_types = ["command"] + log_level = "info" + + match(record.levelno): + case logging.DEBUG: + log_level = None + case logging.WARNING: + log_level = "warning" + case logging.INFO: + log_level = "info" + case logging.ERROR: + log_level = "error" + case logging.CRITICAL: + SystemLib.ogRaiseError() + + print("HERE!") + SystemLib.ogEcho(log_types, log_level, record.getMessage()) + + + def commit_image(disk_num, partition_num, repo, image_name, msg): ntfs_impl = NTFSImplementation.NTFS3G @@ -53,8 +75,7 @@ def commit_image(disk_num, partition_num, repo, image_name, msg): og_git.progress_callback = OgProgressPrinterWeb() device = DiskLib.ogDiskToDev(disk_num, partition_num) - og_git.initRepo(device, image_name) - og_git.commit(device, msg) + og_git.commit(device = device, message = msg) og_git.push() def main(): @@ -72,6 +93,9 @@ def main(): ip_address = NetLib.ogGetIpAddress() logFilePath = f"{opengnsys_log_dir}/{ip_address}.log" + ogLog = OgLogger() + ogLog.setLevel(logging.DEBUG) + fileLog = logging.FileHandler(logFilePath) fileLog.setLevel(logging.DEBUG) @@ -82,6 +106,7 @@ def main(): logger = logging.getLogger(__package__) logger.setLevel(logging.DEBUG) logger.addHandler(fileLog) + logger.addHandler(ogLog) logger.info("Starting ModificarImagenGit") diff --git a/ogclient/interfaceAdm/RestaurarImagenGit.py b/ogclient/interfaceAdm/RestaurarImagenGit.py index d4bc002..df6690f 100755 --- a/ogclient/interfaceAdm/RestaurarImagenGit.py +++ b/ogclient/interfaceAdm/RestaurarImagenGit.py @@ -10,6 +10,27 @@ import DiskLib from GitLib import OpengnsysGitLibrary, NTFSImplementation, OgProgressPrinterWeb +class OgLogger(logging.StreamHandler): + def emit(self, record): + log_types = ["command"] + log_level = "info" + + match(record.levelno): + case logging.DEBUG: + log_level = None + case logging.WARNING: + log_level = "warning" + case logging.INFO: + log_level = "info" + case logging.ERROR: + log_level = "error" + case logging.CRITICAL: + SystemLib.ogRaiseError() + + SystemLib.ogEcho(log_types, log_level, record.getMessage()) + + + if __name__ == "__main__": soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) try: @@ -35,6 +56,9 @@ if __name__ == "__main__": opengnsys_log_dir = "/opt/opengnsys/log" ip_address = NetLib.ogGetIpAddress() logFilePath = f"{opengnsys_log_dir}/{ip_address}.log" + ogLog = OgLogger() + ogLog.setLevel(logging.DEBUG) + fileLog = logging.FileHandler(logFilePath) fileLog.setLevel(logging.DEBUG) @@ -46,6 +70,7 @@ if __name__ == "__main__": logger = logging.getLogger(__package__) logger.setLevel(logging.DEBUG) logger.addHandler(fileLog) + logger.addHandler(ogLog) logger.info("Starting RestaurarImagenGit") diff --git a/ogclient/lib/python3/GitLib/__init__.py b/ogclient/lib/python3/GitLib/__init__.py index f233735..8791b0f 100755 --- a/ogclient/lib/python3/GitLib/__init__.py +++ b/ogclient/lib/python3/GitLib/__init__.py @@ -1567,7 +1567,7 @@ class OpengnsysGitLibrary: if path is None: path = self.fs.ensure_mounted(device) - self.logger.info("Committing changes to repository") + self.logger.info("Committing changes from %s to repository", path) repo = git.Repo(path) self._create_metadata(path, initial_creation=False) From 0d3ffe0180b47d317018a240ca6e7171249ce50d Mon Sep 17 00:00:00 2001 From: Vadim Troshchinskiy Date: Thu, 17 Jul 2025 10:01:46 +0200 Subject: [PATCH 2/2] refs #2385 Fix restoration --- ogclient/interfaceAdm/ModificarImagenGit.py | 19 +-- ogclient/lib/python3/GitLib/filesystem.py | 133 +++++++++++++++++++- 2 files changed, 142 insertions(+), 10 deletions(-) diff --git a/ogclient/interfaceAdm/ModificarImagenGit.py b/ogclient/interfaceAdm/ModificarImagenGit.py index 727d410..1babb39 100755 --- a/ogclient/interfaceAdm/ModificarImagenGit.py +++ b/ogclient/interfaceAdm/ModificarImagenGit.py @@ -63,12 +63,11 @@ class OgLogger(logging.StreamHandler): case logging.CRITICAL: SystemLib.ogRaiseError() - print("HERE!") SystemLib.ogEcho(log_types, log_level, record.getMessage()) -def commit_image(disk_num, partition_num, repo, image_name, msg): +def commit_image(disk_num, partition_num, image_name, msg): ntfs_impl = NTFSImplementation.NTFS3G og_git = OpengnsysGitLibrary(ntfs_implementation = ntfs_impl) @@ -76,17 +75,21 @@ def commit_image(disk_num, partition_num, repo, image_name, msg): device = DiskLib.ogDiskToDev(disk_num, partition_num) og_git.commit(device = device, message = msg) - og_git.push() + og_git.push(device = device) + + return 0 def main(): - if len(sys.argv) != 6: - sys.exit(SystemLib.ogRaiseError([], ogGlobals.OG_ERR_FORMAT, "Incorrect number of arguments. Usage: ModificarImagenGit.py disk_num partition_num image_name repo msg")) + if len(sys.argv) != 5: + import json + args = json.dumps(sys.argv) + sys.exit(SystemLib.ogRaiseError([], ogGlobals.OG_ERR_FORMAT, f"Incorrect number of arguments. Usage: ModificarImagenGit.py disk_num partition_num image_name repo msg. Received args: {args}")) # repo - repositorio, ip address. Opcional porque oglive lo recibe como parametro de kernel # tag - tag a crear automaticamente. Opcional, no necesitamos crear un tag siempre. - disk_num, partition_num, image_name, repo, msg = sys.argv[1:6] + disk_num, partition_num, image_name, msg = sys.argv[1:5] opengnsys_log_dir = "/opt/opengnsys/log" @@ -111,10 +114,10 @@ def main(): logger.info("Starting ModificarImagenGit") - retval = commit_image(disk_num, partition_num, repo, image_name, msg) - + retval = commit_image(disk_num, partition_num, image_name, msg) + logger.info("ModificarImagenGit done, return code %i", retval) sys.exit(retval) if __name__ == "__main__": diff --git a/ogclient/lib/python3/GitLib/filesystem.py b/ogclient/lib/python3/GitLib/filesystem.py index 2f2eb65..fb23619 100644 --- a/ogclient/lib/python3/GitLib/filesystem.py +++ b/ogclient/lib/python3/GitLib/filesystem.py @@ -5,6 +5,7 @@ import os import json import blkid import time +import signal from GitLib.ntfs import * @@ -33,6 +34,120 @@ class FilesystemLibrary: subprocess.run(["/usr/sbin/modprobe", module], check=True) + + def _read_file(self, file): + data = "" + + try: + with open(file, "r", encoding='utf-8') as f: + data = f.read() + except IOError as io_err: + self.logger.debug("IO Error reading file %s: %s", file, io_err) + + return data + + def _read_link(self, file): + data = "" + + try: + data = os.readlink(file) + except IOError as io_err: + self.logger.debug("IO Error reading link %s: %s", file, io_err) + + return data + + + def lsof(self, path): + """ + Lists processes that are using files or directories under the specified path. + + This method inspects the /proc filesystem to find processes whose executable, + current working directory, or open file descriptors reference the given path. + It returns a dictionary mapping process IDs (as strings) to a dictionary + containing the command name. + + Args: + path (str): The file system path to check for usage by running processes. + + Returns: + dict: A dictionary where keys are process IDs (str) and values are dicts + with at least the key "cmd" (the command name or executable path). + + Note: + This method requires sufficient permissions to access /proc and process + information. It may raise exceptions if permissions are insufficient or + if processes terminate during inspection. + """ + proc_path = "/proc" + + pids_using_path = {} + + self.logger.debug("Checking for processes using %s", path) + + + for pid_dir in os.listdir(proc_path): + if not pid_dir.isdigit(): + continue # Not a pid directory + + pid_dir_path = os.path.join(proc_path, pid_dir) + fd_path = os.path.join(pid_dir_path, "fd") + + + command_name = self._read_link(os.path.join(pid_dir_path, "exe")) + working_dir = self._read_link(os.path.join(pid_dir_path, "cwd")) + cmdline = self._read_file(os.path.join(pid_dir_path, "cmdline")).split("\0") + + + + if command_name.startswith(path): + self.logger.debug("PID %s (%s) is running from within %s: %s", pid_dir, command_name, path, command_name) + pids_using_path[pid_dir] = { "cmd" : command_name, "args" : cmdline} + elif working_dir.startswith(path): + self.logger.debug("PID %s (%s) is has a working directory within %s: %s", pid_dir, command_name, path, working_dir) + pids_using_path[pid_dir] = { "cmd" : command_name, "args" : cmdline} + else: + for fd_file in os.listdir(fd_path): + fd_file_full_path = os.path.join(fd_path, fd_file) + target = self._read_link(fd_file_full_path) + + if target.startswith(path): + self.logger.debug("PID %s (%s) is has an open file within %s: %s", pid_dir, command_name, path, target) + pids_using_path[pid_dir] = { "cmd" : command_name, "args" : cmdline} + + return pids_using_path + + def kill_path_users(self, path, use_sigkill = False): + """ + Terminates all processes that are currently using the specified filesystem path. + + This method uses the `lsof` utility to identify processes that have open files or directories + under the given `path`. For each identified process, it sends either a SIGTERM or SIGKILL signal + to terminate the process, depending on the `use_sigkill` flag. + + Args: + path (str): The filesystem path to check for open files. + use_sigkill (bool, optional): If True, sends SIGKILL to forcibly terminate processes. + If False (default), sends SIGTERM to allow processes to terminate gracefully. + + Logs: + Information about each process being terminated, including the PID, command, and arguments. + """ + + self.logger.info("Killing any processes using %s, sigkill = %s", path, str(use_sigkill)) + lsof_data = self.lsof(path) + + for pid, pid_data in lsof_data.items(): + + try: + if use_sigkill: + self.logger.info("Killing process %s with SIGKILL: command %s, args %s", pid, pid_data["cmd"], pid_data["args"]) + os.kill(int(pid), signal.SIGKILL) + else: + self.logger.info("Killing process %s with SIGTERM: command %s, args %s", pid, pid_data["cmd"], pid_data["args"]) + os.kill(int(pid), signal.SIGTERM) + except OSError as os_error: + self.logger.warning("Failed to send signal to pid %s: %s", pid, os_error) + # _parse_mounts def update_mounts(self): """ @@ -94,6 +209,7 @@ class FilesystemLibrary: norm = os.path.normpath(device) self.logger.debug(f"Checking if {device} is mounted") + self.update_mounts() for mountpoint, mount in self.mounts.items(): #self.logger.debug(f"Item: {mount}") #self.logger.debug(f"Checking: " + mount['device']) @@ -172,7 +288,12 @@ class FilesystemLibrary: done = False start_time = time.time() - timeout = 60*15 + + # How long to wait for the filesystem to unmount + timeout = 60 + + # How long to wait before starting to send SIGKILL to users + force_timeout = 5 while not done and (time.time() - start_time) < timeout: @@ -180,8 +301,16 @@ class FilesystemLibrary: if ret.returncode == 0: done=True else: + elapsed = (time.time() - start_time) + if "target is busy" in ret.stderr: - self.logger.debug("Filesystem busy, waiting. %.1f seconds left", timeout - (time.time() - start_time)) + self.logger.debug("Filesystem busy, waiting. %.1f seconds left", timeout - elapsed) + + if elapsed > force_timeout: + self.kill_path_users(mountpoint, use_sigkill=True) + else: + self.kill_path_users(mountpoint, use_sigkill=False) + time.sleep(0.1) else: raise subprocess.CalledProcessError(ret.returncode, ret.args, output=ret.stdout, stderr=ret.stderr)