diff --git a/CHANGELOG.md b/CHANGELOG.md index 54e7306..15816e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.6.0] - 2025-09-23 + +### Added + +- Send gitlib progress to ogcore + ## [1.5.2] - 2025-09-12 ### Fixed diff --git a/ogclient/interfaceAdm/CrearImagenGit b/ogclient/interfaceAdm/CrearImagenGit index a1b54de..694314f 100755 --- a/ogclient/interfaceAdm/CrearImagenGit +++ b/ogclient/interfaceAdm/CrearImagenGit @@ -1,16 +1,14 @@ #!/usr/bin/env python3 + import sys import resource import logging import argparse - import ogGlobals import SystemLib import DiskLib import NetLib - - from GitLib import OpengnsysGitLibrary, NTFSImplementation, OgProgressPrinterWeb @@ -33,10 +31,8 @@ class OgLogger(logging.StreamHandler): SystemLib.ogEcho(log_types, log_level, record.getMessage()) - - def create_image(disk_num, partition_num, repo, image_name, commit_message): - + print ('[10] begin') ntfs_impl = NTFSImplementation.NTFS3G og_git = OpengnsysGitLibrary(ntfs_implementation = ntfs_impl) og_git.progress_callback = OgProgressPrinterWeb() @@ -46,14 +42,15 @@ def create_image(disk_num, partition_num, repo, image_name, commit_message): if device is None: sys.exit(SystemLib.ogRaiseError([], ogGlobals.OG_ERR_FORMAT, f"Failed to translate disk {disk_num} partition {partition_num} to a device")) + print ('[50] initRepo') if og_git.initRepo(device, image_name, message = commit_message): + print ('[100] done') return 0 else: + print ('[100] done') return 1 - def main(): - soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) try: # Usamos el mínimo entre 65536 y el límite hard disponible @@ -63,7 +60,6 @@ def main(): except ValueError as e: print(f"No se pudo aumentar el límite de archivos abiertos: {e}") - parser = argparse.ArgumentParser( prog = "OpenGnsys Git Image Create", description = "Creates a git repository from a partition" @@ -75,14 +71,11 @@ def main(): parser.add_argument("--image-name", type=str, metavar="REPO", required=True, help="Name of the new image at the repository") parser.add_argument("--tag", type=str, metavar="TAG", required=False, help="Tag to automatically create") parser.add_argument("--message", type=str, metavar="MSG", required=False, default="", help="Commit message") - parser.add_help = True args = parser.parse_args() - #disk_num, partition_num, image_name, repo, tag = sys.argv[1:6] - logFilePath = "/var/log/opengnsys.CrearImagenGit.log" ogLog = OgLogger() @@ -100,7 +93,6 @@ def main(): logger.addHandler(fileLog) logger.addHandler(ogLog) - logger.info("Starting CrearImagenGit") # TODO: @@ -108,9 +100,6 @@ def main(): # image = nombre repo retval = create_image(args.disk, args.partition, args.repository, args.image_name, args.message) - - - sys.exit(retval) if __name__ == "__main__": diff --git a/ogclient/interfaceAdm/ModificarImagenGit b/ogclient/interfaceAdm/ModificarImagenGit index 805e251..63a6674 100755 --- a/ogclient/interfaceAdm/ModificarImagenGit +++ b/ogclient/interfaceAdm/ModificarImagenGit @@ -1,4 +1,5 @@ #!/usr/bin/env python3 + import os import subprocess import sys @@ -6,7 +7,6 @@ import time import resource import argparse - soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) try: # Usamos el mínimo entre 65536 y el límite hard disponible @@ -22,17 +22,14 @@ extra_paths = [ "/opt/opengnsys/ogrepository/oggit/lib/" ] - # Si estás completamente seguro de que esta ruta no interfiere con la stdlib # y contiene todos los módulos necesarios, puedes añadirla AL FINAL del path. -path_maybe_problematic = "/opt/oglive/rootfs/opt/opengnsys/lib/python3/" -if os.path.isdir(path_maybe_problematic): - sys.path.append(path_maybe_problematic) - - for path in extra_paths: - if os.path.isdir(path): - sys.path.append(path) - +#path_maybe_problematic = "/opt/oglive/rootfs/opt/opengnsys/lib/python3/" +#if os.path.isdir(path_maybe_problematic): +# sys.path.append(path_maybe_problematic) +# for path in extra_paths: +# if os.path.isdir(path): +# sys.path.append(path) import NetLib import ogGlobals @@ -40,12 +37,8 @@ import SystemLib import logging import DiskLib import NetLib - - - from GitLib import OpengnsysGitLibrary, NTFSImplementation, OgProgressPrinterWeb - class OgLogger(logging.StreamHandler): def emit(self, record): log_types = ["command"] @@ -65,10 +58,8 @@ class OgLogger(logging.StreamHandler): SystemLib.ogEcho(log_types, log_level, record.getMessage()) - - def commit_image(disk_num, partition_num, repository, branch, options, msg): - + print ('[10] begin') ntfs_impl = NTFSImplementation.NTFS3G og_git = OpengnsysGitLibrary(ntfs_implementation = ntfs_impl) og_git.progress_callback = OgProgressPrinterWeb() @@ -92,17 +83,17 @@ def commit_image(disk_num, partition_num, repository, branch, options, msg): if not og_git.create_branch(device = device, name = branch): sys.exit(SystemLib.ogRaiseError([], ogGlobals.OG_ERR_FORMAT, f"Failed to create branch")) + print ('[30] commit') og_git.commit(device = device, message = msg) + print ('[50] push') og_git.push(device = device, force = force_push) except Exception as ex: sys.exit(SystemLib.ogRaiseError([], ogGlobals.OG_ERR_FORMAT, f"Exception during commit: {ex}")) - + print ('[100] done') return 0 def main(): - - parser = argparse.ArgumentParser( prog = "OpenGnsys Git Image Modify", description = "Commits changes to a partition to a git repository" @@ -111,11 +102,9 @@ def main(): parser.add_argument("--disk", type=int, metavar="DISK", required=True, help="Disk ID") parser.add_argument("--partition", type=int, metavar="PART", required=True, help="Disk partition") parser.add_argument("--repository", type=str, metavar="REPO", required=True, help="Address of the Git repository to clone") - parser.add_argument("--branch", type=str, metavar="BRANCH", required=False, help="Branch to automatically create") parser.add_argument("--options", type=str, metavar="OPTS", required=False, help="Options to branch creation (forcepush)") parser.add_argument("--message", type=str, metavar="MSG", required=False, default="", help="Commit message") - parser.add_help = True args = parser.parse_args() @@ -123,7 +112,7 @@ def main(): logFilePath = "/var/log/opengnsys.ModificarImagenGit.log" ogLog = OgLogger() - ogLog.setLevel(logging.DEBUG) + ogLog.setLevel(logging.INFO) fileLog = logging.FileHandler(logFilePath) fileLog.setLevel(logging.DEBUG) @@ -139,10 +128,8 @@ def main(): logger.info("Starting ModificarImagenGit") - retval = commit_image(args.disk, args.partition, args.repository, args.branch, args.options, args.message) - logger.info("ModificarImagenGit done, return code %i", retval) sys.exit(retval) diff --git a/ogclient/interfaceAdm/RestaurarImagenGit b/ogclient/interfaceAdm/RestaurarImagenGit index cd441e3..47aa79b 100755 --- a/ogclient/interfaceAdm/RestaurarImagenGit +++ b/ogclient/interfaceAdm/RestaurarImagenGit @@ -1,20 +1,17 @@ #!/usr/bin/env python3 + import sys import resource import logging import subprocess +import argparse import NetLib import ogGlobals import SystemLib import DiskLib - from GitLib import OpengnsysGitLibrary, NTFSImplementation, OgProgressPrinterWeb -import argparse - - - class OgLogger(logging.StreamHandler): def emit(self, record): log_types = ["command"] @@ -34,8 +31,6 @@ class OgLogger(logging.StreamHandler): SystemLib.ogEcho(log_types, log_level, record.getMessage()) - - if __name__ == "__main__": soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) try: @@ -46,8 +41,6 @@ if __name__ == "__main__": except ValueError as e: print(f"No se pudo aumentar el límite de archivos abiertos: {e}") - - parser = argparse.ArgumentParser( prog = "OpenGnsys Git Image Restore", description = "Restores an image from Git" @@ -57,19 +50,16 @@ if __name__ == "__main__": parser.add_argument("--partition", type=int, metavar="PART", required=True, help="Disk partition") parser.add_argument("--repository", type=str, metavar="REPO", required=True, help="Address of the Git repository to clone") parser.add_argument("--image-name", type=str, metavar="REPO", required=True, help="Name of the new image at the repository") - parser.add_argument("--branch", type=str, metavar="BRANCH", required=True, help="Branch to check out") parser.add_argument("--commit", type=str, metavar="COMMIT_ID", required=True, help="Commit to check out") parser.add_help = True args = parser.parse_args() - logFilePath = "/var/log/opengnsys.RestaurarImagenGit.log" ogLog = OgLogger() ogLog.setLevel(logging.DEBUG) - fileLog = logging.FileHandler(logFilePath) fileLog.setLevel(logging.DEBUG) @@ -85,8 +75,7 @@ if __name__ == "__main__": logger.info("Starting RestaurarImagenGit") - - + print ('[10] begin') ntfs_impl = NTFSImplementation.NTFS3G og_git = OpengnsysGitLibrary(ntfs_implementation = ntfs_impl) og_git.progress_callback = OgProgressPrinterWeb() @@ -98,10 +87,9 @@ if __name__ == "__main__": if args.repository: og_git.repo_server = args.repository + print ('[50] cloneRepo') og_git.cloneRepo(args.image_name, destination = device, boot_device = device, ref = args.commit, branch = args.branch) - + print ('[80] configureOs') subprocess.run(["/opt/opengnsys/scripts/configureOs", str(args.disk), str(args.partition)], check=True) - + print ('[100] done') logger.info("RestaurarImagenGit Finished.") - - diff --git a/ogclient/lib/python3/GitLib/__init__.py b/ogclient/lib/python3/GitLib/__init__.py index 32e0c21..49a2e10 100755 --- a/ogclient/lib/python3/GitLib/__init__.py +++ b/ogclient/lib/python3/GitLib/__init__.py @@ -8,30 +8,19 @@ # python3.8 # Must have working locales, or unicode strings will fail. Install 'locales', configure /etc/locale.gen, run locale-gen. # + import os import sys - - import shutil - import argparse import tempfile import logging import subprocess - import json - - from pathlib import Path - - - import base64 - import stat import time - - import git import libarchive import xattr @@ -41,33 +30,23 @@ import re import uuid from tqdm import tqdm - from GitLib.filesystem import * from GitLib.disk import * from GitLib.ntfs import * from GitLib.kernel import parse_kernel_cmdline -import CacheLib - -#import requests +from CacheLib import ogMountCache def _git_op_to_string(op): op = op & git.RemoteProgress.OP_MASK - - if op == git.RemoteProgress.COMPRESSING: - return "Compressing", "Obj" - elif op == git.RemoteProgress.CHECKING_OUT: - return "Checking out", "Obj" - elif op == git.RemoteProgress.COUNTING: - return "Counting", "Obj" - elif op == git.RemoteProgress.RECEIVING: - return "Receiving", "B" - elif op == git.RemoteProgress.WRITING: - return "Writing", "B" - elif op == git.RemoteProgress.RESOLVING: - return "Resolving deltas", "Obj" + if op == git.RemoteProgress.COMPRESSING: return "Compressing", "Obj" + elif op == git.RemoteProgress.CHECKING_OUT: return "Checking out", "Obj" + elif op == git.RemoteProgress.COUNTING: return "Counting", "Obj" + elif op == git.RemoteProgress.RECEIVING: return "Receiving", "B" + elif op == git.RemoteProgress.WRITING: return "Writing", "B" + elif op == git.RemoteProgress.RESOLVING: return "Resolving deltas", "Obj" return "Unknown", "?" @@ -97,60 +76,44 @@ class OgProgressPrinterWeb(git.RemoteProgress): """ def __init__(self): super().__init__() - - #self.ogcore_ip = get_IPcore() - #self.endpoint_url = f"https://{ogcore_ip}:8443/og-repository/webhook" self.logger = logging.getLogger(f"{__package__}.{self.__class__.__name__}") - #self.jobId = jobId + self.last_ts = time.time() """Total stages of the git process. This comes from Git itself.""" self.total_stages = 6 self.simple_progress = 0 - self.prev_progress = -1 - def _get_stage_number(self, op): op = op & git.RemoteProgress.OP_MASK + if op == git.RemoteProgress.COUNTING: return 0 + elif op == git.RemoteProgress.COMPRESSING: return 1 + elif op == git.RemoteProgress.WRITING: return 2 + elif op == git.RemoteProgress.CHECKING_OUT: return 3 + elif op == git.RemoteProgress.RECEIVING: return 4 + elif op == git.RemoteProgress.RESOLVING: return 5 - if op == git.RemoteProgress.COUNTING: - return 0 - elif op == git.RemoteProgress.COMPRESSING: - return 1 - elif op == git.RemoteProgress.WRITING: - return 2 - elif op == git.RemoteProgress.CHECKING_OUT: - return 3 - elif op == git.RemoteProgress.RECEIVING: - return 4 - elif op == git.RemoteProgress.RESOLVING: - return 5 + def _send_progress(self, p): + i = int (100*p) + print (f'[{i}] {i}% done') def update(self, op_code, cur_count, max_count=None, message=""): # This shouldn't normally fail, but if it ever does, we don't want to fail the entire # Git operation just because we're unable to report the status. + now = time.time() + if now - self.last_ts < 1: return stage_portion = (100 / self.total_stages) if max_count: - self.simple_progress = int((stage_portion * self._get_stage_number(op_code)) + ((stage_portion / max_count) * cur_count)) - - if self.prev_progress != self.simple_progress: - print(f"[{self.simple_progress}]") - - self.prev_progress = self.simple_progress - + self.simple_progress = stage_portion * self._get_stage_number(op_code) + (stage_portion * cur_count/max_count) op_text, _ = _git_op_to_string(op_code) self.logger.debug(f"Progress: {op_text} ({op_code}) {cur_count}/{max_count}: {message}") + self._send_progress (self.simple_progress) + self.last_ts = now - - #try: - # data = json.dumps({"job_id" : self.jobId, "progress" : progress}) - # headers = {'Content-Type': 'application/json'} - # - # response = requests.post(self.endpoint_url, data=data, headers=headers, verify=False) - #except Exception as e: - # self.logger.error("Exception %s while trying to report progress", e) + def __del__(self): + self._send_progress (1.0) class OgProgressPrinterConsole(git.RemoteProgress): """ @@ -304,7 +267,7 @@ class OpengnsysGitLibrary: self.repo_image_path = "oggit" self.ntfs_implementation = ntfs_implementation - self.cache_dir = CacheLib.ogMountCache() + self.cache_dir = ogMountCache() # Si no hay cache, se va a crear el .git en el FS directamente @@ -1763,6 +1726,7 @@ class OpengnsysGitLibrary: origin = repo.remotes["origin"] repo.heads.main.set_tracking_branch(origin.refs.main) + #repo.heads["main"].set_tracking_branch(origin.refs["main"]) ret = origin.push(progress=self.progress_callback, force = force) ret.raise_if_error()