opengnsys
/
ogclone-engine
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refs #2229 send gitlib progress to ogcore

pull/122/head
Natalia Serrano 2025-09-23 13:13:06 +02:00
parent dc2251d771
commit f338f950e4
5 changed files with 56 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CHANGELOG.md
View File

@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [1.6.0] - 2025-09-23
### Added
- Send gitlib progress to ogcore
## [1.5.2] - 2025-09-12
### Fixed

21
ogclient/interfaceAdm/CrearImagenGit
View File

@ -1,16 +1,14 @@
#!/usr/bin/env python3
import sys
import resource
import logging
import argparse
import ogGlobals
import SystemLib
import DiskLib
import NetLib
from GitLib import OpengnsysGitLibrary, NTFSImplementation, OgProgressPrinterWeb
@ -33,10 +31,8 @@ class OgLogger(logging.StreamHandler):
SystemLib.ogEcho(log_types, log_level, record.getMessage())
def create_image(disk_num, partition_num, repo, image_name, commit_message):
print ('[10] begin')
ntfs_impl = NTFSImplementation.NTFS3G
og_git = OpengnsysGitLibrary(ntfs_implementation = ntfs_impl)
og_git.progress_callback = OgProgressPrinterWeb()
@ -46,14 +42,15 @@ def create_image(disk_num, partition_num, repo, image_name, commit_message):
if device is None:
sys.exit(SystemLib.ogRaiseError([], ogGlobals.OG_ERR_FORMAT, f"Failed to translate disk {disk_num} partition {partition_num} to a device"))
print ('[50] initRepo')
if og_git.initRepo(device, image_name, message = commit_message):
print ('[100] done')
return 0
else:
print ('[100] done')
return 1
def main():
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
try:
# Usamos el mínimo entre 65536 y el límite hard disponible
@ -63,7 +60,6 @@ def main():
except ValueError as e:
print(f"No se pudo aumentar el límite de archivos abiertos: {e}")
parser = argparse.ArgumentParser(
prog = "OpenGnsys Git Image Create",
description = "Creates a git repository from a partition"
@ -75,14 +71,11 @@ def main():
parser.add_argument("--image-name", type=str, metavar="REPO", required=True, help="Name of the new image at the repository")
parser.add_argument("--tag", type=str, metavar="TAG", required=False, help="Tag to automatically create")
parser.add_argument("--message", type=str, metavar="MSG", required=False, default="", help="Commit message")
parser.add_help = True
args = parser.parse_args()
#disk_num, partition_num, image_name, repo, tag = sys.argv[1:6]
logFilePath = "/var/log/opengnsys.CrearImagenGit.log"
ogLog = OgLogger()
@ -100,7 +93,6 @@ def main():
logger.addHandler(fileLog)
logger.addHandler(ogLog)
logger.info("Starting CrearImagenGit")
# TODO:
@ -108,9 +100,6 @@ def main():
# image = nombre repo
retval = create_image(args.disk, args.partition, args.repository, args.image_name, args.message)
sys.exit(retval)
if __name__ == "__main__":

37
ogclient/interfaceAdm/ModificarImagenGit
View File

@ -1,4 +1,5 @@
#!/usr/bin/env python3
import os
import subprocess
import sys
@ -6,7 +7,6 @@ import time
import resource
import argparse
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
try:
# Usamos el mínimo entre 65536 y el límite hard disponible
@ -22,17 +22,14 @@ extra_paths = [
"/opt/opengnsys/ogrepository/oggit/lib/"
]
# Si estás completamente seguro de que esta ruta no interfiere con la stdlib
# y contiene todos los módulos necesarios, puedes añadirla AL FINAL del path.
path_maybe_problematic = "/opt/oglive/rootfs/opt/opengnsys/lib/python3/"
if os.path.isdir(path_maybe_problematic):
sys.path.append(path_maybe_problematic)
for path in extra_paths:
if os.path.isdir(path):
sys.path.append(path)
#path_maybe_problematic = "/opt/oglive/rootfs/opt/opengnsys/lib/python3/"
#if os.path.isdir(path_maybe_problematic):
# sys.path.append(path_maybe_problematic)
# for path in extra_paths:
# if os.path.isdir(path):
# sys.path.append(path)
import NetLib
import ogGlobals
@ -40,12 +37,8 @@ import SystemLib
import logging
import DiskLib
import NetLib
from GitLib import OpengnsysGitLibrary, NTFSImplementation, OgProgressPrinterWeb
class OgLogger(logging.StreamHandler):
def emit(self, record):
log_types = ["command"]
@ -65,10 +58,8 @@ class OgLogger(logging.StreamHandler):
SystemLib.ogEcho(log_types, log_level, record.getMessage())
def commit_image(disk_num, partition_num, repository, branch, options, msg):
print ('[10] begin')
ntfs_impl = NTFSImplementation.NTFS3G
og_git = OpengnsysGitLibrary(ntfs_implementation = ntfs_impl)
og_git.progress_callback = OgProgressPrinterWeb()
@ -92,17 +83,17 @@ def commit_image(disk_num, partition_num, repository, branch, options, msg):
if not og_git.create_branch(device = device, name = branch):
sys.exit(SystemLib.ogRaiseError([], ogGlobals.OG_ERR_FORMAT, f"Failed to create branch"))
print ('[30] commit')
og_git.commit(device = device, message = msg)
print ('[50] push')
og_git.push(device = device, force = force_push)
except Exception as ex:
sys.exit(SystemLib.ogRaiseError([], ogGlobals.OG_ERR_FORMAT, f"Exception during commit: {ex}"))
print ('[100] done')
return 0
def main():
parser = argparse.ArgumentParser(
prog = "OpenGnsys Git Image Modify",
description = "Commits changes to a partition to a git repository"
@ -111,11 +102,9 @@ def main():
parser.add_argument("--disk", type=int, metavar="DISK", required=True, help="Disk ID")
parser.add_argument("--partition", type=int, metavar="PART", required=True, help="Disk partition")
parser.add_argument("--repository", type=str, metavar="REPO", required=True, help="Address of the Git repository to clone")
parser.add_argument("--branch", type=str, metavar="BRANCH", required=False, help="Branch to automatically create")
parser.add_argument("--options", type=str, metavar="OPTS", required=False, help="Options to branch creation (forcepush)")
parser.add_argument("--message", type=str, metavar="MSG", required=False, default="", help="Commit message")
parser.add_help = True
args = parser.parse_args()
@ -123,7 +112,7 @@ def main():
logFilePath = "/var/log/opengnsys.ModificarImagenGit.log"
ogLog = OgLogger()
ogLog.setLevel(logging.DEBUG)
ogLog.setLevel(logging.INFO)
fileLog = logging.FileHandler(logFilePath)
fileLog.setLevel(logging.DEBUG)
@ -139,10 +128,8 @@ def main():
logger.info("Starting ModificarImagenGit")
retval = commit_image(args.disk, args.partition, args.repository, args.branch, args.options, args.message)
logger.info("ModificarImagenGit done, return code %i", retval)
sys.exit(retval)

24
ogclient/interfaceAdm/RestaurarImagenGit
View File

@ -1,20 +1,17 @@
#!/usr/bin/env python3
import sys
import resource
import logging
import subprocess
import argparse
import NetLib
import ogGlobals
import SystemLib
import DiskLib
from GitLib import OpengnsysGitLibrary, NTFSImplementation, OgProgressPrinterWeb
import argparse
class OgLogger(logging.StreamHandler):
def emit(self, record):
log_types = ["command"]
@ -34,8 +31,6 @@ class OgLogger(logging.StreamHandler):
SystemLib.ogEcho(log_types, log_level, record.getMessage())
if __name__ == "__main__":
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
try:
@ -46,8 +41,6 @@ if __name__ == "__main__":
except ValueError as e:
print(f"No se pudo aumentar el límite de archivos abiertos: {e}")
parser = argparse.ArgumentParser(
prog = "OpenGnsys Git Image Restore",
description = "Restores an image from Git"
@ -57,19 +50,16 @@ if __name__ == "__main__":
parser.add_argument("--partition", type=int, metavar="PART", required=True, help="Disk partition")
parser.add_argument("--repository", type=str, metavar="REPO", required=True, help="Address of the Git repository to clone")
parser.add_argument("--image-name", type=str, metavar="REPO", required=True, help="Name of the new image at the repository")
parser.add_argument("--branch", type=str, metavar="BRANCH", required=True, help="Branch to check out")
parser.add_argument("--commit", type=str, metavar="COMMIT_ID", required=True, help="Commit to check out")
parser.add_help = True
args = parser.parse_args()
logFilePath = "/var/log/opengnsys.RestaurarImagenGit.log"
ogLog = OgLogger()
ogLog.setLevel(logging.DEBUG)
fileLog = logging.FileHandler(logFilePath)
fileLog.setLevel(logging.DEBUG)
@ -85,8 +75,7 @@ if __name__ == "__main__":
logger.info("Starting RestaurarImagenGit")
print ('[10] begin')
ntfs_impl = NTFSImplementation.NTFS3G
og_git = OpengnsysGitLibrary(ntfs_implementation = ntfs_impl)
og_git.progress_callback = OgProgressPrinterWeb()
@ -98,10 +87,9 @@ if __name__ == "__main__":
if args.repository:
og_git.repo_server = args.repository
print ('[50] cloneRepo')
og_git.cloneRepo(args.image_name, destination = device, boot_device = device, ref = args.commit, branch = args.branch)
print ('[80] configureOs')
subprocess.run(["/opt/opengnsys/scripts/configureOs", str(args.disk), str(args.partition)], check=True)
print ('[100] done')
logger.info("RestaurarImagenGit Finished.")

90
ogclient/lib/python3/GitLib/__init__.py
View File

@ -8,30 +8,19 @@
# python3.8
# Must have working locales, or unicode strings will fail. Install 'locales', configure /etc/locale.gen, run locale-gen.
#
import os
import sys
import shutil
import argparse
import tempfile
import logging
import subprocess
import json
from pathlib import Path
import base64
import stat
import time
import git
import libarchive
import xattr
@ -41,33 +30,23 @@ import re
import uuid
from tqdm import tqdm
from GitLib.filesystem import *
from GitLib.disk import *
from GitLib.ntfs import *
from GitLib.kernel import parse_kernel_cmdline
import CacheLib
#import requests
from CacheLib import ogMountCache
def _git_op_to_string(op):
op = op & git.RemoteProgress.OP_MASK
if op == git.RemoteProgress.COMPRESSING:
return "Compressing", "Obj"
elif op == git.RemoteProgress.CHECKING_OUT:
return "Checking out", "Obj"
elif op == git.RemoteProgress.COUNTING:
return "Counting", "Obj"
elif op == git.RemoteProgress.RECEIVING:
return "Receiving", "B"
elif op == git.RemoteProgress.WRITING:
return "Writing", "B"
elif op == git.RemoteProgress.RESOLVING:
return "Resolving deltas", "Obj"
if op == git.RemoteProgress.COMPRESSING: return "Compressing", "Obj"
elif op == git.RemoteProgress.CHECKING_OUT: return "Checking out", "Obj"
elif op == git.RemoteProgress.COUNTING: return "Counting", "Obj"
elif op == git.RemoteProgress.RECEIVING: return "Receiving", "B"
elif op == git.RemoteProgress.WRITING: return "Writing", "B"
elif op == git.RemoteProgress.RESOLVING: return "Resolving deltas", "Obj"
return "Unknown", "?"
@ -97,60 +76,44 @@ class OgProgressPrinterWeb(git.RemoteProgress):
"""
def __init__(self):
super().__init__()
#self.ogcore_ip = get_IPcore()
#self.endpoint_url = f"https://{ogcore_ip}:8443/og-repository/webhook"
self.logger = logging.getLogger(f"{__package__}.{self.__class__.__name__}")
#self.jobId = jobId
self.last_ts = time.time()
"""Total stages of the git process. This comes from Git itself."""
self.total_stages = 6
self.simple_progress = 0
self.prev_progress = -1
def _get_stage_number(self, op):
op = op & git.RemoteProgress.OP_MASK
if op == git.RemoteProgress.COUNTING: return 0
elif op == git.RemoteProgress.COMPRESSING: return 1
elif op == git.RemoteProgress.WRITING: return 2
elif op == git.RemoteProgress.CHECKING_OUT: return 3
elif op == git.RemoteProgress.RECEIVING: return 4
elif op == git.RemoteProgress.RESOLVING: return 5
if op == git.RemoteProgress.COUNTING:
return 0
elif op == git.RemoteProgress.COMPRESSING:
return 1
elif op == git.RemoteProgress.WRITING:
return 2
elif op == git.RemoteProgress.CHECKING_OUT:
return 3
elif op == git.RemoteProgress.RECEIVING:
return 4
elif op == git.RemoteProgress.RESOLVING:
return 5
def _send_progress(self, p):
i = int (100*p)
print (f'[{i}] {i}% done')
def update(self, op_code, cur_count, max_count=None, message=""):
# This shouldn't normally fail, but if it ever does, we don't want to fail the entire
# Git operation just because we're unable to report the status.
now = time.time()
if now - self.last_ts < 1: return
stage_portion = (100 / self.total_stages)
if max_count:
self.simple_progress = int((stage_portion * self._get_stage_number(op_code)) + ((stage_portion / max_count) * cur_count))
if self.prev_progress != self.simple_progress:
print(f"[{self.simple_progress}]")
self.prev_progress = self.simple_progress
self.simple_progress = stage_portion * self._get_stage_number(op_code) + (stage_portion * cur_count/max_count)
op_text, _ = _git_op_to_string(op_code)
self.logger.debug(f"Progress: {op_text} ({op_code}) {cur_count}/{max_count}: {message}")
self._send_progress (self.simple_progress)
self.last_ts = now
#try:
# data = json.dumps({"job_id" : self.jobId, "progress" : progress})
# headers = {'Content-Type': 'application/json'}
#
# response = requests.post(self.endpoint_url, data=data, headers=headers, verify=False)
#except Exception as e:
# self.logger.error("Exception %s while trying to report progress", e)
def __del__(self):
self._send_progress (1.0)
class OgProgressPrinterConsole(git.RemoteProgress):
"""
@ -304,7 +267,7 @@ class OpengnsysGitLibrary:
self.repo_image_path = "oggit"
self.ntfs_implementation = ntfs_implementation
self.cache_dir = CacheLib.ogMountCache()
self.cache_dir = ogMountCache()
# Si no hay cache, se va a crear el .git en el FS directamente
@ -1763,6 +1726,7 @@ class OpengnsysGitLibrary:
origin = repo.remotes["origin"]
repo.heads.main.set_tracking_branch(origin.refs.main)
#repo.heads["main"].set_tracking_branch(origin.refs["main"])
ret = origin.push(progress=self.progress_callback, force = force)
ret.raise_if_error()