Merge pull request 'refs #2229 send gitlib progress to ogcore' (#122) from gitlib-progress into main
Reviewed-on: #122pull/123/head 1.6.0
commit
b9e3559039
|
|
@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
|
|||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [1.6.0] - 2025-09-23
|
||||
|
||||
### Added
|
||||
|
||||
- Send gitlib progress to ogcore
|
||||
|
||||
## [1.5.2] - 2025-09-12
|
||||
|
||||
### Fixed
|
||||
|
|
|
|||
|
|
@ -1,16 +1,14 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import sys
|
||||
import resource
|
||||
import logging
|
||||
import argparse
|
||||
|
||||
|
||||
import ogGlobals
|
||||
import SystemLib
|
||||
import DiskLib
|
||||
import NetLib
|
||||
|
||||
|
||||
from GitLib import OpengnsysGitLibrary, NTFSImplementation, OgProgressPrinterWeb
|
||||
|
||||
|
||||
|
|
@ -33,10 +31,8 @@ class OgLogger(logging.StreamHandler):
|
|||
|
||||
SystemLib.ogEcho(log_types, log_level, record.getMessage())
|
||||
|
||||
|
||||
|
||||
def create_image(disk_num, partition_num, repo, image_name, commit_message):
|
||||
|
||||
print ('[10] begin')
|
||||
ntfs_impl = NTFSImplementation.NTFS3G
|
||||
og_git = OpengnsysGitLibrary(ntfs_implementation = ntfs_impl)
|
||||
og_git.progress_callback = OgProgressPrinterWeb()
|
||||
|
|
@ -46,14 +42,15 @@ def create_image(disk_num, partition_num, repo, image_name, commit_message):
|
|||
if device is None:
|
||||
sys.exit(SystemLib.ogRaiseError([], ogGlobals.OG_ERR_FORMAT, f"Failed to translate disk {disk_num} partition {partition_num} to a device"))
|
||||
|
||||
print ('[50] initRepo')
|
||||
if og_git.initRepo(device, image_name, message = commit_message):
|
||||
print ('[100] done')
|
||||
return 0
|
||||
else:
|
||||
print ('[100] done')
|
||||
return 1
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
|
||||
try:
|
||||
# Usamos el mínimo entre 65536 y el límite hard disponible
|
||||
|
|
@ -63,7 +60,6 @@ def main():
|
|||
except ValueError as e:
|
||||
print(f"No se pudo aumentar el límite de archivos abiertos: {e}")
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog = "OpenGnsys Git Image Create",
|
||||
description = "Creates a git repository from a partition"
|
||||
|
|
@ -75,14 +71,11 @@ def main():
|
|||
parser.add_argument("--image-name", type=str, metavar="REPO", required=True, help="Name of the new image at the repository")
|
||||
parser.add_argument("--tag", type=str, metavar="TAG", required=False, help="Tag to automatically create")
|
||||
parser.add_argument("--message", type=str, metavar="MSG", required=False, default="", help="Commit message")
|
||||
|
||||
parser.add_help = True
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
#disk_num, partition_num, image_name, repo, tag = sys.argv[1:6]
|
||||
|
||||
|
||||
logFilePath = "/var/log/opengnsys.CrearImagenGit.log"
|
||||
|
||||
ogLog = OgLogger()
|
||||
|
|
@ -100,7 +93,6 @@ def main():
|
|||
logger.addHandler(fileLog)
|
||||
logger.addHandler(ogLog)
|
||||
|
||||
|
||||
logger.info("Starting CrearImagenGit")
|
||||
|
||||
# TODO:
|
||||
|
|
@ -108,9 +100,6 @@ def main():
|
|||
# image = nombre repo
|
||||
|
||||
retval = create_image(args.disk, args.partition, args.repository, args.image_name, args.message)
|
||||
|
||||
|
||||
|
||||
sys.exit(retval)
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
|
@ -6,7 +7,6 @@ import time
|
|||
import resource
|
||||
import argparse
|
||||
|
||||
|
||||
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
|
||||
try:
|
||||
# Usamos el mínimo entre 65536 y el límite hard disponible
|
||||
|
|
@ -22,17 +22,14 @@ extra_paths = [
|
|||
"/opt/opengnsys/ogrepository/oggit/lib/"
|
||||
]
|
||||
|
||||
|
||||
# Si estás completamente seguro de que esta ruta no interfiere con la stdlib
|
||||
# y contiene todos los módulos necesarios, puedes añadirla AL FINAL del path.
|
||||
path_maybe_problematic = "/opt/oglive/rootfs/opt/opengnsys/lib/python3/"
|
||||
if os.path.isdir(path_maybe_problematic):
|
||||
sys.path.append(path_maybe_problematic)
|
||||
|
||||
for path in extra_paths:
|
||||
if os.path.isdir(path):
|
||||
sys.path.append(path)
|
||||
|
||||
#path_maybe_problematic = "/opt/oglive/rootfs/opt/opengnsys/lib/python3/"
|
||||
#if os.path.isdir(path_maybe_problematic):
|
||||
# sys.path.append(path_maybe_problematic)
|
||||
# for path in extra_paths:
|
||||
# if os.path.isdir(path):
|
||||
# sys.path.append(path)
|
||||
|
||||
import NetLib
|
||||
import ogGlobals
|
||||
|
|
@ -40,12 +37,8 @@ import SystemLib
|
|||
import logging
|
||||
import DiskLib
|
||||
import NetLib
|
||||
|
||||
|
||||
|
||||
from GitLib import OpengnsysGitLibrary, NTFSImplementation, OgProgressPrinterWeb
|
||||
|
||||
|
||||
class OgLogger(logging.StreamHandler):
|
||||
def emit(self, record):
|
||||
log_types = ["command"]
|
||||
|
|
@ -65,10 +58,8 @@ class OgLogger(logging.StreamHandler):
|
|||
|
||||
SystemLib.ogEcho(log_types, log_level, record.getMessage())
|
||||
|
||||
|
||||
|
||||
def commit_image(disk_num, partition_num, repository, branch, options, msg):
|
||||
|
||||
print ('[10] begin')
|
||||
ntfs_impl = NTFSImplementation.NTFS3G
|
||||
og_git = OpengnsysGitLibrary(ntfs_implementation = ntfs_impl)
|
||||
og_git.progress_callback = OgProgressPrinterWeb()
|
||||
|
|
@ -92,17 +83,17 @@ def commit_image(disk_num, partition_num, repository, branch, options, msg):
|
|||
if not og_git.create_branch(device = device, name = branch):
|
||||
sys.exit(SystemLib.ogRaiseError([], ogGlobals.OG_ERR_FORMAT, f"Failed to create branch"))
|
||||
|
||||
print ('[30] commit')
|
||||
og_git.commit(device = device, message = msg)
|
||||
print ('[50] push')
|
||||
og_git.push(device = device, force = force_push)
|
||||
except Exception as ex:
|
||||
sys.exit(SystemLib.ogRaiseError([], ogGlobals.OG_ERR_FORMAT, f"Exception during commit: {ex}"))
|
||||
|
||||
|
||||
print ('[100] done')
|
||||
return 0
|
||||
|
||||
def main():
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog = "OpenGnsys Git Image Modify",
|
||||
description = "Commits changes to a partition to a git repository"
|
||||
|
|
@ -111,11 +102,9 @@ def main():
|
|||
parser.add_argument("--disk", type=int, metavar="DISK", required=True, help="Disk ID")
|
||||
parser.add_argument("--partition", type=int, metavar="PART", required=True, help="Disk partition")
|
||||
parser.add_argument("--repository", type=str, metavar="REPO", required=True, help="Address of the Git repository to clone")
|
||||
|
||||
parser.add_argument("--branch", type=str, metavar="BRANCH", required=False, help="Branch to automatically create")
|
||||
parser.add_argument("--options", type=str, metavar="OPTS", required=False, help="Options to branch creation (forcepush)")
|
||||
parser.add_argument("--message", type=str, metavar="MSG", required=False, default="", help="Commit message")
|
||||
|
||||
parser.add_help = True
|
||||
|
||||
args = parser.parse_args()
|
||||
|
|
@ -123,7 +112,7 @@ def main():
|
|||
logFilePath = "/var/log/opengnsys.ModificarImagenGit.log"
|
||||
|
||||
ogLog = OgLogger()
|
||||
ogLog.setLevel(logging.DEBUG)
|
||||
ogLog.setLevel(logging.INFO)
|
||||
|
||||
fileLog = logging.FileHandler(logFilePath)
|
||||
fileLog.setLevel(logging.DEBUG)
|
||||
|
|
@ -139,10 +128,8 @@ def main():
|
|||
|
||||
logger.info("Starting ModificarImagenGit")
|
||||
|
||||
|
||||
retval = commit_image(args.disk, args.partition, args.repository, args.branch, args.options, args.message)
|
||||
|
||||
|
||||
logger.info("ModificarImagenGit done, return code %i", retval)
|
||||
sys.exit(retval)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,20 +1,17 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
import sys
|
||||
import resource
|
||||
import logging
|
||||
import subprocess
|
||||
import argparse
|
||||
|
||||
import NetLib
|
||||
import ogGlobals
|
||||
import SystemLib
|
||||
import DiskLib
|
||||
|
||||
from GitLib import OpengnsysGitLibrary, NTFSImplementation, OgProgressPrinterWeb
|
||||
|
||||
import argparse
|
||||
|
||||
|
||||
|
||||
class OgLogger(logging.StreamHandler):
|
||||
def emit(self, record):
|
||||
log_types = ["command"]
|
||||
|
|
@ -34,8 +31,6 @@ class OgLogger(logging.StreamHandler):
|
|||
|
||||
SystemLib.ogEcho(log_types, log_level, record.getMessage())
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
|
||||
try:
|
||||
|
|
@ -46,8 +41,6 @@ if __name__ == "__main__":
|
|||
except ValueError as e:
|
||||
print(f"No se pudo aumentar el límite de archivos abiertos: {e}")
|
||||
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
prog = "OpenGnsys Git Image Restore",
|
||||
description = "Restores an image from Git"
|
||||
|
|
@ -57,19 +50,16 @@ if __name__ == "__main__":
|
|||
parser.add_argument("--partition", type=int, metavar="PART", required=True, help="Disk partition")
|
||||
parser.add_argument("--repository", type=str, metavar="REPO", required=True, help="Address of the Git repository to clone")
|
||||
parser.add_argument("--image-name", type=str, metavar="REPO", required=True, help="Name of the new image at the repository")
|
||||
|
||||
parser.add_argument("--branch", type=str, metavar="BRANCH", required=True, help="Branch to check out")
|
||||
parser.add_argument("--commit", type=str, metavar="COMMIT_ID", required=True, help="Commit to check out")
|
||||
parser.add_help = True
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
logFilePath = "/var/log/opengnsys.RestaurarImagenGit.log"
|
||||
ogLog = OgLogger()
|
||||
ogLog.setLevel(logging.DEBUG)
|
||||
|
||||
|
||||
fileLog = logging.FileHandler(logFilePath)
|
||||
fileLog.setLevel(logging.DEBUG)
|
||||
|
||||
|
|
@ -85,8 +75,7 @@ if __name__ == "__main__":
|
|||
logger.info("Starting RestaurarImagenGit")
|
||||
|
||||
|
||||
|
||||
|
||||
print ('[10] begin')
|
||||
ntfs_impl = NTFSImplementation.NTFS3G
|
||||
og_git = OpengnsysGitLibrary(ntfs_implementation = ntfs_impl)
|
||||
og_git.progress_callback = OgProgressPrinterWeb()
|
||||
|
|
@ -98,10 +87,9 @@ if __name__ == "__main__":
|
|||
if args.repository:
|
||||
og_git.repo_server = args.repository
|
||||
|
||||
print ('[50] cloneRepo')
|
||||
og_git.cloneRepo(args.image_name, destination = device, boot_device = device, ref = args.commit, branch = args.branch)
|
||||
|
||||
print ('[80] configureOs')
|
||||
subprocess.run(["/opt/opengnsys/scripts/configureOs", str(args.disk), str(args.partition)], check=True)
|
||||
|
||||
print ('[100] done')
|
||||
logger.info("RestaurarImagenGit Finished.")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -8,30 +8,19 @@
|
|||
# python3.8
|
||||
# Must have working locales, or unicode strings will fail. Install 'locales', configure /etc/locale.gen, run locale-gen.
|
||||
#
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
|
||||
import shutil
|
||||
|
||||
import argparse
|
||||
import tempfile
|
||||
import logging
|
||||
import subprocess
|
||||
|
||||
import json
|
||||
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
|
||||
import base64
|
||||
|
||||
import stat
|
||||
import time
|
||||
|
||||
|
||||
import git
|
||||
import libarchive
|
||||
import xattr
|
||||
|
|
@ -41,33 +30,23 @@ import re
|
|||
import uuid
|
||||
from tqdm import tqdm
|
||||
|
||||
|
||||
from GitLib.filesystem import *
|
||||
from GitLib.disk import *
|
||||
from GitLib.ntfs import *
|
||||
from GitLib.kernel import parse_kernel_cmdline
|
||||
|
||||
import CacheLib
|
||||
|
||||
#import requests
|
||||
from CacheLib import ogMountCache
|
||||
|
||||
|
||||
|
||||
def _git_op_to_string(op):
|
||||
op = op & git.RemoteProgress.OP_MASK
|
||||
|
||||
if op == git.RemoteProgress.COMPRESSING:
|
||||
return "Compressing", "Obj"
|
||||
elif op == git.RemoteProgress.CHECKING_OUT:
|
||||
return "Checking out", "Obj"
|
||||
elif op == git.RemoteProgress.COUNTING:
|
||||
return "Counting", "Obj"
|
||||
elif op == git.RemoteProgress.RECEIVING:
|
||||
return "Receiving", "B"
|
||||
elif op == git.RemoteProgress.WRITING:
|
||||
return "Writing", "B"
|
||||
elif op == git.RemoteProgress.RESOLVING:
|
||||
return "Resolving deltas", "Obj"
|
||||
if op == git.RemoteProgress.COMPRESSING: return "Compressing", "Obj"
|
||||
elif op == git.RemoteProgress.CHECKING_OUT: return "Checking out", "Obj"
|
||||
elif op == git.RemoteProgress.COUNTING: return "Counting", "Obj"
|
||||
elif op == git.RemoteProgress.RECEIVING: return "Receiving", "B"
|
||||
elif op == git.RemoteProgress.WRITING: return "Writing", "B"
|
||||
elif op == git.RemoteProgress.RESOLVING: return "Resolving deltas", "Obj"
|
||||
|
||||
return "Unknown", "?"
|
||||
|
||||
|
|
@ -97,60 +76,44 @@ class OgProgressPrinterWeb(git.RemoteProgress):
|
|||
"""
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
#self.ogcore_ip = get_IPcore()
|
||||
#self.endpoint_url = f"https://{ogcore_ip}:8443/og-repository/webhook"
|
||||
self.logger = logging.getLogger(f"{__package__}.{self.__class__.__name__}")
|
||||
#self.jobId = jobId
|
||||
self.last_ts = time.time()
|
||||
|
||||
"""Total stages of the git process. This comes from Git itself."""
|
||||
self.total_stages = 6
|
||||
self.simple_progress = 0
|
||||
self.prev_progress = -1
|
||||
|
||||
|
||||
def _get_stage_number(self, op):
|
||||
op = op & git.RemoteProgress.OP_MASK
|
||||
if op == git.RemoteProgress.COUNTING: return 0
|
||||
elif op == git.RemoteProgress.COMPRESSING: return 1
|
||||
elif op == git.RemoteProgress.WRITING: return 2
|
||||
elif op == git.RemoteProgress.CHECKING_OUT: return 3
|
||||
elif op == git.RemoteProgress.RECEIVING: return 4
|
||||
elif op == git.RemoteProgress.RESOLVING: return 5
|
||||
|
||||
if op == git.RemoteProgress.COUNTING:
|
||||
return 0
|
||||
elif op == git.RemoteProgress.COMPRESSING:
|
||||
return 1
|
||||
elif op == git.RemoteProgress.WRITING:
|
||||
return 2
|
||||
elif op == git.RemoteProgress.CHECKING_OUT:
|
||||
return 3
|
||||
elif op == git.RemoteProgress.RECEIVING:
|
||||
return 4
|
||||
elif op == git.RemoteProgress.RESOLVING:
|
||||
return 5
|
||||
def _send_progress(self, p):
|
||||
i = int (100*p)
|
||||
print (f'[{i}] {i}% done')
|
||||
|
||||
def update(self, op_code, cur_count, max_count=None, message=""):
|
||||
# This shouldn't normally fail, but if it ever does, we don't want to fail the entire
|
||||
# Git operation just because we're unable to report the status.
|
||||
now = time.time()
|
||||
if now - self.last_ts < 1: return
|
||||
|
||||
stage_portion = (100 / self.total_stages)
|
||||
|
||||
if max_count:
|
||||
self.simple_progress = int((stage_portion * self._get_stage_number(op_code)) + ((stage_portion / max_count) * cur_count))
|
||||
|
||||
if self.prev_progress != self.simple_progress:
|
||||
print(f"[{self.simple_progress}]")
|
||||
|
||||
self.prev_progress = self.simple_progress
|
||||
|
||||
self.simple_progress = stage_portion * self._get_stage_number(op_code) + (stage_portion * cur_count/max_count)
|
||||
|
||||
op_text, _ = _git_op_to_string(op_code)
|
||||
self.logger.debug(f"Progress: {op_text} ({op_code}) {cur_count}/{max_count}: {message}")
|
||||
self._send_progress (self.simple_progress)
|
||||
self.last_ts = now
|
||||
|
||||
|
||||
#try:
|
||||
# data = json.dumps({"job_id" : self.jobId, "progress" : progress})
|
||||
# headers = {'Content-Type': 'application/json'}
|
||||
#
|
||||
# response = requests.post(self.endpoint_url, data=data, headers=headers, verify=False)
|
||||
#except Exception as e:
|
||||
# self.logger.error("Exception %s while trying to report progress", e)
|
||||
def __del__(self):
|
||||
self._send_progress (1.0)
|
||||
|
||||
class OgProgressPrinterConsole(git.RemoteProgress):
|
||||
"""
|
||||
|
|
@ -304,7 +267,7 @@ class OpengnsysGitLibrary:
|
|||
self.repo_image_path = "oggit"
|
||||
self.ntfs_implementation = ntfs_implementation
|
||||
|
||||
self.cache_dir = CacheLib.ogMountCache()
|
||||
self.cache_dir = ogMountCache()
|
||||
|
||||
|
||||
# Si no hay cache, se va a crear el .git en el FS directamente
|
||||
|
|
@ -1763,6 +1726,7 @@ class OpengnsysGitLibrary:
|
|||
|
||||
origin = repo.remotes["origin"]
|
||||
repo.heads.main.set_tracking_branch(origin.refs.main)
|
||||
#repo.heads["main"].set_tracking_branch(origin.refs["main"])
|
||||
|
||||
ret = origin.push(progress=self.progress_callback, force = force)
|
||||
ret.raise_if_error()
|
||||
|
|
|
|||
Loading…
Reference in New Issue