Make --pull work like the other commands

Improve performance, better progress display

gitlib/README.md

@@ -73,7 +73,9 @@ Esto clona un repositorio del ogrepository. El destino es un dispositivo físico
 
 * Windows debe haber sido apagado completamente, sin hibernar. Ver https://learn.microsoft.com/en-us/troubleshoot/windows-client/setup-upgrade-and-drivers/disable-and-re-enable-hibernation
 * Windows debe haber sido apagado limpiamente, usando "Apagar sistema". Es posible que gitlib no pueda montar un disco de un sistema apagado incorrectamente. En ese caso hay que volver a iniciar Windows, y apagarlo.
-* No se puede usar cifrado de disco (Bitlocker)
+* No se puede usar cifrado de disco (Bitlocker). Es posible desactivarlo: https://answers.microsoft.com/en-us/windows/forum/all/how-to-disable-bitlocker-in-windows-10/fc9e12d6-a8cd-4515-ab8f-379c6409aa56
+* Es recomendable ejecutar un proceso de limpieza de disco.
+* Es recomendable compactar WinSxS con `Dism.exe /online /Cleanup-Image /StartComponentCleanup /ResetBase`
 
 ## Restauración
 

gitlib/bcd.py

@@ -224,6 +224,7 @@ def format_value(bcd, bcd_value):
 
     return (typename, length, str_value)
 
+
 def dump_all(root, depth = 0):
 
     padding = "\t" * depth

gitlib/filesystem.py

@@ -4,6 +4,7 @@ import subprocess
 import os
 import json
 import blkid
+import time
 
 from ntfs import *
 
@@ -112,6 +113,7 @@ class FilesystemLibrary:
             str or None: The device corresponding to the mount point if found,
                          otherwise None.
         """
+        self.update_mounts()
         self.logger.debug("Finding device corresponding to mount point %s", mountpoint)
         if mountpoint in self.mounts:
             return self.mounts[mountpoint]['device']
@@ -167,7 +169,22 @@ class FilesystemLibrary:
 
         if not mountpoint is None:
             self.logger.debug(f"Unmounting {mountpoint}")
-            subprocess.run(["/usr/bin/umount", mountpoint], check=True)
+
+            done = False
+            start_time = time.time()
+            timeout = 60
+
+
+            while not done and (time.time() - start_time) < timeout:
+                ret = subprocess.run(["/usr/bin/umount", mountpoint], check=False, capture_output=True, encoding='utf-8')
+                if ret.returncode == 0:
+                    done=True
+                else:
+                    if "target is busy" in ret.stderr:
+                        self.logger.debug("Filesystem busy, waiting. %.1f seconds left", timeout - (time.time() - start_time))
+                        time.sleep(0.1)
+                    else:
+                        raise subprocess.CalledProcessError(ret.returncode, ret.args, output=ret.stdout, stderr=ret.stderr)
 
             # We've unmounted a new filesystem, update our filesystems list
             self.update_mounts()

gitlib/gitlib.py

@@ -43,6 +43,8 @@ from disk import *
 from ntfs import *
 import re
 import uuid
+from tqdm import tqdm
+from kernel import parse_kernel_cmdline
 
 class OgProgressPrinter(git.RemoteProgress):
     """
@@ -68,18 +70,53 @@ class OgProgressPrinter(git.RemoteProgress):
     def __init__(self, parentLogger):
         super().__init__()
         self.logger = parentLogger
-        self.prev_len = 0
 
-        print("\n", file=sys.stderr)
+        if sys.stdin.isatty():
+            self.progress = tqdm()
+            self.progress.miniters = 1
+            #self.progress.ascii = False
 
     def update(self, op_code, cur_count, max_count=None, message=""):
+        op = op_code & git.RemoteProgress.OP_MASK
+        stage = op_code & git.RemoteProgress.STAGE_MASK
+
+
+        op_text = "Unknown"
+        op_unit = "?"
+
+        if op == git.RemoteProgress.COMPRESSING:
+            op_text = "Compressing"
+            op_unit = "Obj"
+        elif op == git.RemoteProgress.CHECKING_OUT:
+            op_text = "Checking out"
+            op_unit = "Obj"
+        elif op == git.RemoteProgress.COUNTING:
+            op_text = "Counting"
+            op_unit = "Obj"
+        elif op == git.RemoteProgress.RECEIVING:
+            op_text = "Receiving"
+            op_unit = "B"
+        elif op == git.RemoteProgress.WRITING:
+            op_text = "Writing"
+            op_unit = "B"
+        elif op == git.RemoteProgress.RESOLVING:
+            op_text = "Resolving deltas"
+            op_unit = "Obj"
+
+
+
         self.logger.debug(f"Progress: {op_code} {cur_count}/{max_count}: {message}")
 
-        status_string = "Progress: %s %s/%s: %s" % (op_code, cur_count, max_count, message)
-        padded_string = status_string.rjust(self.prev_len, " ")
-        self.prev_len = len(status_string)
+        if max_count is None:
+            return
 
-        print(f"\r{padded_string}", file=sys.stderr, end="")
+        if not self.progress is None:
+            self.progress.total = max_count
+            self.progress.n = cur_count
+            self.progress.desc = op_text #message
+            self.progress.unit = op_unit
+            self.progress.unit_scale = True
+            self.progress.refresh()
 
     def __del__(self):
         print("\n", file=sys.stderr)
@@ -216,13 +253,16 @@ class OpengnsysGitLibrary:
             '.gitattributes'
         ]
 
-        self.kernel_args = self._parse_kernel_cmdline()
+        self.kernel_args = parse_kernel_cmdline()
         self.repo_server = self.kernel_args["ogrepo"]
+        self.ip_address = self.kernel_args["ip"]
+
 
         if not self.repo_server:
             self.logger.warning("ogrepo kernel argument wasn't passed, or was empty. Defaulting to oglive.")
             self.repo_server = self.kernel_args["oglive"]
 
+
         """Add any untracked files the code might have missed.
         This is a workaround for a bug and it comes with a significant
         performance penalty.
@@ -251,28 +291,7 @@ class OpengnsysGitLibrary:
             f.write("\n".join(self.default_ignore_list))
             f.write("\n")
 
-    def _parse_kernel_cmdline(self):
-        """Parse the kernel arguments to obtain configuration parameters in Oglive
 
-        OpenGnsys passes data in the kernel arguments, for example:
-            [...] group=Aula_virtual ogrepo=192.168.2.1 oglive=192.168.2.1 [...]
-
-        Returns:
-            dict: Dict of configuration parameters and their values.
-        """
-        params = {}
-        self.logger.debug("Parsing kernel parameters")
-
-        with open("/proc/cmdline", encoding='utf-8') as cmdline:
-            line = cmdline.readline()
-            parts = line.split()
-            for part in parts:
-                if "=" in part:
-                    key, value = part.split("=")
-                    params[key] = value
-
-        self.logger.debug("%i parameters found", len(params))
-        return params
 
 
 
@@ -283,7 +302,9 @@ class OpengnsysGitLibrary:
         metadata_file =  os.path.join(data["metadata_dir"], "ntfs_secaudit.txt")
 
         self.logger.debug(f"Unmounting {data['mountpoint']}...")
-        subprocess.run(["/usr/bin/umount", data["mountpoint"]], check = True)
+        self.fs.unmount(mountpoint=data["mountpoint"])
+        #subprocess.run(["/usr/bin/umount", data["mountpoint"]], check = True)
+
         result = subprocess.run(["/usr/bin/ntfssecaudit", "-b", data["device"]], check=True, capture_output=True)
 
         self.logger.debug(f"Remounting {data['device']} on {data['mountpoint']}...")
@@ -547,6 +568,7 @@ class OpengnsysGitLibrary:
     def _getOgRepository(self, name):
         return f"{self.repo_user}@{self.repo_server}:{self.repo_image_path}/{name}.git"
 
+
     def _ogGetOsType(self):
         return "Linux"
 
@@ -1187,6 +1209,8 @@ class OpengnsysGitLibrary:
         repo.config_writer().add_value("user", "name", "OpenGnsys").release()
         repo.config_writer().add_value("user", "email", "OpenGnsys@opengnsys.com").release()
         repo.config_writer().add_value("core", "filemode",  "false").release()
+        repo.config_writer().add_value("push", "autoSetupRemote", "true").release()
+        repo.config_writer().add_value("maintenance", "autoDetach", "false").release()
 
     def initRepo(self, device, repo_name):
         """
@@ -1209,12 +1233,16 @@ class OpengnsysGitLibrary:
             - The method sets up a remote origin and pushes the initial commit.
         """
 
+        if not self.check_remote_exists(repo_name):
+            self.logger.error("Specified repository can't be used, aborting.")
+            return
+
         path = self.fs.ensure_mounted(device)
         self.logger.info("Initializing repository: %s", path)
 
         git_dir = os.path.join(path, ".git")
         real_git_dir = os.path.join(self.cache_dir, f"git-{repo_name}")
-
+        repo_url = self._getOgRepository(repo_name)
 
         if os.path.exists(real_git_dir):
             self.logger.debug(f"Removing existing repository {real_git_dir}")
@@ -1245,11 +1273,35 @@ class OpengnsysGitLibrary:
         os.symlink(real_git_dir, git_dir)
 
 
-        repo = git.Repo.init(path)
-        self._configure_repo(repo)
-        self._write_ignore_list(path)
+        with git.Repo.init(path) as repo:
+            # On NTFS, we have to unmount the filesystem to do secaudit.
+            # Gitpython objects existing at that time may mean a dangling git process that prevents
+            # the required unmounting.
+            #
+            # So we make sure we destroy gitpython after this initial stage, to recreate it
+            # right after _create_metadata.
+            self._configure_repo(repo)
+            self._write_ignore_list(path)
+
+
+            # Adding the gitignore and doing the manual --force saves us an expensive fetch if
+            # the repo already had data in it, and allows us to use the gitpython functions with
+            # progress reports for doing the full push later.
+            origin = repo.create_remote("origin", repo_url)
+            repo.index.add(f"{path}/.gitignore")
+            repo.index.commit("Initial commit")
+            repo.git.push("--force") # Obliterate whatever might have been there
+
+            self.logger.debug("Fetching origin")
+            origin.fetch(progress=OgProgressPrinter(self.logger))
+
+            repo.heads.master.set_tracking_branch(origin.refs.master)
+
+
         metadata_ret = self._create_metadata(path, initial_creation=True)
 
+        repo = git.Repo(path)
+
         self.logger.debug(f"Building list of files to add from path {path}")
 
         add_files = []
@@ -1315,21 +1367,17 @@ class OpengnsysGitLibrary:
             self.fs.unload_ntfs()
 
 
-        repo_url = self._getOgRepository(repo_name)
-        self.logger.debug(f"Creating remote origin: {repo_url}")
 
-        if "origin" in repo.remotes:
-            repo.delete_remote("origin")
 
-        origin = repo.create_remote("origin", repo_url)
 
-        self.logger.debug("Fetching origin")
-        origin.fetch()
      #   repo.create_head
-     #   repo.heads.master.set_tracking_branch(origin.refs.master)
+
+       # repo.heads.master.set_tracking_branch(origin.refs.master)
 
         self.logger.info("Uploading to ogrepository")
-        repo.git.push("--set-upstream", "origin", repo.head.ref, "--force")
+        origin.push(progress=OgProgressPrinter(self.logger))
+
+        #repo.git.push("--set-upstream", "origin", repo.head.ref, "--force")
 
     def cloneRepo(self, repo_name, destination, boot_device):
         """
@@ -1459,7 +1507,15 @@ class OpengnsysGitLibrary:
         repo = git.Repo(path)
 
         self.logger.info("Uploading to ogrepository")
-        repo.git.push("--set-upstream", "origin", repo.head.ref, "--force") # force = True)
+        if not "origin" in repo.remotes:
+            self.logger.critical("'origin' remote not found!")
+            return
+
+        origin = repo.remotes["origin"]
+        repo.heads.master.set_tracking_branch(origin.refs.master)
+        origin.push(progress=OgProgressPrinter(self.logger))
+
+        #repo.git.push("--set-upstream", "origin", repo.head.ref, "--force") # force = True)
 
 
     def fetch(self, path = None, device = None):
@@ -1478,16 +1534,21 @@ class OpengnsysGitLibrary:
 
         if origin:
             self.logger.debug("Fetching from origin")
-            origin.fetch()
+            origin.fetch(progress=OgProgressPrinter(self.logger))
         else:
             self.logger.error("Origin not found, can't fetch")
 
-    def pullRepo(self, path):
+    def pull(self, path = None, device = None):
         """
         Pull changes from ogrepository
 
         This unconditionally overwrites remote changes. There is no conflict resolution.
         """
+
+
+        if path is None:
+            path = self.fs.ensure_mounted(device)
+
         repo = git.Repo(path)
 
         self.logger.debug("Downloading from ogrepository")
@@ -1497,6 +1558,21 @@ class OpengnsysGitLibrary:
         # Restaurar cosas modificadas para git
         self._restore_metadata(path, destructive_only=True)
 
+    def check_remote_exists(self, repo_name):
+        repo_url = self._getOgRepository(repo_name)
+
+        self.logger.info("Checking whether %s exists and is accessible", repo_url)
+
+        ret = subprocess.run(["/usr/bin/git", "ls-remote", repo_url], encoding='utf-8', capture_output=True, check=False)
+        if ret.returncode == 0:
+            return True
+        else:
+            self.logger.warning("Remote can't be accessed, git said: %s", ret.stderr)
+            return False
+
+
+
+
 
 
 if __name__ == '__main__':
@@ -1504,6 +1580,9 @@ if __name__ == '__main__':
     # esto arregla las tildes y las eñes
     sys.stdout.reconfigure(encoding='utf-8')
 
+    kernel_args = parse_kernel_cmdline()
+
+
     opengnsys_log_dir = "/opt/opengnsys/log"
 
     logger = logging.getLogger(__package__)
@@ -1515,7 +1594,13 @@ if __name__ == '__main__':
     if not os.path.exists(opengnsys_log_dir):
         os.mkdir(opengnsys_log_dir)
 
-    logFilePath = f"{opengnsys_log_dir}/gitlib.log"
+    ip_address = "unknown"
+    if "ip" in kernel_args:
+        ip_address = kernel_args["ip"].split(":")[0]
+
+
+    logFilePath = f"{opengnsys_log_dir}/{ip_address}.gitlib.log"
+
     fileLog = logging.FileHandler(logFilePath)
     fileLog.setLevel(logging.DEBUG)
 
@@ -1546,6 +1631,7 @@ if __name__ == '__main__':
     parser.add_argument("--pull", type=str, metavar='DEV', help="Bajar cambios de ogrepository")
     parser.add_argument("--fetch", type=str, metavar='DEV', help="Fetch changes from ogrepository")
     parser.add_argument("--efi-config", type=str, metavar="NAME", help="Name of the custom EFI configuration to deploy")
+    parser.add_argument("--verify-repo", action='store_true', help="Verify whether the indicated repository exists and can be used")
 
 
     parser.add_argument("--ntfs-type", type=str, metavar="FS", help="Tipo de NTFS, 'kernel' o 'fuse'")
@@ -1613,7 +1699,13 @@ if __name__ == '__main__':
             og_git.fetch(device = args.fetch)
     elif args.pull:
         with OperationTimer(og_git, "git pull"):
-            og_git.pullRepo(args.pull)
+            og_git.pull(device = args.pull)
+    elif args.verify_repo:
+        if og_git.check_remote_exists(args.repo):
+            print("Remote checks OK")
+        else:
+            print("Check failed")
+
     elif args.test_create_metadata:
         og_git._create_metadata(args.test_create_metadata, initial_creation=False) # pylint: disable=protected-access
     elif args.test_restore_metadata:
@@ -1636,4 +1728,4 @@ if __name__ == '__main__':
     #
 
     # Make sure all filesystem changes are written, just in case the oglive is rebooted without an unmount
-    os.sync()
+    os.sync()

gitlib/kernel.py

@@ -0,0 +1,22 @@
+
+
+def parse_kernel_cmdline():
+    """Parse the kernel arguments to obtain configuration parameters in Oglive
+
+    OpenGnsys passes data in the kernel arguments, for example:
+        [...] group=Aula_virtual ogrepo=192.168.2.1 oglive=192.168.2.1 [...]
+
+    Returns:
+        dict: Dict of configuration parameters and their values.
+    """
+    params = {}
+
+    with open("/proc/cmdline", encoding='utf-8') as cmdline:
+        line = cmdline.readline()
+        parts = line.split()
+        for part in parts:
+            if "=" in part:
+                key, value = part.split("=")
+                params[key] = value
+
+    return params

gitlib/requirements.txt

@@ -1,9 +1,11 @@
 gitdb==4.0.11
 GitPython==3.1.43
-libarchive==0.4.7
 libarchive-c==5.1
 nose==1.3.7
+pathlib==1.0.1
+pkg_resources==0.0.0
 pylibacl==0.7.0
 pylibblkid==0.3
 pyxattr==0.8.1
 smmap==5.0.1
+tqdm==4.66.5