1440 lines
54 KiB
Python
1440 lines
54 KiB
Python
#!/usr/bin/env python3
|
|
|
|
### NOTES:
|
|
# Install:
|
|
# python3-git
|
|
# python3.8
|
|
# Must have working locales, or unicode strings will fail. Install 'locales', configure /etc/locale.gen, run locale-gen.
|
|
#
|
|
import os
|
|
import shutil
|
|
import git
|
|
import argparse
|
|
import tempfile
|
|
import shutil
|
|
import logging
|
|
import subprocess
|
|
import libarchive
|
|
import pwd
|
|
import grp
|
|
import datetime
|
|
import json
|
|
import sys
|
|
|
|
from pathlib import Path
|
|
|
|
import xattr
|
|
import posix1e
|
|
import base64
|
|
import blkid
|
|
import stat
|
|
import time
|
|
|
|
from enum import Enum
|
|
|
|
|
|
class OgProgressPrinter(git.RemoteProgress):
|
|
def __init(self, logger):
|
|
self.logger = logger
|
|
|
|
def update(self, op_code, cur_count, max_count=None, message=""):
|
|
self.logger.debug(f"Progress: {op_code} {cur_count}/{max_count}: {message}")
|
|
|
|
class OperationTimer:
|
|
def __init__(self, parent, operation_name):
|
|
self.operation_name = operation_name
|
|
self.parent = parent
|
|
self.start = time.time()
|
|
|
|
def __enter__(self):
|
|
self.start = time.time()
|
|
|
|
def __exit__(self, *args):
|
|
elapsed = round(time.time() - self.start, 3)
|
|
|
|
self.parent.logger.info(f"{self.operation_name} took {elapsed}s")
|
|
|
|
class RequirementException(Exception):
|
|
"""Excepción que indica que nos falta algún requisito
|
|
Duplicado de git_installer.
|
|
|
|
Attributes:
|
|
message (str): Mensaje de error mostrado al usuario
|
|
"""
|
|
|
|
def __init__(self, message):
|
|
"""Inicializar RequirementException.
|
|
|
|
Args:
|
|
message (str): Mensaje de error mostrado al usuario
|
|
"""
|
|
super().__init__(message)
|
|
self.message = message
|
|
|
|
|
|
|
|
class NTFSImplementation(Enum):
|
|
Kernel = 1
|
|
NTFS3G = 2
|
|
|
|
|
|
class NTFSLibrary:
|
|
def __init__(self, implementation):
|
|
self.logger = logging.getLogger("NTFSLibrary")
|
|
self.logger.setLevel(logging.DEBUG)
|
|
self.implementation = implementation
|
|
|
|
self.logger.debug("Initializing")
|
|
None
|
|
|
|
def create_filesystem(self, device, label):
|
|
self.logger.info(f"Creating NTFS in {device} with label {label}")
|
|
|
|
subprocess.run(["/usr/sbin/mkntfs", device, "-Q", "-L", label])
|
|
|
|
def mount_filesystem(self, device, mountpoint):
|
|
self.logger.info(f"Mounting {device} in {mountpoint} using implementation {self.implementation}")
|
|
if self.implementation == NTFSImplementation.Kernel:
|
|
subprocess.run(["/usr/bin/mount", "-t", "ntfs3", device, mountpoint], check = True)
|
|
elif self.implementation == NTFSImplementation.NTFS3G:
|
|
subprocess.run(["/usr/bin/ntfs-3g", device, mountpoint], check = True)
|
|
else:
|
|
raise ValueError("Unknown NTFS implementation: {self.implementation}")
|
|
|
|
def _hex_to_bin(self, hex_str):
|
|
|
|
while len(hex_str) != 16:
|
|
hex_str = "0" + hex_str
|
|
|
|
hex_int = int(hex_str, 16)
|
|
binary = bin(hex_int)[2:].zfill(64)
|
|
|
|
return binary
|
|
|
|
def _bin_to_hex(self, binary_data):
|
|
binary_int = int(binary_data, 256)
|
|
hex_str = hex(binary_int)[2:]
|
|
hex_str = hex_str.zfill((len(binary_data) + 3) // 4)
|
|
return hex_str
|
|
|
|
def modify_uuid(self, device, uuid):
|
|
"""Modifica el UUID de un sistema de archivos NTFS
|
|
|
|
Args:
|
|
device (_type_): Dispositivo
|
|
uuid (_type_): UUID nuevo (8 bytes)
|
|
"""
|
|
|
|
ntfs_uuid_offset = 48
|
|
ntfs_uuid_length = 8
|
|
|
|
binary_uuid = self._hex_to_bin(uuid)
|
|
|
|
self.logger.info(f"Changing UUID on {device} to {uuid}")
|
|
with open(device, 'r+b') as ntfs_dev:
|
|
ntfs_dev.seek(ntfs_uuid_offset)
|
|
prev_uuid = ntfs_dev.read(ntfs_uuid_length)
|
|
#prev_uuid_hex = self._bin_to_hex(prev_uuid)
|
|
#self.logger.debug(f"Previous UUID: {prev_uuid_hex}")
|
|
|
|
ntfs_dev.seek(ntfs_uuid_offset)
|
|
ntfs_dev.write(binary_uuid)
|
|
|
|
class OpengnsysGitLibrary:
|
|
|
|
"""Libreria de git"""
|
|
def __init__(self, require_cache = True, ntfs_implementation = NTFSImplementation.Kernel):
|
|
self.logger = logging.getLogger("OpengnsysGitLibrary")
|
|
self.logger.setLevel(logging.DEBUG)
|
|
self.logger.debug(f"Initializing. Cache = {require_cache}, ntfs = {ntfs_implementation}")
|
|
|
|
#self.repo_server = "192.168.2.1"
|
|
self.mounts = self._parse_mounts()
|
|
self.repo_user = "opengnsys"
|
|
self.repo_image_path = "/opt/opengnsys/images"
|
|
self.ntfs_implementation = ntfs_implementation
|
|
|
|
self.cache_dir = self._runBashFunction("ogMountCache", [])
|
|
|
|
# Si no hay cache, se va a crear el .git en el FS directamente
|
|
if (not self.cache_dir) and require_cache:
|
|
raise RequirementException("Failed to mount cache partition. Cache partition may be missing.")
|
|
|
|
self.default_ignore_list = [
|
|
'/proc/*', # filesystem virtual
|
|
'!/proc/.opengnsys-keep',
|
|
'/sys/*', # filesystem virtual
|
|
'!/sys/.opengnsys-keep',
|
|
'/dev/*', # dispositivos -- no soportados por git
|
|
'!/dev/.opengnsys-keep',
|
|
'/run/*', # info temporal
|
|
'!/run/.opengnsys-keep',
|
|
'/var/run/*', # info temporal
|
|
'!/var/run/.opengnsys-keep',
|
|
'/tmp/*', # archivos temporales
|
|
'!/tmp/.opengnsys-keep',
|
|
'/var/tmp/*', # archivos temporales
|
|
'!/var/tmp/.opengnsys-keep',
|
|
'/mnt/*', # otros sistemas de archivos
|
|
'!/mnt/.opengnsys-keep',
|
|
"/lost+found", # lost+found es un directorio especial. Si es necesario lo recreamos
|
|
"/$Recycle.Bin",
|
|
"/$WinREAgent",
|
|
'/PerfLogs'
|
|
"/DumpStack.log.tmp",
|
|
"/pagefile.sys",
|
|
"/swapfile.sys",
|
|
"/Recovery",
|
|
"/System Volume Information"
|
|
|
|
]
|
|
|
|
self.fully_ignored_dirs=[
|
|
'proc',
|
|
'sys',
|
|
'dev',
|
|
'run',
|
|
'var/run',
|
|
'tmp',
|
|
'var/tmp',
|
|
'mnt',
|
|
'$Recycle.Bin',
|
|
'$WinREAgent',
|
|
'PerfLogs',
|
|
'Recovery',
|
|
'System Volume Information'
|
|
]
|
|
|
|
self.kernel_args = self._parse_kernel_cmdline()
|
|
self.repo_server = self.kernel_args["ogrepo"]
|
|
|
|
"""Add any untracked files the code might have missed.
|
|
This is a workaround for a bug and it comes with a significant
|
|
performance penalty.
|
|
"""
|
|
self.debug_check_for_untracked_files = True
|
|
|
|
self.logger.debug(f"Git repository: {self.repo_server}")
|
|
|
|
def _is_efi(self):
|
|
"""Determina si hemos arrancado con EFI
|
|
|
|
Returns:
|
|
Bool: Si hemos arrancado con EFI
|
|
"""
|
|
return os.path.exists("/sys/firmware/efi")
|
|
|
|
def _parse_mounts(self):
|
|
filesystems = {}
|
|
|
|
self.logger.debug("Parsing /proc/mounts")
|
|
|
|
with open("/proc/mounts", 'r') as mounts:
|
|
for line in mounts:
|
|
parts = line.split()
|
|
data = {}
|
|
data['device'] = parts[0]
|
|
data['mountpoint'] = parts[1]
|
|
data['type'] = parts[2]
|
|
data['options'] = parts[3]
|
|
data['dump_freq'] = parts[4]
|
|
data['passno'] = parts[5]
|
|
|
|
filesystems[data["mountpoint"]] = data
|
|
filesystems[data["mountpoint"] + "/"] = data
|
|
|
|
return filesystems
|
|
|
|
def _find_mountpoint(self, device):
|
|
norm = os.path.normpath(device)
|
|
|
|
self.logger.debug(f"Checking if {device} is mounted")
|
|
for mountpoint, mount in self.mounts.items():
|
|
#self.logger.debug(f"Item: {mount}")
|
|
#self.logger.debug(f"Checking: " + mount['device'])
|
|
if mount['device'] == norm:
|
|
return mountpoint
|
|
|
|
return None
|
|
|
|
def _is_device_mounted(self, device):
|
|
return not self._find_mountpoint(device) is None
|
|
|
|
def _unmount_device(self, device):
|
|
mountpoint = self._find_mountpoint(device)
|
|
|
|
if not mountpoint is None:
|
|
self.logger.debug(f"Unmounting {mountpoint}")
|
|
subprocess.run(["/usr/bin/umount", mountpoint], check=True)
|
|
else:
|
|
self.logger.debug(f"{device} is not mounted")
|
|
|
|
|
|
def _rmmod(self, module):
|
|
self.logger.debug("Trying to unload module {module}...")
|
|
# TODO: modprobe not working on oglive
|
|
subprocess.run(["/usr/sbin/rmmod", module], check=False)
|
|
|
|
def _modprobe(self, module):
|
|
self.logger.debug("Trying to load module {module}...")
|
|
# TODO: modprobe not working on oglive
|
|
subprocess.run(["/usr/sbin/modprobe", module], check=False)
|
|
|
|
def _mount(self, device, mountpoint, filesystem):
|
|
self.logger.debug(f"Mounting {device} at {mountpoint}")
|
|
|
|
mount_cmd = ["/usr/bin/mount"]
|
|
|
|
if not filesystem is None:
|
|
mount_cmd = mount_cmd + ["-t", filesystem]
|
|
|
|
mount_cmd = mount_cmd + [mountpoint]
|
|
|
|
self.logger.debug(f"Mount command: {mount_cmd}")
|
|
subprocess.run(mount_cmd, check=True)
|
|
|
|
|
|
def _ntfsfix(self, device):
|
|
self.logger.debug(f"Running ntfsfix on {device}")
|
|
subprocess.run(["/usr/bin/ntfsfix", "-d", device], check=True)
|
|
|
|
def _filesystem_type(self, device):
|
|
self.logger.debug(f"Probing {device}")
|
|
|
|
pr = blkid.Probe()
|
|
pr.set_device(device)
|
|
pr.enable_superblocks(True)
|
|
pr.set_superblocks_flags(blkid.SUBLKS_TYPE | blkid.SUBLKS_USAGE | blkid.SUBLKS_UUID | blkid.SUBLKS_UUIDRAW | blkid.SUBLKS_LABELRAW)
|
|
pr.do_safeprobe()
|
|
|
|
fstype = pr["TYPE"].decode('utf-8')
|
|
self.logger.debug(f"FS type is {fstype}")
|
|
|
|
return fstype
|
|
|
|
def _write_ignore_list(self, base_path):
|
|
ignore_file = base_path + "/.gitignore"
|
|
|
|
self.logger.debug("Creating ignore list: " + ignore_file)
|
|
with open(ignore_file, 'w') as f:
|
|
f.write("\n".join(self.default_ignore_list))
|
|
f.write("\n")
|
|
|
|
def _parse_kernel_cmdline(self):
|
|
"""Obtener parámetros de configuración de la linea de comandos del kernel
|
|
|
|
Opengnsys nos pasa parametros por linea de comando del kernel, por ejemplo:
|
|
[...] group=Aula_virtual ogrepo=192.168.2.1 oglive=192.168.2.1 [...]
|
|
|
|
Returns:
|
|
dict: Diccionario de clave/valor de parámetros
|
|
"""
|
|
params = {}
|
|
self.logger.debug("Parsing kernel parameters")
|
|
|
|
with open("/proc/cmdline") as cmdline:
|
|
line = cmdline.readline()
|
|
parts = line.split()
|
|
for part in parts:
|
|
if "=" in part:
|
|
key, value = part.split("=")
|
|
params[key] = value
|
|
|
|
self.logger.debug("%i parameters found" % len(params))
|
|
return params
|
|
|
|
def _is_filesystem(self, path):
|
|
with open('/proc/mounts', 'r') as mounts:
|
|
for mount in mounts:
|
|
parts = mount.split()
|
|
self.logger.debug(f"Parts: {parts}")
|
|
mountpoint = parts[1]
|
|
|
|
if os.path.normpath(mountpoint) == os.path.normpath(path):
|
|
self.logger.debug(f"Directory {path} is a filesystem root")
|
|
return True
|
|
|
|
self.logger.debug(f"Directory {path} is not a filesystem root")
|
|
return False
|
|
|
|
|
|
def _mklostandfound(self, path):
|
|
"""Recrear el lost+found si es necesario.
|
|
|
|
Cuando clonamos en el raíz de un sistema de archivos, al limpiar los contenidos,
|
|
eliminamos el lost+found. Este es un directorio especial que requiere el uso de
|
|
una herramienta para recrearlo.
|
|
|
|
Puede fallar en caso de que el sistema de archivos no lo necesite.
|
|
"""
|
|
if self._is_filesystem(path):
|
|
curdir = os.getcwd()
|
|
result = None
|
|
|
|
try:
|
|
self.logger.debug(f"Re-creating lost+found in {path}")
|
|
os.chdir(path)
|
|
result = subprocess.run(["/usr/sbin/mklost+found"], check=True, capture_output=True)
|
|
except Exception as e:
|
|
self.logger.warning(f"Error running mklost+found: {e}")
|
|
|
|
if result:
|
|
self.logger.debug(f"retorno: {result.returncode}")
|
|
self.logger.debug(f"stdout: {result.stdout}")
|
|
self.logger.debug(f"stderr: {result.stderr}")
|
|
|
|
os.chdir(curdir)
|
|
|
|
|
|
def _ntfs_secaudit(self, data):
|
|
self.logger.debug(f"Saving NTFS metadata for {data['device']}")
|
|
|
|
metadata_file = os.path.join(data["metadata_dir"], "ntfs_secaudit.txt")
|
|
|
|
self.logger.debug(f"Unmounting {data['mountpoint']}...")
|
|
subprocess.run(["/usr/bin/umount", data["mountpoint"]], check = True)
|
|
result = subprocess.run(["/usr/bin/ntfssecaudit", "-b", data["device"]], check=True, capture_output=True)
|
|
|
|
self.logger.debug(f"Remounting {data['device']} on {data['mountpoint']}...")
|
|
if data["mount_fs"] == "fuseblk":
|
|
self.logger.debug("Mount was FUSE")
|
|
subprocess.run(["/usr/bin/mount", data["device"], data["mountpoint"]], check=True)
|
|
else:
|
|
self.logger.debug(f"Mount was {data['mount_fs']}")
|
|
subprocess.run(["/usr/bin/mount", data["device"], "-t", data["mount_fs"], data["mountpoint"]], check=True)
|
|
|
|
self.logger.debug("Writing NTFS audit metadata...")
|
|
with open(metadata_file + ".new", "w", encoding='utf-8') as meta:
|
|
meta.write(result.stdout.decode('utf-8'))
|
|
|
|
os.rename(metadata_file + ".new", metadata_file)
|
|
|
|
def _create_filesystems(self, fs_data, fs_map):
|
|
for mountpoint in fs_map:
|
|
dest_device = fs_map[mountpoint]
|
|
data = fs_data[mountpoint]
|
|
|
|
fs_type = data["type"]
|
|
fs_uuid = data["uuid"]
|
|
|
|
self.logger.debug(f"Creating filesystem {fs_type} with UUID {fs_uuid} in {dest_device}")
|
|
|
|
if fs_type == "ntfs" or fs_type == "ntfs3":
|
|
self.logger.debug("Creating NTFS filesystem")
|
|
ntfs = NTFSLibrary(self.ntfs_implementation)
|
|
ntfs.create_filesystem(dest_device, "NTFS")
|
|
#ntfs.modify_uuid(dest_device, fs_uuid)
|
|
else:
|
|
command = [f"/usr/sbin/mkfs.{fs_type}"]
|
|
command_args = []
|
|
|
|
if fs_type == "ext4" or fs_type == "ext3":
|
|
command_args = ["-U", fs_uuid, "-F", dest_device]
|
|
elif fs_type == "xfs":
|
|
command_args = ["-m", f"uuid={fs_uuid}", "-f", dest_device]
|
|
elif fs_type == "btrfs":
|
|
command_args = ["-U", fs_uuid, "-f", dest_device]
|
|
else:
|
|
raise RuntimeError(f"Don't know how to create filesystem of type {fs_type}")
|
|
|
|
command = command + command_args
|
|
|
|
self.logger.debug(f"Creating Linux filesystem of type {fs_type} on {dest_device}, command {command}")
|
|
result = subprocess.run(command, check = True, capture_output=True)
|
|
|
|
self.logger.debug(f"retorno: {result.returncode}")
|
|
self.logger.debug(f"stdout: {result.stdout}")
|
|
self.logger.debug(f"stderr: {result.stderr}")
|
|
|
|
def _mount(self, device, directory):
|
|
self.logger.debug(f"Mounting {device} in {directory}")
|
|
|
|
if not os.path.exists(directory):
|
|
os.mkdir(directory)
|
|
|
|
|
|
result = subprocess.run(["/usr/bin/mount", device, directory], check = True, capture_output=True)
|
|
|
|
self.logger.debug(f"retorno: {result.returncode}")
|
|
self.logger.debug(f"stdout: {result.stdout}")
|
|
self.logger.debug(f"stderr: {result.stderr}")
|
|
|
|
def _grub_install(self, boot_device, root_directory):
|
|
"""Instalar grub
|
|
|
|
Args:
|
|
device (str): Dispositivo de arranque
|
|
root_directory (str): Punto de montaje de sistema raiz
|
|
|
|
Raises:
|
|
RequirementException: Si falta binario de GRUB
|
|
"""
|
|
|
|
if os.path.exists("/usr/sbin/grub2-install"):
|
|
self.logger.debug("Installing Grub 2.x (NOT IMPLEMENTED)")
|
|
elif os.path.exists("/usr/sbin/grub-install"):
|
|
self.logger.debug("Installing Grub 1.x")
|
|
|
|
root = ""
|
|
result = subprocess.run(["/usr/sbin/grub-install", "--force", "--root-directory", root_directory, boot_device], check = True, capture_output=True)
|
|
self.logger.debug(f"retorno: {result.returncode}")
|
|
self.logger.debug(f"stdout: {result.stdout}")
|
|
self.logger.debug(f"stderr: {result.stderr}")
|
|
else:
|
|
raise RequirementException("Couldn't find /usr/sbin/grub2-install or /usr/sbin/grub-install")
|
|
|
|
def _efi_install(self, boot_device, root_directory):
|
|
"""Instalar EFI"""
|
|
|
|
self.logger.info(f"Instalando datos EFI en {boot_device}")
|
|
meta_dir = os.path.join(root_directory, ".opengnsys-metadata")
|
|
efi_files_dir = os.path.join(meta_dir, "efi_data")
|
|
|
|
shutil.copytree(efi_files_dir, boot_device)
|
|
|
|
|
|
def _find_boot_device(self):
|
|
disks = []
|
|
|
|
self.logger.debug("Looking for EFI partition")
|
|
with open("/proc/partitions", "r") as partitions_file:
|
|
line_num=0
|
|
for line in partitions_file:
|
|
if line_num >=2:
|
|
data = line.split()
|
|
disk = data[3]
|
|
disks.append(disk)
|
|
self.logger.debug(f"Disk: {disk}")
|
|
|
|
line_num = line_num + 1
|
|
|
|
for disk in disks:
|
|
self.logger.debug("Loading partitions for disk {disk}")
|
|
disk_json_data = subprocess.run(["/usr/sbin/sfdisk", "-J", disk], check=True, capture_output=True)
|
|
disk_data = json.loads(disk_json_data)
|
|
|
|
for part in disk_data["partitiontable"]["partitions"]:
|
|
self.logger.debug("Checking partition {part}")
|
|
if part["type"] == "C12A7328-F81F-11D2-BA4B-00A0C93EC93B":
|
|
return part["node"]
|
|
|
|
|
|
self.logger.warning("Failed to find EFI partition!")
|
|
|
|
|
|
def _delete_contents(self, path):
|
|
self.logger.info(f"Deleting contents of {path}")
|
|
|
|
|
|
for filename in os.listdir(path):
|
|
file_path = os.path.join(path, filename)
|
|
try:
|
|
self.logger.debug(f"Deleting {file_path}")
|
|
|
|
if os.path.isfile(file_path) or os.path.islink(file_path):
|
|
os.unlink(file_path)
|
|
elif os.path.isdir(file_path):
|
|
shutil.rmtree(file_path)
|
|
except Exception as e:
|
|
self.logger.warning('Failed to delete %s. Error: %s' % (file_path, e))
|
|
|
|
|
|
def _runBashFunction(self, function, arguments):
|
|
# Create a temporary file
|
|
self.logger.debug(f"Running bash function: {function} {arguments}")
|
|
|
|
with tempfile.NamedTemporaryFile(mode='w', delete=False) as temp_file:
|
|
temp_file.write("#!/bin/bash\n")
|
|
temp_file.write("for lib in /opt/oglive/rootfs/opt/opengnsys/lib/engine/bin/*.lib ; do\n")
|
|
temp_file.write(" source $lib\n")
|
|
temp_file.write("done\n")
|
|
|
|
#temp_file.write("source /opt/oglive/rootfs/opt/opengnsys/lib/engine/bin/Cache.lib")
|
|
#temp_file.write("source /opt/oglive/rootfs/opt/opengnsys/lib/engine/bin/Git.lib")
|
|
|
|
temp_file.write(f"{function} \"$@\"\n")
|
|
|
|
# Make the temporary file executable
|
|
os.chmod(temp_file.name, 0o755)
|
|
|
|
self.logger.debug(f"File: {temp_file.name}")
|
|
|
|
|
|
# Run the temporary file
|
|
command = [temp_file.name] + arguments
|
|
self.logger.debug(f"Running: {command} {arguments}")
|
|
result = subprocess.run(command, shell=False, capture_output=True, text=True)
|
|
output = result.stdout.strip()
|
|
|
|
self.logger.debug(f"STDOUT: {output}")
|
|
self.logger.debug(f"STDERR: {result.stderr}")
|
|
|
|
# temp_file.delete()
|
|
|
|
return output
|
|
|
|
# os.system(temp_file.name)
|
|
|
|
|
|
def _getOgRepository(self, name):
|
|
return f"{self.repo_user}@{self.repo_server}:{self.repo_image_path}/{name}.git"
|
|
|
|
def _ogGetOsType(self):
|
|
return "Linux"
|
|
|
|
|
|
def _get_repo_metadata(self, repo):
|
|
"""Obtiene metadatos de un repositorio remoto sin clonar el repo entero.
|
|
|
|
Esto resuelve el problema de tener metadatos del sistema de archivos en el propio repo,
|
|
sobre los que necesitamos actuar antes de poder clonarlo.
|
|
|
|
Args:
|
|
repo (str): Nombre del repositorio (linux, windows, mac)
|
|
|
|
Returns:
|
|
dict: Clave/valor con contenidos de filesystems.json y metadata.json
|
|
"""
|
|
results = {}
|
|
|
|
tempdir = tempfile.TemporaryDirectory()
|
|
repo_url = self._getOgRepository(repo)
|
|
wanted_files = ["filesystems.json", "metadata.json"]
|
|
|
|
self.logger.debug(f"Cloning metadata for repository {repo_url}")
|
|
|
|
result = subprocess.run(["git", "archive", "--remote", repo_url, "HEAD:.opengnsys-metadata/"], capture_output=True, check=True)
|
|
tar_data = result.stdout
|
|
|
|
|
|
with libarchive.memory_reader(tar_data) as metadata:
|
|
self.logger.debug(f"Archive: {metadata}")
|
|
|
|
for entry in metadata:
|
|
self.logger.debug(entry)
|
|
if entry.pathname in wanted_files:
|
|
self.logger.debug(f"Extracting {entry}")
|
|
data = bytearray()
|
|
for block in entry.get_blocks():
|
|
data = data + block
|
|
|
|
text = data.decode('utf-8')
|
|
results[entry.pathname] = json.loads(text)
|
|
self.logger.debug(f"Contents: {text}")
|
|
|
|
return results
|
|
|
|
def _create_metadata(self, path):
|
|
"""Calcular metadatos para un filesystem
|
|
|
|
Aquí recorremos todo el sistema de archivos para:
|
|
|
|
1. Encontrar directorios vacíos y rellenarlos para que git los conserve.
|
|
2. Obtener todas las ACLs
|
|
3. Obtener todos los atributos extendidos.
|
|
4. Renombrar archivos .gitignore
|
|
5. Buscar puntos de montaje y obtener información sobre ellos
|
|
6. Metadatos adicionales, como el tipo de arranque
|
|
7. NTFS secaudit, que debe realizarse al final del proceso porque hay
|
|
que desmontar el filesystem.
|
|
|
|
Para archivos vacíos, generamos una lista que podemos usar después para eliminar los archivos
|
|
.opengnsys-keep. Esto se hace porque hay casos en los que un archivo inesperado puede causar
|
|
problemas. Por ejemplo, sshfs por defecto se niega a montar cosas en un directorio que contiene
|
|
archivos.
|
|
|
|
Renombramos los archivos .gitignore en subdirectorios porque git los aplicaría a nuestro proceso.
|
|
|
|
Escribimos todos los datos en JSON para asegurarnos de que no hay problemas con espacios, fines de
|
|
linea ni otros caracteres especiales. Esto también asegura una entrada por linea, lo que podemos
|
|
usar para acelerar el rendimiento, usando el git para obtener la diferencia entre un estado anterior
|
|
y el actual.
|
|
|
|
Args:
|
|
path (str): Ruta base del sistema de archivos
|
|
"""
|
|
|
|
self.logger.info(f"Creating metadata for {path}")
|
|
|
|
return_data = { 'symlinks': [] }
|
|
|
|
seen_roots = {}
|
|
filesystems_data = {}
|
|
ntfs_secaudit_list = []
|
|
|
|
|
|
git_dir = os.path.normpath(os.path.join(path, ".git"))
|
|
|
|
meta_dir = os.path.join(path, ".opengnsys-metadata")
|
|
|
|
if not os.path.exists(meta_dir):
|
|
os.mkdir(meta_dir, mode=0o700)
|
|
|
|
# https://jsonlines.org/
|
|
|
|
metadata_file = open(os.path.join(meta_dir, "metadata.json.new"), "w")
|
|
metadata = {}
|
|
metadata["efi_boot"] = self._is_efi()
|
|
|
|
if self._is_efi():
|
|
self.logger.debug("Copying EFI data")
|
|
efi_files_dir = os.path.join(meta_dir, "efi_data")
|
|
if os.path.exists(efi_files_dir):
|
|
shutil.rmtree(efi_files_dir)
|
|
|
|
boot_device = self._find_boot_device
|
|
shutil.copytree(boot_device, efi_files_dir)
|
|
|
|
|
|
|
|
|
|
empties_file = open(os.path.join(meta_dir, "empty_directories.jsonl.new"), "w")
|
|
specials_file = open(os.path.join(meta_dir, "special_files.jsonl.new"), "w")
|
|
acls_file = open(os.path.join(meta_dir, "acls.jsonl.new"), "w")
|
|
xattrs_file = open(os.path.join(meta_dir, "xattrs.jsonl.new"), "w")
|
|
gitignores_file = open(os.path.join(meta_dir, "gitignores.jsonl.new"), "w")
|
|
filesystems_file = open(os.path.join(meta_dir, "filesystems.json.new"), "w")
|
|
|
|
ntfs = False
|
|
|
|
for root, subdirs, files in os.walk(path):
|
|
#print(f"ROOT: {root}, subdirs: {subdirs}, files: {files}")
|
|
|
|
root_norm = os.path.normpath(root)
|
|
root_rel = root[len(path):None]
|
|
|
|
#print(f"A: {root}")
|
|
#print(f"B: {git_dir}")
|
|
|
|
if root_norm.startswith(git_dir):
|
|
self.logger.debug(f"Ignoring git directory: {root_norm}")
|
|
|
|
|
|
# No examinamos el .git del raíz
|
|
continue
|
|
|
|
if not root in seen_roots:
|
|
seen_roots[root]=1
|
|
|
|
if root in self.mounts:
|
|
mount = self.mounts[root]
|
|
root_path_rel = root[len(path):None]
|
|
if len(root_path_rel) == 0 or root_path_rel[0] != "/":
|
|
root_path_rel = "/" + root_path_rel
|
|
|
|
|
|
|
|
|
|
|
|
pr = blkid.Probe()
|
|
pr.set_device(mount['device'])
|
|
pr.enable_superblocks(True)
|
|
pr.set_superblocks_flags(blkid.SUBLKS_TYPE | blkid.SUBLKS_USAGE | blkid.SUBLKS_UUID | blkid.SUBLKS_UUIDRAW | blkid.SUBLKS_LABELRAW)
|
|
pr.do_safeprobe()
|
|
|
|
|
|
fs = mount['type']
|
|
orig_fs = fs
|
|
|
|
if mount['type'] == 'fuseblk':
|
|
fs = pr["TYPE"].decode('utf-8')
|
|
self.logger.warning(f"FUSE mount detected, replacing type with blkid detected type: {fs}")
|
|
self.logger.warning("FUSE has lower performance, a native mount is recommended.")
|
|
|
|
filesystems_data[root_path_rel] = {}
|
|
filesystems_data[root_path_rel]['type'] = fs
|
|
filesystems_data[root_path_rel]['size'] = pr.size
|
|
filesystems_data[root_path_rel]['sectors'] = pr.sectors
|
|
filesystems_data[root_path_rel]['sector_size'] = pr.sector_size
|
|
filesystems_data[root_path_rel]['uuid'] = str(pr["UUID"], 'utf-8')
|
|
filesystems_data[root_path_rel]['uuid_raw'] = str(base64.b64encode(pr["UUID_RAW"]), 'utf-8')
|
|
|
|
# TODO: Esto de momento no funciona -- LABEL no se encuentra
|
|
#filesystems_data[root_path_rel]['label'] = pr["LABEL"]
|
|
#filesystems_data[root_path_rel]['label_raw'] = pr["LABEL_RAW"]
|
|
|
|
self.logger.debug("Filesystem: {0}, relative {1}. Type {2}, size {3}, UUID {4}".format(root, root_path_rel, mount['type'], pr.size, str(pr["UUID"], 'utf-8')))
|
|
|
|
if fs == 'ntfs' or fs == 'ntfs3':
|
|
self.logger.info("NTFS detected, will do a secaudit")
|
|
ntfs_secaudit_list.append({
|
|
'device' : mount['device'],
|
|
'mountpoint' : root,
|
|
'relative_path' : root_path_rel,
|
|
'mount_fs' : orig_fs,
|
|
'metadata_dir' : meta_dir
|
|
})
|
|
|
|
#self._ntfs_secaudit(root, os.path.join(meta_dir, "ntfs_secaudit.txt"))
|
|
ntfs = True
|
|
|
|
|
|
#self.logger.debug(f"Root rel: {root_rel}")
|
|
if len(files) == 1 and files[0] == ".opengnsys-keep":
|
|
# Si ya tenemos un .opengnsys-keep, lo ignoramos
|
|
files.pop(0)
|
|
|
|
# Ignoramos todo el contenido en directorios como /dev, pero el directorio
|
|
# debe existir.
|
|
if (len(subdirs) == 0 and len(files) == 0) or (root_rel in self.fully_ignored_dirs):
|
|
keep_file = os.path.join(root, ".opengnsys-keep")
|
|
Path(keep_file).touch(mode=0o644, exist_ok = True)
|
|
|
|
self.logger.debug(f"Empty directory: {root}")
|
|
#root_rel = root[len(path):None]
|
|
empties_file.write(json.dumps({"dir" : root_rel}) + "\n")
|
|
|
|
for file in files:
|
|
|
|
full_path = os.path.join(root,file)
|
|
full_path_rel = full_path[len(path):None]
|
|
|
|
# Relative path can't start with a /, git will take it as an
|
|
# absolute path pointing to /, and not a file within the repo.
|
|
while full_path_rel[0] == '/':
|
|
full_path_rel = full_path_rel[1:None]
|
|
|
|
#self.logger.debug(f"Checking {full_path}:")
|
|
|
|
if not ntfs and os.path.isfile(full_path) and not os.path.islink(full_path):
|
|
# docs: https://pylibacl.k1024.org/module.html#posix1e.ACL.to_any_text
|
|
|
|
xattrs = str(xattr.get_all(full_path))
|
|
acls = posix1e.ACL(file=full_path)
|
|
|
|
xattrs_json = json.dumps({"file": full_path_rel, "xattrs" : xattrs})
|
|
#acls_json = json.dumps({"file": full_path_rel, "acl" : str(acls.to_any_text())})
|
|
|
|
# __getstate__ nos permite exportar el estado entero de la ACL
|
|
# TODO: posiblemente formato de texto mejor?
|
|
acl_data = str(base64.b64encode(acls.__getstate__()), 'utf-8')
|
|
acls_json = json.dumps({"file": full_path_rel, "acl" : acl_data })
|
|
|
|
xattrs_file.write(xattrs_json + "\n")
|
|
acls_file.write(acls_json + "\n")
|
|
|
|
if os.path.isfile(full_path) and file == ".gitignore" and root != path:
|
|
# TODO: tener en cuenta archivos ya renombrados
|
|
|
|
logger.debug(f"Found .gitignore: {full_path}")
|
|
renamed_file_path = full_path + "-opengnsys-renamed"
|
|
|
|
|
|
gitignores_json = json.dumps({"file": full_path_rel})
|
|
gitignores_file.write(gitignores_json + "\n")
|
|
|
|
os.rename(full_path, renamed_file_path)
|
|
|
|
#print(f"\tXATTRS: {xattrs}")
|
|
#print(f"\tACLs: {acls_json}")
|
|
|
|
if os.path.exists(full_path):
|
|
if not os.path.islink(full_path):
|
|
stat_data = os.stat(full_path)
|
|
stat_mode = stat_data.st_mode
|
|
|
|
if not (stat.S_ISDIR(stat_mode) or stat.S_ISREG(stat_mode) or stat.S_ISLNK(stat_mode)):
|
|
# Si no es un directorio o un archivo, conservamos los datos necesarios para recrearlo
|
|
|
|
stat_json_data = {
|
|
"file" : full_path_rel,
|
|
"mode" : stat_mode,
|
|
"uid" : stat_data.st_uid,
|
|
"gid" : stat_data.st_gid,
|
|
"rdev" : stat_data.st_rdev
|
|
}
|
|
|
|
stat_json = json.dumps(stat_json_data)
|
|
specials_file.write(stat_json + "\n")
|
|
|
|
# Git falla al encontrarse con archivos especiales como dispositivos. De momento debemos
|
|
# eliminarlos.
|
|
os.unlink(full_path)
|
|
if os.path.islink(full_path):
|
|
self.logger.debug(f"Symlink: {full_path_rel}")
|
|
return_data['symlinks'].append(full_path_rel)
|
|
|
|
|
|
self.logger.debug("Finishing...")
|
|
|
|
filesystems_file.write(json.dumps(filesystems_data, indent=4) + "\n")
|
|
metadata_file.write(json.dumps(metadata, indent=4) + "\n")
|
|
|
|
empties_file.close()
|
|
specials_file.close()
|
|
xattrs_file.close()
|
|
acls_file.close()
|
|
gitignores_file.close()
|
|
filesystems_file.close()
|
|
metadata_file.close()
|
|
|
|
os.rename(os.path.join(meta_dir, "empty_directories.jsonl.new"), os.path.join(meta_dir, "empty_directories.jsonl"))
|
|
os.rename(os.path.join(meta_dir, "special_files.jsonl.new"), os.path.join(meta_dir, "special_files.jsonl"))
|
|
os.rename(os.path.join(meta_dir, "acls.jsonl.new"), os.path.join(meta_dir, "acls.jsonl"))
|
|
os.rename(os.path.join(meta_dir, "xattrs.jsonl.new"), os.path.join(meta_dir, "xattrs.jsonl"))
|
|
os.rename(os.path.join(meta_dir, "gitignores.jsonl.new"), os.path.join(meta_dir, "gitignores.jsonl"))
|
|
os.rename(os.path.join(meta_dir, "filesystems.json.new"), os.path.join(meta_dir, "filesystems.json"))
|
|
os.rename(os.path.join(meta_dir, "metadata.json.new"), os.path.join(meta_dir, "metadata.json"))
|
|
|
|
self.logger.debug("Processing pending NTFS secaudits...")
|
|
for audit in ntfs_secaudit_list:
|
|
self._ntfs_secaudit(audit)
|
|
|
|
|
|
self.logger.debug("Metadata updated")
|
|
return return_data
|
|
|
|
def _restore_metadata(self, path, destructive_only=False):
|
|
"""Restrauracion de metadatos creados por _createmetadata
|
|
|
|
|
|
|
|
Args:
|
|
path (str): Ruta destino
|
|
destructive_only (bool): Solo restaurar lo que se modifique durante un commit
|
|
|
|
Notes:
|
|
El git no maneja archivos de tipo dispositivo o socket correctamente. Por tanto,
|
|
debemos guardar datos sobre ellos antes del commit, y eliminarlos antes de que
|
|
git pueda verlos y confundirse.
|
|
|
|
destructive_only=True solo restaura este tipo de metadatos, los que modificamos
|
|
en el sistema de archivos real antes del commit. Esto se hace para dejar el
|
|
sistema de archivos en el mismo estado que tenia antes del commit.
|
|
"""
|
|
|
|
self.logger.debug("Initializing")
|
|
mounts = self._parse_mounts()
|
|
|
|
self.logger.debug(f"Restoring metadata in {path}")
|
|
meta_dir = os.path.join(path, ".opengnsys-metadata")
|
|
|
|
if not os.path.exists(meta_dir):
|
|
self.logger.error(f"Metadata directory not found: {meta_dir}")
|
|
return
|
|
|
|
if not destructive_only:
|
|
self.logger.debug("Processing empty_directories.jsonl")
|
|
with open(os.path.join(meta_dir, "empty_directories.jsonl"), "r") as empties_file:
|
|
for line in empties_file:
|
|
empties_data = json.loads(line)
|
|
empty_dir = empties_data['dir']
|
|
|
|
# os.path.join no acepta /foo como una ruta relativa para concatenar
|
|
if empty_dir.startswith("/"):
|
|
empty_dir = empty_dir[1:]
|
|
|
|
empty_dir_keep = os.path.join(path, empty_dir, ".opengnsys-keep")
|
|
|
|
self.logger.debug(f"Empty directory: {empty_dir}")
|
|
|
|
if os.path.exists(empty_dir_keep):
|
|
self.logger.debug(f"Deleting: {empty_dir_keep}")
|
|
os.unlink(empty_dir_keep)
|
|
|
|
if not destructive_only:
|
|
self.logger.debug("Processing acls.jsonl")
|
|
with open(os.path.join(meta_dir, "acls.jsonl"), "r") as acls_file:
|
|
for line in acls_file:
|
|
|
|
# docs: https://pylibacl.k1024.org/module.html#posix1e.ACL.to_any_text
|
|
|
|
acls_data = json.loads(line)
|
|
#self.logger.debug(f"Data: {acls_data}")
|
|
|
|
acl_file = acls_data['file']
|
|
acl_text = base64.b64decode(bytes(acls_data['acl'], 'utf-8'))
|
|
|
|
if acl_file.startswith("/"):
|
|
acl_file = acl_file[1:]
|
|
|
|
acl_file_path = os.path.join(path, acl_file)
|
|
#self.logger.debug(f"TXT: {acl_text}" )
|
|
acl = posix1e.ACL(data = acl_text)
|
|
#self.logger.debug(f"ACL: {acl_text}" )
|
|
|
|
self.logger.debug(f"Applying ACL to {acl_file_path}")
|
|
#acl.applyto(acl_file_path)
|
|
|
|
if not destructive_only:
|
|
self.logger.debug("Processing xattrs.jsonl")
|
|
with open(os.path.join(meta_dir, "xattrs.jsonl"), "r") as xattrs_file:
|
|
for line in xattrs_file:
|
|
xattrs_data = json.loads(line)
|
|
xattrs_file = xattrs_data['file']
|
|
|
|
if xattrs_file.startswith("/"):
|
|
xattrs_file = xattrs_file[1:]
|
|
|
|
xattrs_file_path = os.path.join(path, xattrs_file)
|
|
|
|
#self.logger.debug(f"Line: {line}")
|
|
|
|
self.logger.debug("Processing gitignores.jsonl")
|
|
with open(os.path.join(meta_dir, "gitignores.jsonl"), "r") as gitignores_file:
|
|
for line in gitignores_file:
|
|
#self.logger.debug(f"Line: {line}")
|
|
gitignores_data = json.loads(line)
|
|
gitignores_file = gitignores_data['file']
|
|
|
|
if gitignores_file.startswith("/"):
|
|
gitignores_file = gitignores_file[1:]
|
|
|
|
orig_file_path = os.path.join(path, gitignores_file)
|
|
renamed_file_path = orig_file_path + "-opengnsys-renamed"
|
|
|
|
#self.logger.debug(f"Checking: {renamed_file_path}")
|
|
if os.path.exists(renamed_file_path):
|
|
self.logger.debug(f"Renaming {renamed_file_path} => {orig_file_path}")
|
|
os.rename(renamed_file_path, orig_file_path)
|
|
|
|
self.logger.debug("Processing special_files.jsonl")
|
|
with open(os.path.join(meta_dir, "special_files.jsonl"), "r") as specials_file:
|
|
for line in specials_file:
|
|
#self.logger.debug(f"Line: {line}")
|
|
data = json.loads(line)
|
|
filename = data['file']
|
|
full_path = os.path.join(path, filename)
|
|
file_mode = data['mode']
|
|
|
|
try:
|
|
if stat.S_ISSOCK(file_mode):
|
|
self.logger.debug(f"Restoring socket {filename}")
|
|
os.mknod(full_path, mode = file_mode)
|
|
elif stat.S_ISFIFO(file_mode):
|
|
self.logger.debug(f"Restoring FIFO {filename}")
|
|
os.mknod(full_path, mode = file_mode)
|
|
elif stat.S_ISBLK(file_mode):
|
|
self.logger.debug(f"Restoring block device {filename}")
|
|
os.mknod(full_path, mode = file_mode, device = data['rdev'])
|
|
elif stat.S_ISCHR(file_mode):
|
|
self.logger.debug(f"Restoring character device {filename}")
|
|
os.mknod(full_path, mode = file_mode, device = data['rdev'])
|
|
else:
|
|
self.logger.warning(f"Unknown file type for {filename}: {file_mode}")
|
|
except FileExistsError as exists:
|
|
self.logger.debug(f"Exists: {full_path}")
|
|
|
|
os.chown(full_path, data['uid'], data['gid'])
|
|
|
|
self.logger.debug("Metadata restoration completed.")
|
|
|
|
def _configure_repo(self, repo):
|
|
"""
|
|
#ogGitConfig
|
|
#@brief Configura usuario y permisos de git.
|
|
#@return
|
|
"""
|
|
|
|
self.logger.debug(f"Configuring repository {repo}")
|
|
repo.config_writer().add_value("user", "name", "OpenGnsys").release()
|
|
repo.config_writer().add_value("user", "email", "OpenGnsys@opengnsys.com").release()
|
|
repo.config_writer().add_value("core", "filemode", "false").release()
|
|
|
|
|
|
|
|
|
|
|
|
def ogUuidUpdate(str_repo, str_path_image, int_ndisk, int_part):
|
|
"""
|
|
#ogUuidUpdate str_repo str_path_image int_ndisk str_repo
|
|
|
|
#@brief Actualiza el UUID de un sistema de ficheros en los archivos de configuración.
|
|
#@param str_repo repositorio de imágenes o caché local
|
|
#@param str_path_image camino dOk, you can convince me that I'm not an "artist". But so what then? I'll keep generating pictures anyway. I don't make them because I yearn for the coveted status of "artist", but because I enjoy making whatever you want to call I'm making, and the exact terminology is about the least interesting part of the entire thing.e la imagen
|
|
#@param int_ndisk nº de orden del disco
|
|
#@param int_part nº de partición
|
|
#@return (nada, por determinar)
|
|
#@exception OG_ERR_FORMAT formato incorrecto.
|
|
#@exception OG_ERR_NOTFOUND fichero o dispositivo no encontrado.
|
|
"""
|
|
|
|
# Comprobamos que sea un sistema linux
|
|
if ogGetOsType(str_repo, str_path_image) != "Linux":
|
|
ogRaiseError(OG_ERR_NOTFOUND, "Linux system.")
|
|
return
|
|
|
|
# Comprobamos que existe archivo de información de la imagen
|
|
INFOFILE = ogGetPath(str_repo, f".{str_path_image}.img.jsonl")
|
|
if INFOFILE == "":
|
|
return ogRaiseError(OG_ERR_NOTFOUND, INFOFILE)
|
|
|
|
# Comprobamos que exista la partición
|
|
MNTDIR = ogMount(int_ndisk, int_part)
|
|
if MNTDIR is None:
|
|
ogRaiseError(OG_ERR_NOTFOUND, f"Device {str_repo} {str_path_image}")
|
|
return
|
|
|
|
DEVICE = ogDiskToDev(int_ndisk, int_part)
|
|
UUID = subprocess.check_output(["blkid", "-o", "value", "-s", "UUID", DEVICE]).decode().strip()
|
|
OLDUUID = subprocess.check_output(["jq", ".uuid", INFOFILE]).decode().strip().replace('"', '')
|
|
|
|
# Para sistemas UEFI también cambio los archivos de la ESP
|
|
if ogIsEfiActive():
|
|
GRUBEFI = ogMount(ogGetEsp())
|
|
GRUBEFI = f"{GRUBEFI}/boot/grubMBR/boot/grub/grub.cfg"
|
|
else:
|
|
GRUBEFI = ""
|
|
|
|
# Obtenemos el dispositivo en formato del grub
|
|
grub_probe = os.environ.get("grub_probe", f"{OGBIN}/grub-probe1.99_{os.uname().machine}")
|
|
ROOT = subprocess.check_output([grub_probe, "--device", ogDiskToDev(int_ndisk, int_part), "--target=drive"]).decode().strip().replace('()', "''")
|
|
|
|
# Cambiamos UUID en la configuración (fstab y grub)
|
|
files = [
|
|
f"{MNTDIR}/etc/fstab",
|
|
f"{MNTDIR}/{{,boot/}}{GRUBEFI}",
|
|
f"{MNTDIR}/{{,boot/}}{{grubMBR,grubPARTITION}}/boot/grub{{,2}}/{{menu.lst,grub.cfg}}",
|
|
f"{GRUBEFI}"
|
|
]
|
|
for f in files:
|
|
if os.path.isfile(f):
|
|
with open(f, "r+") as file:
|
|
content = file.read()
|
|
content = content.replace(OLDUUID, UUID).replace("'hd.,gpt.'", ROOT).replace("hd.,msdos.", ROOT)
|
|
file.seek(0)
|
|
file.write(content)
|
|
file.truncate()
|
|
|
|
def initRepo(self, device, repo_name):
|
|
|
|
self._unmount_device(device)
|
|
path = os.path.join("/mnt", os.path.basename(device))
|
|
|
|
|
|
self.logger.debug(f"Will mount repo at {path}")
|
|
if not os.path.exists(path):
|
|
os.path.mkdir(path)
|
|
|
|
|
|
|
|
if self._filesystem_type(device) == "ntfs":
|
|
self.logger.debug("Handing a NTFS filesystem")
|
|
|
|
self._modprobe("ntfs3")
|
|
self._ntfsfix(device)
|
|
|
|
ntfs = NTFSLibrary(self.ntfs_implementation)
|
|
ntfs.mount_filesystem(device, path)
|
|
|
|
else:
|
|
self._mount(device, path)
|
|
|
|
|
|
self.logger.info("Initializing repository: " + path)
|
|
|
|
git_dir = os.path.join(path, ".git")
|
|
real_git_dir = os.path.join(self.cache_dir, f"git-{repo_name}")
|
|
|
|
|
|
if os.path.exists(real_git_dir):
|
|
self.logger.debug(f"Removing existing repository {real_git_dir}")
|
|
shutil.rmtree(real_git_dir)
|
|
|
|
if os.path.exists(git_dir) or os.path.islink(git_dir):
|
|
if os.path.islink(git_dir) or os.path.isfile(git_dir):
|
|
self.logger.debug(f"Removing gitdir: {git_dir}")
|
|
os.unlink(git_dir)
|
|
elif os.path.isdir(git_dir):
|
|
# We want to host git in the cache partition, .git shouldn't be a directory under the
|
|
# filesystem.
|
|
self.logger.warning(f"Removing directory-type gitdir, this should be a link or a file: {git_dir}")
|
|
shutil.rmtree(git_dir)
|
|
else:
|
|
raise RuntimeError("Git dir is of an unrecognized file type!")
|
|
|
|
# if not os.path.exists(git_dir):
|
|
#self.logger.debug("Creating " + git_dir)
|
|
#with open(git_dir, "w") as git_dir:
|
|
# git_dir.write(f"gitdir: {real_git_dir}\n")
|
|
|
|
self.logger.debug(f"Initializing repo in cache at {real_git_dir}")
|
|
#os.mkdir(real_git_dir)
|
|
#with git.Repo.init(real_git_dir, bare=True) as temprepo:
|
|
# self._configure_repo(temprepo)
|
|
|
|
os.symlink(real_git_dir, git_dir)
|
|
|
|
|
|
repo = git.Repo.init(path)
|
|
self._configure_repo(repo)
|
|
self._write_ignore_list(path)
|
|
metadata_ret = self._create_metadata(path)
|
|
|
|
self.logger.debug(f"Building list of files to add from path {path}")
|
|
|
|
add_files = []
|
|
|
|
# Nota: repo.index.add(".") agrega archivos pero git después cree que
|
|
# no han sido agregados?
|
|
for ent in os.listdir(path):
|
|
if repo.ignored(ent) or ent == ".git" or ent == "." or ent == "..":
|
|
self.logger.debug(f"Ignoring: {ent}")
|
|
elif ent in self.fully_ignored_dirs:
|
|
# FIXME: repo.index.add tiene un bug que ignora Force=true
|
|
#repo.index.add("dev/.opengnsys-keep")
|
|
self.logger.debug("Fully ignored dir: {ent}")
|
|
add_files.append(f"{ent}/.opengnsys-keep")
|
|
else:
|
|
self.logger.debug(f"Adding: {ent}")
|
|
add_files.append(ent)
|
|
#repo.index.add(ent, force=False)
|
|
|
|
|
|
for lnk in metadata_ret['symlinks']:
|
|
self.logger.debug(f"Adding symlink: {lnk}")
|
|
add_files.append(lnk)
|
|
|
|
self.logger.debug("Adding %d files" % len(add_files))
|
|
with OperationTimer(self, "add all files"):
|
|
subprocess.run(["git", "add"] + add_files, check=True, cwd=path)
|
|
#repo.index.add(items = add_files, force=True, )
|
|
|
|
# FIXME: This shouldn't actually happen, we shouldn't have any untracked files
|
|
if self.debug_check_for_untracked_files:
|
|
self.logger.info("Checking for untracked files...")
|
|
|
|
with OperationTimer(self, "add untracked files"):
|
|
untracked_list = repo.untracked_files
|
|
if untracked_list:
|
|
self.logger.warning(f"Untracked files: {untracked_list}")
|
|
self.logger.warning("Adding %d untracked files" % len(untracked_list))
|
|
#repo.index.add(items = untracked_list, force=True)
|
|
subprocess.run(["git", "add"] + untracked_list, check=True, cwd=path)
|
|
|
|
|
|
self.logger.debug("Committing")
|
|
repo.index.commit("Initial commit")
|
|
|
|
# Restaurar cosas modificadas para git
|
|
self._restore_metadata(path, destructive_only=True)
|
|
|
|
|
|
#self.logger.debug("Commit done, will unmount now")
|
|
#self._umount_device(device)
|
|
self._rmmod("ntfs3")
|
|
|
|
|
|
repo_url = self._getOgRepository(repo_name)
|
|
self.logger.debug(f"Creating remote origin: {repo_url}")
|
|
|
|
if "origin" in repo.remotes:
|
|
repo.delete_remote("origin")
|
|
|
|
origin = repo.create_remote("origin", repo_url)
|
|
|
|
self.logger.debug("Fetching origin")
|
|
origin.fetch()
|
|
# repo.create_head
|
|
# repo.heads.master.set_tracking_branch(origin.refs.master)
|
|
|
|
self.logger.debug("Uploading to ogrepository")
|
|
repo.git.push("--set-upstream", "origin", repo.head.ref, "--force") # force = True)
|
|
|
|
def cloneRepo(self, repo_name, destination, boot_device):
|
|
self.logger.info(f"Cloning repo: {repo_name} => {destination}")
|
|
|
|
|
|
repo_url = self._getOgRepository(repo_name)
|
|
self.logger.debug(f"URL: {repo_url}")
|
|
|
|
all_metadata = self._get_repo_metadata(repo_name)
|
|
metadata = all_metadata["metadata.json"]
|
|
fs_data = all_metadata["filesystems.json"]
|
|
|
|
|
|
if len(fs_data.keys()) == 0:
|
|
raise RequirementException("El repositorio contiene metadatos incorrectos. Falta información sobre sistemas de archivos en filesystems.json.")
|
|
|
|
if not "efi_boot" in metadata:
|
|
raise RequirementException("El repositorio contiene metadatos incorrectos. Falta información de metadatos en metadata.json")
|
|
|
|
repo_is_efi = metadata["efi_boot"]
|
|
efi = self._is_efi()
|
|
|
|
self.logger.debug(f"Repository made for EFI: {repo_is_efi}")
|
|
self.logger.debug(f"Our system using EFI : {efi}")
|
|
|
|
if repo_is_efi != efi:
|
|
raise RequirementException("Repositorio usa sistema de arranque incompatible con sistema actual")
|
|
|
|
|
|
filesystem_map = {"/" : destination}
|
|
|
|
self._create_filesystems(fs_data, filesystem_map)
|
|
|
|
destination_dir = "/mnt/repo-" + repo_name
|
|
|
|
self._mount(destination, destination_dir)
|
|
|
|
self._delete_contents(destination_dir)
|
|
|
|
repo = git.Repo.clone_from(repo_url, destination_dir)
|
|
|
|
if repo_is_efi:
|
|
self._efi_install(boot_device, destination_dir)
|
|
else:
|
|
self._grub_install(boot_device, destination_dir)
|
|
|
|
self._mklostandfound(destination_dir)
|
|
self._restore_metadata(destination_dir)
|
|
|
|
|
|
def commitRepo(self, path):
|
|
"""
|
|
Commit all current changes to the local data
|
|
"""
|
|
repo = git.Repo(path)
|
|
|
|
metadata_ret = self._create_metadata(path)
|
|
|
|
# Restaurar cosas modificadas para git
|
|
self._restore_metadata(path, destructive_only=True)
|
|
|
|
|
|
def restoreRepo(self, path):
|
|
"""
|
|
Restore the repository to the state it had before the non-committed modifications
|
|
"""
|
|
repo = git.Repo(path)
|
|
|
|
repo.head.reset(index=True, working_tree=True)
|
|
|
|
# Restaurar cosas modificadas para git
|
|
self._restore_metadata(path, destructive_only=True)
|
|
|
|
def pushRepo(self, path):
|
|
"""
|
|
Push local changes to ogrepository
|
|
|
|
Use commitRepo() first to save local changes.
|
|
"""
|
|
repo = git.Repo(path)
|
|
|
|
self.logger.debug("Uploading to ogrepository")
|
|
repo.git.push("--set-upstream", "origin", repo.head.ref, "--force") # force = True)
|
|
|
|
def pullRepo(self, path):
|
|
"""
|
|
Pull changes from ogrepository
|
|
|
|
This unconditionally overwrites remote changes. There is no conflict resolution.
|
|
"""
|
|
repo = git.Repo(path)
|
|
|
|
self.logger.debug("Downloading from ogrepository")
|
|
repo.git.fetch()
|
|
repo.head.reset(index=True, working_tree=True)
|
|
|
|
# Restaurar cosas modificadas para git
|
|
self._restore_metadata(path, destructive_only=True)
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
# python no cree que nuestra consola usa utf-8.
|
|
# esto arregla las tildes y las eñes
|
|
sys.stdout.reconfigure(encoding='utf-8')
|
|
|
|
logger = logging.getLogger(__package__)
|
|
logger.setLevel(logging.DEBUG)
|
|
|
|
streamLog = logging.StreamHandler()
|
|
streamLog.setLevel(logging.DEBUG)
|
|
|
|
fileLog = logging.FileHandler("gitlib.log")
|
|
fileLog.setLevel(logging.DEBUG)
|
|
|
|
formatter = logging.Formatter('%(asctime)s - %(name)24s - [%(levelname)5s] - %(message)s')
|
|
|
|
streamLog.setFormatter(formatter)
|
|
fileLog.setFormatter(formatter)
|
|
|
|
logger.addHandler(streamLog)
|
|
logger.addHandler(fileLog)
|
|
|
|
logger.info("Inicio del programa")
|
|
|
|
parser = argparse.ArgumentParser(
|
|
prog="OpenGnsys Git Library",
|
|
description="Funciones de Git",
|
|
)
|
|
|
|
#parser.add_argument("--init-repo", type=str, metavar='DIR', help="Inicializar repositorio desde DIR")
|
|
parser.add_argument("--init-repo-from", type=str, metavar='DEV', help="Inicializar repositorio desde DEV")
|
|
parser.add_argument("--clone-repo-to", type=str, metavar='DIR', help="Clonar repositorio a DIR. Elimina todos los datos en ruta destino!")
|
|
parser.add_argument("--repo", type=str, help="Repositorio en ogrepository (linux, windows, mac)")
|
|
parser.add_argument("--boot-device", type=str, help="Dispositivo de arranque")
|
|
parser.add_argument("--commit", type=str, metavar='DIR', help="Commit de cambios en el directorio")
|
|
parser.add_argument("--restore", type=str, metavar='DIR', help="Eliminar cambios en el directorio")
|
|
parser.add_argument("--push", type=str, metavar='DIR', help="Subir cambios a ogrepository")
|
|
parser.add_argument("--pull", type=str, metavar='DIR', help="Bajar cambios de ogrepository")
|
|
|
|
parser.add_argument("--ntfs-type", type=str, metavar="FS", help="Tipo de NTFS, 'kernel' o 'fuse'")
|
|
parser.add_argument("--test-metadata", type=str, metavar="DIR", help="Test metadata generation")
|
|
parser.add_argument("--test-restore-metadata", type=str, metavar="DIR", help="Test metadata restoration")
|
|
parser.add_argument("--test-restore-metadata-destructive", type=str, metavar="DIR", help="Test metadata restoration, destructive parts only")
|
|
parser.add_argument("--test-clone-metadata", type=str, metavar="REPO", help="Test metadata cloning")
|
|
|
|
|
|
args = parser.parse_args()
|
|
|
|
|
|
|
|
logger.debug("Inicio")
|
|
|
|
ntfs_impl = NTFSImplementation.Kernel
|
|
|
|
if args.ntfs_type == "kernel":
|
|
ntfs_impl = NTFSImplementation.Kernel
|
|
elif args.ntfs_type == "fuse":
|
|
ntfs_impl = NTFSImplementation.NTFS3G
|
|
else:
|
|
raise ValueError("Unknown NTFS implementation: {args.ntfs_type}")
|
|
|
|
|
|
og_git = OpengnsysGitLibrary(ntfs_implementation = ntfs_impl)
|
|
# og_git._runBashFunction("ogMountCache", [])
|
|
|
|
|
|
# if args.init_repo:
|
|
# #og_git.initRepo("/mnt/sda1", "linux")
|
|
# with OperationTimer(og_git, "git init"):
|
|
# og_git.initRepo(args.init_repo, args.repo)
|
|
if args.init_repo_from:
|
|
with OperationTimer(og_git, "git init"):
|
|
og_git.initRepo(args.init_repo_from, args.repo)
|
|
elif args.clone_repo_to:
|
|
#og_git.cloneRepo("linux", "/opt/opengnsys/cache/cloned")
|
|
with OperationTimer(og_git, "git clone"):
|
|
og_git.cloneRepo(args.repo, args.clone_repo_to, args.boot_device)
|
|
#og_git._restore_metadata("/opt/opengnsys/cache/cloned")
|
|
#og_git._restore_metadata(args.clone_repo_to)
|
|
elif args.commit:
|
|
with OperationTimer(og_git, "git commit"):
|
|
og_git.commitRepo(args.commit)
|
|
elif args.restore:
|
|
with OperationTimer(og_git, "git restore"):
|
|
og_git.restoreRepo(args.restore)
|
|
elif args.push:
|
|
with OperationTimer(og_git, "git push"):
|
|
og_git.pushRepo(args.push)
|
|
elif args.pull:
|
|
with OperationTimer(og_git, "git pull"):
|
|
og_git.pullRepo(args.pull)
|
|
elif args.test_metadata:
|
|
og_git._create_metadata(args.test_metadata)
|
|
elif args.test_restore_metadata:
|
|
og_git._restore_metadata(args.test_restore_metadata)
|
|
elif args.test_restore_metadata_destructive:
|
|
og_git._restore_metadata(path = args.test_restore_metadata_destructive, destructive_only=True)
|
|
elif args.test_clone_metadata:
|
|
og_git._get_repo_metadata(args.test_clone_metadata)
|
|
else:
|
|
print("Debe especificar una acción")
|
|
#
|
|
|
|
|
|
|
|
|