From 594314161e0bc7d34f3b5836fbbb97a820832a1e Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Mon, 16 Jun 2025 15:58:54 +0200 Subject: [PATCH] refs #2227 add oggit scripts and libs --- CHANGELOG.md | 6 + ogclient/interfaceAdm/CrearImagenGit.py | 38 + ogclient/interfaceAdm/RestaurarImagenGit.py | 32 + ogclient/lib/python3/bcd.py | 345 ++++ ogclient/lib/python3/disk.py | 124 ++ ogclient/lib/python3/filesystem.py | 544 ++++++ ogclient/lib/python3/gitlib-tests.py | 52 + ogclient/lib/python3/gitlib.py | 1731 +++++++++++++++++++ ogclient/lib/python3/kernel.py | 22 + ogclient/lib/python3/ntfs.py | 111 ++ ogclient/lib/python3/test.py | 13 + 11 files changed, 3018 insertions(+) create mode 100755 ogclient/interfaceAdm/CrearImagenGit.py create mode 100755 ogclient/interfaceAdm/RestaurarImagenGit.py create mode 100755 ogclient/lib/python3/bcd.py create mode 100644 ogclient/lib/python3/disk.py create mode 100644 ogclient/lib/python3/filesystem.py create mode 100755 ogclient/lib/python3/gitlib-tests.py create mode 100755 ogclient/lib/python3/gitlib.py create mode 100644 ogclient/lib/python3/kernel.py create mode 100644 ogclient/lib/python3/ntfs.py create mode 100644 ogclient/lib/python3/test.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f2c2910..8818ebe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.18.0] - 2025-06-16 + +### Added + +- Added scripts and libs for oggit + ## [0.17.0] - 2025-06-16 ### Changed diff --git a/ogclient/interfaceAdm/CrearImagenGit.py b/ogclient/interfaceAdm/CrearImagenGit.py new file mode 100755 index 0000000..06a5c45 --- /dev/null +++ b/ogclient/interfaceAdm/CrearImagenGit.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +import os +import subprocess +import sys +import time + + +sys.path.insert(0, "/opt/oglive/rootfs/opt/opengnsys/lib/python3/") +sys.path.insert(0, "/opt/opengnsys/interfaceAdm/git/") +sys.path.insert(0, "/opt/opengnsys/ogrepository/oggit/lib/") + +import NetLib +import ogGlobals +import SystemLib + + +from gitlib import OpengnsysGitLibrary, NTFSImplementation + + +def create_image(disk_num, partition_num, repo, image_name): + + ntfs_impl = NTFSImplementation.NTFS3G + og_git = OpengnsysGitLibrary(ntfs_implementation = ntfs_impl) + device = og_git._runBashFunction("ogDiskToDev", [str(disk_num), str(partition_num)]) + og_git.initRepo(device, image_name) + +def main(): + if len(sys.argv) != 6: + sys.exit(SystemLib.ogRaiseError(OG_ERR_FORMAT, "Incorrect number of arguments")) + + disk_num, partition_num, image_name, repo, tag = sys.argv[1:6] + + retval = create_image(disk_num, partition_num, repo, image_name) + + sys.exit(retval) + +if __name__ == "__main__": + main() diff --git a/ogclient/interfaceAdm/RestaurarImagenGit.py b/ogclient/interfaceAdm/RestaurarImagenGit.py new file mode 100755 index 0000000..1ab9f37 --- /dev/null +++ b/ogclient/interfaceAdm/RestaurarImagenGit.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +import sys +import subprocess + + +sys.path.insert(0, "/opt/oglive/rootfs/opt/opengnsys/lib/python3/") +sys.path.insert(0, "/opt/opengnsys/interfaceAdm/git/") +sys.path.insert(0, "/opt/opengnsys/ogrepository/oggit/lib/") + +import NetLib +import ogGlobals +import SystemLib +from gitlib import OpengnsysGitLibrary, NTFSImplementation + +if __name__ == "__main__": + if len(sys.argv) < 4: + print("Usage: python RestaurarImagenGit.py ") + sys.exit(1) + + disk = sys.argv[1] + partition = sys.argv[2] + repo = sys.argv[3] + boot_device = sys.argv[4] + + ntfs_impl = NTFSImplementation.NTFS3G + og_git = OpengnsysGitLibrary(ntfs_implementation = ntfs_impl) + + device = og_git._runBashFunction("ogDiskToDev", [str(disk), str(partition)]) + + og_git.cloneRepo(repo, device, boot_device) + + diff --git a/ogclient/lib/python3/bcd.py b/ogclient/lib/python3/bcd.py new file mode 100755 index 0000000..40d1280 --- /dev/null +++ b/ogclient/lib/python3/bcd.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python3 +import hivex +import argparse +import struct + +from hivex import Hivex +from hivex.hive_types import * + + +# Docs: +# +# https://www.geoffchappell.com/notes/windows/boot/bcd/objects.htm +# https://learn.microsoft.com/en-us/previous-versions/windows/desktop/bcd/bcdbootmgrelementtypes + +#print(f"Root: {root}") + + +BCD_Enumerations = { + "BcdLibraryDevice_ApplicationDevice" : 0x11000001, + "BcdLibraryString_ApplicationPath" : 0x12000002, + "BcdLibraryString_Description" : 0x12000004, + "BcdLibraryString_PreferredLocale" : 0x12000005, + "BcdLibraryObjectList_InheritedObjects" : 0x14000006, + "BcdLibraryInteger_TruncatePhysicalMemory" : 0x15000007, + "BcdLibraryObjectList_RecoverySequence" : 0x14000008, + "BcdLibraryBoolean_AutoRecoveryEnabled" : 0x16000009, + "BcdLibraryIntegerList_BadMemoryList" : 0x1700000a, + "BcdLibraryBoolean_AllowBadMemoryAccess" : 0x1600000b, + "BcdLibraryInteger_FirstMegabytePolicy" : 0x1500000c, + "BcdLibraryInteger_RelocatePhysicalMemory" : 0x1500000D, + "BcdLibraryInteger_AvoidLowPhysicalMemory" : 0x1500000E, + "BcdLibraryBoolean_DebuggerEnabled" : 0x16000010, + "BcdLibraryInteger_DebuggerType" : 0x15000011, + "BcdLibraryInteger_SerialDebuggerPortAddress" : 0x15000012, + "BcdLibraryInteger_SerialDebuggerPort" : 0x15000013, + "BcdLibraryInteger_SerialDebuggerBaudRate" : 0x15000014, + "BcdLibraryInteger_1394DebuggerChannel" : 0x15000015, + "BcdLibraryString_UsbDebuggerTargetName" : 0x12000016, + "BcdLibraryBoolean_DebuggerIgnoreUsermodeExceptions" : 0x16000017, + "BcdLibraryInteger_DebuggerStartPolicy" : 0x15000018, + "BcdLibraryString_DebuggerBusParameters" : 0x12000019, + "BcdLibraryInteger_DebuggerNetHostIP" : 0x1500001A, + "BcdLibraryInteger_DebuggerNetPort" : 0x1500001B, + "BcdLibraryBoolean_DebuggerNetDhcp" : 0x1600001C, + "BcdLibraryString_DebuggerNetKey" : 0x1200001D, + "BcdLibraryBoolean_EmsEnabled" : 0x16000020, + "BcdLibraryInteger_EmsPort" : 0x15000022, + "BcdLibraryInteger_EmsBaudRate" : 0x15000023, + "BcdLibraryString_LoadOptionsString" : 0x12000030, + "BcdLibraryBoolean_DisplayAdvancedOptions" : 0x16000040, + "BcdLibraryBoolean_DisplayOptionsEdit" : 0x16000041, + "BcdLibraryDevice_BsdLogDevice" : 0x11000043, + "BcdLibraryString_BsdLogPath" : 0x12000044, + "BcdLibraryBoolean_GraphicsModeDisabled" : 0x16000046, + "BcdLibraryInteger_ConfigAccessPolicy" : 0x15000047, + "BcdLibraryBoolean_DisableIntegrityChecks" : 0x16000048, + "BcdLibraryBoolean_AllowPrereleaseSignatures" : 0x16000049, + "BcdLibraryString_FontPath" : 0x1200004A, + "BcdLibraryInteger_SiPolicy" : 0x1500004B, + "BcdLibraryInteger_FveBandId" : 0x1500004C, + "BcdLibraryBoolean_ConsoleExtendedInput" : 0x16000050, + "BcdLibraryInteger_GraphicsResolution" : 0x15000052, + "BcdLibraryBoolean_RestartOnFailure" : 0x16000053, + "BcdLibraryBoolean_GraphicsForceHighestMode" : 0x16000054, + "BcdLibraryBoolean_IsolatedExecutionContext" : 0x16000060, + "BcdLibraryBoolean_BootUxDisable" : 0x1600006C, + "BcdLibraryBoolean_BootShutdownDisabled" : 0x16000074, + "BcdLibraryIntegerList_AllowedInMemorySettings" : 0x17000077, + "BcdLibraryBoolean_ForceFipsCrypto" : 0x16000079, + + + "BcdBootMgrObjectList_DisplayOrder" : 0x24000001, + "BcdBootMgrObjectList_BootSequence" : 0x24000002, + "BcdBootMgrObject_DefaultObject" : 0x23000003, + "BcdBootMgrInteger_Timeout" : 0x25000004, + "BcdBootMgrBoolean_AttemptResume" : 0x26000005, + "BcdBootMgrObject_ResumeObject" : 0x23000006, + "BcdBootMgrObjectList_ToolsDisplayOrder" : 0x24000010, + "BcdBootMgrBoolean_DisplayBootMenu" : 0x26000020, + "BcdBootMgrBoolean_NoErrorDisplay" : 0x26000021, + "BcdBootMgrDevice_BcdDevice" : 0x21000022, + "BcdBootMgrString_BcdFilePath" : 0x22000023, + "BcdBootMgrBoolean_ProcessCustomActionsFirst" : 0x26000028, + "BcdBootMgrIntegerList_CustomActionsList" : 0x27000030, + "BcdBootMgrBoolean_PersistBootSequence" : 0x26000031, + + "BcdDeviceInteger_RamdiskImageOffset" : 0x35000001, + "BcdDeviceInteger_TftpClientPort" : 0x35000002, + "BcdDeviceInteger_SdiDevice" : 0x31000003, + "BcdDeviceInteger_SdiPath" : 0x32000004, + "BcdDeviceInteger_RamdiskImageLength" : 0x35000005, + "BcdDeviceBoolean_RamdiskExportAsCd" : 0x36000006, + "BcdDeviceInteger_RamdiskTftpBlockSize" : 0x36000007, + "BcdDeviceInteger_RamdiskTftpWindowSize" : 0x36000008, + "BcdDeviceBoolean_RamdiskMulticastEnabled" : 0x36000009, + "BcdDeviceBoolean_RamdiskMulticastTftpFallback" : 0x3600000A, + "BcdDeviceBoolean_RamdiskTftpVarWindow" : 0x3600000B, + + "BcdMemDiagInteger_PassCount" : 0x25000001, + "BcdMemDiagInteger_FailureCount" : 0x25000003, + + "Reserved1" : 0x21000001, + "Reserved2" : 0x22000002, + "BcdResumeBoolean_UseCustomSettings" : 0x26000003, + "BcdResumeDevice_AssociatedOsDevice" : 0x21000005, + "BcdResumeBoolean_DebugOptionEnabled" : 0x26000006, + "BcdResumeInteger_BootMenuPolicy" : 0x25000008, + + "BcdOSLoaderDevice_OSDevice" : 0x21000001, + "BcdOSLoaderString_SystemRoot" : 0x22000002, + "BcdOSLoaderObject_AssociatedResumeObject" : 0x23000003, + "BcdOSLoaderBoolean_DetectKernelAndHal" : 0x26000010, + "BcdOSLoaderString_KernelPath" : 0x22000011, + "BcdOSLoaderString_HalPath" : 0x22000012, + "BcdOSLoaderString_DbgTransportPath" : 0x22000013, + "BcdOSLoaderInteger_NxPolicy" : 0x25000020, + "BcdOSLoaderInteger_PAEPolicy" : 0x25000021, + "BcdOSLoaderBoolean_WinPEMode" : 0x26000022, + "BcdOSLoaderBoolean_DisableCrashAutoReboot" : 0x26000024, + "BcdOSLoaderBoolean_UseLastGoodSettings" : 0x26000025, + "BcdOSLoaderBoolean_AllowPrereleaseSignatures" : 0x26000027, + "BcdOSLoaderBoolean_NoLowMemory" : 0x26000030, + "BcdOSLoaderInteger_RemoveMemory" : 0x25000031, + "BcdOSLoaderInteger_IncreaseUserVa" : 0x25000032, + "BcdOSLoaderBoolean_UseVgaDriver" : 0x26000040, + "BcdOSLoaderBoolean_DisableBootDisplay" : 0x26000041, + "BcdOSLoaderBoolean_DisableVesaBios" : 0x26000042, + "BcdOSLoaderBoolean_DisableVgaMode" : 0x26000043, + "BcdOSLoaderInteger_ClusterModeAddressing" : 0x25000050, + "BcdOSLoaderBoolean_UsePhysicalDestination" : 0x26000051, + "BcdOSLoaderInteger_RestrictApicCluster" : 0x25000052, + "BcdOSLoaderBoolean_UseLegacyApicMode" : 0x26000054, + "BcdOSLoaderInteger_X2ApicPolicy" : 0x25000055, + "BcdOSLoaderBoolean_UseBootProcessorOnly" : 0x26000060, + "BcdOSLoaderInteger_NumberOfProcessors" : 0x25000061, + "BcdOSLoaderBoolean_ForceMaximumProcessors" : 0x26000062, + "BcdOSLoaderBoolean_ProcessorConfigurationFlags" : 0x25000063, + "BcdOSLoaderBoolean_MaximizeGroupsCreated" : 0x26000064, + "BcdOSLoaderBoolean_ForceGroupAwareness" : 0x26000065, + "BcdOSLoaderInteger_GroupSize" : 0x25000066, + "BcdOSLoaderInteger_UseFirmwarePciSettings" : 0x26000070, + "BcdOSLoaderInteger_MsiPolicy" : 0x25000071, + "BcdOSLoaderInteger_SafeBoot" : 0x25000080, + "BcdOSLoaderBoolean_SafeBootAlternateShell" : 0x26000081, + "BcdOSLoaderBoolean_BootLogInitialization" : 0x26000090, + "BcdOSLoaderBoolean_VerboseObjectLoadMode" : 0x26000091, + "BcdOSLoaderBoolean_KernelDebuggerEnabled" : 0x260000a0, + "BcdOSLoaderBoolean_DebuggerHalBreakpoint" : 0x260000a1, + "BcdOSLoaderBoolean_UsePlatformClock" : 0x260000A2, + "BcdOSLoaderBoolean_ForceLegacyPlatform" : 0x260000A3, + "BcdOSLoaderInteger_TscSyncPolicy" : 0x250000A6, + "BcdOSLoaderBoolean_EmsEnabled" : 0x260000b0, + "BcdOSLoaderInteger_DriverLoadFailurePolicy" : 0x250000c1, + "BcdOSLoaderInteger_BootMenuPolicy" : 0x250000C2, + "BcdOSLoaderBoolean_AdvancedOptionsOneTime" : 0x260000C3, + "BcdOSLoaderInteger_BootStatusPolicy" : 0x250000E0, + "BcdOSLoaderBoolean_DisableElamDrivers" : 0x260000E1, + "BcdOSLoaderInteger_HypervisorLaunchType" : 0x250000F0, + "BcdOSLoaderBoolean_HypervisorDebuggerEnabled" : 0x260000F2, + "BcdOSLoaderInteger_HypervisorDebuggerType" : 0x250000F3, + "BcdOSLoaderInteger_HypervisorDebuggerPortNumber" : 0x250000F4, + "BcdOSLoaderInteger_HypervisorDebuggerBaudrate" : 0x250000F5, + "BcdOSLoaderInteger_HypervisorDebugger1394Channel" : 0x250000F6, + "BcdOSLoaderInteger_BootUxPolicy" : 0x250000F7, + "BcdOSLoaderString_HypervisorDebuggerBusParams" : 0x220000F9, + "BcdOSLoaderInteger_HypervisorNumProc" : 0x250000FA, + "BcdOSLoaderInteger_HypervisorRootProcPerNode" : 0x250000FB, + "BcdOSLoaderBoolean_HypervisorUseLargeVTlb" : 0x260000FC, + "BcdOSLoaderInteger_HypervisorDebuggerNetHostIp" : 0x250000FD, + "BcdOSLoaderInteger_HypervisorDebuggerNetHostPort" : 0x250000FE, + "BcdOSLoaderInteger_TpmBootEntropyPolicy" : 0x25000100, + "BcdOSLoaderString_HypervisorDebuggerNetKey" : 0x22000110, + "BcdOSLoaderBoolean_HypervisorDebuggerNetDhcp" : 0x26000114, + "BcdOSLoaderInteger_HypervisorIommuPolicy" : 0x25000115, + "BcdOSLoaderInteger_XSaveDisable" : 0x2500012b +} + + +def format_value(bcd, bcd_value): + + name = bcd.value_key(bcd_value) + (type, length) = bcd.value_type(bcd_value) + + typename = "" + str_value = "" + if type == REG_SZ: + typename = "SZ" + str_value = bcd.value_string(bcd_value) + elif type == REG_DWORD: + typename = "DWORD" + dval = bcd.value_dword(bcd_value) + + str_value = hex(dval) + " (" + str(bcd.value_dword(bcd_value)) + ")" + elif type == REG_BINARY: + typename = "BIN" + (length, value) = bcd.value_value(bcd_value) + str_value = value.hex() + elif type == REG_DWORD_BIG_ENDIAN: + typename = "DWORD_BE" + elif type == REG_EXPAND_SZ: + typename = "EXPAND SZ" + elif type == REG_FULL_RESOURCE_DESCRIPTOR: + typename = "RES DESC" + elif type == REG_LINK: + typename = "LINK" + elif type == REG_MULTI_SZ: + typename = "MULTISZ" + (length, str_value) = bcd.value_value(bcd_value) + str_value = str_value.decode('utf-16le') + str_value = str_value.replace("\0", ";") + #value = ";".join("\0".split(value)) + elif type == REG_NONE: + typename = "NONE" + elif type == REG_QWORD: + typename = "QWORD" + elif type == REG_RESOURCE_LIST: + typename = "RES LIST" + elif type == REG_RESOURCE_REQUIREMENTS_LIST: + typename = "REQ LIST" + else: + typename = str(type) + str_value = "???" + + + return (typename, length, str_value) + +def dump_all(root, depth = 0): + + padding = "\t" * depth + + children = bcd.node_children(root) + + if len(children) > 0: + + for child in children: + name = bcd.node_name(child) + print(f"{padding}{name}") + + dump_all(child, depth + 1) + # print(f"Child: {child}") + + #print(f"Values: {num_vals}") + return + + + + + + values = bcd.node_values(root) + #print(f"Value list: {values}") + + for v in values: + (type_name, length, str_value) = format_value(bcd, v) + name = bcd.value_key(v) + + print(f"{padding}{name: <16}: [{type_name: <10}]; ({length: < 4}) {str_value}") + + +class WindowsBCD: + def __init__(self, filename): + self.filename = filename + self.bcd = Hivex(filename) + + def dump(self, root=None, depth = 0): + padding = "\t" * depth + + if root is None: + root = self.bcd.root() + + children = self.bcd.node_children(root) + + if len(children) > 0: + for child in children: + name = self.bcd.node_name(child) + print(f"{padding}{name}") + + self.dump(child, depth + 1) + return + + values = self.bcd.node_values(root) + + for v in values: + (type_name, length, str_value) = format_value(self.bcd, v) + name = self.bcd.value_key(v) + + print(f"{padding}{name: <16}: [{type_name: <10}]; ({length: < 4}) {str_value}") + + def list(self): + root = self.bcd.root() + objects = self.bcd.node_get_child(root, "Objects") + + for child in self.bcd.node_children(objects): + entry_id = self.bcd.node_name(child) + + elements = self.bcd.node_get_child(child, "Elements") + description_entry = self.bcd.node_get_child(elements, "12000004") + + if description_entry: + values = self.bcd.node_values(description_entry) + if values: + (type_name, length, str_value) = format_value(self.bcd, values[0]) + print(f"{entry_id}: {str_value}") + else: + print(f"{entry_id}: [no description value!?]") + + + appdevice_entry = self.bcd.node_get_child(elements, "11000001") + + if appdevice_entry: + values = self.bcd.node_values(appdevice_entry) + (length, data) = self.bcd.value_value(values[0]) + hex = data.hex() + print(f"LEN: {length}, HEX: {hex}, RAW: {data}") + if len(data) > 10: + etype = struct.unpack_from('=2: + data = line.split() + disk = data[3] + disks.append(disk) + self.logger.debug(f"Disk: {disk}") + + line_num = line_num + 1 + + for disk in disks: + self.logger.debug("Loading partitions for disk %s", disk) + #disk_json_data = subprocess.run(["/usr/sbin/sfdisk", "-J", f"/dev/{disk}"], check=False, capture_output=True) + sfdisk_out = subprocess.run(["/usr/sbin/sfdisk", "-J", f"/dev/{disk}"], check=False, capture_output=True) + + if sfdisk_out.returncode == 0: + disk_json_data = sfdisk_out.stdout + disk_data = json.loads(disk_json_data) + + for part in disk_data["partitiontable"]["partitions"]: + self.logger.debug("Checking partition %s", part) + if part["type"] == "C12A7328-F81F-11D2-BA4B-00A0C93EC93B": + self.logger.debug("EFI partition found at %s", part["node"]) + return part["node"] + else: + self.logger.debug("sfdisk returned with code %i, error %s", sfdisk_out.returncode, sfdisk_out.stderr) + + + self.logger.warning("Failed to find EFI partition!") + + def temp_unmount(self, mountpoint): + """ + Temporarily unmounts the filesystem at the given mountpoint. + + This method finds the device associated with the specified mountpoint, + and returns the information to remount it with temp_remount. + + The purpose of this function is to temporarily unmount a filesystem for + actions like fsck, and to mount it back afterwards. + + Args: + mountpoint (str): The mountpoint of the filesystem to unmount. + + Returns: + dict: A dictionary containing the information needed to remount the filesystem. + """ + device = self.find_device(mountpoint) + fs = self.filesystem_type(mountpoint = mountpoint) + + data = {"mountpoint" : mountpoint, "device" :device, "filesystem" : fs} + + self.logger.debug("Temporarily unmounting device %s, mounted on %s, fs type %s", mountpoint, device, fs) + + self.unmount(mountpoint = mountpoint) + return data + + def temp_remount(self, unmount_data): + """ + Remounts a filesystem unmounted with temp_unmount + + This method remounts a filesystem using the data provided by temp_unmount + + Args: + unmount_data (dict): A dictionary containing the data needed to remount the filesystem. + + Returns: + None + """ + + self.logger.debug("Remounting temporarily unmounted device %s on %s, fs type %s", unmount_data["device"], unmount_data["mountpoint"], unmount_data["filesystem"]) + self.mount(device = unmount_data["device"], mountpoint=unmount_data["mountpoint"], filesystem=unmount_data["filesystem"]) diff --git a/ogclient/lib/python3/gitlib-tests.py b/ogclient/lib/python3/gitlib-tests.py new file mode 100755 index 0000000..09ee851 --- /dev/null +++ b/ogclient/lib/python3/gitlib-tests.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +import unittest +import logging +import os +import sys +import urllib.request +import tarfile +import subprocess +from shutil import rmtree +from pathlib import Path + + +parent_dir = str(Path(__file__).parent.parent.absolute()) +sys.path.append(parent_dir) +sys.path.append("/opengnsys/installer") +print(parent_dir) + +from gitlib import OpengnsysGitLibrary + + + + + +class GitTests(unittest.TestCase): + def setUp(self): + self.logger = logging.getLogger("OpengnsysTest") + self.oggit = OpengnsysGitLibrary() + + self.logger.info("setUp()") + if not hasattr(self, 'init_complete'): + self.init_complete = True + def test_init(self): + self.assertIsNotNone(self.oggit) + def test_acls(self): + self.oggit.ogCreateAcl() + + def test_sync_local(self): + # self.oggit.ogSyncLocalGitImage() + None + +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(name)20s - [%(levelname)5s] - %(message)s') + logger = logging.getLogger(__name__) + logger.setLevel(logging.DEBUG) + logger.info("Inicio del programa") + + unittest.main() + + + + + diff --git a/ogclient/lib/python3/gitlib.py b/ogclient/lib/python3/gitlib.py new file mode 100755 index 0000000..48dad07 --- /dev/null +++ b/ogclient/lib/python3/gitlib.py @@ -0,0 +1,1731 @@ +#!/usr/bin/env python3 + +# pylint: disable=locally-disabled, line-too-long, logging-fstring-interpolation, too-many-lines + +### NOTES: +# Install: +# python3-git +# python3.8 +# Must have working locales, or unicode strings will fail. Install 'locales', configure /etc/locale.gen, run locale-gen. +# +import os +import sys +sys.path.insert(0, "/usr/share/opengnsys-modules/python3/dist-packages") + + +import shutil + +import argparse +import tempfile +import logging +import subprocess + +import json + + +from pathlib import Path + + + +import base64 + +import stat +import time + + +import git +import libarchive +import xattr +import posix1e +import blkid +from filesystem import * +from disk import * +from ntfs import * +import re +import uuid +from tqdm import tqdm +from kernel import parse_kernel_cmdline + +class OgProgressPrinter(git.RemoteProgress): + """ + A class to print progress updates for Git operations. + + This class extends `git.RemoteProgress` to provide custom logging and + printing of progress updates to the standard error stream. + + Attributes: + logger (Logger): The logger instance used to log debug messages. + prev_len (int): The length of the previous status string printed. + + Methods: + __init__(parentLogger): + Initializes the OgProgressPrinter with a logger instance. + + update(op_code, cur_count, max_count=None, message=""): + Updates the progress status and prints it to the standard error stream. + + __del__(): + Ensures a newline is printed when the instance is deleted. + """ + def __init__(self, parentLogger): + super().__init__() + self.logger = parentLogger + + if sys.stdin.isatty(): + self.progress = tqdm() + self.progress.miniters = 1 + #self.progress.ascii = False + + def update(self, op_code, cur_count, max_count=None, message=""): + op = op_code & git.RemoteProgress.OP_MASK + stage = op_code & git.RemoteProgress.STAGE_MASK + + + op_text = "Unknown" + op_unit = "?" + + if op == git.RemoteProgress.COMPRESSING: + op_text = "Compressing" + op_unit = "Obj" + elif op == git.RemoteProgress.CHECKING_OUT: + op_text = "Checking out" + op_unit = "Obj" + elif op == git.RemoteProgress.COUNTING: + op_text = "Counting" + op_unit = "Obj" + elif op == git.RemoteProgress.RECEIVING: + op_text = "Receiving" + op_unit = "B" + elif op == git.RemoteProgress.WRITING: + op_text = "Writing" + op_unit = "B" + elif op == git.RemoteProgress.RESOLVING: + op_text = "Resolving deltas" + op_unit = "Obj" + + + + self.logger.debug(f"Progress: {op_code} {cur_count}/{max_count}: {message}") + + if max_count is None: + return + + if not self.progress is None: + self.progress.total = max_count + self.progress.n = cur_count + self.progress.desc = op_text #message + self.progress.unit = op_unit + self.progress.unit_scale = True + self.progress.refresh() + + def __del__(self): + print("\n", file=sys.stderr) + + +class OperationTimer: + def __init__(self, parent, operation_name): + self.operation_name = operation_name + self.parent = parent + self.start = time.time() + + def __enter__(self): + self.start = time.time() + + def __exit__(self, *args): + elapsed = round(time.time() - self.start, 3) + + self.parent.logger.info(f"{self.operation_name} took {elapsed}s") + +class RequirementException(Exception): + """Excepción que indica que nos falta algún requisito + Duplicado de git_installer. + + Attributes: + message (str): Mensaje de error mostrado al usuario + """ + + def __init__(self, message): + """Inicializar RequirementException. + + Args: + message (str): Mensaje de error mostrado al usuario + """ + super().__init__(message) + self.message = message + +class OpengnsysGitLibrary: + + """OpenGnsys Git Library""" + def __init__(self, require_cache = True, ntfs_implementation = NTFSImplementation.KERNEL): + """ + Initializes the Git library for OpenGnsys. + + Args: + require_cache (bool): Indicates whether a cache partition is required. Defaults to True. + ntfs_implementation (NTFSImplementation): Specifies the NTFS implementation to use. Defaults to NTFSImplementation.KERNEL. + + Raises: + RequirementException: If the cache partition is required but cannot be mounted. + + Attributes: + logger (logging.Logger): Logger instance for the Git library. + mounts (list): List of mounted filesystems. + repo_user (str): Username for the repository. + repo_image_path (str): Path to the repository images. + ntfs_implementation (NTFSImplementation): NTFS implementation being used. + cache_dir (str): Directory for the cache. + default_ignore_list (list): List of default paths to ignore. + fully_ignored_dirs (list): List of directories to fully ignore. + kernel_args (dict): Parsed kernel command line arguments. + repo_server (str): Server address for the repository. + debug_check_for_untracked_files (bool): Flag to check for untracked files for debugging purposes. + """ + + self.logger = logging.getLogger("OpengnsysGitLibrary") + self.logger.setLevel(logging.DEBUG) + self.logger.debug(f"Initializing. Cache = {require_cache}, ntfs = {ntfs_implementation}") + + self.fs = FilesystemLibrary(ntfs_implementation = ntfs_implementation) + self.disk = DiskLibrary() + #self.ntfs = NTFSLibrary() + + + #self.repo_server = "192.168.2.1" + self.repo_user = "oggit" + self.repo_image_path = "oggit" + self.ntfs_implementation = ntfs_implementation + + self.cache_dir = self._runBashFunction("ogMountCache", []) + + # Si no hay cache, se va a crear el .git en el FS directamente + if (not self.cache_dir) and require_cache: + raise RequirementException("Failed to mount cache partition. Cache partition may be missing.") + + self.default_ignore_list = [ + '/proc/*', # filesystem virtual + '!/proc/.opengnsys-keep', + '/sys/*', # filesystem virtual + '!/sys/.opengnsys-keep', + '/dev/*', # dispositivos -- no soportados por git + '!/dev/.opengnsys-keep', + '/run/*', # info temporal + '!/run/.opengnsys-keep', + '/var/run/*', # info temporal + '!/var/run/.opengnsys-keep', + '/tmp/*', # archivos temporales + '!/tmp/.opengnsys-keep', + '/var/tmp/*', # archivos temporales + '!/var/tmp/.opengnsys-keep', + '/mnt/*', # otros sistemas de archivos + '!/mnt/.opengnsys-keep', + "/lost+found", # lost+found es un directorio especial. Si es necesario lo recreamos + "/$Recycle.Bin", + "/$WinREAgent", + '/PerfLogs' + "/DumpStack.log.tmp", + "/pagefile.sys", + "/swapfile.sys", + "/Recovery", + "/System Volume Information" + + ] + + self.fully_ignored_dirs=[ + 'proc', + 'sys', + 'dev', + 'run', + 'var/run', + 'tmp', + 'var/tmp', + 'mnt', + '$Recycle.Bin', + '$WinREAgent', + 'PerfLogs', + 'Recovery', + 'System Volume Information' + ] + + """List of files to rename before commit. This is done for files that may interfere with Git, such as inner git repositories.""" + self.rename_list = [ + '.git', + '.gitignore', + '.gitattributes' + ] + + self.kernel_args = parse_kernel_cmdline() + self.repo_server = self.kernel_args["ogrepo"] + self.ip_address = self.kernel_args["ip"] + + + if not self.repo_server: + self.logger.warning("ogrepo kernel argument wasn't passed, or was empty. Defaulting to oglive.") + self.repo_server = self.kernel_args["oglive"] + + + """Add any untracked files the code might have missed. + This is a workaround for a bug and it comes with a significant + performance penalty. + """ + self.debug_check_for_untracked_files = True + + if not self.repo_server: + self.logger.error("Git repository not known, we may not have booted correctly? Check ogrepo and oglive kernel arguments.") + else: + self.logger.debug(f"Git repository: {self.repo_server}") + + def _is_efi(self): + """Determina si hemos arrancado con EFI + + Returns: + Bool: Si hemos arrancado con EFI + """ + return os.path.exists("/sys/firmware/efi") + + + def _write_ignore_list(self, base_path): + ignore_file = base_path + "/.gitignore" + + self.logger.debug("Creating ignore list: %s", ignore_file) + with open(ignore_file, 'w', encoding='utf-8') as f: + f.write("\n".join(self.default_ignore_list)) + f.write("\n") + + + + + + + def _ntfs_secaudit(self, data): + self.logger.debug(f"Saving NTFS metadata for {data['device']}") + + metadata_file = os.path.join(data["metadata_dir"], "ntfs_secaudit.txt") + + self.logger.debug(f"Unmounting {data['mountpoint']}...") + self.fs.unmount(mountpoint=data["mountpoint"]) + #subprocess.run(["/usr/bin/umount", data["mountpoint"]], check = True) + + result = subprocess.run(["/usr/bin/ntfssecaudit", "-b", data["device"]], check=True, capture_output=True) + + self.logger.debug(f"Remounting {data['device']} on {data['mountpoint']}...") + if data["mount_fs"] == "fuseblk": + self.logger.debug("Mount was FUSE") + subprocess.run(["/usr/bin/mount", data["device"], data["mountpoint"]], check=True) + else: + self.logger.debug(f"Mount was {data['mount_fs']}") + subprocess.run(["/usr/bin/mount", data["device"], "-t", data["mount_fs"], data["mountpoint"]], check=True) + + self.logger.debug("Writing NTFS audit metadata...") + with open(metadata_file + ".new", "w", encoding='utf-8') as meta: + meta.write(result.stdout.decode('utf-8')) + + os.rename(metadata_file + ".new", metadata_file) + + + def _ntfs_restore_secaudit(self, path): + self.logger.debug("Restoring NTFS metadata for %s", path) + + if not self.fs.is_filesystem(path): + self.logger.error("Path %s is not a filesystem!") + return + + if self.fs.filesystem_type(mountpoint = path) != "ntfs": + self.logger.error("Path %s is not NTFS!", path) + return + + metadata_file = os.path.join(path,".opengnsys-metadata", "ntfs_secaudit.txt") + secaudit_data = "" + + self.logger.debug("Reading audit metadata from %s...", metadata_file) + with open(metadata_file, "rb") as meta: + secaudit_data = meta.read() + self.logger.debug("Read %i bytes", len(secaudit_data)) + + device = self.fs.find_device(path) + mountdata = self.fs.temp_unmount(path) + + self.logger.info("Restoring secaudit data...") + result = subprocess.run(["/usr/bin/ntfssecaudit", "-se", device], check=False, capture_output=True, input=secaudit_data) + + if result.returncode == 0: + self.logger.debug("Completed, return code %i", result.returncode) + self.logger.debug("STDOUT: %s", result.stdout) + self.logger.debug("STDERR: %s", result.stderr) + else: + # An error return code can be returned for reasons like missing files, so we deal this + # as non-fatal. + self.logger.error("Completed, return code %i", result.returncode) + self.logger.error("STDOUT: %s", result.stdout) + self.logger.error("STDERR: %s", result.stderr) + + + self.fs.temp_remount(mountdata) + + + def _create_filesystems(self, fs_data, fs_map): + for mountpoint in fs_map: + dest_device = fs_map[mountpoint] + data = fs_data[mountpoint] + + self.fs.create_filesystem(device = dest_device, fs_type = data["type"], fs_uuid = data["uuid"]) + + + def _grub_install(self, root_directory, boot_device): + """ + Install GRUB boot loader on the specified boot device. + + This method checks for the presence of GRUB 2.x and GRUB 1.x installers + and attempts to install the appropriate version. If neither installer is + found, a RequirementException is raised. + + Args: + boot_device (str): The device on which to install the GRUB boot loader (e.g., '/dev/sda'). + root_directory (str): The root directory where GRUB files should be installed. + + Raises: + RequirementException: If neither GRUB 2.x nor GRUB 1.x installer is found. + subprocess.CalledProcessError: If the GRUB installation command fails. + + Logs: + Debug information about the installation process, including the return code, + stdout, and stderr of the GRUB installation command. + """ + if os.path.exists("/usr/sbin/grub2-install"): + self.logger.debug("Installing Grub 2.x (NOT IMPLEMENTED)") + elif os.path.exists("/usr/sbin/grub-install"): + self.logger.debug("Installing Grub 1.x") + + result = subprocess.run(["/usr/sbin/grub-install", "--force", "--root-directory", root_directory, boot_device], check = True, capture_output=True) + self.logger.debug(f"retorno: {result.returncode}") + self.logger.debug(f"stdout: {result.stdout}") + self.logger.debug(f"stderr: {result.stderr}") + else: + raise RequirementException("Couldn't find /usr/sbin/grub2-install or /usr/sbin/grub-install") + + def _get_system_uuid(self): + self.logger.debug("Obtaining system UUID...") + result = subprocess.run(["/usr/sbin/dmidecode", "-s", "system-uuid"], check=True, capture_output=True, encoding='utf-8') + uuid = result.stdout.strip() + self.logger.debug("UUID is %s", uuid) + return uuid + + def _efi_install(self, root_directory, config_name = None): + """ + Install EFI data on the specified boot device. + + Copies EFI data from a metadata directory within the root directory + to the specified boot device. It logs the process of installing the EFI data. + + Boot device is detected automatically + Args: + root_directory (str): The root directory containing the metadata and EFI data. + + Raises: + shutil.Error: If an error occurs during the copying of the EFI data. + """ + + boot_device = self.fs.find_boot_device() + boot_mount = self.fs.find_mountpoint(boot_device) + + self.logger.info(f"Installing EFI files in {boot_mount}") + meta_dir = os.path.join(root_directory, ".opengnsys-metadata") + efi_files_dir = os.path.join(meta_dir, "efi_data") + + if os.path.exists(efi_files_dir): + self.logger.debug("Copying EFI files") + shutil.copytree(efi_files_dir, boot_mount, dirs_exist_ok=True) + else: + self.logger.error("No general EFI files found") + + sys_efi_files_dir = "" + + if config_name: + self.logger.debug("Custom EFI config %s specified...", config_name) + sys_efi_files_dir = os.path.join(meta_dir, f"efi_data.{config_name}") + else: + uuid = self._get_system_uuid() + self.logger.debug("Checking if we have system-specific EFI data for system id %s...", uuid) + sys_efi_files_dir = os.path.join(meta_dir, f"efi_data.{uuid}") + + + if os.path.exists(sys_efi_files_dir): + self.logger.info("This system has specific EFI data, overriding default...") + shutil.copytree(sys_efi_files_dir, boot_mount, dirs_exist_ok=True) + else: + self.logger.debug("No system-specific EFI data.") + + def _efi_copy(self, root_directory, system_specific = False, config_name = None): + meta_dir = os.path.join(root_directory, ".opengnsys-metadata") + boot_device = self.fs.find_boot_device() + boot_mount = self.fs.find_mountpoint(boot_device) + efi_files_dir = "" + + + + if not system_specific: + self.logger.debug("Copying default EFI data") + efi_files_dir = os.path.join(meta_dir, "efi_data") + if os.path.exists(efi_files_dir): + shutil.rmtree(efi_files_dir) + + shutil.copytree(boot_mount, efi_files_dir) + else: + if config_name: + self.logger.debug("Copying EFI data for preset %s", config_name) + efi_files_dir = os.path.join(meta_dir, f"efi_data.{config_name}") + else: + uuid = self._get_system_uuid() + self.logger.debug("Copying EFI data for system %s", uuid) + efi_files_dir = os.path.join(meta_dir, f"efi_data.{uuid}") + + # TODO: On Windows we can probably get away with just copying: + # EFI/Microsoft/Boot/BCD* + + + if os.path.exists(efi_files_dir): + shutil.rmtree(efi_files_dir) + + shutil.copytree(boot_mount, efi_files_dir) + + + + + + def _delete_contents(self, path): + self.logger.info(f"Deleting contents of {path}") + + + for filename in os.listdir(path): + file_path = os.path.join(path, filename) + try: + self.logger.debug(f"Deleting {file_path}") + + if os.path.isfile(file_path) or os.path.islink(file_path): + os.unlink(file_path) + elif os.path.isdir(file_path): + shutil.rmtree(file_path) + except OSError as e: + self.logger.warning('Failed to delete %s. Error: %s', file_path, e) + + + def _runBashFunction(self, function, arguments): + """ + Executes an OpenGnsys bash function with given arguments. + + This method creates a temporary bash script that sources all `.lib` files in a specific directory, + writes the specified bash function and its arguments to the script, makes the script executable, + and then runs the script. The output and errors from the script execution are captured and logged. + + This is a temporary migration convenience function, it won't be present once the rest of the + code is migrated to Python. + + Args: + function (str): The name of the bash function to execute. + arguments (list): A list of arguments to pass to the bash function. + + Returns: + str: The standard output from the executed bash function. + + Logs: + - Debug information about the bash function and arguments being run. + - The path of the temporary file created. + - The command being executed. + - The standard output and standard error from the script execution. + """ + # Create a temporary file + self.logger.debug(f"Running bash function: {function} {arguments}") + + with tempfile.NamedTemporaryFile(mode='w', delete=False) as temp_file: + temp_file.write("#!/bin/bash\n") + temp_file.write("for lib in /opt/opengnsys/client/lib/engine/bin/*.lib ; do\n") + temp_file.write(" source $lib\n") + temp_file.write("done\n") + + #temp_file.write("source /opt/oglive/rootfs/opt/opengnsys/lib/engine/bin/Cache.lib") + #temp_file.write("source /opt/oglive/rootfs/opt/opengnsys/lib/engine/bin/Git.lib") + + temp_file.write(f"{function} \"$@\"\n") + + # Make the temporary file executable + os.chmod(temp_file.name, 0o755) + + self.logger.debug(f"File: {temp_file.name}") + + + # Run the temporary file + command = [temp_file.name] + arguments + self.logger.debug(f"Running: {command} {arguments}") + result = subprocess.run(command, shell=False, capture_output=True, text=True, check=True) + output = result.stdout.strip() + + self.logger.debug(f"STDOUT: {output}") + self.logger.debug(f"STDERR: {result.stderr}") + + return output + + + + def _getOgRepository(self, name): + return f"{self.repo_user}@{self.repo_server}:{self.repo_image_path}/{name}.git" + + + def _ogGetOsType(self): + return "Linux" + + def _get_repo_metadata(self, repo): + """Obtiene metadatos de un repositorio remoto sin clonar el repo entero. + + Esto resuelve el problema de tener metadatos del sistema de archivos en el propio repo, + sobre los que necesitamos actuar antes de poder clonarlo. + + Args: + repo (str): Nombre del repositorio (linux, windows, mac) + + Returns: + dict: Clave/valor con contenidos de filesystems.json y metadata.json + """ + results = {} + + tempdir = tempfile.TemporaryDirectory() + repo_url = self._getOgRepository(repo) + wanted_files = ["filesystems.json", "metadata.json"] + + self.logger.debug(f"Cloning metadata for repository {repo_url}") + + result = subprocess.run(["git", "archive", "--remote", repo_url, "HEAD:.opengnsys-metadata/"], capture_output=True, check=True) + tar_data = result.stdout + + + with libarchive.memory_reader(tar_data) as metadata: + self.logger.debug(f"Archive: {metadata}") + + for entry in metadata: + self.logger.debug(entry) + if entry.pathname in wanted_files: + self.logger.debug(f"Extracting {entry}") + data = bytearray() + for block in entry.get_blocks(): + data = data + block + + text = data.decode('utf-8') + results[entry.pathname] = json.loads(text) + self.logger.debug(f"Contents: {text}") + + return results + + def _create_metadata(self, path, initial_creation=False): + """Calculate metadata for a filesystem + + Here we traverse the entire filesystem to: + + 1. Find empty directories and fill them so that git preserves them. + 2. Obtain all ACLs. + 3. Obtain all extended attributes. + 4. Rename .gitignore files. + 5. Find mount points and obtain information about them. + 6. Additional metadata, such as the boot type. + 7. NTFS secaudit, which must be performed at the end of the process because the filesystem needs to be unmounted. + + For empty files, we generate a list that we can use later to delete the + .opengnsys-keep files. This is done because there are cases where an unexpected + file can cause problems. For example, sshfs by default refuses to mount things + in a directory that contains files. + + We rename the .gitignore files in subdirectories because git would apply them + to our process. + + We write all data in JSON to ensure there are no issues with spaces, line endings, + or other special characters. This also ensures one entry per line, which we can use + to speed up performance by using git to get the difference between a previous state + and the current one. + + Args: + path (str): Base path of the filesystem + initial_creation (bool): This is being called from the initial repository creation + """ + + self.logger.info(f"Creating metadata for {path}") + + return_data = { 'symlinks': [] } + + seen_roots = {} + filesystems_data = {} + ntfs_secaudit_list = [] + path_norm = os.path.normpath(path) + + + git_dir = os.path.normpath(os.path.join(path, ".git")) + + meta_dir = os.path.join(path, ".opengnsys-metadata") + + if not os.path.exists(meta_dir): + os.mkdir(meta_dir, mode=0o700) + + # https://jsonlines.org/ + + metadata_file = open(os.path.join(meta_dir, "metadata.json.new"), "w", encoding='utf-8') + metadata = {} + metadata["efi_boot"] = self._is_efi() + + if self._is_efi(): + # If we're doing the initial repository creation, then we're creating the initial, + # non-system-specific EFI data. + + self._efi_copy(root_directory=path, system_specific=not initial_creation) + + empties_file = open(os.path.join(meta_dir, "empty_directories.jsonl.new"), "w", encoding='utf-8') + specials_file = open(os.path.join(meta_dir, "special_files.jsonl.new"), "w", encoding='utf-8') + acls_file = open(os.path.join(meta_dir, "acls.jsonl.new"), "w", encoding='utf-8') + perms_file = open(os.path.join(meta_dir, "unix_permissions.jsonl.new"), "w", encoding='utf-8') + xattrs_file = open(os.path.join(meta_dir, "xattrs.jsonl.new"), "w", encoding='utf-8') + renamed_file = open(os.path.join(meta_dir, "renamed.jsonl.new"), "w", encoding='utf-8') + filesystems_file = open(os.path.join(meta_dir, "filesystems.json.new"), "w", encoding='utf-8') + partitions_file = open(os.path.join(meta_dir, "partitions.json.new"), "w", encoding='utf-8') + + + partitions_file.write(self.disk.get_disk_json_data(self.fs.find_device(mountpoint=path))) + + ntfs = False + + for root, subdirs, files in os.walk(path): + #print(f"ROOT: {root}, subdirs: {subdirs}, files: {files}") + + root_norm = os.path.normpath(root) + root_rel = root[len(path):None] + #print(f"A: {root}") + #print(f"B: {git_dir}") + + if root_norm.startswith(git_dir): + self.logger.debug(f"Ignoring git directory: {root_norm}") + # No examinamos el .git del raíz + continue + + if root_rel.startswith("/.opengnsys-metadata"): + self.logger.debug("Ignoring metadata directory: %s", root_norm) + continue + + + if not root in seen_roots: + seen_roots[root]=1 + + mount_found = False + + if root in self.fs.mounts: + mount = self.fs.mounts[root] + root_path_rel = root[len(path):None] + if len(root_path_rel) == 0 or root_path_rel[0] != "/": + root_path_rel = "/" + root_path_rel + + + + + + pr = blkid.Probe() + pr.set_device(mount['device']) + pr.enable_superblocks(True) + pr.set_superblocks_flags(blkid.SUBLKS_TYPE | blkid.SUBLKS_USAGE | blkid.SUBLKS_UUID | blkid.SUBLKS_UUIDRAW | blkid.SUBLKS_LABELRAW) + pr.do_safeprobe() + + + fs = mount['type'] + orig_fs = fs + + if mount['type'] == 'fuseblk': + fs = pr["TYPE"].decode('utf-8') + self.logger.warning(f"FUSE mount detected, replacing type with blkid detected type: {fs}") + self.logger.warning("FUSE has lower performance, a native mount is recommended.") + + filesystems_data[root_path_rel] = {} + filesystems_data[root_path_rel]['type'] = fs + filesystems_data[root_path_rel]['size'] = pr.size + filesystems_data[root_path_rel]['sectors'] = pr.sectors + filesystems_data[root_path_rel]['sector_size'] = pr.sector_size + filesystems_data[root_path_rel]['uuid'] = str(pr["UUID"], 'utf-8') + filesystems_data[root_path_rel]['uuid_raw'] = str(base64.b64encode(pr["UUID_RAW"]), 'utf-8') + filesystems_data[root_path_rel]["part_uuid"] = self.disk.get_partition_uuid(mount['device']) + filesystems_data[root_path_rel]["part_type"] = self.disk.get_partition_type(mount['device']) + filesystems_data[root_path_rel]["disk_uuid"] = self.disk.get_disk_uuid(mount['device']) + + + + # TODO: Esto de momento no funciona -- LABEL no se encuentra + #filesystems_data[root_path_rel]['label'] = pr["LABEL"] + #filesystems_data[root_path_rel]['label_raw'] = pr["LABEL_RAW"] + + self.logger.debug("Filesystem: {0}, relative {1}. Type {2}, size {3}, UUID {4}".format(root, root_path_rel, mount['type'], pr.size, str(pr["UUID"], 'utf-8'))) + + if fs == 'ntfs' or fs == 'ntfs3': + self.logger.info("NTFS detected, will do a secaudit") + ntfs_secaudit_list.append({ + 'device' : mount['device'], + 'mountpoint' : root, + 'relative_path' : root_path_rel, + 'mount_fs' : orig_fs, + 'metadata_dir' : meta_dir + }) + + #self._ntfs_secaudit(root, os.path.join(meta_dir, "ntfs_secaudit.txt")) + ntfs = True + else: + if root_norm == path_norm: + errstr = f"""We've failed to find metadata for the root filesystem! + Root: {root} + FS data: """ + errstr = errstr + json.dumps(self.fs.mounts, indent=4) + + self.logger.error(errstr) + raise RuntimeError(errstr) + + #self.logger.debug(f"Root rel: {root_rel}") + if len(files) == 1 and files[0] == ".opengnsys-keep": + # Si ya tenemos un .opengnsys-keep, lo ignoramos + files.pop(0) + + # Ignoramos todo el contenido en directorios como /dev, pero el directorio + # debe existir. + if (len(subdirs) == 0 and len(files) == 0) or (root_rel in self.fully_ignored_dirs): + keep_file = os.path.join(root, ".opengnsys-keep") + Path(keep_file).touch(mode=0o644, exist_ok = True) + + self.logger.debug(f"Empty directory: {root}") + #root_rel = root[len(path):None] + empties_file.write(json.dumps({"dir" : root_rel}) + "\n") + + for file in files: + + full_path = os.path.join(root,file) + full_path_rel = full_path[len(path):None] + + # Relative path can't start with a /, git will take it as an + # absolute path pointing to /, and not a file within the repo. + while full_path_rel[0] == '/': + full_path_rel = full_path_rel[1:None] + + #self.logger.debug(f"Checking {full_path}:") + + if not ntfs and os.path.isfile(full_path) and not os.path.islink(full_path): + # docs: https://pylibacl.k1024.org/module.html#posix1e.ACL.to_any_text + + # Git doesn't preserve setuid, we've got to store it separately + stat_data = os.stat(full_path) + + perms_json = json.dumps({ + "path" : full_path_rel, + "mode" : stat_data.st_mode, + "uid" : stat_data.st_uid, + "gid" : stat_data.st_gid + }) + + xattrs = str(xattr.get_all(full_path)) + acls = posix1e.ACL(file=full_path) + + xattrs_json = json.dumps({"file": full_path_rel, "xattrs" : xattrs}) + #acls_json = json.dumps({"file": full_path_rel, "acl" : str(acls.to_any_text())}) + + # __getstate__ nos permite exportar el estado entero de la ACL + # TODO: posiblemente formato de texto mejor? + acl_data = str(base64.b64encode(acls.__getstate__()), 'utf-8') + acls_json = json.dumps({"file": full_path_rel, "acl" : acl_data }) + + perms_file.write(perms_json + "\n") + xattrs_file.write(xattrs_json + "\n") + acls_file.write(acls_json + "\n") + + #self.logger.debug(f"Checking if {file} is ignored") + + + if os.path.exists(full_path): + if not os.path.islink(full_path): + stat_data = os.stat(full_path) + stat_mode = stat_data.st_mode + + if not (stat.S_ISDIR(stat_mode) or stat.S_ISREG(stat_mode) or stat.S_ISLNK(stat_mode)): + # Si no es un directorio o un archivo, conservamos los datos necesarios para recrearlo + + stat_json_data = { + "file" : full_path_rel, + "mode" : stat_mode, + "uid" : stat_data.st_uid, + "gid" : stat_data.st_gid, + "rdev" : stat_data.st_rdev + } + + stat_json = json.dumps(stat_json_data) + specials_file.write(stat_json + "\n") + + # Git falla al encontrarse con archivos especiales como dispositivos. De momento debemos + # eliminarlos. + os.unlink(full_path) + if os.path.islink(full_path): + self.logger.debug(f"Symlink: {full_path_rel}") + return_data['symlinks'].append(full_path_rel) + + if os.path.isfile(full_path) and file in self.rename_list and root != path: + # Process this last so that all the metadata references the real names. + + self.logger.debug(f"Found file to rename: {full_path}") + renamed_file_path = full_path + "-opengnsys-renamed" + renamed_file_path_rel = full_path_rel + "-opengnsys-renamed" + + + renamed_json = json.dumps({"path": full_path_rel, "renamed" : renamed_file_path_rel}) + renamed_file.write(renamed_json + "\n") + + os.rename(full_path, renamed_file_path) + + for subdir in subdirs: + full_path = os.path.join(root, subdir) + full_path_rel = full_path[len(path):None] + + # Relative path can't start with a /, git will take it as an + # absolute path pointing to /, and not a file within the repo. + while full_path_rel[0] == '/': + full_path_rel = full_path_rel[1:None] + + if os.path.isdir(full_path) and subdir in self.rename_list and root != path: + self.logger.debug(f"Found directory to rename: {full_path}") + renamed_dir_path = full_path + "-opengnsys-renamed" + renamed_dir_path_rel = full_path_rel + "-opengnsys-renamed" + + renamed_json = json.dumps({"path": full_path_rel, "renamed" : renamed_dir_path_rel}) + renamed_file.write(renamed_json + "\n") + + os.rename(full_path, renamed_dir_path) + + if not ntfs and os.path.isdir(full_path) and not os.path.islink(full_path): + stat_data = os.stat(full_path) + + perms_json = json.dumps({ + "path" : full_path_rel, + "mode" : stat_data.st_mode, + "uid" : stat_data.st_uid, + "gid" : stat_data.st_gid + }) + + xattrs = str(xattr.get_all(full_path)) + acls = posix1e.ACL(file=full_path) + + xattrs_json = json.dumps({"file": full_path_rel, "xattrs" : xattrs}) + + # __getstate__ nos permite exportar el estado entero de la ACL + acl_data = str(base64.b64encode(acls.__getstate__()), 'utf-8') + acls_json = json.dumps({"file": full_path_rel, "acl" : acl_data }) + + perms_file.write(perms_json + "\n") + xattrs_file.write(xattrs_json + "\n") + acls_file.write(acls_json + "\n") + + self.logger.debug("Finishing...") + + if len(filesystems_data) < 1: + self.logger.error("Filesystems data doesn't contain anything, this is probably a bug!") + + + filesystems_file.write(json.dumps(filesystems_data, indent=4) + "\n") + metadata_file.write(json.dumps(metadata, indent=4) + "\n") + + empties_file.close() + specials_file.close() + xattrs_file.close() + acls_file.close() + perms_file.close() + renamed_file.close() + filesystems_file.close() + metadata_file.close() + partitions_file.close() + + os.rename(os.path.join(meta_dir, "empty_directories.jsonl.new"), os.path.join(meta_dir, "empty_directories.jsonl")) + os.rename(os.path.join(meta_dir, "special_files.jsonl.new"), os.path.join(meta_dir, "special_files.jsonl")) + os.rename(os.path.join(meta_dir, "acls.jsonl.new"), os.path.join(meta_dir, "acls.jsonl")) + os.rename(os.path.join(meta_dir, "unix_permissions.jsonl.new"), os.path.join(meta_dir, "unix_permissions.jsonl")) + os.rename(os.path.join(meta_dir, "xattrs.jsonl.new"), os.path.join(meta_dir, "xattrs.jsonl")) + os.rename(os.path.join(meta_dir, "renamed.jsonl.new"), os.path.join(meta_dir, "renamed.jsonl")) + os.rename(os.path.join(meta_dir, "filesystems.json.new"), os.path.join(meta_dir, "filesystems.json")) + os.rename(os.path.join(meta_dir, "metadata.json.new"), os.path.join(meta_dir, "metadata.json")) + os.rename(os.path.join(meta_dir, "partitions.json.new"), os.path.join(meta_dir, "partitions.json")) + + + self.logger.debug("Processing pending NTFS secaudits...") + for audit in ntfs_secaudit_list: + self._ntfs_secaudit(audit) + + + self.logger.debug("Metadata updated") + return return_data + + def _restore_metadata(self, path, destructive_only=False, set_device_uuids=False): + """Restore the metadata created by _create_metadata + + Args: + path (str): Destination path + destructive_only (bool): Only restore what is modified during a commit + + Notes: + Git does not handle device or socket type files correctly. Therefore, + we must save data about them before the commit, and delete them before + git can see them and get confused. + + destructive_only=True only restores the metadata that we modify + in the real file system before the commit. This is done to leave the + file system in the same state it had before the commit. + """ + + self.logger.debug("Initializing") + + self.logger.info(f"Restoring metadata in {path}") + meta_dir = os.path.join(path, ".opengnsys-metadata") + + if not os.path.exists(meta_dir): + self.logger.error(f"Metadata directory not found: {meta_dir}") + return + + + if set_device_uuids: + # Windows boot manager uses partition UUIDs in at least some cases. One option to make booting work + # is to store all such UUIDs and restore them on the destination machine. + + self.logger.info("Processing filesystems.json") + with open(os.path.join(meta_dir, "filesystems.json"), "r", encoding='utf-8') as filesystems_file: + filesystems = json.loads(filesystems_file.read()) + + disk_device = self.fs.find_device(path) + + if "disk_uuid" in filesystems["/"]: + self.logger.info("Restoring device and partition UUIDs on %s", disk_device) + + prev_uuid = self.disk.get_disk_uuid(disk_device) + new_uuid = filesystems["/"]["disk_uuid"] + if new_uuid != prev_uuid: + self.logger.info("Setting disk UUID to %s (was %s)", new_uuid, prev_uuid) + self.disk.set_disk_uuid(disk_device, new_uuid) + else: + self.logger.info("Not setting disk UUID, already was correct") + + prev_uuid = self.disk.get_partition_uuid(disk_device) + new_uuid = filesystems["/"]["part_uuid"] + if new_uuid != prev_uuid: + self.logger.info("Setting partition UUID to %s (was %s)", new_uuid, prev_uuid) + self.disk.set_partition_uuid(disk_device, new_uuid) + else: + self.logger.info("Not setting partition UUID, already was correct") + + prev_uuid = self.disk.get_partition_type(disk_device) + new_uuid = filesystems["/"]["part_type"] + if new_uuid != prev_uuid: + self.logger.info("Setting partition type to %s (was %s)", new_uuid, prev_uuid) + self.disk.set_partition_type(disk_device, new_uuid) + else: + self.logger.info("Not setting partition type, already was correct") + + self.logger.info("Done setting disk UUIDs") + else: + self.logger.warning("Partition UUID data not present in metadata, skipping") + + + + # Process renames first so that all the filenames are as they should be + # for the following steps. + self.logger.info("Processing renamed.jsonl") + with open(os.path.join(meta_dir, "renamed.jsonl"), "r", encoding='utf-8') as gitignores_file: + for line in gitignores_file: + if line.isspace(): + self.logger.debug("Empty line, skipping") + continue + + #self.logger.debug(f"Line: {line}") + renamed_data = json.loads(line) + orig_file = renamed_data['path'] + renamed_file = renamed_data['renamed'] + + if renamed_file.startswith("/"): + renamed_file = renamed_file[1:] + + orig_file_path = os.path.join(path, orig_file) + renamed_file_path = os.path.join(path, renamed_file) + + #self.logger.debug(f"Checking: {renamed_file_path}") + if os.path.exists(renamed_file_path): + self.logger.debug(f"Renaming {renamed_file_path} => {orig_file_path}") + os.rename(renamed_file_path, orig_file_path) + else: + if os.path.exists(orig_file_path): + self.logger.warning(f"Can't rename {renamed_file_path} => {orig_file_path}: Already renamed") + else: + self.logger.warning(f"Can't rename {renamed_file_path} => {orig_file_path}: Source file not found") + + if not destructive_only: + self.logger.info("Processing empty_directories.jsonl") + with open(os.path.join(meta_dir, "empty_directories.jsonl"), "r", encoding='utf-8') as empties_file: + for line in empties_file: + if line.isspace(): + self.logger.debug("Empty line, skipping") + continue + + empties_data = json.loads(line) + empty_dir = empties_data['dir'] + + # os.path.join no acepta /foo como una ruta relativa para concatenar + if empty_dir.startswith("/"): + empty_dir = empty_dir[1:] + + empty_dir_keep = os.path.join(path, empty_dir, ".opengnsys-keep") + + self.logger.debug(f"Empty directory: {empty_dir}") + full_empty_dir = os.path.join(path, empty_dir) + Path(full_empty_dir).mkdir(parents=True, exist_ok=True) + + if os.path.exists(empty_dir_keep): + self.logger.debug(f"Deleting: {empty_dir_keep}") + os.unlink(empty_dir_keep) + + if not destructive_only: + self.logger.info("Processing unix_permissions.jsonl") + with open(os.path.join(meta_dir, "unix_permissions.jsonl"), "r", encoding='utf-8') as acls_file: + for line in acls_file: + if line.isspace(): + self.logger.debug("Empty line, skipping") + continue + + perms_data = json.loads(line) + #self.logger.debug(f"Data: {acls_data}") + + perms_path = perms_data['path'] + file_perms = perms_data['mode'] + file_uid = perms_data['uid'] + file_gid = perms_data['gid'] + + if perms_path.startswith("/"): + perms_path = perms_path[1:] + + perms_file_path = os.path.join(path, perms_path) + + if os.path.exists(perms_file_path): + self.logger.debug(f"Applying permissions {file_perms}, owner {file_uid}, group {file_gid} to {perms_file_path}") + # chown clears suid bits, must come first + os.chown(perms_file_path, file_uid, file_gid) + os.chmod(perms_file_path, file_perms) + else: + self.logger.warning(f"Can't apply permissions to {perms_file_path}, file doesn't exist.") + + + if not destructive_only: + self.logger.info("Processing acls.jsonl") + with open(os.path.join(meta_dir, "acls.jsonl"), "r", encoding='utf-8') as acls_file: + for line in acls_file: + if line.isspace(): + self.logger.debug("Empty line, skipping") + continue + + # docs: https://pylibacl.k1024.org/module.html#posix1e.ACL.to_any_text + + acls_data = json.loads(line) + #self.logger.debug(f"Data: {acls_data}") + + acl_file = acls_data['file'] + acl_text = base64.b64decode(bytes(acls_data['acl'], 'utf-8')) + + if acl_file.startswith("/"): + acl_file = acl_file[1:] + + acl_file_path = os.path.join(path, acl_file) + #self.logger.debug(f"TXT: {acl_text}" ) + acl = posix1e.ACL(data = acl_text) + #self.logger.debug(f"ACL: {acl_text}" ) + + self.logger.debug(f"Applying ACL to {acl_file_path}") + if os.path.exists(acl_file_path): + acl.applyto(acl_file_path) + + if not destructive_only: + self.logger.info("Processing xattrs.jsonl") + with open(os.path.join(meta_dir, "xattrs.jsonl"), "r", encoding='utf-8') as xattrs_file: + for line in xattrs_file: + if line.isspace(): + self.logger.debug("Empty line, skipping") + continue + + xattrs_data = json.loads(line) + xattrs_file = xattrs_data['file'] + + if xattrs_file.startswith("/"): + xattrs_file = xattrs_file[1:] + + xattrs_file_path = os.path.join(path, xattrs_file) + + #self.logger.debug(f"Line: {line}") + + self.logger.info("Processing special_files.jsonl") + with open(os.path.join(meta_dir, "special_files.jsonl"), "r", encoding='utf-8') as specials_file: + for line in specials_file: + if line.isspace(): + self.logger.debug("Empty line, skipping") + continue + + #self.logger.debug(f"Line: {line}") + data = json.loads(line) + filename = data['file'] + full_path = os.path.join(path, filename) + file_mode = data['mode'] + + try: + if stat.S_ISSOCK(file_mode): + self.logger.debug(f"Restoring socket {filename}") + os.mknod(full_path, mode = file_mode) + elif stat.S_ISFIFO(file_mode): + self.logger.debug(f"Restoring FIFO {filename}") + os.mknod(full_path, mode = file_mode) + elif stat.S_ISBLK(file_mode): + self.logger.debug(f"Restoring block device {filename}") + os.mknod(full_path, mode = file_mode, device = data['rdev']) + elif stat.S_ISCHR(file_mode): + self.logger.debug(f"Restoring character device {filename}") + os.mknod(full_path, mode = file_mode, device = data['rdev']) + else: + self.logger.warning(f"Unknown file type for {filename}: {file_mode}") + + # chown clears suid bit, so even though it's redundant in most cases and already + # done above, set the full perms on the file again anyway. + os.chown(full_path, data['uid'], data['gid']) + os.chmod(full_path, file_mode) + except FileExistsError as exists: + self.logger.debug(f"Exists: {full_path}") + except OSError as oserr: + self.logger.warning(f"Failed to create special file {full_path}: Error {oserr.errno}: {oserr.strerror}") + + + + + + self.logger.info("Metadata restoration completed.") + + def _configure_repo(self, repo): + """ + #ogGitConfig + #@brief Configura usuario y permisos de git. + #@return + """ + + self.logger.debug(f"Configuring repository {repo}") + repo.config_writer().add_value("user", "name", "OpenGnsys").release() + repo.config_writer().add_value("user", "email", "OpenGnsys@opengnsys.com").release() + repo.config_writer().add_value("core", "filemode", "false").release() + repo.config_writer().add_value("push", "autoSetupRemote", "true").release() + repo.config_writer().add_value("maintenance", "autoDetach", "false").release() + + def initRepo(self, device, repo_name): + """ + Initialize a Git repository on a specified device. + + This method mounts the device, initializes a Git repository, configures it, + and sets up a remote origin. It handles both NTFS and other filesystem types. + + Args: + device (str): The device path to initialize the repository on. + repo_name (str): The name of the repository to be created. + + Raises: + RuntimeError: If the .git directory is of an unrecognized file type. + + Notes: + - The method mounts the device to /mnt/{device_basename}. + - The .git directory is created in a cache partition and symlinked to the device. + - The repository is initialized and configured, and an initial commit is made. + - The method sets up a remote origin and pushes the initial commit. + """ + + if not self.check_remote_exists(repo_name): + self.logger.error("Specified repository can't be used, aborting.") + return + + path = self.fs.ensure_mounted(device) + self.logger.info("Initializing repository: %s", path) + + git_dir = os.path.join(path, ".git") + real_git_dir = os.path.join(self.cache_dir, f"git-{repo_name}") + repo_url = self._getOgRepository(repo_name) + + if os.path.exists(real_git_dir): + self.logger.debug(f"Removing existing repository {real_git_dir}") + shutil.rmtree(real_git_dir) + + if os.path.exists(git_dir) or os.path.islink(git_dir): + if os.path.islink(git_dir) or os.path.isfile(git_dir): + self.logger.debug(f"Removing gitdir: {git_dir}") + os.unlink(git_dir) + elif os.path.isdir(git_dir): + # We want to host git in the cache partition, .git shouldn't be a directory under the + # filesystem. + self.logger.warning(f"Removing directory-type gitdir, this should be a link or a file: {git_dir}") + shutil.rmtree(git_dir) + else: + raise RuntimeError("Git dir is of an unrecognized file type!") + +# if not os.path.exists(git_dir): + #self.logger.debug("Creating " + git_dir) + #with open(git_dir, "w") as git_dir: + # git_dir.write(f"gitdir: {real_git_dir}\n") + + self.logger.debug(f"Initializing repo in cache at {real_git_dir}") + #os.mkdir(real_git_dir) + #with git.Repo.init(real_git_dir, bare=True) as temprepo: + # self._configure_repo(temprepo) + + os.symlink(real_git_dir, git_dir) + + + with git.Repo.init(path) as repo: + # On NTFS, we have to unmount the filesystem to do secaudit. + # Gitpython objects existing at that time may mean a dangling git process that prevents + # the required unmounting. + # + # So we make sure we destroy gitpython after this initial stage, to recreate it + # right after _create_metadata. + self._configure_repo(repo) + self._write_ignore_list(path) + + + # Adding the gitignore and doing the manual --force saves us an expensive fetch if + # the repo already had data in it, and allows us to use the gitpython functions with + # progress reports for doing the full push later. + origin = repo.create_remote("origin", repo_url) + repo.index.add(f"{path}/.gitignore") + repo.index.commit("Initial commit") + repo.git.push("--force") # Obliterate whatever might have been there + + self.logger.debug("Fetching origin") + origin.fetch(progress=OgProgressPrinter(self.logger)) + + repo.heads.master.set_tracking_branch(origin.refs.master) + + + metadata_ret = self._create_metadata(path, initial_creation=True) + + repo = git.Repo(path) + + self.logger.debug(f"Building list of files to add from path {path}") + + add_files = [] + + # Nota: repo.index.add(".") agrega archivos pero git después cree que + # no han sido agregados? + for ent in os.listdir(path): + if repo.ignored(ent) or ent == ".git" or ent == "." or ent == "..": + self.logger.debug(f"Ignoring: {ent}") + elif ent in self.fully_ignored_dirs: + # FIXME: repo.index.add tiene un bug que ignora Force=true + #repo.index.add("dev/.opengnsys-keep") + self.logger.debug("Fully ignored dir: {ent}") + add_files.append(f"{ent}/.opengnsys-keep") + else: + self.logger.debug(f"Adding: {ent}") + add_files.append(ent) + #repo.index.add(ent, force=False) + + + for lnk in metadata_ret['symlinks']: + self.logger.debug(f"Adding symlink: {lnk}") + add_files.append(lnk) + + add_files_new = [] + for file in add_files: + if os.path.exists(os.path.join(path, file)): + add_files_new = add_files_new + [file] + else: + self.logger.warning(f"We wanted to add {file} but it wasn't found. Please debug.") + + add_files = add_files_new + + self.logger.info("Adding %d files", len(add_files)) + with OperationTimer(self, "add all files"): + #subprocess.run(["git", "add"] + add_files, check=True, cwd=path) + repo.index.add(items = add_files, force=True ) + + # FIXME: This shouldn't actually happen, we shouldn't have any untracked files + if self.debug_check_for_untracked_files: + self.logger.info("Checking for untracked files...") + + with OperationTimer(self, "add untracked files"): + untracked_list = repo.untracked_files + if untracked_list: + self.logger.warning(f"Untracked files: {untracked_list}") + self.logger.warning("Adding %d untracked files", len(untracked_list)) + #repo.index.add(items = untracked_list, force=True) + subprocess.run(["git", "add"] + untracked_list, check=True, cwd=path) + + + self.logger.info("Committing") + repo.index.commit("Initial commit") + + # Restaurar cosas modificadas para git + self._restore_metadata(path, destructive_only=True, set_device_uuids=False) + + + #self.logger.debug("Commit done, will unmount now") + #self._umount_device(device) + + if self.fs.filesystem_type(mountpoint = path) == "ntfs": + self.fs.unload_ntfs() + + + + + + # repo.create_head + + # repo.heads.master.set_tracking_branch(origin.refs.master) + + self.logger.info("Uploading to ogrepository") + origin.push(progress=OgProgressPrinter(self.logger)) + + #repo.git.push("--set-upstream", "origin", repo.head.ref, "--force") + + def cloneRepo(self, repo_name, destination, boot_device): + """ + Clones a repository to a specified destination and sets up the bootloader. + + Args: + repo_name (str): The name of the repository to clone. + destination (str): The destination directory where the repository will be cloned. + boot_device (str): The boot device to install the bootloader. + + Raises: + RequirementException: If the repository metadata is incorrect or if the repository's + boot system is incompatible with the current system. + + Logs: + Info: Logs the start of the cloning process. + Debug: Logs the repository URL, EFI compatibility of the repository and the system. + """ + self.logger.info(f"Cloning repo: {repo_name} => {destination}") + + + repo_url = self._getOgRepository(repo_name) + real_git_dir = os.path.join(self.cache_dir, f"git-{repo_name}") + + if os.path.exists(real_git_dir): + self.logger.debug(f"Removing existing repository {real_git_dir}") + shutil.rmtree(real_git_dir) + + self.logger.debug(f"URL: {repo_url}") + + all_metadata = self._get_repo_metadata(repo_name) + metadata = all_metadata["metadata.json"] + fs_data = all_metadata["filesystems.json"] + + + if len(fs_data.keys()) == 0: + raise RequirementException("El repositorio contiene metadatos incorrectos. Falta información sobre sistemas de archivos en filesystems.json.") + + if not "efi_boot" in metadata: + raise RequirementException("El repositorio contiene metadatos incorrectos. Falta información de metadatos en metadata.json") + + repo_is_efi = metadata["efi_boot"] + efi = self._is_efi() + + self.logger.debug(f"Repository made for EFI: {repo_is_efi}") + self.logger.debug(f"Our system using EFI : {efi}") + + if repo_is_efi != efi: + raise RequirementException("Repositorio usa sistema de arranque incompatible con sistema actual") + + self.fs.unmount(device = destination) + + filesystem_map = {"/" : destination} + + self._create_filesystems(fs_data, filesystem_map) + + destination_dir = "/mnt/repo-" + repo_name + + self.fs.mount(destination, destination_dir) + + self._delete_contents(destination_dir) + + self.logger.info("Cloning repository from %s", repo_url) + + repo = git.Repo.clone_from(repo_url, destination_dir, multi_options = [f"--separate-git-dir={real_git_dir}"], progress=OgProgressPrinter(self.logger)) + + if repo_is_efi: + self._efi_install(root_directory=destination_dir) + else: + self._grub_install(root_directory=destination_dir, boot_device=boot_device) + + self.fs.mklostandfound(destination_dir) + self._restore_metadata(destination_dir, set_device_uuids=True) + + if self.fs.filesystem_type(mountpoint = destination_dir) == "ntfs": + self._ntfs_restore_secaudit(destination_dir) + + self.logger.info("Clone completed.") + + + def commit(self, path = None, device = None, message = None): + """ + Commit all current changes to the local data + """ + + if path is None: + path = self.fs.ensure_mounted(device) + + self.logger.info("Committing changes to repository") + repo = git.Repo(path) + + self._create_metadata(path, initial_creation=False) + + self.logger.info("Adding files") + repo.index.add("*") + + self.logger.info("Creating commit") + repo.index.commit(message) + + # Restaurar cosas modificadas para git + self._restore_metadata(path, destructive_only=True) + + + def restoreRepo(self, path): + """ + Restore the repository to the state it had before the non-committed modifications + """ + self.logger.info("Undoing any user changes to the filesystem") + repo = git.Repo(path) + + repo.head.reset(index=True, working_tree=True) + + # Restaurar cosas modificadas para git + self._restore_metadata(path, destructive_only=True) + + def push(self, path = None, device = None): + """ + Push local changes to ogrepository + + Use commit() first to save local changes. + """ + + + if path is None: + path = self.fs.ensure_mounted(device) + + repo = git.Repo(path) + + self.logger.info("Uploading to ogrepository") + if not "origin" in repo.remotes: + self.logger.critical("'origin' remote not found!") + return + + origin = repo.remotes["origin"] + repo.heads.master.set_tracking_branch(origin.refs.master) + origin.push(progress=OgProgressPrinter(self.logger)) + + #repo.git.push("--set-upstream", "origin", repo.head.ref, "--force") # force = True) + + + def fetch(self, path = None, device = None): + """ + Fetch updates from ogrepository. Doesn't change the filesystem. + """ + + + if path is None: + path = self.fs.ensure_mounted(device) + + repo = git.Repo(path) + + self.logger.info("Fetching from ogrepository") + origin = repo.remotes.origin + + if origin: + self.logger.debug("Fetching from origin") + origin.fetch(progress=OgProgressPrinter(self.logger)) + else: + self.logger.error("Origin not found, can't fetch") + + def pull(self, path = None, device = None): + """ + Pull changes from ogrepository + + This unconditionally overwrites remote changes. There is no conflict resolution. + """ + + + if path is None: + path = self.fs.ensure_mounted(device) + + repo = git.Repo(path) + + self.logger.debug("Downloading from ogrepository") + repo.git.fetch() + repo.head.reset(index=True, working_tree=True) + + # Restaurar cosas modificadas para git + self._restore_metadata(path, destructive_only=True) + + def check_remote_exists(self, repo_name): + repo_url = self._getOgRepository(repo_name) + + self.logger.info("Checking whether %s exists and is accessible", repo_url) + + ret = subprocess.run(["/usr/bin/git", "ls-remote", repo_url], encoding='utf-8', capture_output=True, check=False) + if ret.returncode == 0: + return True + else: + self.logger.warning("Remote can't be accessed, git said: %s", ret.stderr) + return False + + + + + + +if __name__ == '__main__': + # python no cree que nuestra consola usa utf-8. + # esto arregla las tildes y las eñes + sys.stdout.reconfigure(encoding='utf-8') + + kernel_args = parse_kernel_cmdline() + + + opengnsys_log_dir = "/opt/opengnsys/log" + + logger = logging.getLogger(__package__) + logger.setLevel(logging.DEBUG) + + streamLog = logging.StreamHandler() + streamLog.setLevel(logging.INFO) + + if not os.path.exists(opengnsys_log_dir): + os.mkdir(opengnsys_log_dir) + + ip_address = "unknown" + if "ip" in kernel_args: + ip_address = kernel_args["ip"].split(":")[0] + + + logFilePath = f"{opengnsys_log_dir}/{ip_address}.gitlib.log" + + fileLog = logging.FileHandler(logFilePath) + fileLog.setLevel(logging.DEBUG) + + formatter = logging.Formatter('%(asctime)s - %(name)24s - [%(levelname)5s] - %(message)s') + + streamLog.setFormatter(formatter) + fileLog.setFormatter(formatter) + + logger.addHandler(streamLog) + logger.addHandler(fileLog) + + + logger.info("Program start, logging details to %s", logFilePath) + + parser = argparse.ArgumentParser( + prog="OpenGnsys Git Library", + description="Funciones de Git", + ) + + #parser.add_argument("--init-repo", type=str, metavar='DIR', help="Inicializar repositorio desde DIR") + parser.add_argument("--init-repo-from", type=str, metavar='DEV', help="Inicializar repositorio desde DEV") + parser.add_argument("--clone-repo-to", type=str, metavar='DEV', help="Clonar repositorio a DIR. Elimina todos los datos en ruta destino!") + parser.add_argument("--repo", type=str, help="Repositorio en ogrepository (linux, windows, mac)") + parser.add_argument("--boot-device", type=str, help="Dispositivo de arranque") + parser.add_argument("--commit", type=str, metavar='DEV', help="Commit de cambios en el directorio") + parser.add_argument("--restore", type=str, metavar='DEV', help="Eliminar cambios en el directorio") + parser.add_argument("--push", type=str, metavar='DEV', help="Subir cambios a ogrepository") + parser.add_argument("--pull", type=str, metavar='DEV', help="Bajar cambios de ogrepository") + parser.add_argument("--fetch", type=str, metavar='DEV', help="Fetch changes from ogrepository") + parser.add_argument("--efi-config", type=str, metavar="NAME", help="Name of the custom EFI configuration to deploy") + parser.add_argument("--verify-repo", action='store_true', help="Verify whether the indicated repository exists and can be used") + + + parser.add_argument("--ntfs-type", type=str, metavar="FS", help="Tipo de NTFS, 'kernel' o 'fuse'") + parser.add_argument("--test-create-metadata", type=str, metavar="DIR", help="Test metadata generation") + parser.add_argument("--test-restore-metadata", type=str, metavar="DIR", help="Test metadata restoration") + parser.add_argument("--test-restore-metadata-destructive", type=str, metavar="DIR", help="Test metadata restoration, destructive parts only") + parser.add_argument("--test-clone-metadata", type=str, metavar="REPO", help="Test metadata cloning") + parser.add_argument("--test-efi-install", type=str, metavar="DIR", help = "Install EFI") + parser.add_argument("-m", "--message", type=str, metavar="MSG", help="Commit message") + parser.add_argument("--test-set-ntfsid", type=str, metavar="ID", help="Set NTFS ID") + parser.add_argument("--test-restore-secaudit",type=str, metavar="DIR", help="Test restoring NTFS secaudit") + parser.add_argument("--test-get-part-uuid", type=str, metavar="PART", help="Get partition UUID") + + parser.add_argument("--device", type=str, metavar="DEV", help="Device to set the UUID on") + + parser.add_argument("-v", "--verbose", action="store_true", help = "Verbose console output") + + args = parser.parse_args() + + if args.verbose: + streamLog.setLevel(logging.DEBUG) + + + logger.debug("Starting") + + ntfs_impl = NTFSImplementation.NTFS3G + + if not args.ntfs_type is None: + if args.ntfs_type == "kernel": + ntfs_impl = NTFSImplementation.KERNEL + elif args.ntfs_type == "fuse": + ntfs_impl = NTFSImplementation.NTFS3G + else: + raise ValueError(f"Unknown NTFS implementation: {args.ntfs_type}") + + + og_git = OpengnsysGitLibrary(ntfs_implementation = ntfs_impl) + # og_git._runBashFunction("ogMountCache", []) + + +# if args.init_repo: + # #og_git.initRepo("/mnt/sda1", "linux") + # with OperationTimer(og_git, "git init"): + # og_git.initRepo(args.init_repo, args.repo) + if args.init_repo_from: + with OperationTimer(og_git, "git init"): + og_git.initRepo(args.init_repo_from, args.repo) + elif args.clone_repo_to: + #og_git.cloneRepo("linux", "/opt/opengnsys/cache/cloned") + with OperationTimer(og_git, "git clone"): + og_git.cloneRepo(args.repo, args.clone_repo_to, args.boot_device) + #og_git._restore_metadata("/opt/opengnsys/cache/cloned") + #og_git._restore_metadata(args.clone_repo_to) + elif args.commit: + with OperationTimer(og_git, "git commit"): + og_git.commit(device = args.commit, message = args.message) + elif args.restore: + with OperationTimer(og_git, "git restore"): + og_git.restoreRepo(args.restore) + elif args.push: + with OperationTimer(og_git, "git push"): + og_git.push(device = args.push) + elif args.fetch: + with OperationTimer(og_git, "git fetch"): + og_git.fetch(device = args.fetch) + elif args.pull: + with OperationTimer(og_git, "git pull"): + og_git.pull(device = args.pull) + elif args.verify_repo: + if og_git.check_remote_exists(args.repo): + print("Remote checks OK") + else: + print("Check failed") + + elif args.test_create_metadata: + og_git._create_metadata(args.test_create_metadata, initial_creation=False) # pylint: disable=protected-access + elif args.test_restore_metadata: + og_git._restore_metadata(args.test_restore_metadata, set_device_uuids=True) # pylint: disable=protected-access + elif args.test_restore_metadata_destructive: + og_git._restore_metadata(path = args.test_restore_metadata_destructive, destructive_only=True) # pylint: disable=protected-access + elif args.test_clone_metadata: + og_git._get_repo_metadata(args.test_clone_metadata) # pylint: disable=protected-access + elif args.test_set_ntfsid: + ntfs = NTFSLibrary(ntfs_impl) + ntfs.modify_uuid(args.device, args.test_set_ntfsid) + elif args.test_efi_install: + og_git._efi_install(root_directory=args.test_efi_install, config_name = args.efi_config) # pylint: disable=protected-access + elif args.test_restore_secaudit: + og_git._ntfs_restore_secaudit(args.test_restore_secaudit) # pylint: disable=protected-access + else: + print("Please specify an action.") + parser.print_help() + sys.exit(1) + # + + # Make sure all filesystem changes are written, just in case the oglive is rebooted without an unmount + os.sync() diff --git a/ogclient/lib/python3/kernel.py b/ogclient/lib/python3/kernel.py new file mode 100644 index 0000000..f0a7eb2 --- /dev/null +++ b/ogclient/lib/python3/kernel.py @@ -0,0 +1,22 @@ + + +def parse_kernel_cmdline(): + """Parse the kernel arguments to obtain configuration parameters in Oglive + + OpenGnsys passes data in the kernel arguments, for example: + [...] group=Aula_virtual ogrepo=192.168.2.1 oglive=192.168.2.1 [...] + + Returns: + dict: Dict of configuration parameters and their values. + """ + params = {} + + with open("/proc/cmdline", encoding='utf-8') as cmdline: + line = cmdline.readline() + parts = line.split() + for part in parts: + if "=" in part: + key, value = part.split("=") + params[key] = value + + return params \ No newline at end of file diff --git a/ogclient/lib/python3/ntfs.py b/ogclient/lib/python3/ntfs.py new file mode 100644 index 0000000..d3c3a57 --- /dev/null +++ b/ogclient/lib/python3/ntfs.py @@ -0,0 +1,111 @@ + +import logging +import subprocess + +from enum import Enum + + +class NTFSImplementation(Enum): + KERNEL = 1 + NTFS3G = 2 + + +class NTFSLibrary: + """ + A library for managing NTFS filesystems. + + Attributes: + logger (logging.Logger): Logger for the class. + implementation (NTFSImplementation): The implementation to use for mounting NTFS filesystems. + """ + + def __init__(self, implementation): + """ + Initializes the instance with the given implementation. + + Args: + implementation: The implementation to be used by the instance. + + Attributes: + logger (logging.Logger): Logger instance for the class, set to debug level. + implementation: The implementation provided during initialization. + """ + self.logger = logging.getLogger("NTFSLibrary") + self.logger.setLevel(logging.DEBUG) + self.implementation = implementation + + self.logger.debug("Initializing") + + def create_filesystem(self, device, label): + """ + Creates an NTFS filesystem on the specified device with the given label. + + Args: + device (str): The device path where the NTFS filesystem will be created. + label (str): The label to assign to the NTFS filesystem. + + Returns: + None + + Logs: + Logs the creation process with the device and label information. + """ + self.logger.info(f"Creating NTFS in {device} with label {label}") + + subprocess.run(["/usr/sbin/mkntfs", device, "-Q", "-L", label], check=True) + + + def mount_filesystem(self, device, mountpoint): + """ + Mounts a filesystem on the specified mountpoint using the specified NTFS implementation. + + Args: + device (str): The device path to be mounted (e.g., '/dev/sda1'). + mountpoint (str): The directory where the device will be mounted. + + Raises: + ValueError: If the NTFS implementation is unknown. + + """ + self.logger.info(f"Mounting {device} in {mountpoint} using implementation {self.implementation}") + if self.implementation == NTFSImplementation.KERNEL: + subprocess.run(["/usr/bin/mount", "-t", "ntfs3", device, mountpoint], check = True) + elif self.implementation == NTFSImplementation.NTFS3G: + subprocess.run(["/usr/bin/ntfs-3g", device, mountpoint], check = True) + else: + raise ValueError("Unknown NTFS implementation: {self.implementation}") + + def modify_uuid(self, device, uuid): + """ + Modify the UUID of an NTFS device. + + This function changes the UUID of the specified NTFS device to the given UUID. + It reads the current UUID from the device, logs the change, and writes the new UUID. + + Args: + device (str): The path to the NTFS device file. + uuid (str): The new UUID to be set, in hexadecimal string format. + + Raises: + IOError: If there is an error opening or writing to the device file. + """ + + ntfs_uuid_offset = 0x48 + ntfs_uuid_length = 8 + + binary_uuid = bytearray.fromhex(uuid) + binary_uuid.reverse() + + self.logger.info(f"Changing UUID on {device} to {uuid}") + with open(device, 'r+b') as ntfs_dev: + self.logger.debug("Reading %i bytes from offset %i", ntfs_uuid_length, ntfs_uuid_offset) + + ntfs_dev.seek(ntfs_uuid_offset) + prev_uuid = bytearray(ntfs_dev.read(ntfs_uuid_length)) + prev_uuid.reverse() + prev_uuid_hex = bytearray.hex(prev_uuid) + self.logger.debug(f"Previous UUID: {prev_uuid_hex}") + + self.logger.debug("Writing...") + ntfs_dev.seek(ntfs_uuid_offset) + ntfs_dev.write(binary_uuid) diff --git a/ogclient/lib/python3/test.py b/ogclient/lib/python3/test.py new file mode 100644 index 0000000..37e14a0 --- /dev/null +++ b/ogclient/lib/python3/test.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +def _hex_to_bin( hex_str): + + while len(hex_str) != 16: + hex_str = "0" + hex_str + + hex_int = int(hex_str, 16) + binary = bin(hex_int)[2:].zfill(64) + + return binary + + +print( _hex_to_bin("0ACA")) \ No newline at end of file