diff --git a/client/lib/engine/bin/FileLib.py b/client/lib/engine/bin/FileLib.py index e9de16f..f25414b 100755 --- a/client/lib/engine/bin/FileLib.py +++ b/client/lib/engine/bin/FileLib.py @@ -1,31 +1,81 @@ +#/** +#@file FileLib.py +#@brief Librería o clase File +#@class File +#@brief Funciones para gestión de archivos y directorios. +#@warning License: GNU GPLv3+ +#*/ + import subprocess import os import shutil +import hashlib import ogGlobals import SystemLib import CacheLib import FileSystemLib -def ogCalculateChecksum(*args): - # Check if help is requested - if "help" in args: - print("ogCalculateChecksum [ str_repo | int_ndisk int_npartition ] path_filepath") - print("ogCalculateChecksum REPO ubuntu.img ==> ef899299caf8b517ce36f1157a93d8bf") +#/** +# ogCalculateChecksum [ str_repo | int_ndisk int_npart ] path_filepath +#@brief Devuelve la suma de comprobación (checksum) de un fichero. +#@param path_filepath camino del fichero (independiente de mayúsculas) +#@param str_repo repositorio de ficheros +#@param int_ndisk nº de orden del disco +#@param int_npartition nº de orden de la partición +#@return hex_checksum Checksum del fichero +#*/ ## +#ogCalculateChecksum ([ str_repo | int_ndisk int_npartition ] path_filepath") +#ogCalculateChecksum (container='REPO', file='ubuntu.img') ==> ef899299caf8b517ce36f1157a93d8bf +#ogCalculateChecksum (disk=1, par=1, file='ubuntu.img') ==> ef899299caf8b517ce36f1157a93d8bf +def ogCalculateChecksum (disk=None, par=None, container=None, file=None): + if file is None: + raise TypeError ('missing required argument: "file"') + + if container is not None: + if disk is None and par is None: + ## we were given container= + f = ogGetPath (src=container, file=file) + dev_err = f'{container} {file}' + print (f'ogGetPath (src=({container}), file=({file})) = f ({f})') + else: + raise TypeError ('argument "container" can be specified along neither "disk" nor "par"') + + else: + if disk is not None and par is not None: + ## we were given disk= par= + f = ogGetPath (src=f'{disk} {par}', file=file) + dev_err = f'{disk} {par} {file}' + print (f'ogGetPath (src=({disk} {par}), file=({file})) = f ({f})') + elif disk is None and par is None: + ## we were given nothing + f = ogGetPath (file=file) + dev_err = file + print (f'ogGetPath (file=({file})) = f ({f})') + else: + raise TypeError ('if one of "disk" and "par" are specified, then both must be') + + if not f: + SystemLib.ogRaiseError ([], ogGlobals.OG_ERR_NOTFOUND, dev_err) return - # Get the file path - file_path = ogGetPath(*args) - if not file_path: - SystemLib.ogRaiseError(OG_ERR_NOTFOUND, *args) - return + last_n_bytes = 1024*1024 + if last_n_bytes >= os.stat ('/bin/ls').st_size: + return ogCalculateFullChecksum (disk, par, container, file) + with open (f, 'rb') as fd: + fd.seek (-last_n_bytes, os.SEEK_END) + data = file.read() + md5 = hashlib.md5(data).hexdigest() - # Calculate the checksum - result = subprocess.run(["tail", "-c1M", file_path], capture_output=True) - checksum = result.stdout.decode().split()[0] + return md5 - return checksum +#/** +# ogCompareChecksumFiles [ str_repo | int_ndisk int_npart ] path_source [ str_repo | int_ndisk int_npart ] path_target +#@brief Metafunción que compara las sumas de comprobación almacenadas de 2 ficheros. +#@return bool_compare Valor de comparación. +#@warning No es necesario especificar la extensión ".sum". +#*/ ## def ogCompareChecksumFiles(*args): # Variables locales. ARGS = args @@ -59,24 +109,63 @@ def ogCompareChecksumFiles(*args): except FileNotFoundError: return False -def ogCalculateFullChecksum(*args): - # Variables locales. - FILE = None - if "help" in args: - SystemLib.ogHelp("$FUNCNAME", "$FUNCNAME [ str_repo | int_ndisk int_npartition ] path_filepath", "$FUNCNAME REPO ubuntu.img ==> ef899299caf8b517ce36f1157a93d8bf") + +#/** +# ogCalculateFullChecksum [ str_repo | int_ndisk int_npart ] path_filepath +#@brief Devuelve la suma COMPLETA de comprobación (checksum) de un fichero. +#@param path_filepath camino del fichero (independiente de mayúsculas) +#@param str_repo repositorio de ficheros +#@param int_ndisk nº de orden del disco +#@param int_npartition nº de orden de la partición +#@return hex_checksum Checksum del fichero +#*/ ## +def ogCalculateFullChecksum (disk=None, par=None, container=None, file=None): + if file is None: + raise TypeError ('missing required argument: "file"') + + if container is not None: + if disk is None and par is None: + ## we were given container= + f = ogGetPath (src=container, file=file) + dev_err = f'{container} {file}' + print (f'ogGetPath (src=({container}), file=({file})) = f ({f})') + else: + raise TypeError ('argument "container" can be specified along neither "disk" nor "par"') + + else: + if disk is not None and par is not None: + ## we were given disk= par= + f = ogGetPath (src=f'{disk} {par}', file=file) + dev_err = f'{disk} {par} {file}' + print (f'ogGetPath (src=({disk} {par}), file=({file})) = f ({f})') + elif disk is None and par is None: + ## we were given nothing + f = ogGetPath (file=file) + dev_err = file + print (f'ogGetPath (file=({file})) = f ({f})') + else: + raise TypeError ('if one of "disk" and "par" are specified, then both must be') + + if not f: + SystemLib.ogRaiseError ([], ogGlobals.OG_ERR_NOTFOUND, dev_err) return - # Comprobar que existe el fichero y devolver sus datos. - FILE = ogGetPath(*args) - if not FILE: - SystemLib.ogRaiseError(OG_ERR_NOTFOUND, *args) - return + md5 = hashlib.md5() + with open (f, 'rb') as fd: + for chunk in iter (lambda: fd.read (64*1024), b''): + md5.update (chunk) + return md5.hexdigest() - # Calculate the checksum - result = subprocess.run(["md5sum", FILE, "-b"], capture_output=True) - checksum = result.stdout.decode().split()[0] - return checksum + + +#/** +# ogCopyFile [ str_repo | int_ndisk int_npart ] path_source [ str_repo | int_ndisk int_npart ] path_target +#@brief Metafunción para copiar un fichero de sistema OpenGnSys a un directorio. +#@see ogGetPath +#@return Progreso de la copia. +#@warning Deben existir tanto el fichero origen como el directorio destino. +#*/ ## def ogCopyFile(*args): # Variables locales. @@ -113,22 +202,45 @@ def ogCopyFile(*args): result = subprocess.run(["rsync", "--progress", "--inplace", "-avh", SOURCE, TARGET]) return result.returncode -def ogDeleteFile(*args): - # Variables locales. - FILE = None - if "help" in args: - SystemLib.ogHelp("$FUNCNAME", "$FUNCNAME [ str_repo | int_ndisk int_npartition ] path_file", "$FUNCNAME 1 2 /tmp/newfile.txt") + +#/** +# ogDeleteFile [ str_repo | int_ndisk int_npartition ] path_filepath +#@brief Metafunción que borra un fichero de un dispositivo. +#@see ogGetPath +#@version 0.9 - Pruebas con OpenGnSys. +#@author Ramon Gomez, ETSII Universidad de Sevilla +#@date 2009-09-29 +#*/ ## +#ogDeleteFile ([ str_repo | int_ndisk int_npartition ] path_file) +#ogDeleteFile (container='REPO', file='/tmp/newfile.txt') +#ogDeleteFile (disk=1, par=2, file='/tmp/newfile.txt') +def ogDeleteFile (disk=None, par=None, container=None, file=None): + if file is None: + raise TypeError ('missing required argument: "file"') + + if container is not None: + if disk is None and par is None: + ## we were given container= + src = container + else: + raise TypeError ('argument "container" can be specified along neither "disk" nor "par"') + else: + if disk is not None and par is not None: + ## we were given disk= par= + src = f'{disk} {par}' + else: + ## we were given nothing + raise TypeError ('either "container" or both "disk" and "par" must be specified') + + f = ogGetPath (src=src, file=file) + if not f: + SystemLib.ogRaiseError ([], ogGlobals.OG_ERR_NOTFOUND, f'{src} {file}') return - # Comprobar que existe el fichero y borrarlo. - FILE = ogGetPath(*args) - if not FILE: - SystemLib.ogRaiseError(OG_ERR_NOTFOUND, *args) - return try: - os.remove(FILE) + os.remove (f) except OSError as e: - SystemLib.ogRaiseError(OG_ERR_NOTFOUND, *args) + SystemLib.ogRaiseError ([], ogGlobals.OG_ERR_GENERIC, str (e)) return def ogDeleteTree(*args): diff --git a/client/lib/engine/bin/ProtocolLib.py b/client/lib/engine/bin/ProtocolLib.py index b070ac8..b421307 100644 --- a/client/lib/engine/bin/ProtocolLib.py +++ b/client/lib/engine/bin/ProtocolLib.py @@ -4,6 +4,7 @@ import subprocess import re import json import os.path +import shutil import ogGlobals import SystemLib @@ -852,10 +853,104 @@ def ogCreateTorrent (disk=None, par=None, container=None, file=None, ip_bttrack= #@param 1 str_REPO #@param 2 str_Relative_path_file_OGIMG_with_/ #@param 3 md5 to check: use full to check download image torrent -#@return 0 (true) cache sin imagen, SI es necesario actualizar el fichero. -#@return 1 (false) imagen en la cache, NO es necesario actualizar el fichero -#@return >1 (false) error de sintaxis (TODO) +#@return True cache sin imagen, SI es necesario actualizar el fichero. +#@return False imagen en la cache, NO es necesario actualizar el fichero +#@return None error de sintaxis (TODO) #@note #@todo: Proceso en el caso de que el fichero tenga el mismo nombre, pero su contenido sea distinto. #@todo: Se dejan mensajes mientras se confirma su funcionamiento. #*/ ## +#ogUpdateCacheIsNecesary ('REPO', '/PS1_PH1.img', 'UNICAST') +#ogUpdateCacheIsNecesary ('REPO', '/ogclient.sqfs', 'FULL') +#ogUpdateCacheIsNecesary ('REPO', '/ogclient.sqfs', 'TORRENT') +def ogUpdateCacheIsNecesary (repo, file, proto): + if not CacheLib.ogFindCache(): + SystemLib.ogRaiseError ([], ogGlobals.OG_ERR_NOTCACHE, '') + return None + + if repo.lower() != 'repo': + SystemLib.ogRaiseError ([], ogGlobals.OG_ERR_NOTFOUND, f' {repo} {file}') + return None + + filesource = FileLib.ogGetPath (src=repo, file=file) + print (f'filesource ({filesource})') + if not filesource: + SystemLib.ogRaiseError ([], ogGlobals.OG_ERR_NOTFOUND, f' {repo} {file}') + return None + + # paso 1. si no existe la imagen, confirmar que es necesario actualizar la cache. + filetarget = FileLib.ogGetPath (src='CACHE', file=file) + print (f'filetarget ({filetarget})') + if not filetarget: + # borramos el fichero bf del torrent, en el caso de que se hubiese quedado de algun proceso fallido + if FileLib.ogGetPath (src='CACHE', file=f'/{file}.torrent.bf'): ogDeleteFile ('CACHE', f'{file}.torrent.bf') + if FileLib.ogGetPath (src='CACHE', file=f'/{file}.sum'): ogDeleteFile ('CACHE', f'{file}.sum') + if FileLib.ogGetPath (src='CACHE', file=f'/{file}.full.sum'): ogDeleteFile ('CACHE', f'{file}.full.sum') + print ('TRUE(0), es necesario actualizar. Paso 1, la cache no contiene esa imagen ') + return True + + # Paso 2. Comprobamos que la imagen no estuviese en un proceso previo torrent + if FileLib.ogGetPath (file=f'{filetarget}.torrent.bf'): + #TODO: comprobar los md5 del fichero .torrent para asegurarnos que la imagen a descarga es la misma. + print ('TRUE(0), es necesario actualizar. Paso 2, la imagen esta en un estado de descarga torrent interrumpido') + return True + + ## En este punto la imagen en el repo y en la cache se llaman igual, + # paso 4. Obtener los md5 del fichero imagen en la cacha segun PROTOCOLO $3 + if proto.lower() in ['full', 'torrent']: + #Buscamos MD5 en el REPO SOURCE + if os.path.exists (f'{filesource}.full.sum'): + with open (f'{filesource}.full.sum', 'r') as fd: + md5source = fd.read().strip() + else: + md5source = FileLib.ogCalculateFullChecksum (file=filesource) + + # Generamos el MD5 (full) en la CACHE + if not os.path.exists (f'{filetarget}.full.sum'): + fullck = FileLib.ogCalculateFullChecksum (file=filetarget) + with open (f'{filetarget}.full.sum', 'w') as fd: + fd.write (fullck + '\n') + with open (f'{filetarget}.full.sum', 'r') as fd: + md5target = fd.read().strip() + # Generamos el MD5 (little) en la CACHE para posteriores usos del protocolo MULTICAST + if not os.path.exists (f'{filetarget}.sum'): + ck = FileLib.ogCalculateChecksum (file=filetarget) + with open (f'{filetarget}.sum', 'w') as fd: + fd.write (ck + '\n') + else: + #Buscamos MD5 en el REPO SOURCE + if os.path.exists (f'{filesource}.sum'): + with open (f'{filesource}.sum', 'r') as fd: + md5source = fd.read().strip() + else: + md5source = FileLib.ogCalculateChecksum (file=filesource) + + # Generamos el MD5 (little) en la CACHE + if not os.path.exists (f'{filetarget}.sum'): + ck = FileLib.ogCalculateChecksum (file=filetarget) + with open (f'{filetarget}.sum', 'w') as fd: + fd.write (ck + '\n') + with open (f'{filetarget}.sum', 'r') as fd: + md5target = fd.read().strip() + #Generamos o copiamos MD5 (full) en la CACHE para posteriores usos con Torrent + # Si no existe el full.sum y si existe el .sum es porque el upateCACHE multicast o unicast ha sido correcto. + if not os.path.exists (f'{filetarget}.full.sum') and os.path.exists (f'{filetarget}.sum'): + if os.path.exists (f'{filesource}.full.sum'): + #Existe el .full.sum en REPO realizamos COPIA + shutil.copy2 (f'{filesource}.full.sum', f'{filetarget}.full.sum') + else: + #No existe .full.sum no en REPO LO GENERAMOS en la cache: situacion dificil que ocurra + fullck = FileLib.ogCalculateFullChecksum (file=filetarget) + with open (f'{filetarget}.full.sum', 'w') as fd: + fd.write (fullck + '\n') + + # Paso 5. comparar los md5 + if md5source == md5target: + print ('FALSE (1), No es neceario actualizar. Paso5.A la imagen esta en cache') + return False + else: + print ('imagen en cache distinta, borramos la imagen anterior') + for f in [f'{filetarget}', f'{filetarget}.sum', f'{filetarget}.torrent', f'{filetarget}.full.sum']: + os.unlink (f) + print ('TRUE (0), Si es necesario actualizar.') + return True