refs #1101 add ogUpdateCacheIsNecesary() and dependencies
Natalia Serrano
parent
bc3d68e8fb
commit
646d1da734
|
|
@ -1,31 +1,81 @@
|
|||
#/**
|
||||
#@file FileLib.py
|
||||
#@brief Librería o clase File
|
||||
#@class File
|
||||
#@brief Funciones para gestión de archivos y directorios.
|
||||
#@warning License: GNU GPLv3+
|
||||
#*/
|
||||
|
||||
import subprocess
|
||||
import os
|
||||
import shutil
|
||||
import hashlib
|
||||
|
||||
import ogGlobals
|
||||
import SystemLib
|
||||
import CacheLib
|
||||
import FileSystemLib
|
||||
|
||||
def ogCalculateChecksum(*args):
|
||||
# Check if help is requested
|
||||
if "help" in args:
|
||||
print("ogCalculateChecksum [ str_repo | int_ndisk int_npartition ] path_filepath")
|
||||
print("ogCalculateChecksum REPO ubuntu.img ==> ef899299caf8b517ce36f1157a93d8bf")
|
||||
#/**
|
||||
# ogCalculateChecksum [ str_repo | int_ndisk int_npart ] path_filepath
|
||||
#@brief Devuelve la suma de comprobación (checksum) de un fichero.
|
||||
#@param path_filepath camino del fichero (independiente de mayúsculas)
|
||||
#@param str_repo repositorio de ficheros
|
||||
#@param int_ndisk nº de orden del disco
|
||||
#@param int_npartition nº de orden de la partición
|
||||
#@return hex_checksum Checksum del fichero
|
||||
#*/ ##
|
||||
#ogCalculateChecksum ([ str_repo | int_ndisk int_npartition ] path_filepath")
|
||||
#ogCalculateChecksum (container='REPO', file='ubuntu.img') ==> ef899299caf8b517ce36f1157a93d8bf
|
||||
#ogCalculateChecksum (disk=1, par=1, file='ubuntu.img') ==> ef899299caf8b517ce36f1157a93d8bf
|
||||
def ogCalculateChecksum (disk=None, par=None, container=None, file=None):
|
||||
if file is None:
|
||||
raise TypeError ('missing required argument: "file"')
|
||||
|
||||
if container is not None:
|
||||
if disk is None and par is None:
|
||||
## we were given container=
|
||||
f = ogGetPath (src=container, file=file)
|
||||
dev_err = f'{container} {file}'
|
||||
print (f'ogGetPath (src=({container}), file=({file})) = f ({f})')
|
||||
else:
|
||||
raise TypeError ('argument "container" can be specified along neither "disk" nor "par"')
|
||||
|
||||
else:
|
||||
if disk is not None and par is not None:
|
||||
## we were given disk= par=
|
||||
f = ogGetPath (src=f'{disk} {par}', file=file)
|
||||
dev_err = f'{disk} {par} {file}'
|
||||
print (f'ogGetPath (src=({disk} {par}), file=({file})) = f ({f})')
|
||||
elif disk is None and par is None:
|
||||
## we were given nothing
|
||||
f = ogGetPath (file=file)
|
||||
dev_err = file
|
||||
print (f'ogGetPath (file=({file})) = f ({f})')
|
||||
else:
|
||||
raise TypeError ('if one of "disk" and "par" are specified, then both must be')
|
||||
|
||||
if not f:
|
||||
SystemLib.ogRaiseError ([], ogGlobals.OG_ERR_NOTFOUND, dev_err)
|
||||
return
|
||||
|
||||
# Get the file path
|
||||
file_path = ogGetPath(*args)
|
||||
if not file_path:
|
||||
SystemLib.ogRaiseError(OG_ERR_NOTFOUND, *args)
|
||||
return
|
||||
last_n_bytes = 1024*1024
|
||||
if last_n_bytes >= os.stat ('/bin/ls').st_size:
|
||||
return ogCalculateFullChecksum (disk, par, container, file)
|
||||
with open (f, 'rb') as fd:
|
||||
fd.seek (-last_n_bytes, os.SEEK_END)
|
||||
data = file.read()
|
||||
md5 = hashlib.md5(data).hexdigest()
|
||||
|
||||
# Calculate the checksum
|
||||
result = subprocess.run(["tail", "-c1M", file_path], capture_output=True)
|
||||
checksum = result.stdout.decode().split()[0]
|
||||
return md5
|
||||
|
||||
return checksum
|
||||
|
||||
#/**
|
||||
# ogCompareChecksumFiles [ str_repo | int_ndisk int_npart ] path_source [ str_repo | int_ndisk int_npart ] path_target
|
||||
#@brief Metafunción que compara las sumas de comprobación almacenadas de 2 ficheros.
|
||||
#@return bool_compare Valor de comparación.
|
||||
#@warning No es necesario especificar la extensión ".sum".
|
||||
#*/ ##
|
||||
def ogCompareChecksumFiles(*args):
|
||||
# Variables locales.
|
||||
ARGS = args
|
||||
|
|
@ -59,24 +109,63 @@ def ogCompareChecksumFiles(*args):
|
|||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
def ogCalculateFullChecksum(*args):
|
||||
# Variables locales.
|
||||
FILE = None
|
||||
if "help" in args:
|
||||
SystemLib.ogHelp("$FUNCNAME", "$FUNCNAME [ str_repo | int_ndisk int_npartition ] path_filepath", "$FUNCNAME REPO ubuntu.img ==> ef899299caf8b517ce36f1157a93d8bf")
|
||||
|
||||
#/**
|
||||
# ogCalculateFullChecksum [ str_repo | int_ndisk int_npart ] path_filepath
|
||||
#@brief Devuelve la suma COMPLETA de comprobación (checksum) de un fichero.
|
||||
#@param path_filepath camino del fichero (independiente de mayúsculas)
|
||||
#@param str_repo repositorio de ficheros
|
||||
#@param int_ndisk nº de orden del disco
|
||||
#@param int_npartition nº de orden de la partición
|
||||
#@return hex_checksum Checksum del fichero
|
||||
#*/ ##
|
||||
def ogCalculateFullChecksum (disk=None, par=None, container=None, file=None):
|
||||
if file is None:
|
||||
raise TypeError ('missing required argument: "file"')
|
||||
|
||||
if container is not None:
|
||||
if disk is None and par is None:
|
||||
## we were given container=
|
||||
f = ogGetPath (src=container, file=file)
|
||||
dev_err = f'{container} {file}'
|
||||
print (f'ogGetPath (src=({container}), file=({file})) = f ({f})')
|
||||
else:
|
||||
raise TypeError ('argument "container" can be specified along neither "disk" nor "par"')
|
||||
|
||||
else:
|
||||
if disk is not None and par is not None:
|
||||
## we were given disk= par=
|
||||
f = ogGetPath (src=f'{disk} {par}', file=file)
|
||||
dev_err = f'{disk} {par} {file}'
|
||||
print (f'ogGetPath (src=({disk} {par}), file=({file})) = f ({f})')
|
||||
elif disk is None and par is None:
|
||||
## we were given nothing
|
||||
f = ogGetPath (file=file)
|
||||
dev_err = file
|
||||
print (f'ogGetPath (file=({file})) = f ({f})')
|
||||
else:
|
||||
raise TypeError ('if one of "disk" and "par" are specified, then both must be')
|
||||
|
||||
if not f:
|
||||
SystemLib.ogRaiseError ([], ogGlobals.OG_ERR_NOTFOUND, dev_err)
|
||||
return
|
||||
|
||||
# Comprobar que existe el fichero y devolver sus datos.
|
||||
FILE = ogGetPath(*args)
|
||||
if not FILE:
|
||||
SystemLib.ogRaiseError(OG_ERR_NOTFOUND, *args)
|
||||
return
|
||||
md5 = hashlib.md5()
|
||||
with open (f, 'rb') as fd:
|
||||
for chunk in iter (lambda: fd.read (64*1024), b''):
|
||||
md5.update (chunk)
|
||||
return md5.hexdigest()
|
||||
|
||||
# Calculate the checksum
|
||||
result = subprocess.run(["md5sum", FILE, "-b"], capture_output=True)
|
||||
checksum = result.stdout.decode().split()[0]
|
||||
|
||||
return checksum
|
||||
|
||||
|
||||
#/**
|
||||
# ogCopyFile [ str_repo | int_ndisk int_npart ] path_source [ str_repo | int_ndisk int_npart ] path_target
|
||||
#@brief Metafunción para copiar un fichero de sistema OpenGnSys a un directorio.
|
||||
#@see ogGetPath
|
||||
#@return Progreso de la copia.
|
||||
#@warning Deben existir tanto el fichero origen como el directorio destino.
|
||||
#*/ ##
|
||||
|
||||
def ogCopyFile(*args):
|
||||
# Variables locales.
|
||||
|
|
@ -113,22 +202,45 @@ def ogCopyFile(*args):
|
|||
result = subprocess.run(["rsync", "--progress", "--inplace", "-avh", SOURCE, TARGET])
|
||||
return result.returncode
|
||||
|
||||
def ogDeleteFile(*args):
|
||||
# Variables locales.
|
||||
FILE = None
|
||||
if "help" in args:
|
||||
SystemLib.ogHelp("$FUNCNAME", "$FUNCNAME [ str_repo | int_ndisk int_npartition ] path_file", "$FUNCNAME 1 2 /tmp/newfile.txt")
|
||||
|
||||
#/**
|
||||
# ogDeleteFile [ str_repo | int_ndisk int_npartition ] path_filepath
|
||||
#@brief Metafunción que borra un fichero de un dispositivo.
|
||||
#@see ogGetPath
|
||||
#@version 0.9 - Pruebas con OpenGnSys.
|
||||
#@author Ramon Gomez, ETSII Universidad de Sevilla
|
||||
#@date 2009-09-29
|
||||
#*/ ##
|
||||
#ogDeleteFile ([ str_repo | int_ndisk int_npartition ] path_file)
|
||||
#ogDeleteFile (container='REPO', file='/tmp/newfile.txt')
|
||||
#ogDeleteFile (disk=1, par=2, file='/tmp/newfile.txt')
|
||||
def ogDeleteFile (disk=None, par=None, container=None, file=None):
|
||||
if file is None:
|
||||
raise TypeError ('missing required argument: "file"')
|
||||
|
||||
if container is not None:
|
||||
if disk is None and par is None:
|
||||
## we were given container=
|
||||
src = container
|
||||
else:
|
||||
raise TypeError ('argument "container" can be specified along neither "disk" nor "par"')
|
||||
else:
|
||||
if disk is not None and par is not None:
|
||||
## we were given disk= par=
|
||||
src = f'{disk} {par}'
|
||||
else:
|
||||
## we were given nothing
|
||||
raise TypeError ('either "container" or both "disk" and "par" must be specified')
|
||||
|
||||
f = ogGetPath (src=src, file=file)
|
||||
if not f:
|
||||
SystemLib.ogRaiseError ([], ogGlobals.OG_ERR_NOTFOUND, f'{src} {file}')
|
||||
return
|
||||
|
||||
# Comprobar que existe el fichero y borrarlo.
|
||||
FILE = ogGetPath(*args)
|
||||
if not FILE:
|
||||
SystemLib.ogRaiseError(OG_ERR_NOTFOUND, *args)
|
||||
return
|
||||
try:
|
||||
os.remove(FILE)
|
||||
os.remove (f)
|
||||
except OSError as e:
|
||||
SystemLib.ogRaiseError(OG_ERR_NOTFOUND, *args)
|
||||
SystemLib.ogRaiseError ([], ogGlobals.OG_ERR_GENERIC, str (e))
|
||||
return
|
||||
|
||||
def ogDeleteTree(*args):
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import subprocess
|
|||
import re
|
||||
import json
|
||||
import os.path
|
||||
import shutil
|
||||
|
||||
import ogGlobals
|
||||
import SystemLib
|
||||
|
|
@ -852,10 +853,104 @@ def ogCreateTorrent (disk=None, par=None, container=None, file=None, ip_bttrack=
|
|||
#@param 1 str_REPO
|
||||
#@param 2 str_Relative_path_file_OGIMG_with_/
|
||||
#@param 3 md5 to check: use full to check download image torrent
|
||||
#@return 0 (true) cache sin imagen, SI es necesario actualizar el fichero.
|
||||
#@return 1 (false) imagen en la cache, NO es necesario actualizar el fichero
|
||||
#@return >1 (false) error de sintaxis (TODO)
|
||||
#@return True cache sin imagen, SI es necesario actualizar el fichero.
|
||||
#@return False imagen en la cache, NO es necesario actualizar el fichero
|
||||
#@return None error de sintaxis (TODO)
|
||||
#@note
|
||||
#@todo: Proceso en el caso de que el fichero tenga el mismo nombre, pero su contenido sea distinto.
|
||||
#@todo: Se dejan mensajes mientras se confirma su funcionamiento.
|
||||
#*/ ##
|
||||
#ogUpdateCacheIsNecesary ('REPO', '/PS1_PH1.img', 'UNICAST')
|
||||
#ogUpdateCacheIsNecesary ('REPO', '/ogclient.sqfs', 'FULL')
|
||||
#ogUpdateCacheIsNecesary ('REPO', '/ogclient.sqfs', 'TORRENT')
|
||||
def ogUpdateCacheIsNecesary (repo, file, proto):
|
||||
if not CacheLib.ogFindCache():
|
||||
SystemLib.ogRaiseError ([], ogGlobals.OG_ERR_NOTCACHE, '')
|
||||
return None
|
||||
|
||||
if repo.lower() != 'repo':
|
||||
SystemLib.ogRaiseError ([], ogGlobals.OG_ERR_NOTFOUND, f' {repo} {file}')
|
||||
return None
|
||||
|
||||
filesource = FileLib.ogGetPath (src=repo, file=file)
|
||||
print (f'filesource ({filesource})')
|
||||
if not filesource:
|
||||
SystemLib.ogRaiseError ([], ogGlobals.OG_ERR_NOTFOUND, f' {repo} {file}')
|
||||
return None
|
||||
|
||||
# paso 1. si no existe la imagen, confirmar que es necesario actualizar la cache.
|
||||
filetarget = FileLib.ogGetPath (src='CACHE', file=file)
|
||||
print (f'filetarget ({filetarget})')
|
||||
if not filetarget:
|
||||
# borramos el fichero bf del torrent, en el caso de que se hubiese quedado de algun proceso fallido
|
||||
if FileLib.ogGetPath (src='CACHE', file=f'/{file}.torrent.bf'): ogDeleteFile ('CACHE', f'{file}.torrent.bf')
|
||||
if FileLib.ogGetPath (src='CACHE', file=f'/{file}.sum'): ogDeleteFile ('CACHE', f'{file}.sum')
|
||||
if FileLib.ogGetPath (src='CACHE', file=f'/{file}.full.sum'): ogDeleteFile ('CACHE', f'{file}.full.sum')
|
||||
print ('TRUE(0), es necesario actualizar. Paso 1, la cache no contiene esa imagen ')
|
||||
return True
|
||||
|
||||
# Paso 2. Comprobamos que la imagen no estuviese en un proceso previo torrent
|
||||
if FileLib.ogGetPath (file=f'{filetarget}.torrent.bf'):
|
||||
#TODO: comprobar los md5 del fichero .torrent para asegurarnos que la imagen a descarga es la misma.
|
||||
print ('TRUE(0), es necesario actualizar. Paso 2, la imagen esta en un estado de descarga torrent interrumpido')
|
||||
return True
|
||||
|
||||
## En este punto la imagen en el repo y en la cache se llaman igual,
|
||||
# paso 4. Obtener los md5 del fichero imagen en la cacha segun PROTOCOLO $3
|
||||
if proto.lower() in ['full', 'torrent']:
|
||||
#Buscamos MD5 en el REPO SOURCE
|
||||
if os.path.exists (f'{filesource}.full.sum'):
|
||||
with open (f'{filesource}.full.sum', 'r') as fd:
|
||||
md5source = fd.read().strip()
|
||||
else:
|
||||
md5source = FileLib.ogCalculateFullChecksum (file=filesource)
|
||||
|
||||
# Generamos el MD5 (full) en la CACHE
|
||||
if not os.path.exists (f'{filetarget}.full.sum'):
|
||||
fullck = FileLib.ogCalculateFullChecksum (file=filetarget)
|
||||
with open (f'{filetarget}.full.sum', 'w') as fd:
|
||||
fd.write (fullck + '\n')
|
||||
with open (f'{filetarget}.full.sum', 'r') as fd:
|
||||
md5target = fd.read().strip()
|
||||
# Generamos el MD5 (little) en la CACHE para posteriores usos del protocolo MULTICAST
|
||||
if not os.path.exists (f'{filetarget}.sum'):
|
||||
ck = FileLib.ogCalculateChecksum (file=filetarget)
|
||||
with open (f'{filetarget}.sum', 'w') as fd:
|
||||
fd.write (ck + '\n')
|
||||
else:
|
||||
#Buscamos MD5 en el REPO SOURCE
|
||||
if os.path.exists (f'{filesource}.sum'):
|
||||
with open (f'{filesource}.sum', 'r') as fd:
|
||||
md5source = fd.read().strip()
|
||||
else:
|
||||
md5source = FileLib.ogCalculateChecksum (file=filesource)
|
||||
|
||||
# Generamos el MD5 (little) en la CACHE
|
||||
if not os.path.exists (f'{filetarget}.sum'):
|
||||
ck = FileLib.ogCalculateChecksum (file=filetarget)
|
||||
with open (f'{filetarget}.sum', 'w') as fd:
|
||||
fd.write (ck + '\n')
|
||||
with open (f'{filetarget}.sum', 'r') as fd:
|
||||
md5target = fd.read().strip()
|
||||
#Generamos o copiamos MD5 (full) en la CACHE para posteriores usos con Torrent
|
||||
# Si no existe el full.sum y si existe el .sum es porque el upateCACHE multicast o unicast ha sido correcto.
|
||||
if not os.path.exists (f'{filetarget}.full.sum') and os.path.exists (f'{filetarget}.sum'):
|
||||
if os.path.exists (f'{filesource}.full.sum'):
|
||||
#Existe el .full.sum en REPO realizamos COPIA
|
||||
shutil.copy2 (f'{filesource}.full.sum', f'{filetarget}.full.sum')
|
||||
else:
|
||||
#No existe .full.sum no en REPO LO GENERAMOS en la cache: situacion dificil que ocurra
|
||||
fullck = FileLib.ogCalculateFullChecksum (file=filetarget)
|
||||
with open (f'{filetarget}.full.sum', 'w') as fd:
|
||||
fd.write (fullck + '\n')
|
||||
|
||||
# Paso 5. comparar los md5
|
||||
if md5source == md5target:
|
||||
print ('FALSE (1), No es neceario actualizar. Paso5.A la imagen esta en cache')
|
||||
return False
|
||||
else:
|
||||
print ('imagen en cache distinta, borramos la imagen anterior')
|
||||
for f in [f'{filetarget}', f'{filetarget}.sum', f'{filetarget}.torrent', f'{filetarget}.full.sum']:
|
||||
os.unlink (f)
|
||||
print ('TRUE (0), Si es necesario actualizar.')
|
||||
return True
|
||||
|
|
|
|||
Loading…
Reference in New Issue