From d5279eba424702cefab85a62606f14089e761bd1 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Tue, 24 Jun 2025 15:03:52 +0200 Subject: [PATCH] refs #2305 have ogExecAndLog do subprocess.Popen --- CHANGELOG.md | 6 +++ ogclient/functions/ogCopyFile | 30 +++++++++++- ogclient/interfaceAdm/Configurar.py | 12 ++--- ogclient/lib/python3/DiskLib.py | 2 +- ogclient/lib/python3/ImageLib.py | 4 +- ogclient/lib/python3/ProtocolLib.py | 8 +-- ogclient/lib/python3/SystemLib.py | 75 +++++++++++++---------------- ogclient/scripts/initCache.py | 1 - ogclient/scripts/restoreImage.py | 4 +- ogclient/scripts/updateCache.py | 10 ++-- 10 files changed, 88 insertions(+), 64 deletions(-) mode change 100644 => 100755 ogclient/scripts/restoreImage.py mode change 100644 => 100755 ogclient/scripts/updateCache.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ad309c3..9cf2cb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.20.0] - 2025-06-24 + +### Changed + +- In order to have /tmp/command.log updated in realtime, ogExecAndLog now spawns a subprocess rather than calling functions directly + ## [0.19.2] - 2025-06-24 ### Changed diff --git a/ogclient/functions/ogCopyFile b/ogclient/functions/ogCopyFile index f65663b..522b9e7 100755 --- a/ogclient/functions/ogCopyFile +++ b/ogclient/functions/ogCopyFile @@ -19,6 +19,7 @@ if 5 == len (sys.argv): args = parser.parse_args() src = { 'container': args.src_container, 'file': args.src_file } dst = { 'container': args.dst_container, 'file': args.dst_file } + elif 7 == len (sys.argv): parser.add_argument ('src_disk') parser.add_argument ('src_par') @@ -29,6 +30,31 @@ elif 7 == len (sys.argv): args = parser.parse_args() src = { 'disk': args.src_disk, 'par': args.src_par, 'file': args.src_file } dst = { 'disk': args.dst_disk, 'par': args.dst_par, 'file': args.dst_file } + +elif 4 == len (sys.argv): + ## can be either: + ## - src_container src_file dst_file + ## - src_file dst_container dst_file + if 'REPO' == sys.argv[1] or 'CACHE' == sys.argv[1]: + ## we are in "src_container src_file dst_file" + parser.add_argument ('src_container') + parser.add_argument ('src_file') + parser.add_argument ('dst_file') + args = parser.parse_args() + src = { 'container': args.src_container, 'file': args.src_file } + dst = { 'file': args.dst_file } + elif 'REPO' == sys.argv[2] or 'CACHE' == sys.argv[2]: + ## we are in "src_file dst_container dst_file" + parser.add_argument ('src_file') + parser.add_argument ('dst_container') + parser.add_argument ('dst_file') + args = parser.parse_args() + src = { 'file': args.src_file } + dst = { 'container': args.dst_container, 'file': args.dst_file } + else: + ogHelp ('ogCopyFile', 'ogCopyFile [ str_repo | int_ndisk int_npartition ] path_source [ str_repo | int_ndisk int_npartition ] path_target', ['ogCopyFile REPO newfile.txt 1 2 /tmp/newfile.txt']) + sys.exit (1) + elif 6 == len (sys.argv): ## can be either: ## - src_disk src_par src_file dst_container dst_file @@ -36,7 +62,7 @@ elif 6 == len (sys.argv): try: num = int (sys.argv[1]) ## raises ValueError if sys.argv[1] doesn't look like a number - ## "src_disk src_par src_file dst_container dst_file" + ## we are in "src_disk src_par src_file dst_container dst_file" parser.add_argument ('src_disk') parser.add_argument ('src_par') parser.add_argument ('src_file') @@ -46,7 +72,7 @@ elif 6 == len (sys.argv): src = { 'disk': args.src_disk, 'par': args.src_par, 'file': args.src_file } dst = { 'container': args.dst_container, 'file': args.dst_file } except: - ## "src_container src_file dst_disk dst_par dst_file" + ## we are in "src_container src_file dst_disk dst_par dst_file" parser.add_argument ('src_container') parser.add_argument ('src_file') parser.add_argument ('dst_disk') diff --git a/ogclient/interfaceAdm/Configurar.py b/ogclient/interfaceAdm/Configurar.py index 731a611..b6fb12f 100755 --- a/ogclient/interfaceAdm/Configurar.py +++ b/ogclient/interfaceAdm/Configurar.py @@ -143,14 +143,14 @@ cur_ptt = DiskLib.ogGetPartitionTableType (dis) ptt = 'GPT' if InventoryLib.ogIsEfiActive() else 'MSDOS' if not cur_ptt or ptt != cur_ptt: DiskLib.ogDeletePartitionTable (dis) - SystemLib.ogExecAndLog ('command', DiskLib.ogUpdatePartitionTable) + SystemLib.ogExecAndLog ('command', [f'{ogGlobals.OGPYFUNCS}/ogUpdatePartitionTable']) DiskLib.ogCreatePartitionTable (dis, ptt) # Inicia la cache. if 'CACHE' in sparam: SystemLib.ogEcho (['session', 'log'], None, f'[30] {ogGlobals.lang.MSG_HELP_ogCreateCache}') SystemLib.ogEcho (['session', 'log'], None, f' initCache {tch}') - rc = SystemLib.ogExecAndLog ('command', CacheLib.initCache, tch) + rc = SystemLib.ogExecAndLog ('command', [f'{ogGlobals.OGSCRIPTS}/initCache', tch]) if not rc: SystemLib.ogRaiseError (['log', 'session'], ogGlobals.OG_ERR_CACHE, f'initCache failed') coproc.kill() @@ -159,12 +159,12 @@ if 'CACHE' in sparam: # Definir particionado. SystemLib.ogEcho (['session', 'log'], None, f'[50] {ogGlobals.lang.MSG_HELP_ogCreatePartitions}') SystemLib.ogEcho (['session', 'log'], None, f' ogCreatePartitions {dis} {' '.join (tbp)}') -res = SystemLib.ogExecAndLog ('command', DiskLib.ogCreatePartitions, dis, tbp) +res = SystemLib.ogExecAndLog ('command', [f'{ogGlobals.OGPYFUNCS}/ogCreatePartitions', dis] + tbp) if not res: coproc.kill() SystemLib.ogRaiseError (['log', 'session'], ogGlobals.OG_ERR_GENERIC, f'ogCreatePartitions {dis} {' '.join (tbp)}') sys.exit (1) -SystemLib.ogExecAndLog ('command', DiskLib.ogUpdatePartitionTable) +SystemLib.ogExecAndLog ('command', [f'{ogGlobals.OGPYFUNCS}/ogUpdatePartitionTable']) # Formatear particiones SystemLib.ogEcho (['session', 'log'], None, f'[70] {ogGlobals.lang.MSG_HELP_ogFormat}') @@ -174,10 +174,10 @@ for p in range (1, maxp+1): if 'CACHE' == tbf[p]: if CACHESIZE == tch: # Si el tamaño es distinto ya se ha formateado. SystemLib.ogEcho (['session', 'log'], None, ' ogFormatCache') - retval = SystemLib.ogExecAndLog ('command', CacheLib.ogFormatCache) + retval = SystemLib.ogExecAndLog ('command', [f'{ogGlobals.OGPYFUNCS}/ogFormatCache']) else: SystemLib.ogEcho (['session', 'log'], None, f' ogFormatFs {dis} {p} {tbf[p]}') - retval = SystemLib.ogExecAndLog ('command', FileSystemLib.ogFormatFs, dis, str(p), tbf[p]) + retval = SystemLib.ogExecAndLog ('command', [f'{ogGlobals.OGPYFUNCS}/ogFormatFs', dis, str(p), tbf[p]]) if retval: coproc.kill() SystemLib.ogRaiseError (['session', 'log'], ogGlobals.OG_ERR_GENERIC, f'ogFormatFs {dis} {p} {tbf[p]}') diff --git a/ogclient/lib/python3/DiskLib.py b/ogclient/lib/python3/DiskLib.py index 0977520..4a4f344 100644 --- a/ogclient/lib/python3/DiskLib.py +++ b/ogclient/lib/python3/DiskLib.py @@ -177,7 +177,7 @@ def ogCreatePartitions (disk, parts): ogCreatePartitionTable (ND) # Definir particiones y notificar al kernel. - p = subprocess.run (['sfdisk', DISK], input=sfdisk_input, capture_output=True, text=True) + p = subprocess.run (['sfdisk', DISK], input=sfdisk_input, text=True) subprocess.run (['partprobe', DISK]) if CACHESIZE: CacheLib.ogMountCache() return not p.returncode diff --git a/ogclient/lib/python3/ImageLib.py b/ogclient/lib/python3/ImageLib.py index 4c826ec..f590e38 100644 --- a/ogclient/lib/python3/ImageLib.py +++ b/ogclient/lib/python3/ImageLib.py @@ -515,9 +515,7 @@ def ogRestoreImage (repo, imgpath, disk, par): rc = None try: - p = subprocess.run (program, shell=True, capture_output=True, text=True) - print (p.stdout) - print (p.stderr) + p = subprocess.run (program, shell=True, text=True) rc = not p.returncode if not rc: SystemLib.ogRaiseError ([], ogGlobalsOG_ERR_IMAGE, f'{imgfile}, {disk}, {par}') diff --git a/ogclient/lib/python3/ProtocolLib.py b/ogclient/lib/python3/ProtocolLib.py index 4f1243b..83b3307 100644 --- a/ogclient/lib/python3/ProtocolLib.py +++ b/ogclient/lib/python3/ProtocolLib.py @@ -726,20 +726,22 @@ def ogTorrentStart (disk=None, par=None, container=None, torrentfile=None, torre # Creamos el fichero de resumen por defecto open (f'{source}.bf', 'w').close() # ctorrent controla otro fichero -b ${SOURCE}.bfog - subprocess.run (['ctorrent', '-f', '-c', '-X', f'sleep {time}; kill -2 $(pidof ctorrent)', '-C', '100', source, '-s', target, '-b', f'{source}.bfog']) + ctorrent_cmd = ['ctorrent', '-f', '-c', '-X', f'sleep {time}; kill -2 $(pidof ctorrent)', '-C', '100', source, '-s', target, '-b', f'{source}.bfog'] elif 'leecher' == mode: print ('Donwloading Torrent as leecher') - subprocess.run (['ctorrent', '${SOURCE}', '-X', 'sleep 30; kill -2 $(pidof ctorrent)', '-C', '100', '-U', '0']) + ctorrent_cmd = ['ctorrent', source, '-X', 'sleep 30; kill -2 $(pidof ctorrent)', '-C', '100', '-U', '0'] elif 'seeder' == mode: print ('MODE seeder ctorrent') # Creamos el fichero de resumen por defecto open (f'{source}.bf', 'w').close() # ctorrent controla otro fichero -b ${SOURCE}.bfog - subprocess.run (['ctorrent', '-f', '-c', '-X', f'sleep {time}; kill -2 $(pidof ctorrent)', '-C', '100', source, '-s', target, '-b', f'{source}.bfog']) + ctorrent_cmd = ['ctorrent', '-f', '-c', '-X', f'sleep {time}; kill -2 $(pidof ctorrent)', '-C', '100', source, '-s', target, '-b', f'{source}.bfog'] else: print ('this should not happen') return None + print (f'executing ctorrent: {' '.join(ctorrent_cmd)}') + subprocess.run (ctorrent_cmd) os.chdir (cwd) diff --git a/ogclient/lib/python3/SystemLib.py b/ogclient/lib/python3/SystemLib.py index d86b635..29884e2 100644 --- a/ogclient/lib/python3/SystemLib.py +++ b/ogclient/lib/python3/SystemLib.py @@ -3,22 +3,15 @@ import datetime from zoneinfo import ZoneInfo import sys import os +import select import json import shutil import inspect import glob -## for ogExecAndLog -from io import StringIO -from contextlib import redirect_stdout, redirect_stderr - import ogGlobals import StringLib -#NODEBUGFUNCTIONS, OGIMG, OG_ERR_CACHESIZE, OG_ERR_NOTCACHE, OG_ERR_NOTWRITE, OG_ERR_FILESYS -#OG_ERR_REPO, OG_ERR_NOTOS, OG_ERR_NOGPT, OG_ERR_OUTOFLIMIT, OG_ERR_IMAGE, OG_ERR_CACHE -#OGLOGSESSION, OGLOGCOMMAND, OGLOGFILE, OG_ERR_LOCKED, OG_ERR_PARTITION, OG_ERR_FORMAT, OG_ERR_NOTEXEC, OG_ERR_NOTFOUND - def _logtype2logfile (t): if 'log' == t.lower(): return ogGlobals.OGLOGFILE if 'jsonlog' == t.lower(): return ogGlobals.OGJSONLOGFILE @@ -84,11 +77,11 @@ def ogEcho (logtypes, loglevel, msg): #@note str_logfile = { LOG, SESSION, COMMAND } #*/ #ogExecAndLog (str_logfile ... str_command ...", -#ogExecAndLog ([], ogMyLib.ogSomeMethod, *args, **kwargs) -#ogExecAndLog ('command', ogMyLib.ogSomeMethod, *args, **kwargs) -#ogExecAndLog (['command'], ogMyLib.ogSomeMethod, *args, **kwargs) -#ogExecAndLog (['log', 'command'], ogMyLib.ogSomeMethod, *args, **kwargs) -def ogExecAndLog (logtypes, fun, *args, **kwargs): +#ogExecAndLog ([], ['/path/to/script', *args]) +#ogExecAndLog ('command', ['/path/to/script', *args]) +#ogExecAndLog (['command'], ['/path/to/script', *args]) +#ogExecAndLog (['log', 'command'], ['/path/to/script', *args]) +def ogExecAndLog (logtypes, script_and_args): logfiles = ['/dev/stdout'] if type (logtypes) is list: for l in logtypes: @@ -106,7 +99,7 @@ def ogExecAndLog (logtypes, fun, *args, **kwargs): else: logfiles.append (_logtype2logfile (logtypes)) - if not fun: + if not script_and_args: ogRaiseError ([], ogGlobals.OG_ERR_FORMAT, 'no function provided') return @@ -129,38 +122,38 @@ def ogExecAndLog (logtypes, fun, *args, **kwargs): # ## redirect stdout only # eval $COMMAND | tee -a $FILES - sout = serr = '' + capture_stderr = False if 'command' in logtypes: os.unlink (ogGlobals.OGLOGCOMMAND) open (ogGlobals.OGLOGCOMMAND, 'w').close() - with redirect_stdout (StringIO()) as r_stdout, redirect_stderr (StringIO()) as r_stderr: - rc = fun (*args, **kwargs) - sout = r_stdout.getvalue() - serr = r_stderr.getvalue() - else: - with redirect_stdout (StringIO()) as r_stdout: - rc = fun (*args, **kwargs) - sout = r_stdout.getvalue() + capture_stderr = True - rc_str = str (rc) - if sout or serr or ('True' != rc_str and 'False' != rc_str and 'None' != rc_str): - for f in logfiles: - with open (f, 'a') as fd: - if ogGlobals.OGJSONLOGFILE == f: - if sout: fd.write (json.dumps ({'message':sout}) + '\n') - if serr: fd.write (json.dumps ({'message':serr}) + '\n') - if rc_str: fd.write (json.dumps ({'message':rc_str}) + '\n') - else: - if sout: fd.write (f'{sout}\n') - if serr: fd.write (f'{serr}\n') - if rc_str: fd.write (f'{rc_str}\n') - #fd.write (f"ogExecAndLog: {fun.__name__} rc:\n{rc_str}\n") - #if sout: fd.write (f"ogExecAndLog: {fun.__name__} stdout:\n{sout}\n") - #else: fd.write (f"ogExecAndLog: {fun.__name__} stdout: (none)\n") - #if serr: fd.write (f"ogExecAndLog: {fun.__name__} stderr:\n{serr}\n") - #else: fd.write (f"ogExecAndLog: {fun.__name__} stderr: (none)\n") + p = subprocess.Popen (script_and_args, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + while True: + ready_to_read, _, _ = select.select ([p.stdout, p.stderr], [], [], 1) - return rc + partial_out = '' + if p.stdout in ready_to_read: + l = p.stdout.readline() + partial_out += l + if p.stderr in ready_to_read: + l = p.stderr.readline() ## always read from stderr even if we're discarding it, to prevent buffers from filling up + if capture_stderr: + partial_out += l + + if partial_out: + for f in logfiles: + with open (f, 'a') as fd: + if ogGlobals.OGJSONLOGFILE == f: + fd.write (json.dumps ({'message':partial_out})) + else: + fd.write (partial_out) + + if p.poll() is not None: + break + + rc = p.returncode + return not rc ## negate shell return code #/** # ogGetCaller diff --git a/ogclient/scripts/initCache.py b/ogclient/scripts/initCache.py index a0e038a..f564924 100755 --- a/ogclient/scripts/initCache.py +++ b/ogclient/scripts/initCache.py @@ -5,7 +5,6 @@ import sys import time -import subprocess import ogGlobals import SystemLib diff --git a/ogclient/scripts/restoreImage.py b/ogclient/scripts/restoreImage.py old mode 100644 new mode 100755 index 468b693..5e62c79 --- a/ogclient/scripts/restoreImage.py +++ b/ogclient/scripts/restoreImage.py @@ -90,7 +90,7 @@ if not imgfile or not imgdir: retval = 0 if proto in ['UNICAST', 'UNICAST-DIRECT']: SystemLib.ogEcho (['log', 'session'], None, f'[40] ogRestoreImage {repo} {imgname} {disk} {par} UNICAST') - retval = SystemLib.ogExecAndLog ('command', ImageLib.ogRestoreImage, repo, imgname, disk, par) + retval = SystemLib.ogExecAndLog ('command', [f'{ogGlobals.OGPYFUNCS}/ogRestoreImage', repo, imgname, disk, par]) elif proto in ['MULTICAST', 'MULTICAST-DIRECT']: tool = ImageLib.ogGetImageProgram ('REPO', imgname) if not tool: @@ -103,7 +103,7 @@ elif proto in ['MULTICAST', 'MULTICAST-DIRECT']: sys.exit (1) SystemLib.ogEcho (['log', 'session'], None, f'[40] ogMcastReceiverPartition {disk} {par} {port} {tool} {compress}') - retval = SystemLib.ogExecAndLog ('command', ProtocolLib.ogMcastReceiverPartition, disk, par, port, tool, compress) + retval = SystemLib.ogExecAndLog ('command', [f'{ogGlobals.OGPYFUNCS}/ogMcastReceiverPartition', disk, par, port, tool, compress]) else: SystemLib.ogRaiseError ('session', ogGlobals.OG_ERR_FORMAT, f'{ogGlobals.lang.MSG_FORMAT}: {prog} REPO|CACHE imagen ndisco nparticion [ UNICAST|MULTICAST opciones protocolo]') sys.exit (1) diff --git a/ogclient/scripts/updateCache.py b/ogclient/scripts/updateCache.py old mode 100644 new mode 100755 index 62be432..f0f7364 --- a/ogclient/scripts/updateCache.py +++ b/ogclient/scripts/updateCache.py @@ -167,13 +167,13 @@ if 'TORRENT' == protocolo: SystemLib.ogEcho (['log', 'session'], None, f'ogCopyFile {repositorio} {path}.torrent absolute {ogGlobals.OGCAC}/{ogGlobals.OGIMG}') mac_digits = NetLib.ogGetMacAddress().split (':') timewait = int ('0x' + mac_digits[4] + mac_digits[5], 16) * 120 / 65535 - if not SystemLib.ogExecAndLog ('command', FileLib.ogCopyFile, {'container':repositorio, 'file':f'{path}.torrent'}, {'file':imgdir}): + if not SystemLib.ogExecAndLog ('command', [f'{ogGlobals.OGPYFUNCS}/ogCopyFile', repositorio, f'{path}.torrent', imgdir]): sys.exit (1) p2pwait = random.randint (1, 121) SystemLib.ogEcho (['log', 'session'], None, f' [ ] {ogGlobals.lang.MSG_SCRIPTS_TASK_SLEEP} : {p2pwait} seconds') time.sleep (p2pwait) SystemLib.ogEcho (['log', 'session'], None, f' [ ] {ogGlobals.lang.MSG_SCRIPTS_TASK_START}: ogTorrentStart CACHE {path}.torrent {optprotocolo}') - SystemLib.ogExecAndLog ('command', ProtocolLib.ogTorrentStart, container='CACHE', torrentfile=f'{path}.torrent', torrentsess=optprotocolo) + SystemLib.ogExecAndLog ('command', [f'{ogGlobals.OGPYFUNCS}/ogTorrentStart', 'CACHE', f'{path}.torrent', optprotocolo]) resumeupdatecache = subprocess.run (['grep', '--max-count', '1', '--before-context', '1', 'Download', ogGlobals.OGLOGCOMMAND], capture_output=True, text=True).stdout resumeupdatecachebf = subprocess.run (['grep', '--max-count', '1', 'Download', ogGlobals.OGLOGCOMMAND], capture_output=True, text=True).stdout if 'Download complete.' == resumeupdatecachebf: @@ -183,14 +183,14 @@ elif 'MULTICAST' == protocolo: time.sleep (random.randint (1, 31)) SystemLib.ogEcho (['log', 'session'], None, f'ogMcastReceiverFile {port} CACHE {path}') - if not SystemLib.ogExecAndLog ('command', ProtocolLib.ogMcastReceiverFile, sess=port, container='CACHE', file=path): + if not SystemLib.ogExecAndLog ('command', [f'{ogGlobals.OGPYFUNCS}/ogMcastReceiverFile', port, 'CACHE', path]): sys.exit (1) resumeupdatecache = subprocess.run (['grep', '--max-count', '1', '--before-context', '1', 'Transfer complete', f'{ogGlobals.OGLOGCOMMAND}.tmp'], capture_output=True, text=True).stdout elif 'UNICAST' == protocolo: - print (f'ogExecAndLog ("command", FileLib.ogCopyFile, {{"container":{repositorio}, "file":{path}}}, {{"file":{imgdir}}})') - SystemLib.ogExecAndLog ('command', FileLib.ogCopyFile, {'container':repositorio, 'file':path}, {'file':imgdir}) + print (f'ogExecAndLog ("command", {ogGlobals.OGPYFUNCS}/ogCopyFile {repositorio} {path} {imgdir}') + SystemLib.ogExecAndLog ('command', [f'{ogGlobals.OGPYFUNCS}/ogCopyFile', repositorio, path, imgdir]) time.sleep (5) resumeupdatecache = subprocess.run (['grep', '--max-count', '1', '100%', f'{ogGlobals.OGLOGCOMMAND}.tmp'], capture_output=True, text=True).stdout -- 2.40.1