refs #2301 updates changelog 0.19.3

Reviewed-on: #68

CHANGELOG.md

@@ -5,12 +5,18 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [0.19.3] - 2025-06-24
+## [0.20.1] - 2025-06-24
 
 ### Changed
 
 - Fix mark for ogboot on linux
 
+## [0.20.0] - 2025-06-24
+
+### Changed
+
+- In order to have /tmp/command.log updated in realtime, ogExecAndLog now spawns a subprocess rather than calling functions directly
+
 ## [0.19.2] - 2025-06-24
 
 ### Changed

ogclient/functions/ogCopyFile

@@ -19,6 +19,7 @@ if 5 == len (sys.argv):
     args = parser.parse_args()
     src = { 'container': args.src_container, 'file': args.src_file }
     dst = { 'container': args.dst_container, 'file': args.dst_file }
+
 elif 7 == len (sys.argv):
     parser.add_argument ('src_disk')
     parser.add_argument ('src_par')
@@ -29,6 +30,31 @@ elif 7 == len (sys.argv):
     args = parser.parse_args()
     src = { 'disk': args.src_disk, 'par': args.src_par, 'file': args.src_file }
     dst = { 'disk': args.dst_disk, 'par': args.dst_par, 'file': args.dst_file }
+
+elif 4 == len (sys.argv):
+    ## can be either:
+    ## - src_container src_file       dst_file
+    ## - src_file                     dst_container dst_file
+    if 'REPO' == sys.argv[1] or 'CACHE' == sys.argv[1]:
+        ## we are in "src_container src_file       dst_file"
+        parser.add_argument ('src_container')
+        parser.add_argument ('src_file')
+        parser.add_argument ('dst_file')
+        args = parser.parse_args()
+        src = { 'container': args.src_container, 'file': args.src_file }
+        dst = { 'file': args.dst_file }
+    elif 'REPO' == sys.argv[2] or 'CACHE' == sys.argv[2]:
+        ## we are in "src_file                     dst_container dst_file"
+        parser.add_argument ('src_file')
+        parser.add_argument ('dst_container')
+        parser.add_argument ('dst_file')
+        args = parser.parse_args()
+        src = { 'file': args.src_file }
+        dst = { 'container': args.dst_container, 'file': args.dst_file }
+    else:
+        ogHelp ('ogCopyFile', 'ogCopyFile [ str_repo | int_ndisk int_npartition ] path_source [ str_repo | int_ndisk int_npartition ] path_target', ['ogCopyFile REPO newfile.txt 1 2 /tmp/newfile.txt'])
+        sys.exit (1)
+
 elif 6 == len (sys.argv):
     ## can be either:
     ## - src_disk src_par src_file    dst_container dst_file
@@ -36,7 +62,7 @@ elif 6 == len (sys.argv):
 
     try:
         num = int (sys.argv[1])     ## raises ValueError if sys.argv[1] doesn't look like a number
-        ## "src_disk src_par src_file    dst_container dst_file"
+        ## we are in "src_disk src_par src_file    dst_container dst_file"
         parser.add_argument ('src_disk')
         parser.add_argument ('src_par')
         parser.add_argument ('src_file')
@@ -46,7 +72,7 @@ elif 6 == len (sys.argv):
         src = { 'disk': args.src_disk, 'par': args.src_par, 'file': args.src_file }
         dst = { 'container': args.dst_container, 'file': args.dst_file }
     except:
-        ## "src_container src_file       dst_disk dst_par dst_file"
+        ## we are in "src_container src_file       dst_disk dst_par dst_file"
         parser.add_argument ('src_container')
         parser.add_argument ('src_file')
         parser.add_argument ('dst_disk')

ogclient/interfaceAdm/Configurar.py

@@ -143,14 +143,14 @@ cur_ptt = DiskLib.ogGetPartitionTableType (dis)
 ptt = 'GPT' if InventoryLib.ogIsEfiActive() else 'MSDOS'
 if not cur_ptt or ptt != cur_ptt:
     DiskLib.ogDeletePartitionTable (dis)
-    SystemLib.ogExecAndLog ('command', DiskLib.ogUpdatePartitionTable)
+    SystemLib.ogExecAndLog ('command', [f'{ogGlobals.OGPYFUNCS}/ogUpdatePartitionTable'])
     DiskLib.ogCreatePartitionTable (dis, ptt)
 
 # Inicia la cache.
 if 'CACHE' in sparam:
     SystemLib.ogEcho (['session', 'log'], None, f'[30] {ogGlobals.lang.MSG_HELP_ogCreateCache}')
     SystemLib.ogEcho (['session', 'log'], None, f'   initCache {tch}')
-    rc = SystemLib.ogExecAndLog ('command', CacheLib.initCache, tch)
+    rc = SystemLib.ogExecAndLog ('command', [f'{ogGlobals.OGSCRIPTS}/initCache', tch])
     if not rc:
         SystemLib.ogRaiseError (['log', 'session'], ogGlobals.OG_ERR_CACHE, f'initCache failed')
         coproc.kill()
@@ -159,12 +159,12 @@ if 'CACHE' in sparam:
 # Definir particionado.
 SystemLib.ogEcho (['session', 'log'], None, f'[50] {ogGlobals.lang.MSG_HELP_ogCreatePartitions}')
 SystemLib.ogEcho (['session', 'log'], None, f'   ogCreatePartitions {dis} {' '.join (tbp)}')
-res = SystemLib.ogExecAndLog ('command', DiskLib.ogCreatePartitions, dis, tbp)
+res = SystemLib.ogExecAndLog ('command', [f'{ogGlobals.OGPYFUNCS}/ogCreatePartitions', dis] + tbp)
 if not res:
     coproc.kill()
     SystemLib.ogRaiseError (['log', 'session'], ogGlobals.OG_ERR_GENERIC, f'ogCreatePartitions {dis} {' '.join (tbp)}')
     sys.exit (1)
-SystemLib.ogExecAndLog ('command', DiskLib.ogUpdatePartitionTable)
+SystemLib.ogExecAndLog ('command', [f'{ogGlobals.OGPYFUNCS}/ogUpdatePartitionTable'])
 
 #  Formatear particiones
 SystemLib.ogEcho (['session', 'log'], None, f'[70] {ogGlobals.lang.MSG_HELP_ogFormat}')
@@ -174,10 +174,10 @@ for p in range (1, maxp+1):
     if 'CACHE' == tbf[p]:
         if CACHESIZE == tch:    # Si el tamaño es distinto ya se ha formateado.
             SystemLib.ogEcho (['session', 'log'], None, '   ogFormatCache')
-            retval = SystemLib.ogExecAndLog ('command', CacheLib.ogFormatCache)
+            retval = SystemLib.ogExecAndLog ('command', [f'{ogGlobals.OGPYFUNCS}/ogFormatCache'])
     else:
         SystemLib.ogEcho (['session', 'log'], None, f'   ogFormatFs {dis} {p} {tbf[p]}')
-        retval = SystemLib.ogExecAndLog ('command', FileSystemLib.ogFormatFs, dis, str(p), tbf[p])
+        retval = SystemLib.ogExecAndLog ('command', [f'{ogGlobals.OGPYFUNCS}/ogFormatFs', dis, str(p), tbf[p]])
         if retval:
             coproc.kill()
             SystemLib.ogRaiseError (['session', 'log'], ogGlobals.OG_ERR_GENERIC, f'ogFormatFs  {dis} {p} {tbf[p]}')

ogclient/lib/python3/DiskLib.py

@@ -177,7 +177,7 @@ def ogCreatePartitions (disk, parts):
     ogCreatePartitionTable (ND)
 
     # Definir particiones y notificar al kernel.
-    p = subprocess.run (['sfdisk', DISK], input=sfdisk_input, capture_output=True, text=True)
+    p = subprocess.run (['sfdisk', DISK], input=sfdisk_input, text=True)
     subprocess.run (['partprobe', DISK])
     if CACHESIZE: CacheLib.ogMountCache()
     return not p.returncode

ogclient/lib/python3/ImageLib.py

@@ -515,9 +515,7 @@ def ogRestoreImage (repo, imgpath, disk, par):
 
     rc = None
     try:
-        p = subprocess.run (program, shell=True, capture_output=True, text=True)
-        print (p.stdout)
-        print (p.stderr)
+        p = subprocess.run (program, shell=True, text=True)
         rc = not p.returncode
         if not rc:
             SystemLib.ogRaiseError ([], ogGlobalsOG_ERR_IMAGE, f'{imgfile}, {disk}, {par}')

ogclient/lib/python3/ProtocolLib.py

@@ -726,20 +726,22 @@ def ogTorrentStart (disk=None, par=None, container=None, torrentfile=None, torre
         # Creamos el fichero de resumen por defecto
         open (f'{source}.bf', 'w').close()
         # ctorrent controla otro fichero -b ${SOURCE}.bfog
-        subprocess.run (['ctorrent', '-f', '-c', '-X', f'sleep {time}; kill -2 $(pidof ctorrent)', '-C', '100', source, '-s', target, '-b', f'{source}.bfog'])
+        ctorrent_cmd = ['ctorrent', '-f', '-c', '-X', f'sleep {time}; kill -2 $(pidof ctorrent)', '-C', '100', source, '-s', target, '-b', f'{source}.bfog']
     elif 'leecher' == mode:
         print ('Donwloading Torrent as leecher')
-        subprocess.run (['ctorrent', '${SOURCE}', '-X', 'sleep 30; kill -2 $(pidof ctorrent)', '-C', '100', '-U', '0'])
+        ctorrent_cmd = ['ctorrent', source, '-X', 'sleep 30; kill -2 $(pidof ctorrent)', '-C', '100', '-U', '0']
     elif 'seeder' == mode:
         print ('MODE seeder ctorrent')
         # Creamos el fichero de resumen por defecto
         open (f'{source}.bf', 'w').close()
         # ctorrent controla otro fichero -b ${SOURCE}.bfog
-        subprocess.run (['ctorrent', '-f', '-c', '-X', f'sleep {time}; kill -2 $(pidof ctorrent)', '-C', '100', source, '-s', target, '-b', f'{source}.bfog'])
+        ctorrent_cmd = ['ctorrent', '-f', '-c', '-X', f'sleep {time}; kill -2 $(pidof ctorrent)', '-C', '100', source, '-s', target, '-b', f'{source}.bfog']
     else:
         print ('this should not happen')
         return None
 
+    print (f'executing ctorrent: {' '.join(ctorrent_cmd)}')
+    subprocess.run (ctorrent_cmd)
     os.chdir (cwd)
 
 

ogclient/lib/python3/SystemLib.py

@@ -3,22 +3,15 @@ import datetime
 from zoneinfo import ZoneInfo
 import sys
 import os
+import select
 import json
 import shutil
 import inspect
 import glob
 
-## for ogExecAndLog
-from io import StringIO
-from contextlib import redirect_stdout, redirect_stderr
-
 import ogGlobals
 import StringLib
 
-#NODEBUGFUNCTIONS, OGIMG, OG_ERR_CACHESIZE, OG_ERR_NOTCACHE, OG_ERR_NOTWRITE, OG_ERR_FILESYS
-#OG_ERR_REPO, OG_ERR_NOTOS, OG_ERR_NOGPT, OG_ERR_OUTOFLIMIT, OG_ERR_IMAGE, OG_ERR_CACHE
-#OGLOGSESSION, OGLOGCOMMAND, OGLOGFILE, OG_ERR_LOCKED, OG_ERR_PARTITION, OG_ERR_FORMAT, OG_ERR_NOTEXEC, OG_ERR_NOTFOUND
-
 def _logtype2logfile (t):
     if   'log'     == t.lower(): return ogGlobals.OGLOGFILE
     if   'jsonlog' == t.lower(): return ogGlobals.OGJSONLOGFILE
@@ -84,11 +77,11 @@ def ogEcho (logtypes, loglevel, msg):
 #@note    str_logfile = { LOG, SESSION, COMMAND }
 #*/
 #ogExecAndLog (str_logfile ... str_command ...",
-#ogExecAndLog ([],                 ogMyLib.ogSomeMethod, *args, **kwargs)
-#ogExecAndLog ('command',          ogMyLib.ogSomeMethod, *args, **kwargs)
-#ogExecAndLog (['command'],        ogMyLib.ogSomeMethod, *args, **kwargs)
-#ogExecAndLog (['log', 'command'], ogMyLib.ogSomeMethod, *args, **kwargs)
-def ogExecAndLog (logtypes, fun, *args, **kwargs):
+#ogExecAndLog ([],                 ['/path/to/script', *args])
+#ogExecAndLog ('command',          ['/path/to/script', *args])
+#ogExecAndLog (['command'],        ['/path/to/script', *args])
+#ogExecAndLog (['log', 'command'], ['/path/to/script', *args])
+def ogExecAndLog (logtypes, script_and_args):
     logfiles = ['/dev/stdout']
     if type (logtypes) is list:
         for l in logtypes:
@@ -106,7 +99,7 @@ def ogExecAndLog (logtypes, fun, *args, **kwargs):
         else:
             logfiles.append (_logtype2logfile (logtypes))
 
-    if not fun:
+    if not script_and_args:
         ogRaiseError ([], ogGlobals.OG_ERR_FORMAT, 'no function provided')
         return
 
@@ -129,38 +122,38 @@ def ogExecAndLog (logtypes, fun, *args, **kwargs):
     #    ## redirect stdout only
     #    eval $COMMAND      | tee -a $FILES
 
-    sout = serr = ''
+    capture_stderr = False
     if 'command' in logtypes:
         os.unlink (ogGlobals.OGLOGCOMMAND)
         open (ogGlobals.OGLOGCOMMAND, 'w').close()
-        with redirect_stdout (StringIO()) as r_stdout, redirect_stderr (StringIO()) as r_stderr:
-            rc = fun (*args, **kwargs)
-            sout = r_stdout.getvalue()
-            serr = r_stderr.getvalue()
-    else:
-        with redirect_stdout (StringIO()) as r_stdout:
-            rc = fun (*args, **kwargs)
-            sout = r_stdout.getvalue()
+        capture_stderr = True
 
-    rc_str = str (rc)
-    if sout or serr or ('True' != rc_str and 'False' != rc_str and 'None' != rc_str):
-        for f in logfiles:
-            with open (f, 'a') as fd:
-                if ogGlobals.OGJSONLOGFILE == f:
-                    if sout:   fd.write (json.dumps ({'message':sout})   + '\n')
-                    if serr:   fd.write (json.dumps ({'message':serr})   + '\n')
-                    if rc_str: fd.write (json.dumps ({'message':rc_str}) + '\n')
-                else:
-                    if sout:   fd.write (f'{sout}\n')
-                    if serr:   fd.write (f'{serr}\n')
-                    if rc_str: fd.write (f'{rc_str}\n')
-                    #fd.write (f"ogExecAndLog: {fun.__name__} rc:\n{rc_str}\n")
-                    #if sout: fd.write (f"ogExecAndLog: {fun.__name__} stdout:\n{sout}\n")
-                    #else:    fd.write (f"ogExecAndLog: {fun.__name__} stdout: (none)\n")
-                    #if serr: fd.write (f"ogExecAndLog: {fun.__name__} stderr:\n{serr}\n")
-                    #else:    fd.write (f"ogExecAndLog: {fun.__name__} stderr: (none)\n")
+    p = subprocess.Popen (script_and_args, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    while True:
+        ready_to_read, _, _ = select.select ([p.stdout, p.stderr], [], [], 1)
 
-    return rc
+        partial_out = ''
+        if p.stdout in ready_to_read:
+            l = p.stdout.readline()
+            partial_out += l
+        if p.stderr in ready_to_read:
+            l = p.stderr.readline()    ## always read from stderr even if we're discarding it, to prevent buffers from filling up
+            if capture_stderr:
+                partial_out += l
+
+        if partial_out:
+            for f in logfiles:
+                with open (f, 'a') as fd:
+                    if ogGlobals.OGJSONLOGFILE == f:
+                        fd.write (json.dumps ({'message':partial_out}))
+                    else:
+                        fd.write (partial_out)
+
+        if p.poll() is not None:
+            break
+
+    rc = p.returncode
+    return not rc       ## negate shell return code
 
 #/**
 #         ogGetCaller

ogclient/scripts/initCache.py

@@ -5,7 +5,6 @@
 
 import sys
 import time
-import subprocess
 
 import ogGlobals
 import SystemLib

ogclient/scripts/restoreImage.py

@@ -90,7 +90,7 @@ if not imgfile or not imgdir:
 retval = 0
 if proto in ['UNICAST', 'UNICAST-DIRECT']:
     SystemLib.ogEcho (['log', 'session'], None, f'[40] ogRestoreImage {repo} {imgname} {disk} {par} UNICAST')
-    retval = SystemLib.ogExecAndLog ('command', ImageLib.ogRestoreImage, repo, imgname, disk, par)
+    retval = SystemLib.ogExecAndLog ('command', [f'{ogGlobals.OGPYFUNCS}/ogRestoreImage', repo, imgname, disk, par])
 elif proto in ['MULTICAST', 'MULTICAST-DIRECT']:
     tool = ImageLib.ogGetImageProgram ('REPO', imgname)
     if not tool:
@@ -103,7 +103,7 @@ elif proto in ['MULTICAST', 'MULTICAST-DIRECT']:
         sys.exit (1)
 
     SystemLib.ogEcho (['log', 'session'], None, f'[40] ogMcastReceiverPartition {disk} {par} {port} {tool} {compress}')
-    retval = SystemLib.ogExecAndLog ('command', ProtocolLib.ogMcastReceiverPartition, disk, par, port, tool, compress)
+    retval = SystemLib.ogExecAndLog ('command', [f'{ogGlobals.OGPYFUNCS}/ogMcastReceiverPartition', disk, par, port, tool, compress])
 else:
     SystemLib.ogRaiseError ('session', ogGlobals.OG_ERR_FORMAT, f'{ogGlobals.lang.MSG_FORMAT}: {prog} REPO|CACHE imagen ndisco nparticion [ UNICAST|MULTICAST opciones protocolo]')
     sys.exit (1)

ogclient/scripts/updateCache.py

@@ -167,13 +167,13 @@ if 'TORRENT' == protocolo:
     SystemLib.ogEcho (['log', 'session'], None, f'ogCopyFile {repositorio} {path}.torrent absolute {ogGlobals.OGCAC}/{ogGlobals.OGIMG}')
     mac_digits = NetLib.ogGetMacAddress().split (':')
     timewait = int ('0x' + mac_digits[4] + mac_digits[5], 16) * 120 / 65535
-    if not SystemLib.ogExecAndLog ('command', FileLib.ogCopyFile, {'container':repositorio, 'file':f'{path}.torrent'}, {'file':imgdir}):
+    if not SystemLib.ogExecAndLog ('command', [f'{ogGlobals.OGPYFUNCS}/ogCopyFile', repositorio, f'{path}.torrent', imgdir]):
         sys.exit (1)
     p2pwait = random.randint (1, 121)
     SystemLib.ogEcho (['log', 'session'], None, f'   [ ] {ogGlobals.lang.MSG_SCRIPTS_TASK_SLEEP} :  {p2pwait} seconds')
     time.sleep (p2pwait)
     SystemLib.ogEcho (['log', 'session'], None, f'   [  ] {ogGlobals.lang.MSG_SCRIPTS_TASK_START}:  ogTorrentStart CACHE {path}.torrent {optprotocolo}')
-    SystemLib.ogExecAndLog ('command', ProtocolLib.ogTorrentStart, container='CACHE', torrentfile=f'{path}.torrent', torrentsess=optprotocolo)
+    SystemLib.ogExecAndLog ('command', [f'{ogGlobals.OGPYFUNCS}/ogTorrentStart', 'CACHE', f'{path}.torrent', optprotocolo])
     resumeupdatecache   = subprocess.run (['grep', '--max-count', '1', '--before-context', '1', 'Download', ogGlobals.OGLOGCOMMAND], capture_output=True, text=True).stdout
     resumeupdatecachebf = subprocess.run (['grep', '--max-count', '1',                          'Download', ogGlobals.OGLOGCOMMAND], capture_output=True, text=True).stdout
     if 'Download complete.' == resumeupdatecachebf:
@@ -183,14 +183,14 @@ elif 'MULTICAST' == protocolo:
     time.sleep (random.randint (1, 31))
 
     SystemLib.ogEcho (['log', 'session'], None, f'ogMcastReceiverFile {port} CACHE {path}')
-    if not SystemLib.ogExecAndLog ('command', ProtocolLib.ogMcastReceiverFile, sess=port, container='CACHE', file=path):
+    if not SystemLib.ogExecAndLog ('command', [f'{ogGlobals.OGPYFUNCS}/ogMcastReceiverFile', port, 'CACHE', path]):
         sys.exit (1)
 
     resumeupdatecache = subprocess.run (['grep', '--max-count', '1', '--before-context', '1', 'Transfer complete', f'{ogGlobals.OGLOGCOMMAND}.tmp'], capture_output=True, text=True).stdout
 
 elif 'UNICAST' == protocolo:
-    print (f'ogExecAndLog ("command", FileLib.ogCopyFile, {{"container":{repositorio}, "file":{path}}}, {{"file":{imgdir}}})')
-    SystemLib.ogExecAndLog ('command', FileLib.ogCopyFile, {'container':repositorio, 'file':path}, {'file':imgdir})
+    print (f'ogExecAndLog ("command", {ogGlobals.OGPYFUNCS}/ogCopyFile {repositorio} {path} {imgdir}')
+    SystemLib.ogExecAndLog ('command', [f'{ogGlobals.OGPYFUNCS}/ogCopyFile', repositorio, path, imgdir])
     time.sleep (5)
     resumeupdatecache = subprocess.run (['grep', '--max-count', '1', '100%', f'{ogGlobals.OGLOGCOMMAND}.tmp'], capture_output=True, text=True).stdout