refs #597 translate and clean up comments

.dockerignore

@@ -0,0 +1,12 @@
+.git
+.gitignore
+.dockerignore
+__pycache__
+Dockerfile
+INSTALL.es.txt
+Jenkinsfile
+kernelParameterOG.es.txt
+ogclient/
+ogclientImg2Sqfs
+ogclientSqfs2Img
+*.swp

Dockerfile

@@ -0,0 +1,40 @@
+FROM ubuntu:noble
+
+ARG OPENGNSYS_BRANCH=main
+
+RUN apt-get update && apt-get -y install \
+  debootstrap \
+  fdisk \
+  file \
+  genisoimage \
+  git \
+  ipxe \
+  jq \
+  lsof \
+  python3 \
+  python3-apt \
+  qemu-utils \
+  rsync \
+  schroot \
+  squashfs-tools \
+  subversion \
+  syslinux \
+  syslinux-efi \
+  syslinux-utils \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN groupadd opengnsys
+RUN mkdir -p /opt/opengnsys/client /tmp/opengnsys/oglive_builder /var/lib/tftpboot/
+RUN ln -fs /var/lib/tftpboot /opt/opengnsys
+
+## take stuff from the OG repo: a) cloning engine, b) shared stuff and c) gitrelease
+RUN git clone -c http.sslVerify=false --branch ${OPENGNSYS_BRANCH} https://ognproject.evlt.uma.es/gitea/opengnsys/opengnsys.git /tmp/ogrepo && \
+    mv /tmp/ogrepo/client/engine /tmp/ogrepo/client/shared /tmp/opengnsys/ && \
+    git --git-dir /tmp/ogrepo/.git log --date format:r%Y%m%d --format=%ad -1 >/tmp/opengnsys/oglive_builder/gitrelease && \
+    rm -rf /tmp/ogrepo/
+
+#RUN echo 0     ## development: invalidate docker cache at this point, so that the 'COPY' below actually copies files
+COPY . /tmp/opengnsys/oglive_builder/
+RUN cp /tmp/opengnsys/oglive_builder/schroot.conf /tmp/opengnsys/oglive_builder/mount-defaults /etc/schroot/ && echo '' >/etc/schroot/default/nssdatabases && rm -f /etc/schroot/setup.d/*chrootname
+
+ENTRYPOINT ["/tmp/opengnsys/oglive_builder/mkoglive.py"]

INSTALL.es.txt

@@ -4,43 +4,37 @@ OpenGnsys Client boot-tools   INSTALL.es.txt
 
 Advertencia previa:
 -----------------------------
-- No usar este instalador en un sistema OpenGnsys en producción.
 - Se puede ejecutar este script desde un sistema Ubuntu 10.04 o superior.
 - Mas información https://opengnsys.es/trac/wiki/ClienteInitrdDSGenerarloV1.0.2
 
 
 Requisitos iniciales
 --------------------
-- Utilizar el usuario "root" para ejecutar el proceso de generación del cliente.
 - Comprobar la conexión a Internet, sobre todo si usas proxy (el hecho de que funcione apt-get no significa que tengas el proxy configurado).
 
 
-Proceso de instalación
-----------------------
+Proceso de creación de imagen docker
+------------------------------------
 
-#0. Si ya tenemos configurado TFTP, renombrar el directorio /var/lib/tftpboot/ogclient.
-mv /opt/opengnsys/tftpboot/ogclient /opt/opengnsys/tftpboot/ogclient-old;
+Simplemente llamar a docker build sin sorpresas:
 
-#1. Descargar del Subversion las herramientas del cliente
-apt-get install subversion wget
-svn checkout https://opengnsys.es/svn/branches/version1.1/client /tmp/opengnsys_installer/opengnsys/client/;
-find /tmp/opengnsys_installer/ -name .svn -type d -exec rm -fr {} \; 2>/dev/null;
+  docker build -t mkoglive .
 
-#2. Ejecutar el instalador: 
-/tmp/opengnsys_installer/opengnsys/client/boot-tools/boottoolsgenerator.sh;
+Si queremos usar una rama diferente del repo de opengnsys:
 
-Notas para ejecución del instalador en Ubuntu 12.10:
-- Configuración openssh-client:
-  - *** ssh_config (Y/I/N/O/D/Z) [default=N] ?   [intro]
-- Mapa de caracteres:
-  - Pantalla "Configuring console-data": elegir "Select keymap from full list"
-  - Keymap: pc / qwerty / Spanish / Standard / Standard
-- Clave publica del servidor
- - verwrite (y/n)?   [intro]
-- Mapa de caracteres (repetición):
-  - Pantalla "Configuring console-data":
-  - Select keymap from full list
-  - pc / qwerty / Spanish / Standard / Standard,  UTF-8,  Combined - Latin; Slavic Cyrillic; Greek,  Fixed,  16
+  docker build --build-arg OPENGNSYS_BRANCH=mybranch -t mkoglive .
+
+
+Proceso de creación de imagen oglive
+------------------------------------
+
+  docker run --rm --name mkoglive --privileged=true --volume $PWD/ogclient:/var/lib/tftpboot/ogclient mkoglive --codename noble
+
+Hace falta --privileged=true para que mkoglive.py pueda ejecutar, entre otras cosas, losetup.
+
+El parámetro --codename permite especificar la versión de ubuntu.
+
+Las imágenes se crean en ./ogclient.
 
 
 
@@ -48,14 +42,15 @@ Descripción de la estrucutra de boot-tools
 ------------------------------------------
 
 - INSTALL.es.txt          este mismo archivo.
-- boottollsgenerator.sh   proceso de generación del sistema operativo opengnsys.
-- boottoolsfunctions.lib  libreria complementaria del boottolsgenerator.sh
-- includes/               directorio con ficheros específicos para incluir en la distribución.
-
-
-
-NOTA:
- boottollsfuncions.lib  ->  btogSetfsBase
- bootsoftwareinstall ->
- packages ->sw.testing
-
+- mkoglive:
+  - mkoglive.py           script principal para crear la imagen oglive
+  - mkoglive.cfg          configuración de mkoglive.py
+  - boottools/            libreria complementaria del mkoglive.py
+  - includes/             directorio con ficheros específicos para incluir en la distribución
+- schroot:
+  - schroot.conf          configuración de schroot
+  - mount-defaults        configuración de puntos de montaje para schroot
+  - chroot-tasks.py       helper para configurar el oglive estando en schroot
+- docker:
+  - Dockerfile            para crear una imagen docker
+  - .dockerignore         para limitar el contexto de docker-build

boottools/btog.py

@@ -11,49 +11,17 @@ from . import utils
 
 logger = logging.getLogger ('boottools')
 
-schroot_conf_tpl = """
-[IMGogclient]
-type=loopback
-file=/var/lib/tftpboot/ogclient/ogclient.img
-description=ogclient Ubuntu image
-#priority=1
-users=root
-groups=root
-root-groups=root
-mount-options=-o offset=32256
-root-users=root
-""".strip()
-
-mount_defaults_tpl = """
-# mount.defaults: static file system information for chroots.
-# Note that the mount point will be prefixed by the chroot path
-# (CHROOT_PATH)
-#
-# <file system> <mount point>   <type>  <options>       <dump>  <pass>
-proc            /proc           proc    defaults        0       0
-#procbususb      /proc/bus/usb   usbfs   defaults        0       0
-#/dev            /dev            none    rw,bind         0       0
-/dev/pts        /dev/pts        none    rw,bind         0       0
-/dev/shm        /dev/shm        none    rw,bind         0       0
-#/home          /home           none    rw,bind         0       0
-/tmp            /tmp            none    rw,bind         0       0
-""".strip()
-
 def GetVar (osarch):
-    btdir            = '/tmp/opengnsys/oglive_builder'
     bttargetdir      = '/var/lib/tftpboot/ogclient/'
     btrootfsimg      = os.path.join (bttargetdir, 'ogclient.img')
     btrootfsmnt      = os.path.join (bttargetdir, 'ogclientmount')
-    btrootfsimglabel = 'ogClient'
-    log_file         = '/tmp/boot-tools_installation.log'
-    versionboottools = 'ogLive'
     if 'i386' == osarch:
         btvirtualdisksize = '2000'    # tamaño maximo limitado por schroot 2GB para 32 bits
     else:
         btvirtualdisksize = '3G'
-    return btdir, bttargetdir, btrootfsimg, btrootfsmnt, btrootfsimglabel, log_file, versionboottools, btvirtualdisksize
+    return bttargetdir, btrootfsimg, btrootfsmnt, btvirtualdisksize
 
-def GetOsInfo1 (type_client):
+def GetOsInfo (type_client='host'):
     if 'precise' == type_client:	# ogLive 1.0.4-rc2 basado en Ubuntu 12.04 LTS.
             osdistrib='ubuntu'
             oscodename='precise'
@@ -151,16 +119,14 @@ def GetOsInfo1 (type_client):
             osarch, _=utils.run (['dpkg', '--print-architecture'])
             oshttp='http://es.archive.ubuntu.com/ubuntu/'
     else:	# Parámetro desconocido
-            print ('Parámetro no válido.')
-            os.exit (1)
+            logger.error ('Parámetro no válido.')
+            osdistrib=None
+            oscodename=None
+            osrelease=None
+            osarch=None
+            oshttp=None
     return osdistrib, oscodename, osrelease, osarch, oshttp
 
-def GetOsInfo2 (ogrepo_dir, versionboottools, oscodename, osrelease, osarch):
-    gitrelease, _  = utils.run (['git', '--git-dir', f'{ogrepo_dir}/.git', 'log', '--date', 'format:r%Y%m%d', '--format=%ad', '-1'])
-    nameisoclient  ='-'.join ([versionboottools, oscodename, osrelease, osarch, gitrelease])
-    namehostclient = '-'.join ([versionboottools, oscodename, gitrelease])
-    return gitrelease, nameisoclient, namehostclient
-
 def mkrootfs (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, bttargetdir, osarch):
     logger.info (f'Creación y formateo del disco virtual "{btrootfsimg}" "{btvirtualdisksize}" MB')
 
@@ -199,8 +165,6 @@ def mkrootfs (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, btt
 
     logger.info ('Partitioning disk image')
     stdout, _ = utils.run (['bash', '-c', f"echo $'n\np\n1\n\n\nt\n83\nw' |fdisk {diskloop}"])
-    logger.debug (f'fdisk stdout follows:')
-    for i in stdout.strip().split('\n'): logger.debug ('  ' + i)
 
     time.sleep (3)
     logger.debug ('losetup --detach')
@@ -229,30 +193,7 @@ def mkrootfs (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, btt
 
     logger.info (f'"{btrootfsimg}" "{btvirtualdisksize}" MB : OK')
 
-#SetFsAcces: habilita el acceso al sistema root del cliente con  schroot
-def configure_schroot (btrootfsimg):
-    if (utils.grep (btrootfsimg, '/etc/schroot/schroot.conf')):
-        return
-    logger.info ('Iniciando la configuración del schroot')
-
-    if (os.path.exists ('/etc/schroot/schroot.conf')):
-        shutil.copy ('/etc/schroot/schroot.conf', '/etc/schroot/schroot.conf.bak')
-    with open ('/etc/schroot/schroot.conf', 'w') as f:
-        f.write (schroot_conf_tpl + '\n')
-
-    if (os.path.exists ('/etc/schroot/mount-defaults')):
-        shutil.copy ('/etc/schroot/mount-defaults', '/etc/schroot/mount-defaults.bak')
-    with open ('/etc/schroot/mount-defaults', 'w') as f:
-        f.write (mount_defaults_tpl + '\n')
-
-    for i in glob.glob ('/etc/schroot/setup.d/*chrootname'):
-        os.unlink (i)
-
-    logger.info ('Finalizado: OK')
-    return 0
-
-# debootstrap: Genera el sistema root base con debootstrap
-# trabaja sobre un rootfs ya montado
+# works on an already mounted rootfs
 def debootstrap (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp):
     logger.info ('Iniciando la generación del sistema de archivos')
 
@@ -303,23 +244,22 @@ def copy_og_files (builder, og_shared, og_engine, ogclientmount, osdistrib, osco
     if os.path.exists (f'{og_shared}/bin/ogAdmClient'): shutil.copy (f'{og_shared}/bin/ogAdmClient', f'{ogclientmount}/bin/')
 
 def ssh_server (btrootfsmnt):
-    if not os.path.exists ('/root/.ssh/id_rsa'):                     ## crea un par de claves en la VM, no en el chroot
+    if not os.path.exists ('/root/.ssh/id_rsa'):                     ## creates a key pair in the VM (or docker container), not in the chroot
         utils.run (['ssh-keygen', '-q', '-f', '/root/.ssh/id_rsa', '-N', ''])
 
     logger.debug ('comprobando directorio .ssh del root')
-    if not os.path.exists (f'{btrootfsmnt}/root/.ssh'):     ## crea directorio dentro del chroot
+    if not os.path.exists (f'{btrootfsmnt}/root/.ssh'):              ## creates directory within the chroot
         logger.debug ('creando directorio .ssh 600')
         os.mkdir (f'{btrootfsmnt}/root/.ssh')
         os.chmod (f'{btrootfsmnt}/root/.ssh', 0o700)
 
-    logger.debug ('creando el fichero authorized_keys')              ## crea archivo en el chroot
+    logger.debug ('creando el fichero authorized_keys')              ## creates file within the chroot
     if not os.path.exists (f'{btrootfsmnt}/root/.ssh/authorized_keys'):
         open (f'{btrootfsmnt}/root/.ssh/authorized_keys', 'w').close()
     os.chmod (f'{btrootfsmnt}/root/.ssh/authorized_keys', 0o600)
 
     logger.debug ('importando la clave publica del servidor OG')
-    #cat /tmp/id_rsa.pub
-    if os.path.exists ('/root/.ssh/id_rsa.pub'):                     ## coge la publica de la VM y la pone en el authorized_keys del chroot
+    if os.path.exists ('/root/.ssh/id_rsa.pub'):                     ## takes the pubkey from the VM (or docker container) and puts it in the authorized_keys within the chroot
         fdin = open ('/root/.ssh/id_rsa.pub', 'r')
         fdout = open (f'{btrootfsmnt}/root/.ssh/authorized_keys', 'a')
         while True:
@@ -334,9 +274,9 @@ def ssh_server (btrootfsmnt):
 
 def ssh_client (btrootfsmnt):
     if not os.path.exists (f'{btrootfsmnt}/root/.ssh/id_rsa'):
-        utils.run (['ssh-keygen', '-q', '-f', f'{btrootfsmnt}/root/.ssh/id_rsa', '-N', ''])     ## crea un par de claves en el chroot
+        utils.run (['ssh-keygen', '-q', '-f', f'{btrootfsmnt}/root/.ssh/id_rsa', '-N', ''])     ## creates a key pair in the chroot
 
-    #cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys          ## coge la publica y se la autoriza a sí mismo
+    #cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys          ## takes the pubkey and authorises it to itself
     fdin = open (f'{btrootfsmnt}//root/.ssh/id_rsa.pub', 'r')
     fdout = open (f'{btrootfsmnt}/root/.ssh/authorized_keys', 'a')
     while True:
@@ -349,11 +289,7 @@ def ssh_client (btrootfsmnt):
     ## TODO: exportamos la publica a los repos
     #cp /root/.ssh/id_rsa.pub /tmp/rsa.ogclient.pub
 
-#move_initrd  genera un initrd.
 def move_initrd (bttargetdir, osrelease):
-    #echo "cp /tmp/*-${OSRELEASE} ${BTTARGETDIR}"
-    #cp /tmp/*-${OSRELEASE} ${BTTARGETDIR}      ## esto copia algo??
-
     ## backup de oginitrd.img, oginitrd.img.sum, ogvmlinuz y ogvmlinuz.sum
     now = datetime.datetime.now(datetime.timezone.utc).strftime ('%Y%m%d-%H%M%S%z')
     if os.path.exists (f'{bttargetdir}/oginitrd.img'):
@@ -373,21 +309,19 @@ def move_initrd (bttargetdir, osrelease):
     for f in glob.glob (f'{bttargetdir}/oginitrd*') + glob.glob (f'{bttargetdir}/vmlinuz*'):
         os.chmod (f, 0o755)
 
-#mksquashfs   convierte el sistema root en sqfs
 def mksquashfs (bttargetdir, btrootfsmnt):
     logger.info ('Iniciando la creación del sistema de archivos en sqfs')
 
-    # si ya existe un sqfs lo renombramos
     if os.path.exists (f'{bttargetdir}/ogclient.sqfs'):
         now = datetime.datetime.now(datetime.timezone.utc).strftime ('%Y%m%d-%H%M%S%z')
         os.rename (f'{bttargetdir}/ogclient.sqfs', f'{bttargetdir}/ogclient.sqfs.{now}')
 
-    utils.run (['mksquashfs', btrootfsmnt, f'{bttargetdir}/ogclient.sqfs'])
+    ## uses all CPU cores available, even within docker
+    utils.run (['mksquashfs', btrootfsmnt, f'{bttargetdir}/ogclient.sqfs', '-e', 'var/lib/apt/lists', '-e', 'usr/share/doc'])
     os.chmod (f'{bttargetdir}/ogclient.sqfs', 0o744)
 
     utils.write_md5 (f'{bttargetdir}/ogclient.sqfs')
 
-# mkisofs genera la iso del cliente
 def mkisofs (pxepkg, isolinux_tpl, bttargetdir, nameisoclient):
     #Preparamos los gestores de arranque
     try: os.makedirs ('/tmp/iso/isolinux', exist_ok=True)
@@ -430,14 +364,14 @@ def mkisofs (pxepkg, isolinux_tpl, bttargetdir, nameisoclient):
 
     oldpwd = os.getcwd()
     os.chdir ('/tmp')
-    logger.debug (f'mkisofs -V ogClient -o {nameisoclient}.iso -b isolinux/isolinux.bin -c isolinux/boot.cat -J -no-emul-boot -boot-load-size 4 -boot-info-table /tmp/iso')
-    utils.run (['mkisofs', '-V', 'ogClient', '-o', f'{nameisoclient}.iso', '-b', 'isolinux/isolinux.bin', '-c', 'isolinux/boot.cat', '-J', '-no-emul-boot', '-boot-load-size', '4', '-boot-info-table', '/tmp/iso'])
+    logger.debug (f'mkisofs -quiet -V ogClient -o {nameisoclient}.iso -b isolinux/isolinux.bin -c isolinux/boot.cat -J -no-emul-boot -boot-load-size 4 -boot-info-table /tmp/iso')
+    utils.run (['mkisofs', '-quiet', '-V', 'ogClient', '-o', f'{nameisoclient}.iso', '-b', 'isolinux/isolinux.bin', '-c', 'isolinux/boot.cat', '-J', '-no-emul-boot', '-boot-load-size', '4', '-boot-info-table', '/tmp/iso'])
 
     utils.write_md5 (f'{nameisoclient}.iso')
     os.chdir (oldpwd)
 
-    os.rename (f'/tmp/{nameisoclient}.iso'    , f'/var/lib/tftpboot/ogclient/{nameisoclient}.iso')
-    os.rename (f'/tmp/{nameisoclient}.iso.sum', f'/var/lib/tftpboot/ogclient/{nameisoclient}.iso.sum')
+    shutil.move (f'/tmp/{nameisoclient}.iso'    , f'/var/lib/tftpboot/ogclient/{nameisoclient}.iso')
+    shutil.move (f'/tmp/{nameisoclient}.iso.sum', f'/var/lib/tftpboot/ogclient/{nameisoclient}.iso.sum')
 
 #def __unused_boottoolsBootGraphics():
 #    find /tmp/opengnsys_installer/ -name .svn -type d -exec rm -fr {} \; 2>/dev/null;

boottools/utils.py

@@ -3,23 +3,49 @@ import subprocess
 import re
 import os
 import configparser
+import selectors
 
 logger = logging.getLogger ('boottools')
 
 def run (args):
-    cp = subprocess.run (args, text=True, capture_output=True)
-    if cp.returncode:
-        logger.error ('command "{}" failed with rc "{}"'.format (' '.join(args), cp.returncode))
+    stdout = stderr = ''
+    logger.debug ('run 10 args "{}"'.format (' '.join(args)))
+    sel = selectors.DefaultSelector()
+    p = subprocess.Popen (args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+    sel.register (p.stdout.fileno(), selectors.EVENT_READ)
+    sel.register (p.stderr.fileno(), selectors.EVENT_READ)
+    while True:
+        events = sel.select()
+        for key, _ in events:
+            if key.fileobj == p.stdout.fileno():
+                line = p.stdout.readline()
+                if not line: break
+                stdout += line
+                logger.debug (line.rstrip())
+            elif key.fileobj == p.stderr.fileno():
+                line = p.stderr.readline()
+                if not line: break
+                stderr += line
+                logger.warn (line.rstrip())
+        if p.poll() != None:
+            break
+    sel.close()
+    stdout = stdout.strip()
+    stderr = stderr.strip()
 
-        logger.error ('stdout follows:')
-        for i in cp.stdout.strip().split('\n'): logger.error ('  ' + i)
+    logger.debug (f'p.returncode {p.returncode}')
+    if p.returncode:
+        logger.error ('command "{}" failed with rc "{}"'.format (' '.join(args), p.returncode))
+
+        #logger.error ('stdout follows:')
+        #if stdout:
+        #    for i in stdout.split('\n'): logger.error ('  ' + i)
 
         logger.error ('stderr follows:')
-        for i in cp.stderr.strip().split('\n'): logger.error ('  ' + i)
+        if stderr:
+            for i in stderr.split('\n'): logger.error ('  ' + i)
 
-        raise Exception ('command "{}" failed with rc "{}"'.format (' '.join(args), cp.returncode))
-    stdout = cp.stdout.strip()
-    stderr = cp.stderr.strip()
+        raise Exception ('command "{}" failed with rc "{}"'.format (' '.join(args), p.returncode))
     return stdout, stderr
 
 def grep (regex, file):

chroot-tasks.py

@@ -10,11 +10,6 @@ import subprocess
 
 from boottools import utils, apt
 
-config = utils.read_config ('mkoglive.cfg')
-if config is None:
-    sys.exit (1)
-debconf_settings = config['General'].get ('debconf_settings')
-
 def _oghook_deactivate():
     #Desactivamos el hook del oginitrd.img para evitar problemas, al final de este escripts se activará
     os.rename ('/etc/initramfs-tools/hooks/oghooks', '/etc/initramfs-tools/oghooks')
@@ -37,27 +32,28 @@ def boottoolsSoftwareInstall (osarch, osrelease):
     pkgs32 = []
     if 'i386' != osarch:
         utils.run (['dpkg', '--add-architecture', 'i386'])
-        pkgs32 = 'lib32gcc-s1 lib32stdc++6 lib32z1 libc6-i386'.split (' ')         ## he cambiado lib32gcc1 por lib32gcc-s1 pero como queramos crear un oglive viejo, esto va a petar
+        pkgs32 = 'lib32gcc-s1 lib32stdc++6 lib32z1 libc6-i386'.split (' ')         ## nserrano: he cambiado lib32gcc1 por lib32gcc-s1 pero como queramos crear un oglive viejo, esto va a petar
 
     _oghook_deactivate()
 
+    print ('boottoolsSoftwareInstall: debconf-set-selections', file=sys.stderr)
     subprocess.run (['debconf-set-selections'], input=debconf_settings, text=True)
-    utils.run (['dpkg-reconfigure', '--frontend', 'noninteractive', 'console-setup', 'locales'])   ## despues de esto, debconf-get-selections devuelve los valores antiguos, no se por que...
+    utils.run (['dpkg-reconfigure', '--frontend', 'noninteractive', 'console-setup', 'locales'])   ## XXX: despues de esto, debconf-get-selections devuelve los valores antiguos, no se por que...
 
     pkgs = []
     for section in config.options('Packages'):
         pkgs += re.split (r'[ \n]', config['Packages'].get(section).strip())
 
-    apt.install (
-        [f'linux-image-{osrelease}', f'linux-headers-{osrelease}', f'linux-modules-{osrelease}', f'linux-modules-extra-{osrelease}', 'dkms', 'shim-signed', 'openssl', 'sshfs', 'kexec-tools'] + pkgs32 + pkgs,
-        opts={'DPkg::Options::': '--force-confdef'}         ## hace falta --force-confdef para evitar un tema interactivo del /etc/ssh/ssh_config
-    )
+    pkgs = [f'linux-image-{osrelease}', f'linux-headers-{osrelease}', f'linux-modules-{osrelease}', f'linux-modules-extra-{osrelease}', 'dkms', 'shim-signed', 'openssl', 'sshfs', 'kexec-tools'] + pkgs32 + pkgs
+    print (f'boottoolsSoftwareInstall: installing packages: {str(pkgs)}', file=sys.stderr)
+    apt.install (pkgs, opts={'DPkg::Options::': '--force-confdef'})         ## --force-confdef is required to avoid an interactive question regarding /etc/ssh/ssh_config
 
     # Instalar módulos que algunos paquetes puedan tener pendientes de compilar.
+    print ('boottoolsSoftwareInstall: dkms', file=sys.stderr)
     stdout, _ = utils.run (['dkms', 'status'])
     for l in stdout.strip().split ('\n'):
         if not l: continue
-        print (f'l "{l}"')
+        #print (f'l "{l}"')
         mod, vers, status = l.split (',')
         if 'added' in status:
             print (f'dkms installing {mod} {vers}')
@@ -74,10 +70,10 @@ def boottoolsSoftwareCompile():
     os.environ['LANGUAGE'] = os.environ['LC_ALL'] = os.environ['LANG'] = 'C'
     os.chdir ('/tmp')
 
-    print ('ms-sys')
+    print ('boottoolsSoftwareCompile: ms-sys', file=sys.stderr)
     try: utils.run (['which', 'ms-sys'])
     except:
-        utils.run (['wget', 'https://sourceforge.net/projects/ms-sys/files/latest/download', '-O', 'ms-sys.tar.gz'])
+        utils.run (['wget', '--quiet', 'https://sourceforge.net/projects/ms-sys/files/latest/download', '-O', 'ms-sys.tar.gz'])
         utils.run (['tar', '-xpzf', 'ms-sys.tar.gz'])
         mssys_dir = subprocess.run (['tar tzf ms-sys.tar.gz |head -n 1'], shell=True, capture_output=True, text=True).stdout.strip()
         print (f'mssys_dir "{mssys_dir}"')
@@ -85,16 +81,16 @@ def boottoolsSoftwareCompile():
         utils.run (['make', 'install'])
         os.chdir ('..')
 
-    print ('spartlnx')
+    print ('boottoolsSoftwareCompile: spartlnx', file=sys.stderr)
     try: utils.run (['which', 'spartl64.run'])
     except:
-        utils.run (['wget', 'http://damien.guibouret.free.fr/savepart.zip'])
+        utils.run (['wget', '--quiet', 'http://damien.guibouret.free.fr/savepart.zip'])
         utils.run (['unzip', '-o', 'savepart.zip', '-d', '/sbin/', 'spartl64.run'])
         utils.run (['mkdir', '/usr/share/doc/spartlnx'])
         utils.run (['unzip', '-j', '-o', 'savepart.zip', '-d', '/usr/share/doc/spartlnx/', 'doc/en/*'])
 
     if not os.path.exists ('python-libfdisk'):
-        print ('python-libfdisk')
+        print ('boottoolsSoftwareCompile: python-libfdisk', file=sys.stderr)
         apt.install (['python3-psutil', 'python3-dev', 'libfdisk-dev', 'python3-setuptools'])
         utils.run (['git', 'clone', 'git://git.48k.eu/python-libfdisk'])
         os.chdir ('python-libfdisk')
@@ -107,19 +103,27 @@ def boottoolsSoftwareCompile():
     os.environ['LANG']     = env_lang
 
 def boottoolsInitrdGenerate (osrelease):
+    print ('boottoolsInitrdGenerate', file=sys.stderr)
     for f in glob.glob ('/usr/lib/initramfs-tools/bin/*'):
         os.unlink (f)
     shutil.copy ('/bin/busybox', '/usr/lib/initramfs-tools/bin')
 
     os.chdir ('/tmp')
-    utils.run (['mkinitramfs', '-o', f'/tmp/initrd.img-{osrelease}', '-v', osrelease])
+    utils.run (['mkinitramfs', '-o', f'/tmp/initrd.img-{osrelease}', osrelease])
     shutil.copy (f'/boot/vmlinuz-{osrelease}', '/tmp/')
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
-    parser.add_argument ('--osarch',    help='OS architecture', action='store', required=True)
-    parser.add_argument ('--osrelease', help='OS release',      action='store', required=True)
+    parser.add_argument ('--osarch',    help='OS architecture',            action='store', required=True)
+    parser.add_argument ('--osrelease', help='OS release',                 action='store', required=True)
+    parser.add_argument ('--config',    help='Path to configuration file', action='store')
     args = parser.parse_args()
+
+    config = utils.read_config (args.config or 'mkoglive.cfg')
+    if config is None:
+        sys.exit (1)
+    debconf_settings = config['General'].get ('debconf_settings')
+
     boottoolsSoftwareInstall (args.osarch, args.osrelease)
     boottoolsSoftwareCompile()
     boottoolsInitrdGenerate (args.osrelease)

mkoglive.cfg

@@ -1,6 +1,7 @@
 [General]
 
 logging_level = INFO
+rootfs_image_label = ogClient
 
 debconf_settings =
   kexec-tools kexec-tools/load_kexec boolean true

mkoglive.py

@@ -7,6 +7,7 @@ import subprocess
 import glob
 import stat
 import shutil
+import argparse
 
 curdir = os.path.dirname (__file__)
 sys.path.insert (0, curdir)
@@ -14,9 +15,7 @@ from boottools import utils, apt, btog
 
 os.chdir (curdir)
 
-def _logging():
-    #logging.root.handlers = []
-
+def _logging (lvl='INFO', logfile='/tmp/boot-tools_installation.log'):
     numeric_level = getattr (logging, lvl.upper(), None)
     if numeric_level is None:
         numeric_level = getattr (logging, 'INFO')
@@ -25,26 +24,19 @@ def _logging():
         format='%(levelname)s %(asctime)s (%(funcName)s) %(message)s',
         level=numeric_level,
         handlers = [
-            logging.FileHandler ('/tmp/boot-tools_installation.log'),
+            logging.FileHandler (logfile),
             logging.StreamHandler (sys.stdout),
         ],
     )
     return logging.getLogger ('boottools')
 
-def clone_client_dirs (ogrepo_url, ogrepo_branch, ogrepo_dir):
-    if not os.path.exists ('/tmp/opengnsys/engine'):
-        utils.run (['git', 'clone', '-c', 'http.sslVerify=false', '--branch', ogrepo_branch, ogrepo_url, ogrepo_dir])
-        utils.run (['rsync', '-aH', f'{ogrepo_dir}/client/engine', f'{ogrepo_dir}/client/shared', '/tmp/opengnsys/'])
-
-def _mount_rootfs():
-    global btrootfsimg, btrootfsmnt
+def _mount_rootfs (btrootfsimg, btrootfsmnt):
     try: utils.mount (btrootfsimg, btrootfsmnt, opts=['-o', 'loop,offset=32256'])
     except:
         logger.error ('mount failed')
         sys.exit (1)
 
 def _get_pxepkg():
-    #grep "http://free.nchc.org.tw/drbl-core" /etc/apt/sources.list || echo "deb http://free.nchc.org.tw/drbl-core drbl stable" >> /etc/apt/sources.list
     pxepkg = None
     cache = apt.cache_search (['gpxe', 'ipxe'])
     if cache['gpxe']: pxepkg = 'gpxe'
@@ -55,8 +47,8 @@ def _get_pxepkg():
     logger.info (f'PXE package is "{pxepkg}"')
     return pxepkg
 
-def _mkrootfs():
-    logger.info ('Stage 3.1 - create, partition and format the rootfs')
+def _mkrootfs (btrootfsimg, btrootfsmnt, btrootfsimglabel, btvirtualdisksize, bttargetdir, osarch):
+    logger.info ('Stage 1.1 - create, partition and format the rootfs')
     rc = subprocess.run (f'file "{btrootfsimg}" |grep -q "partition 1 *: ID=0x83"', shell=True).returncode
     if (rc):   ## 'file|grep' failed
         try: btog.mkrootfs (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, bttargetdir, osarch)
@@ -64,122 +56,117 @@ def _mkrootfs():
             logger.error (str (e))
             sys.exit (1)
 
-def _schroot (btrootfsimg):
-    logger.info ('Stage 3.2 - configure schroot in the VM')
-    btog.configure_schroot (btrootfsimg)
-
-## para hacer schroot --cosas, el mntpt tiene que estar desmontado
-## si está montado da un pete tal que 'E: 10mount: mount: /run/schroot/mount/IMGogclient-7fbf51a2-e37e-48e5-8e5d-83f8901fc7ed: wrong fs type, bad option, bad superblock on /dev/loop1, missing codepage or helper program, or other error.'
-
-def _debootstrap():
-    logger.info ('Stage 3.3 - debootstrap system')
+def _debootstrap (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp):
+    logger.info ('Stage 1.2 - debootstrap system')
     logger.debug ('Try creation of a file within chroot (this operation may fail with "... etc/resolv.conf: No such file or directory"--that is ok)')
     logger.debug ('Running \'schroot --chroot IMGogclient -- stat /etc\'')
     cp = subprocess.run (['schroot', '--chroot', 'IMGogclient', '--', 'stat', '/etc'])
     if (cp.returncode):
         logger.debug (f'schroot returned code "{cp.returncode}", calling btog.debootstrap()')
-        _mount_rootfs()
+        _mount_rootfs (btrootfsimg, btrootfsmnt)
         try: btog.debootstrap (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp)
         except Exception as e:
             logger.error (str (e))
             sys.exit (1)
 
-# Incluir revisión.
-def _initramfs_version (gitrelease, osrelease, btdir):
-    ## FIXME esto la incluye incondicionalmente, y luego terminamos con "OpenGnsys Client 1.2.0-rc1 gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) ..."
-    #sed -i "1 s/$/ $GITRELEASE ($OSRELEASE)/" ${BTDIR}/includes/etc/initramfs-tools/scripts/VERSION.txt
-    utils.run (['sed', '-i', f'1 s/$/ {gitrelease} ({osrelease})/', f'{btdir}/includes/etc/initramfs-tools/scripts/VERSION.txt'])
+def _initramfs_version (gitrelease, osrelease, curdir):
+    utils.run (['sed', '-i', f'1 s/$/ {gitrelease} ({osrelease})/', f'{curdir}/includes/etc/initramfs-tools/scripts/VERSION.txt'])
 
-def _copy_og_files (btrootfsmnt, osdistrib, oscodename):
-    _mount_rootfs()
+def _copy_og_files (btrootfsimg, btrootfsmnt, osdistrib, oscodename):
+    _mount_rootfs (btrootfsimg, btrootfsmnt)
     builder   = '/tmp/opengnsys/oglive_builder'
     og_shared = '/tmp/opengnsys/shared'
     og_engine = '/tmp/opengnsys/engine'
     btog.copy_og_files (builder, og_shared, og_engine, btrootfsmnt, osdistrib, oscodename)
     utils.umount (btrootfsmnt)
 
-def _chroot_tasks (curdir, osrelease, osarch):
+def _chroot_tasks (cfgfile, curdir, osrelease, osarch):
     logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/chroot-tasks.py --osrelease "{osrelease}" --osarch "{osarch}"\'')
-    stdout, _ = utils.run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/chroot-tasks.py', '--osrelease', osrelease, '--osarch', osarch])
-    logger.debug (f'chroot-tasks.py stdout follows:')
-    ## esto deja initrd.img-6.8.0-31-generic y vmlinuz-6.8.0-31-generic en /tmp
-    for i in stdout.strip().split ('\n'): logger.debug ('  ' + i)
+    stdout, _ = utils.run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/chroot-tasks.py', '--osrelease', osrelease, '--osarch', osarch, '--config', cfgfile])
+    ## this leaves initrd.img-6.8.0-31-generic and vmlinuz-6.8.0-31-generic in /tmp
 
-def _ssh_stuff():
-    _mount_rootfs()
+def _ssh_stuff (btrootfsimg, btrootfsmnt):
+    _mount_rootfs (btrootfsimg, btrootfsmnt)
     btog.ssh_server (btrootfsmnt)
     btog.ssh_client (btrootfsmnt)
     utils.umount (btrootfsmnt)
-    ## el resultado es:
-    ## - hay un nuevo par de claves en la VM /root/.ssh
-    ## - hay otro nuevo par de claves en el rootfs /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh
-    ## - las dos claves públicas (una de cada par) están autorizadan en el rootfs /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh/authorized_keys
+    ## the end result is:
+    ## - there's a new key pair in the VM (or docker container), in /root/.ssh
+    ## - there's another new key pair in the rootfs, in /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh
+    ## - the two pubkeys (one of each pair) end up being authorised in the rootfs, in /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh/authorized_keys
 
-def _mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsmnt, pxepkg, isolinux_tpl, nameisoclient):
-    logger.info ('Stage 6.1 - Put initrd in place')
-    _mount_rootfs()
+def _mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsimg, btrootfsmnt, pxepkg, isolinux_tpl, nameisoclient):
+    logger.info ('Stage 4.1 - Put initrd in place')
+    _mount_rootfs (btrootfsimg, btrootfsmnt)
     btog.move_initrd (bttargetdir, osrelease)
 
-    logger.info ('Stage 6.2 - make squash filesystem')
+    logger.info ('Stage 4.2 - make squash filesystem')
     btog.mksquashfs (bttargetdir, btrootfsmnt)
     utils.umount (btrootfsmnt)
 
-    logger.info ('Stage 6.3 - make iso filesystem')
+    logger.info ('Stage 4.3 - make iso filesystem')
     btog.mkisofs (pxepkg, isolinux_tpl, bttargetdir, nameisoclient)
 
+def _main (cfgfile, config, type_client):
+    isolinux_tpl     = config['General'].get ('isolinux_template')
+    btrootfsimglabel = config['General'].get ('rootfs_image_label')
+
+    logger.info ('OpenGnsys CLIENT installation begins')
+
+    fd = open (f'{curdir}/gitrelease', 'r')     ## per the Dockerfile
+    gitrelease = fd.readline().strip()
+    fd.close()
+
+    osdistrib, oscodename, osrelease, osarch, oshttp = btog.GetOsInfo (type_client)
+    if osdistrib is None:
+        logger.error ('GetOsInfo() failed')
+        sys.exit (1)
+    bttargetdir, btrootfsimg, btrootfsmnt, btvirtualdisksize = btog.GetVar (osarch)
+    logger.info (':'.join ([osdistrib, oscodename, osrelease, osarch, oshttp]))
+
+    ## this is convenient in case the previous run failed and we want to run this program again
+    try: utils.umount (btrootfsmnt)
+    except: pass
+
+    logger.info ('STAGE 1 - create and bootstrap rootfs')
+    _mkrootfs (btrootfsimg, btrootfsmnt, btrootfsimglabel, btvirtualdisksize, bttargetdir, osarch)
+    _debootstrap (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp)
+
+    logger.info ('STAGE 2 - copy files to the rootfs')
+    _initramfs_version (gitrelease, osrelease, curdir)
+    _copy_og_files (btrootfsimg, btrootfsmnt, osdistrib, oscodename)
+
+    logger.info ('STAGE 3 - perform tasks within the chroot')
+    _chroot_tasks (cfgfile, curdir, osrelease, osarch)
+
+    _ssh_stuff (btrootfsimg, btrootfsmnt)
+
+    logger.info ('STAGE 4 - generate distribution files')
+    pxepkg = _get_pxepkg()
+    nameisoclient  = '-'.join (['ogLive', oscodename, osrelease, osarch, gitrelease])
+    _mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsimg, btrootfsmnt, pxepkg, isolinux_tpl, nameisoclient)
+
+    logger.info ('OpenGnsys installation finished')
 
 
-config = utils.read_config ('mkoglive.cfg')
-if config is None:
-    sys.exit (1)
-isolinux_tpl = config['General'].get ('isolinux_template')
-lvl = config['General'].get ('logging_level')
+if __name__ == '__main__':
+    if os.getuid():
+        print ('ERROR: this program must run under root privileges!!', file=sys.stderr)
+        sys.exit (1)
 
-logger = _logging()
-type_client    = sys.argv[1] if len (sys.argv)>1 else 'host'
+    parser = argparse.ArgumentParser()
+    parser.add_argument ('--loglevel', help='Log level',                  action='store')
+    parser.add_argument ('--codename', help='OS codename',                action='store')
+    parser.add_argument ('--config',   help='Path to configuration file', action='store')
+    args = parser.parse_args()
 
-if os.getuid():
-    logger.error ('ERROR: this program must run under root privileges!!')
-    sys.exit (1)
+    cfgfile = args.config or 'mkoglive.cfg'
+    config = utils.read_config (cfgfile)
+    if config is None:
+        print ('ERROR: no configuration found', file=sys.stderr)
+        sys.exit (1)
+    cfg_loglevel = config['General'].get ('logging_level')
 
-## this is convenient in case the previous run failed and we want to run this program again
-try: utils.umount (btrootfsmnt)
-except: pass
+    logger = _logging (args.loglevel or cfg_loglevel)
 
-logger.info ('OpenGnsys CLIENT installation begins')
-
-logger.info ('STAGE 1 - clone opengnsys repo')
-ogrepo_url = 'https://ognproject.evlt.uma.es/gitea/opengnsys/opengnsys.git'
-ogrepo_branch = 'main'
-ogrepo_dir = '/tmp/ogrepo'
-clone_client_dirs (ogrepo_url, ogrepo_branch, ogrepo_dir)
-
-osdistrib, oscodename, osrelease, osarch, oshttp = btog.GetOsInfo1 (type_client)
-btdir, bttargetdir, btrootfsimg, btrootfsmnt, btrootfsimglabel, log_file, versionboottools, btvirtualdisksize = btog.GetVar (osarch)
-gitrelease, nameisoclient, namehostclient = btog.GetOsInfo2 (ogrepo_dir, versionboottools, oscodename, osrelease, osarch)
-logger.info (':'.join ([osdistrib, oscodename, osrelease, osarch, oshttp]))
-
-logger.info ('STAGE 2 - install software in the VM')
-apt.update()
-apt.upgrade()
-pxepkg = _get_pxepkg()
-apt.install (['jq', 'syslinux', 'syslinux-efi', 'syslinux-utils', 'debootstrap', 'subversion', 'schroot', 'squashfs-tools', 'syslinux', 'genisoimage', 'qemu-utils', 'lsof', pxepkg])    ## TODO qemu no existe, hace falta?
-
-logger.info ('STAGE 3 - create and bootstrap rootfs')
-_mkrootfs()
-_schroot (btrootfsimg)
-_debootstrap()
-
-logger.info ('STAGE 4 - copy files to the rootfs')
-_initramfs_version (gitrelease, osrelease, btdir)
-_copy_og_files (btrootfsmnt, osdistrib, oscodename)
-
-logger.info ('STAGE 5 - perform tasks within the chroot')
-_chroot_tasks (curdir, osrelease, osarch)
-
-_ssh_stuff()
-
-logger.info ('STAGE 6 - generate distribution files')
-_mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsmnt, pxepkg, isolinux_tpl, nameisoclient)
-
-logger.info ('OpenGnsys installation finished')
+    _main (cfgfile, config, args.codename)

mount-defaults

@@ -0,0 +1,12 @@
+# mount.defaults: static file system information for chroots.
+# Note that the mount point will be prefixed by the chroot path
+# (CHROOT_PATH)
+#
+# <file system> <mount point>   <type>  <options>       <dump>  <pass>
+proc            /proc           proc    defaults        0       0
+#procbususb      /proc/bus/usb   usbfs   defaults        0       0
+#/dev            /dev            none    rw,bind         0       0
+/dev/pts        /dev/pts        none    rw,bind         0       0
+/dev/shm        /dev/shm        none    rw,bind         0       0
+#/home          /home           none    rw,bind         0       0
+/tmp            /tmp            none    rw,bind         0       0

schroot.conf

@@ -0,0 +1,10 @@
+[IMGogclient]
+type=loopback
+file=/var/lib/tftpboot/ogclient/ogclient.img
+description=ogclient Ubuntu image
+#priority=1
+users=root
+groups=root
+root-groups=root
+mount-options=-o offset=32256
+root-users=root