refs #596 fix bugs, add 03-boottoolsSoftwareInstall.py to install packages

02-boottoolsFsOpengnsys.py

@@ -13,16 +13,12 @@ namehostclientfile = '/tmp/opengnsys_chroot'
 svnclientdir       = '/tmp/opengnsys_installer/opengnsys/client/boot-tools'
 svnclientstructure = '/tmp/opengnsys_installer/opengnsys/client/shared'
 svnclientengine    = '/tmp/opengnsys_installer/opengnsys/client/engine'
- 
-ogclientmount = ''
 
 def boottoolsFsOpengnsys (osdistrib, oscodename, osrelease, osarch, oshttp):
     print (':'.join ([osdistrib, oscodename, osrelease, osarch, oshttp]))
 
     print ('Iniciando la personalización con datos del repositorio')
 
-    ## TODO mount fs, para lo cual necesito helpers
-
     #sed -e "s/OSCODENAME/$OSCODENAME/g" ${SVNCLIENTDIR}/includes/etc/apt/sources.list.${OSDISTRIB,,} > ${SVNCLIENTDIR}/includes/etc/apt/sources.list
     sources_list_in  = '{}/includes/etc/apt/sources.list.{}'.format (svnclientdir, osdistrib.lower())
     sources_list_out = '{}/includes/etc/apt/sources.list'.format (svnclientdir)
@@ -66,8 +62,6 @@ def boottoolsFsOpengnsys (osdistrib, oscodename, osrelease, osarch, oshttp):
     #echo "${VERSIONBOOTTOOLS}-${OSCODENAME}-${OSRELEASE}-${GITRELEASE}" > $nameisoclientfile
     #echo "${VERSIONBOOTTOOLS}-${OSCODENAME}-${GITRELEASE}"              > $namehostclientfile
 
-    ## desmontar de nuevo, ejem...
-
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
     parser.add_argument ('--mntpt',      help='rootfs mount point', action='store', required=True)

03-boottoolsSoftwareInstall.py

@@ -0,0 +1,133 @@
+#!/usr/bin/python3
+
+import argparse
+import os
+import subprocess
+
+def _aptget_clean():
+    subprocess.run (['apt-get', 'clean'])
+
+def _aptget_autoclean():
+    subprocess.run (['apt-get', 'autoclean'])
+
+def _aptget_autoremove():
+    subprocess.run (['apt-get', 'autoremove'])
+
+def _aptget_update():
+    subprocess.run (['apt-get', 'update'])
+
+def _aptget_upgrade():
+    subprocess.run (['apt-get', 'upgrade', '--yes'])
+
+def _aptget_install (pkgs, opts=[]):
+    subprocess.run (['apt-get', '--yes'] + opts + ['install'] + pkgs)
+
+def _oghook_deactivate():
+    #Desactivamos el hook del oginitrd.img para evitar problemas, al final de este escripts se activará
+    #mv /etc/initramfs-tools/hooks/oghooks /etc/initramfs-tools/
+    pass
+
+def _oghook_activate():
+    #Activamos el hook del oginitrd.img
+    #mv /etc/initramfs-tools/oghooks /etc/initramfs-tools/hooks/
+    pass
+
+def _mock_mtab():
+    # Preparamos el mtab  necesario para la instalacion correcta de paquetes.
+    #echo "/dev/sda1 / ext4 rw,errors=remount-ro 0 0" > /etc/mtab          ## nati: falla porque es un symlink a ../proc/self/mounts
+    pass
+
+def _restore_mtab():
+    # Dejamos el mtab como al principio
+    #echo "   " > /etc/mtab
+    pass
+
+def boottoolsSoftwareInstall (osarch, osrelease):
+    os.environ['LANGUAGE'] = 'C'
+    os.environ['LC_ALL'] = 'C'
+    os.environ['LANG'] = 'C'
+    os.environ['DEBIAN_FRONTEND'] = 'noninteractive'
+
+    subprocess.run (['dpkg-divert', '--local', '--rename', '--add', '/sbin/initctl'])
+    os.symlink ('/bin/true', '/sbin/initctl')
+
+    #TEST desactivamos upstart
+    #apt-get update; apt-get install --no-install-recommends dbus; dbus-uuidgen > /var/lib/dbus/machine-id; dpkg-divert --local --rename --add /sbin/initctl; ln -s /bin/true /sbin/initctl
+
+    _aptget_clean()
+    pkgs32 = []
+    if 'i386' != osarch:
+        subprocess.run (['dpkg', '--add-architecture', 'i386'])
+        pkgs32 = 'lib32gcc1 lib32stdc++6 lib32z1 libc6-i386'.split (' ')
+
+    _aptget_update()
+#Ign:13 http://ppa.launchpad.net/zfs-native/stable/ubuntu noble InRelease                                                                             
+#Err:15 http://ppa.launchpad.net/zfs-native/stable/ubuntu noble Release                                                                              
+#  404  Not Found [IP: 185.125.190.80 80]
+#
+#E: The repository 'http://ppa.launchpad.net/zfs-native/stable/ubuntu noble Release' does not have a Release file.
+#N: Updating from such a repository can't be done securely, and is therefore disabled by default.
+#N: See apt-secure(8) manpage for repository creation and user configuration details.
+    _aptget_upgrade()
+    _oghook_deactivate()
+    _mock_mtab()
+    return
+
+
+    """
+    # Instalamos el kernel.
+    _aptget_install (['linux-image-{}'.format (osrelease), 'linux-headers-{}'.format (osrelease), 'dkms', 'shim-signed', 'openssl'])
+    _aptget_install (['linux-modules-{}'.format (osrelease), 'linux-modules-extra-{}'.format (osrelease)])
+
+    # Valores para paquetes interactivos.
+    cat << EOT | debconf-set-selections --
+kexec-tools kexec-tools/load_kexec boolean true
+openssh-server openssh-server/permit-root-login boolean true
+refind refind/install_to_esp boolean false
+EOT
+    ## nati: hace falta --force-confdef para evitar un tema interactivo del /etc/ssh/ssh_config
+    _aptget_install (['sshfs', 'kexec-tools'] + pkgs32, opts=['-o', 'DPkg::Options::=--force-confdef'])
+
+    #comenzamos con la instalación de los paquetes a instalar.
+    to_install = {}
+    for group in `find /usr/bin/boot-tools/listpackages/ -name sw.*`; do
+        echo "Instalando el grupo de paquetes almacenados en $group"
+        for package in ` awk /^install/'{print $2}' $group `; do
+            echo -n $package' ' >&2
+            to_install[package] = 1
+        done    
+        echo >&2
+    done
+    del to_install['refind']       ## nati: este paquete no existe y toda la invocación de apt se muere (no sé cómo hacer que apt ignore los paquetes que no existen)
+    echo =========================================== nati pkgs to install ${!TO_INSTALL[*]}
+    _aptget_install (list (to_install.keys()))
+
+    # Instalar módulos que algunos paquetes puedan tener pendientes de compilar.
+    echo "Instalando módulos adicionales con DKMS"
+    while read -e mod vers; do
+        echo -n "Intalando módulo $mod v$vers"
+        dkms install -m $mod -v $vers &>/dev/null
+        RETVAL=$?
+        if [ $RETVAL == 0 ]; then
+            echo " : OK - Módulo instalado correctamente (codigo interno de dkms $RETVAL)"
+        else
+            echo " : Error módulo $mod (codigo interno de dkms $RETVAL) "
+            echo "Pulse [Intro] para continuar"
+            read
+        fi
+    done < <(dkms status 2>/dev/null | awk -F, '$3~/added/ {print $1,$2}')
+    """
+
+    _oghook_activate()
+    _restore_mtab()
+    _aptget_clean()
+    _aptget_autoclean()
+    _aptget_autoremove()
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument ('--osarch',     help='OS architecture',    action='store', required=True)
+    parser.add_argument ('--osrelease',  help='OS release',         action='store', required=True)
+    args = parser.parse_args()
+    boottoolsSoftwareInstall (args.osarch, args.osrelease)
+

boottoolsfunctions/__init__.py

@@ -211,7 +211,7 @@ def btogSetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisks
 
     time.sleep (5)
     logger.debug ('losetup --detach')
-    try: _run (['losetup', '--detache', diskloop])
+    try: _run (['losetup', '--detach', diskloop])
     except:
         raise Exception ('Liberando disco virtual despues del particionado: ERROR')
 
@@ -293,11 +293,12 @@ def btogSetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp):
         raise Exception ('rootfs should be mounted')
 
     logger.info ('debootstrapping system')
-    try: _run (['debootstrap', '--arch="{}"'.format(osarch), '--components=main,universe', oscodename, btrootfsmnt, oshttp])
+    logger.debug ('debootstrap --arch="{}" --components=main,universe "{}" "{}" "{}"'.format (osarch, oscodename, btrootfsmnt, oshttp)
+    try: _run (['debootstrap', '--arch={}'.format(osarch), '--components=main,universe', oscodename, btrootfsmnt, oshttp])
     except:
         if (_is_mounted (btrootfsmnt)):
             subprocess.run (['umount', btrootfsmnt])
-        raise Exception ('debootstrap --arch={} --components=main,universe "{}" "{}" "{}" : ha fallado!'.format (osarch, oscodename, btrootfsmnt, oshttp))
+        raise Exception ('debootstrap --arch="{}" --components=main,universe "{}" "{}" "{}" : ha fallado!'.format (osarch, oscodename, btrootfsmnt, oshttp))
 
     logger.info ('debootstrap --arch="{}" --components=main,universe "{}" "{}" "{}" : ok'.format (osarch, oscodename, btrootfsmnt, oshttp))
     return 0

boottoolsgenerator.py

@@ -9,7 +9,7 @@ import stat
 
 curdir = os.path.dirname (__file__)
 sys.path.insert (0, curdir)
-from boottoolsfunctions import _run, btogGetOsInfo1, btogGetOsInfo2, btogGetVar, btogSetFsVirtual, btogSetFsAccess, btogSetFsBase
+from boottoolsfunctions import _run, _is_mounted, btogGetOsInfo1, btogGetOsInfo2, btogGetVar, btogSetFsVirtual, btogSetFsAccess, btogSetFsBase
 
 logging.basicConfig (filename='/tmp/boot-tools_installation.log', filemode='a', format='%(levelname)s %(asctime)s (%(funcName)s) %(message)s', level=logging.INFO)
 logger = logging.getLogger ('boottools')
@@ -18,7 +18,7 @@ if not os.path.exists ('/tmp/opengnsys_installer/opengnsys/client/engine'):
     branch = 'main'
     tmpdir = subprocess.run (['mktemp', '--tmpdir', '--directory', 'oggit.XXXXXX'], capture_output=True).stdout.decode ('utf-8').strip()
     subprocess.run (['git', 'clone', '-c', 'http.sslVerify=false', '--branch', branch, 'https://ognproject.evlt.uma.es/gitea/opengnsys/opengnsys.git', tmpdir])
-    subprocess.run (['rsync', '-aHv', '{}/client/engine'.format(tmpdir), '{}/client/shared'.format(tmpdir), '/tmp/opengnsys_installer/opengnsys/client/'])
+    subprocess.run (['rsync', '-aH', '{}/client/engine'.format(tmpdir), '{}/client/shared'.format(tmpdir), '/tmp/opengnsys_installer/opengnsys/client/'])
     subprocess.run (['rm', '-rf', tmpdir])
 
 type_client    = sys.argv[1] if len(sys.argv)>1 else 'host'
@@ -61,7 +61,8 @@ else:
     os._exit (1)
 logger.info ('PXE package is "{}"'.format (pxepkg))
 ## TODO qemu no existe, hace falta?
-subprocess.run (['apt-get', '-y', 'install', 'debootstrap', 'subversion', 'schroot', 'squashfs-tools', 'syslinux', 'genisoimage', 'qemu-utils', 'lsof', pxepkg])  #>>/tmp/fase2.out
+subprocess.run (['apt-get', '-y', 'update'])
+subprocess.run (['apt-get', '-y', 'install', 'jq', 'syslinux', 'syslinux-efi', 'syslinux-utils', 'debootstrap', 'subversion', 'schroot', 'squashfs-tools', 'syslinux', 'genisoimage', 'qemu-utils', 'lsof', pxepkg])  #>>/tmp/fase2.out
 
 
 ###################################################################3
@@ -80,25 +81,27 @@ logger.info ('Fase 3.2 - Configurar acceso schroot al Segundo Sistema de archivo
 #cat /etc/schroot/schroot.conf | grep $BTROOTFSIMG || btogSetFsAccess
 btogSetFsAccess (btrootfsimg)
 
-try: _run (['mount', btrootfsimg, btrootfsmnt, '-o', 'loop,offset=32256'])
-except:
-    logger.error ('mount failed')
-    os._exit (3)
+## para hacer schroot --cosas, el mntpt tiene que estar desmontado
+## si está montado da un pete tal que 'E: 10mount: mount: /run/schroot/mount/IMGogclient-7fbf51a2-e37e-48e5-8e5d-83f8901fc7ed: wrong fs type, bad option, bad superblock on /dev/loop1, missing codepage or helper program, or other error.'
 
 logger.info ('Fase 3.3 Generar sistema de archivos con debootstrap')
-logger.debug ('Try creation of a file within chroot (this operation may fail--that is ok)')
-logger.debug ('Running \'schroot --chroot IMGogclient -- touch /tmp/ogclientOK\'')
-cp = subprocess.run (['schroot', '--chroot', 'IMGogclient', '--', 'touch', '/tmp/ogclientOK'])
+logger.debug ('Try creation of a file within chroot (this operation may fail with "... etc/resolv.conf: No such file or directory"--that is ok)')
+logger.debug ('Running \'schroot --chroot IMGogclient -- stat /etc\'')
+cp = subprocess.run (['schroot', '--chroot', 'IMGogclient', '--', 'stat', '/etc'])
 if (cp.returncode):
     logger.debug ('schroot returned code "{}", calling btogSetFsBase()'.format (cp.returncode))
+    try: _run (['mount', btrootfsimg, btrootfsmnt, '-o', 'loop,offset=32256'])
+    except:
+        logger.error ('mount failed')
+        os._exit (3)
     try: btogSetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp)   #>>/tmp/fase3.out
     except Exception as e:
         logger.error (str (e))
         os._exit (3)
-else:
-    os.unlink ('/tmp/ogclientOK')
 #echo ================= nati after fase 3; ls -la /opt/opengnsys/tftpboot/ogclient/
 
+if (_is_mounted (btrootfsmnt)):
+    subprocess.run (['umount', btrootfsmnt])
 
 ###########################################################################
 logger.info ('FASE 4 - Incorporando ficheros OpenGnsys al sistema raíz rootfs')
@@ -106,23 +109,24 @@ logger.info ('FASE 4 - Incorporando ficheros OpenGnsys al sistema raíz rootfs')
 ## por qué esta copia???
 
 #cp -a ${BTDIR}/includes/usr/bin/* /tmp >>/tmp/fase5.out
-subprocess.run (['cp', '-a'] + glob.glob ('{}/includes/usr/bin/*'.format (btdir)) + ['/tmp'])
+#subprocess.run (['cp', '-a'] + glob.glob ('{}/includes/usr/bin/*'.format (btdir)) + ['/tmp'])
 
 #chmod +x /tmp/boot-tools/*.sh
-for i in glob.glob ('/tmp/boot-tools/*.sh'):
-    st = os.stat (i)
-    os.chmod (i, st.st_mode|stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH)
+#for i in glob.glob ('/tmp/boot-tools/*.sh'):
+#    st = os.stat (i)
+#    os.chmod (i, st.st_mode|stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH)
 
 # Incluir revisión.
+## FIXME esto la incluye incondicionalmente, y luego terminamos con "OpenGnsys Client 1.2.0-rc1 gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) ..."
 #sed -i "1 s/$/ $GITRELEASE ($OSRELEASE)/" ${BTDIR}/includes/etc/initramfs-tools/scripts/VERSION.txt
 subprocess.run (['sed', '-i', '1 s/$/ {} ({})/'.format (gitrelease, osrelease), '{}/includes/etc/initramfs-tools/scripts/VERSION.txt'.format (btdir)])
 
 # En Ubuntu 13.04+ es necesario matar proceso de "udev" antes de desmontar.
 #umount $BTROOTFSMNT 2>/dev/null || (kill -9 $(lsof -t $BTROOTFSMNT); umount $BTROOTFSMNT) 2>/dev/null
 
-logger.info ('running \'{}/02-boottoolsFsOpengnsys.py --mntpt "{}" --osdistrib "{}" --oscodename "{}" --osrelease "{}" --osarch "{}" --oshttp "{}"\''.format (curdir, btrootfsmnt, osdistrib, oscodename, osrelease, osarch, oshttp))
+logger.debug ('running \'{}/02-boottoolsFsOpengnsys.py --mntpt "{}" --osdistrib "{}" --oscodename "{}" --osrelease "{}" --osarch "{}" --oshttp "{}"\''.format (curdir, btrootfsmnt, osdistrib, oscodename, osrelease, osarch, oshttp))
 cp = subprocess.run (['{}/02-boottoolsFsOpengnsys.py'.format (curdir), '--mntpt', btrootfsmnt, '--osdistrib', osdistrib, '--oscodename', oscodename, '--osrelease', osrelease, '--osarch', osarch, '--oshttp', oshttp], capture_output=True)
-logger.info ('02-boottoolsFsOpengnsys stdout follows: {}'.format (cp.stdout.decode ('utf-8')))
+logger.debug ('02-boottoolsFsOpengnsys stdout follows: {}'.format (cp.stdout.decode ('utf-8')))
 ## /tmp/02.py --osdistrib ubuntu --oscodename focal --osrelease 5.4.0-42-generic --osarch amd64 --oshttp http://es.archive.ubuntu.com/ubuntu/
 ## /tmp/opengnsys_installer/opengnsys/client/./boot-tools/02-boottoolsFsOpengnsys.py --mntpt "/var/lib/tftpboot/ogclient/ogclientmount" --osdistrib "Ubuntu" --oscodename "noble" --osrelease "6.8.0-39-generic" --osarch "amd64" --oshttp "http://es.archive.ubuntu.com/ubuntu/"
 
@@ -130,6 +134,10 @@ logger.info ('02-boottoolsFsOpengnsys stdout follows: {}'.format (cp.stdout.deco
 ############################################################################################
 logger.info ('FASE 5 - Instalar software')
 logger.info ('Fase 5.1 instalar paquetes deb con apt-get')
+logger.debug ('running \'schroot --chroot IMGogclient -- {}/03-boottoolsSoftwareInstall.py --osrelease "{}" --osarch "{}"\''.format (curdir, osrelease, osarch))
+cp = subprocess.run (['schroot', '--chroot', 'IMGogclient', '--', '{}/03-boottoolsSoftwareInstall.py'.format (curdir), '--osrelease', osrelease, '--osarch', osarch])
+logger.debug ('03-boottoolsSoftwareInstall stdout follows: {}'.format (cp.stdout.decode ('utf-8')))
+
 """
 schroot --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSoftwareInstall.sh >>/tmp/fase5.out
 logger.info ('Fase 5.2 compilar software.')