refs #596 group chroot tasks in a single script

03-boottoolsSoftwareInstall.py

@@ -1,7 +1,9 @@
 #!/usr/bin/python3
 
 import argparse
+import shutil
 import os
+import glob
 import sys
 import re
 import subprocess
@@ -12,6 +14,7 @@ config = utils.read_config ('boottoolsgenerator.cfg')
 if config is None:
     sys.exit (1)
 debconf_settings = config['General'].get ('debconf_settings')
+debconf_settings2 = config['General'].get ('debconf_settings2')
 
 def _oghook_deactivate():
     #Desactivamos el hook del oginitrd.img para evitar problemas, al final de este escripts se activará
@@ -76,10 +79,63 @@ def boottoolsSoftwareInstall (osarch, osrelease):
     apt.clean()
     apt.autoremove()
 
+def boottoolsSoftwareCompile():
+    os.environ['LANGUAGE'] = os.environ['LC_ALL'] = os.environ['LANG'] = 'C'
+    os.chdir ('/tmp')
+
+    print ('ms-sys')
+    try: utils.run (['which', 'ms-sys'])
+    except:
+        utils.run (['wget', 'https://sourceforge.net/projects/ms-sys/files/latest/download', '-O', 'ms-sys.tar.gz'])
+        utils.run (['tar', '-xpzf', 'ms-sys.tar.gz'])
+        mssys_dir = subprocess.run (['tar tzf ms-sys.tar.gz |head -n 1'], shell=True, capture_output=True, text=True).stdout.strip()
+        print (f'mssys_dir "{mssys_dir}"')
+        os.chdir (mssys_dir)
+        utils.run (['make', 'install'])
+        os.chdir ('..')
+
+    print ('spartlnx')
+    try: utils.run (['which', 'spartl64.run'])
+    except:
+        utils.run (['wget', 'http://damien.guibouret.free.fr/savepart.zip'])
+        utils.run (['unzip', '-o', 'savepart.zip', '-d', '/sbin/', 'spartl64.run'])
+        utils.run (['mkdir', '/usr/share/doc/spartlnx'])
+        utils.run (['unzip', '-j', '-o', 'savepart.zip', '-d', '/usr/share/doc/spartlnx/', 'doc/en/*'])
+
+    if not os.path.exists ('python-libfdisk'):
+        print ('python-libfdisk')
+        apt.install (['python3-psutil', 'python3-dev', 'libfdisk-dev', 'python3-setuptools'])
+        utils.run (['git', 'clone', 'git://git.48k.eu/python-libfdisk'])
+        os.chdir ('python-libfdisk')
+        utils.run (['python3', 'setup.py', 'install'])
+        os.chdir ('..')
+        apt.remove (['python3-dev', 'python3-setuptools'])
+
+    ## TODO restore os.environ and pwd
+    os.environ['LANGUAGE'] = os.environ['LC_ALL'] = os.environ['LANG'] = 'C'
+    os.chdir ('/tmp')
+
+def boottoolsFsLocales():
+    subprocess.run (['debconf-set-selections'], input=debconf_settings2, text=True)
+
+    ## despues de esto, debconf-get-selections devuelve los valores antiguos, no se por que...
+    utils.run (['dpkg-reconfigure', '--frontend', 'noninteractive', 'console-setup', 'locales'])
+
+def boottoolsInitrdGenerate (osrelease):
+    for f in glob.glob ('/usr/lib/initramfs-tools/bin/*'):
+        os.unlink (f)
+    shutil.copy ('/bin/busybox', '/usr/lib/initramfs-tools/bin')
+
+    os.chdir ('/tmp')
+    utils.run (['mkinitramfs', '-o', f'/tmp/initrd.img-{osrelease}', '-v', osrelease])
+    shutil.copy (f'/boot/vmlinuz-{osrelease}', '/tmp/')
+
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
-    parser.add_argument ('--osarch',     help='OS architecture',    action='store', required=True)
+    parser.add_argument ('--osarch',    help='OS architecture', action='store', required=True)
-    parser.add_argument ('--osrelease',  help='OS release',         action='store', required=True)
+    parser.add_argument ('--osrelease', help='OS release',      action='store', required=True)
     args = parser.parse_args()
     boottoolsSoftwareInstall (args.osarch, args.osrelease)
-
+    boottoolsSoftwareCompile()
+    boottoolsFsLocales()
+    boottoolsInitrdGenerate (args.osrelease)

04-boottoolsSoftwareCompile.py

@@ -1,36 +0,0 @@
-#!/usr/bin/python3
-
-import os
-import subprocess
-from boottoolsfunctions import utils, apt
-
-os.environ['LANGUAGE'] = os.environ['LC_ALL'] = os.environ['LANG'] = 'C'
-os.chdir ('/tmp')
-
-print ('ms-sys')
-try: utils.run (['which', 'ms-sys'])
-except:
-    utils.run (['wget', 'https://sourceforge.net/projects/ms-sys/files/latest/download', '-O', 'ms-sys.tar.gz'])
-    utils.run (['tar', '-xpzf', 'ms-sys.tar.gz'])
-    mssys_dir = subprocess.run (['tar tzf ms-sys.tar.gz |head -n 1'], shell=True, capture_output=True, text=True).stdout.strip()
-    print (f'mssys_dir "{mssys_dir}"')
-    os.chdir (mssys_dir)
-    utils.run (['make', 'install'])
-    os.chdir ('..')
-
-print ('spartlnx')
-try: utils.run (['which', 'spartl64.run'])
-except:
-    utils.run (['wget', 'http://damien.guibouret.free.fr/savepart.zip'])
-    utils.run (['unzip', '-o', 'savepart.zip', '-d', '/sbin/', 'spartl64.run'])
-    utils.run (['mkdir', '/usr/share/doc/spartlnx'])
-    utils.run (['unzip', '-j', '-o', 'savepart.zip', '-d', '/usr/share/doc/spartlnx/', 'doc/en/*'])
-
-if not os.path.exists ('python-libfdisk'):
-    print ('python-libfdisk')
-    apt.install (['python3-psutil', 'python3-dev', 'libfdisk-dev', 'python3-setuptools'])
-    utils.run (['git', 'clone', 'git://git.48k.eu/python-libfdisk'])
-    os.chdir ('python-libfdisk')
-    utils.run (['python3', 'setup.py', 'install'])
-    os.chdir ('..')
-    apt.remove (['python3-dev', 'python3-setuptools'])

05-boottoolsFsLocales.py

@@ -1,15 +0,0 @@
-#!/usr/bin/python3
-
-import os
-import subprocess
-from boottoolsfunctions import utils
-
-config = utils.read_config ('boottoolsgenerator.cfg')
-if config is None:
-    sys.exit (1)
-debconf_settings2 = config['General'].get ('debconf_settings2')
-
-subprocess.run (['debconf-set-selections'], input=debconf_settings2, text=True)
-
-## despues de esto, debconf-get-selections devuelve los valores antiguos, no se por que...
-utils.run (['dpkg-reconfigure', '--frontend', 'noninteractive', 'console-setup', 'locales'])

06-boottoolsInitrdGenerate.py

@@ -1,22 +0,0 @@
-#!/usr/bin/python3
-
-import argparse
-import shutil
-import os
-import glob
-from boottoolsfunctions import utils
-
-def boottoolsInitrdGenerate (osrelease):
-    for f in glob.glob ('/usr/lib/initramfs-tools/bin/*'):
-        os.unlink (f)
-    shutil.copy ('/bin/busybox', '/usr/lib/initramfs-tools/bin')
-
-    os.chdir ('/tmp')
-    utils.run (['mkinitramfs', '-o', f'/tmp/initrd.img-{osrelease}', '-v', osrelease])
-    shutil.copy (f'/boot/vmlinuz-{osrelease}', '/tmp/')
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument ('--osrelease', help='OS release', action='store', required=True)
-    args = parser.parse_args()
-    boottoolsInitrdGenerate (args.osrelease)

boottoolsgenerator.py

@@ -41,10 +41,6 @@ def _mount_rootfs():
         logger.error ('mount failed')
         sys.exit (1)
 
-def _apt_update_upgrade():
-    apt.update()
-    apt.upgrade()
-
 def _get_pxepkg():
     #grep "http://free.nchc.org.tw/drbl-core" /etc/apt/sources.list || echo "deb http://free.nchc.org.tw/drbl-core drbl stable" >> /etc/apt/sources.list
     pxepkg = None
@@ -58,7 +54,7 @@ def _get_pxepkg():
     return pxepkg
 
 def _mkrootfs():
-    logger.info ('Fase 3.1 Generar y formatear el disco virtual. Generar el dispositivo loop.')
+    logger.info ('Stage 3.1 - create, partition and format the rootfs')
     rc = subprocess.run (f'file "{btrootfsimg}" |grep -q "partition 1 *: ID=0x83"', shell=True).returncode
     print (rc)
     if (rc):   ## 'file|grep' failed
@@ -68,14 +64,14 @@ def _mkrootfs():
             sys.exit (1)
 
 def _schroot (btrootfsimg):
-    logger.info ('Fase 3.2 - Configurar acceso schroot al Segundo Sistema de archivos (img)')
+    logger.info ('Stage 3.2 - configure schroot in the VM')
     btog.SetFsAccess (btrootfsimg)
 
 ## para hacer schroot --cosas, el mntpt tiene que estar desmontado
 ## si está montado da un pete tal que 'E: 10mount: mount: /run/schroot/mount/IMGogclient-7fbf51a2-e37e-48e5-8e5d-83f8901fc7ed: wrong fs type, bad option, bad superblock on /dev/loop1, missing codepage or helper program, or other error.'
 
 def _debootstrap():
-    logger.info ('Fase 3.3 Generar sistema de archivos con debootstrap')
+    logger.info ('Stage 3.3 - debootstrap system')
     logger.debug ('Try creation of a file within chroot (this operation may fail with "... etc/resolv.conf: No such file or directory"--that is ok)')
     logger.debug ('Running \'schroot --chroot IMGogclient -- stat /etc\'')
     cp = subprocess.run (['schroot', '--chroot', 'IMGogclient', '--', 'stat', '/etc'])
@@ -101,35 +97,14 @@ def _copy_files (btrootfsmnt, osdistrib, oscodename):
     btog.FsOpengnsys (builder, og_shared, og_engine, btrootfsmnt, osdistrib, oscodename)
     utils.umount (btrootfsmnt)
 
-def _install_compile_software (curdir, osrelease, osarch):
+def _chroot_tasks (curdir, osrelease, osarch):
-    logger.info ('Fase 5.1 instalar paquetes deb con apt-get')
     logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/03-boottoolsSoftwareInstall.py --osrelease "{osrelease}" --osarch "{osarch}"\'')
     stdout, _ = utils.run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/03-boottoolsSoftwareInstall.py', '--osrelease', osrelease, '--osarch', osarch])
     logger.debug (f'03-boottoolsSoftwareInstall stdout follows:')
-    for i in stdout.strip().split('\n'): logger.debug ('  ' + i)
-
-    logger.info ('Fase 5.2 compilar software.')
-    logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/04-boottoolsSoftwareCompile.py\'')
-    stdout, _ = utils.run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/04-boottoolsSoftwareCompile.py'])
-    logger.debug (f'04-boottoolsSoftwareCompile stdout follows:')
-    for i in stdout.strip().split('\n'): logger.debug ('  ' + i)
-
-def _debconf2 (curdir):
-    logger.info ('Fase 6.1 Configurar las locales')
-    logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/05-boottoolsFsLocales.py\'')
-    stdout, _ = utils.run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/05-boottoolsFsLocales.py'])
-    logger.debug (f'05-boottoolsFsLocales stdout follows:')
-    for i in stdout.strip().split('\n'): logger.debug ('  ' + i)
-
-    logger.info ('Fase 6.3 Crear initrd')
-    logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/06-boottoolsInitrdGenerate.py --osrelease "{osrelease}"\'')
-    stdout, _ = utils.run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/06-boottoolsInitrdGenerate.py', '--osrelease', osrelease])
-    logger.debug (f'06-boottoolsInitrdGenerate stdout follows:')
-    for i in stdout.strip().split('\n'): logger.debug ('  ' + i)
     ## esto deja initrd.img-6.8.0-31-generic y vmlinuz-6.8.0-31-generic en /tmp
+    for i in stdout.strip().split ('\n'): logger.debug ('  ' + i)
 
 def _ssh_stuff():
-    logger.info ('Fase 6.2 Configurar ssh')
     _mount_rootfs()
     btog.SshServer (btrootfsmnt)
     btog.SshClient (btrootfsmnt)
@@ -140,15 +115,15 @@ def _ssh_stuff():
     ## - las dos claves públicas (una de cada par) están autorizadan en el rootfs /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh/authorized_keys
 
 def _mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsmnt, pxepkg, isolinux_tpl, nameisoclient):
-    logger.info ('Fase 7.1 Copiar el initrd a su sitio')
+    logger.info ('Stage 6.1 - Put initrd in place')
     _mount_rootfs()
     btog.FsInitrd (bttargetdir, osrelease)
 
-    logger.info ('Fase 7.2 Generar fichero sqfs a partir del fichero img')
+    logger.info ('Stage 6.2 - make squash filesystem')
     btog.FsSqfs (bttargetdir, btrootfsmnt)
     utils.umount (btrootfsmnt)
 
-    logger.info ('Fase 7.3 Generar la ISO')
+    logger.info ('Stage 6.3 - make iso filesystem')
     btog.IsoGenerator (pxepkg, isolinux_tpl, bttargetdir, nameisoclient)
 
 
@@ -160,7 +135,7 @@ isolinux_tpl = config['General'].get ('isolinux_template')
 lvl = config['General'].get ('logging_level')
 
 logger = _logging()
-type_client    = sys.argv[1] if len(sys.argv)>1 else 'host'
+type_client    = sys.argv[1] if len (sys.argv)>1 else 'host'
 
 if os.getuid():
     logger.error ('ERROR: this program must run under root privileges!!')
@@ -168,45 +143,44 @@ if os.getuid():
 
 #os.chdir ('/tmp')
 
+## this is convenient in case the previous run failed and we want to run this program again
+try: utils.umount (btrootfsmnt)
+except: pass
+
 logger.info ('OpenGnsys CLIENT installation begins')
+
+logger.info ('STAGE 1 - clone opengnsys repo')
 ogrepo_url = 'https://ognproject.evlt.uma.es/gitea/opengnsys/opengnsys.git'
 ogrepo_branch = 'main'
 ogrepo_dir = '/tmp/ogrepo'
 clone_client_dirs (ogrepo_url, ogrepo_branch, ogrepo_dir)
 
-logger.info ('FASE 1 - Asignación de variables')
+osdistrib, oscodename, osrelease, osarch, oshttp = btog.GetOsInfo1 (type_client)
-osdistrib, oscodename, osrelease, osarch, oshttp = btog.GetOsInfo1(type_client)
+btdir, bttargetdir, btrootfsimg, btrootfsmnt, btrootfsimglabel, log_file, versionboottools, btvirtualdisksize = btog.GetVar (osarch)
-btdir, bttargetdir, btrootfsimg, btrootfsmnt, btrootfsimglabel, log_file, versionboottools, btvirtualdisksize = btog.GetVar(osarch)
+gitrelease, nameisoclient, namehostclient = btog.GetOsInfo2 (ogrepo_dir, versionboottools, oscodename, osrelease, osarch)
-gitrelease, nameisoclient, namehostclient = btog.GetOsInfo2(ogrepo_dir, versionboottools, oscodename, osrelease, osarch)
-
 logger.info (':'.join ([osdistrib, oscodename, osrelease, osarch, oshttp]))
 
-## this is convenient in case the previous run failed and we want to run this program again
+logger.info ('STAGE 2 - install software in the VM')
-try: utils.umount (btrootfsmnt)
+apt.update()
-except: pass
+apt.upgrade()
-
-logger.info ('FASE 2 - Instalación de software adicional.')
-_apt_update_upgrade()
 pxepkg = _get_pxepkg()
-utils.run (['apt-get', '-y', 'install', 'jq', 'syslinux', 'syslinux-efi', 'syslinux-utils', 'debootstrap', 'subversion', 'schroot', 'squashfs-tools', 'syslinux', 'genisoimage', 'qemu-utils', 'lsof', pxepkg])    ## TODO qemu no existe, hace falta?
+apt.install (['jq', 'syslinux', 'syslinux-efi', 'syslinux-utils', 'debootstrap', 'subversion', 'schroot', 'squashfs-tools', 'syslinux', 'genisoimage', 'qemu-utils', 'lsof', pxepkg])    ## TODO qemu no existe, hace falta?
 
-logger.info ('FASE 3 - Creación del Sistema raiz RootFS (Segundo Sistema archivos (img))')
+logger.info ('STAGE 3 - create and bootstrap rootfs')
 _mkrootfs()
 _schroot (btrootfsimg)
 _debootstrap()
 
-logger.info ('FASE 4 - Incorporando ficheros OpenGnsys al sistema raíz rootfs')
+logger.info ('STAGE 4 - copy files to the rootfs')
 _initramfs_version (gitrelease, osrelease, btdir)
 _copy_files (btrootfsmnt, osdistrib, oscodename)
 
-logger.info ('FASE 5 - Instalar software')
+logger.info ('STAGE 5 - perform tasks within the chroot')
-_install_compile_software (curdir, osrelease, osarch)
+_chroot_tasks (curdir, osrelease, osarch)
 
-logger.info ('FASE 6 - Personalizar el sistema creado')
-_debconf2 (curdir)
 _ssh_stuff()
 
-logger.info ('FASE 7 - Generar distribucion')
+logger.info ('STAGE 6 - generate distribution files')
 _mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsmnt, pxepkg, isolinux_tpl, nameisoclient)
 
 logger.info ('OpenGnsys installation finished')