From feae76840596623d5be18ddbfda820422215ef43 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Fri, 16 Aug 2024 12:43:26 +0200 Subject: [PATCH] refs #596 group chroot tasks in a single script --- 03-boottoolsSoftwareInstall.py | 62 ++++++++++++++++++++++++-- 04-boottoolsSoftwareCompile.py | 36 --------------- 05-boottoolsFsLocales.py | 15 ------- 06-boottoolsInitrdGenerate.py | 22 ---------- boottoolsgenerator.py | 80 ++++++++++++---------------------- 5 files changed, 86 insertions(+), 129 deletions(-) diff --git a/03-boottoolsSoftwareInstall.py b/03-boottoolsSoftwareInstall.py index e93cabc..70ea143 100755 --- a/03-boottoolsSoftwareInstall.py +++ b/03-boottoolsSoftwareInstall.py @@ -1,7 +1,9 @@ #!/usr/bin/python3 import argparse +import shutil import os +import glob import sys import re import subprocess @@ -12,6 +14,7 @@ config = utils.read_config ('boottoolsgenerator.cfg') if config is None: sys.exit (1) debconf_settings = config['General'].get ('debconf_settings') +debconf_settings2 = config['General'].get ('debconf_settings2') def _oghook_deactivate(): #Desactivamos el hook del oginitrd.img para evitar problemas, al final de este escripts se activará @@ -76,10 +79,63 @@ def boottoolsSoftwareInstall (osarch, osrelease): apt.clean() apt.autoremove() +def boottoolsSoftwareCompile(): + os.environ['LANGUAGE'] = os.environ['LC_ALL'] = os.environ['LANG'] = 'C' + os.chdir ('/tmp') + + print ('ms-sys') + try: utils.run (['which', 'ms-sys']) + except: + utils.run (['wget', 'https://sourceforge.net/projects/ms-sys/files/latest/download', '-O', 'ms-sys.tar.gz']) + utils.run (['tar', '-xpzf', 'ms-sys.tar.gz']) + mssys_dir = subprocess.run (['tar tzf ms-sys.tar.gz |head -n 1'], shell=True, capture_output=True, text=True).stdout.strip() + print (f'mssys_dir "{mssys_dir}"') + os.chdir (mssys_dir) + utils.run (['make', 'install']) + os.chdir ('..') + + print ('spartlnx') + try: utils.run (['which', 'spartl64.run']) + except: + utils.run (['wget', 'http://damien.guibouret.free.fr/savepart.zip']) + utils.run (['unzip', '-o', 'savepart.zip', '-d', '/sbin/', 'spartl64.run']) + utils.run (['mkdir', '/usr/share/doc/spartlnx']) + utils.run (['unzip', '-j', '-o', 'savepart.zip', '-d', '/usr/share/doc/spartlnx/', 'doc/en/*']) + + if not os.path.exists ('python-libfdisk'): + print ('python-libfdisk') + apt.install (['python3-psutil', 'python3-dev', 'libfdisk-dev', 'python3-setuptools']) + utils.run (['git', 'clone', 'git://git.48k.eu/python-libfdisk']) + os.chdir ('python-libfdisk') + utils.run (['python3', 'setup.py', 'install']) + os.chdir ('..') + apt.remove (['python3-dev', 'python3-setuptools']) + + ## TODO restore os.environ and pwd + os.environ['LANGUAGE'] = os.environ['LC_ALL'] = os.environ['LANG'] = 'C' + os.chdir ('/tmp') + +def boottoolsFsLocales(): + subprocess.run (['debconf-set-selections'], input=debconf_settings2, text=True) + + ## despues de esto, debconf-get-selections devuelve los valores antiguos, no se por que... + utils.run (['dpkg-reconfigure', '--frontend', 'noninteractive', 'console-setup', 'locales']) + +def boottoolsInitrdGenerate (osrelease): + for f in glob.glob ('/usr/lib/initramfs-tools/bin/*'): + os.unlink (f) + shutil.copy ('/bin/busybox', '/usr/lib/initramfs-tools/bin') + + os.chdir ('/tmp') + utils.run (['mkinitramfs', '-o', f'/tmp/initrd.img-{osrelease}', '-v', osrelease]) + shutil.copy (f'/boot/vmlinuz-{osrelease}', '/tmp/') + if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument ('--osarch', help='OS architecture', action='store', required=True) - parser.add_argument ('--osrelease', help='OS release', action='store', required=True) + parser.add_argument ('--osarch', help='OS architecture', action='store', required=True) + parser.add_argument ('--osrelease', help='OS release', action='store', required=True) args = parser.parse_args() boottoolsSoftwareInstall (args.osarch, args.osrelease) - + boottoolsSoftwareCompile() + boottoolsFsLocales() + boottoolsInitrdGenerate (args.osrelease) diff --git a/04-boottoolsSoftwareCompile.py b/04-boottoolsSoftwareCompile.py index 4456066..e69de29 100755 --- a/04-boottoolsSoftwareCompile.py +++ b/04-boottoolsSoftwareCompile.py @@ -1,36 +0,0 @@ -#!/usr/bin/python3 - -import os -import subprocess -from boottoolsfunctions import utils, apt - -os.environ['LANGUAGE'] = os.environ['LC_ALL'] = os.environ['LANG'] = 'C' -os.chdir ('/tmp') - -print ('ms-sys') -try: utils.run (['which', 'ms-sys']) -except: - utils.run (['wget', 'https://sourceforge.net/projects/ms-sys/files/latest/download', '-O', 'ms-sys.tar.gz']) - utils.run (['tar', '-xpzf', 'ms-sys.tar.gz']) - mssys_dir = subprocess.run (['tar tzf ms-sys.tar.gz |head -n 1'], shell=True, capture_output=True, text=True).stdout.strip() - print (f'mssys_dir "{mssys_dir}"') - os.chdir (mssys_dir) - utils.run (['make', 'install']) - os.chdir ('..') - -print ('spartlnx') -try: utils.run (['which', 'spartl64.run']) -except: - utils.run (['wget', 'http://damien.guibouret.free.fr/savepart.zip']) - utils.run (['unzip', '-o', 'savepart.zip', '-d', '/sbin/', 'spartl64.run']) - utils.run (['mkdir', '/usr/share/doc/spartlnx']) - utils.run (['unzip', '-j', '-o', 'savepart.zip', '-d', '/usr/share/doc/spartlnx/', 'doc/en/*']) - -if not os.path.exists ('python-libfdisk'): - print ('python-libfdisk') - apt.install (['python3-psutil', 'python3-dev', 'libfdisk-dev', 'python3-setuptools']) - utils.run (['git', 'clone', 'git://git.48k.eu/python-libfdisk']) - os.chdir ('python-libfdisk') - utils.run (['python3', 'setup.py', 'install']) - os.chdir ('..') - apt.remove (['python3-dev', 'python3-setuptools']) diff --git a/05-boottoolsFsLocales.py b/05-boottoolsFsLocales.py index 02684ca..e69de29 100755 --- a/05-boottoolsFsLocales.py +++ b/05-boottoolsFsLocales.py @@ -1,15 +0,0 @@ -#!/usr/bin/python3 - -import os -import subprocess -from boottoolsfunctions import utils - -config = utils.read_config ('boottoolsgenerator.cfg') -if config is None: - sys.exit (1) -debconf_settings2 = config['General'].get ('debconf_settings2') - -subprocess.run (['debconf-set-selections'], input=debconf_settings2, text=True) - -## despues de esto, debconf-get-selections devuelve los valores antiguos, no se por que... -utils.run (['dpkg-reconfigure', '--frontend', 'noninteractive', 'console-setup', 'locales']) diff --git a/06-boottoolsInitrdGenerate.py b/06-boottoolsInitrdGenerate.py index 02c94b1..e69de29 100755 --- a/06-boottoolsInitrdGenerate.py +++ b/06-boottoolsInitrdGenerate.py @@ -1,22 +0,0 @@ -#!/usr/bin/python3 - -import argparse -import shutil -import os -import glob -from boottoolsfunctions import utils - -def boottoolsInitrdGenerate (osrelease): - for f in glob.glob ('/usr/lib/initramfs-tools/bin/*'): - os.unlink (f) - shutil.copy ('/bin/busybox', '/usr/lib/initramfs-tools/bin') - - os.chdir ('/tmp') - utils.run (['mkinitramfs', '-o', f'/tmp/initrd.img-{osrelease}', '-v', osrelease]) - shutil.copy (f'/boot/vmlinuz-{osrelease}', '/tmp/') - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument ('--osrelease', help='OS release', action='store', required=True) - args = parser.parse_args() - boottoolsInitrdGenerate (args.osrelease) diff --git a/boottoolsgenerator.py b/boottoolsgenerator.py index dc443f9..c18046e 100755 --- a/boottoolsgenerator.py +++ b/boottoolsgenerator.py @@ -41,10 +41,6 @@ def _mount_rootfs(): logger.error ('mount failed') sys.exit (1) -def _apt_update_upgrade(): - apt.update() - apt.upgrade() - def _get_pxepkg(): #grep "http://free.nchc.org.tw/drbl-core" /etc/apt/sources.list || echo "deb http://free.nchc.org.tw/drbl-core drbl stable" >> /etc/apt/sources.list pxepkg = None @@ -58,7 +54,7 @@ def _get_pxepkg(): return pxepkg def _mkrootfs(): - logger.info ('Fase 3.1 Generar y formatear el disco virtual. Generar el dispositivo loop.') + logger.info ('Stage 3.1 - create, partition and format the rootfs') rc = subprocess.run (f'file "{btrootfsimg}" |grep -q "partition 1 *: ID=0x83"', shell=True).returncode print (rc) if (rc): ## 'file|grep' failed @@ -68,14 +64,14 @@ def _mkrootfs(): sys.exit (1) def _schroot (btrootfsimg): - logger.info ('Fase 3.2 - Configurar acceso schroot al Segundo Sistema de archivos (img)') + logger.info ('Stage 3.2 - configure schroot in the VM') btog.SetFsAccess (btrootfsimg) ## para hacer schroot --cosas, el mntpt tiene que estar desmontado ## si está montado da un pete tal que 'E: 10mount: mount: /run/schroot/mount/IMGogclient-7fbf51a2-e37e-48e5-8e5d-83f8901fc7ed: wrong fs type, bad option, bad superblock on /dev/loop1, missing codepage or helper program, or other error.' def _debootstrap(): - logger.info ('Fase 3.3 Generar sistema de archivos con debootstrap') + logger.info ('Stage 3.3 - debootstrap system') logger.debug ('Try creation of a file within chroot (this operation may fail with "... etc/resolv.conf: No such file or directory"--that is ok)') logger.debug ('Running \'schroot --chroot IMGogclient -- stat /etc\'') cp = subprocess.run (['schroot', '--chroot', 'IMGogclient', '--', 'stat', '/etc']) @@ -101,35 +97,14 @@ def _copy_files (btrootfsmnt, osdistrib, oscodename): btog.FsOpengnsys (builder, og_shared, og_engine, btrootfsmnt, osdistrib, oscodename) utils.umount (btrootfsmnt) -def _install_compile_software (curdir, osrelease, osarch): - logger.info ('Fase 5.1 instalar paquetes deb con apt-get') +def _chroot_tasks (curdir, osrelease, osarch): logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/03-boottoolsSoftwareInstall.py --osrelease "{osrelease}" --osarch "{osarch}"\'') stdout, _ = utils.run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/03-boottoolsSoftwareInstall.py', '--osrelease', osrelease, '--osarch', osarch]) logger.debug (f'03-boottoolsSoftwareInstall stdout follows:') - for i in stdout.strip().split('\n'): logger.debug (' ' + i) - - logger.info ('Fase 5.2 compilar software.') - logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/04-boottoolsSoftwareCompile.py\'') - stdout, _ = utils.run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/04-boottoolsSoftwareCompile.py']) - logger.debug (f'04-boottoolsSoftwareCompile stdout follows:') - for i in stdout.strip().split('\n'): logger.debug (' ' + i) - -def _debconf2 (curdir): - logger.info ('Fase 6.1 Configurar las locales') - logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/05-boottoolsFsLocales.py\'') - stdout, _ = utils.run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/05-boottoolsFsLocales.py']) - logger.debug (f'05-boottoolsFsLocales stdout follows:') - for i in stdout.strip().split('\n'): logger.debug (' ' + i) - - logger.info ('Fase 6.3 Crear initrd') - logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/06-boottoolsInitrdGenerate.py --osrelease "{osrelease}"\'') - stdout, _ = utils.run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/06-boottoolsInitrdGenerate.py', '--osrelease', osrelease]) - logger.debug (f'06-boottoolsInitrdGenerate stdout follows:') - for i in stdout.strip().split('\n'): logger.debug (' ' + i) ## esto deja initrd.img-6.8.0-31-generic y vmlinuz-6.8.0-31-generic en /tmp + for i in stdout.strip().split ('\n'): logger.debug (' ' + i) def _ssh_stuff(): - logger.info ('Fase 6.2 Configurar ssh') _mount_rootfs() btog.SshServer (btrootfsmnt) btog.SshClient (btrootfsmnt) @@ -140,15 +115,15 @@ def _ssh_stuff(): ## - las dos claves públicas (una de cada par) están autorizadan en el rootfs /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh/authorized_keys def _mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsmnt, pxepkg, isolinux_tpl, nameisoclient): - logger.info ('Fase 7.1 Copiar el initrd a su sitio') + logger.info ('Stage 6.1 - Put initrd in place') _mount_rootfs() btog.FsInitrd (bttargetdir, osrelease) - logger.info ('Fase 7.2 Generar fichero sqfs a partir del fichero img') + logger.info ('Stage 6.2 - make squash filesystem') btog.FsSqfs (bttargetdir, btrootfsmnt) utils.umount (btrootfsmnt) - logger.info ('Fase 7.3 Generar la ISO') + logger.info ('Stage 6.3 - make iso filesystem') btog.IsoGenerator (pxepkg, isolinux_tpl, bttargetdir, nameisoclient) @@ -160,7 +135,7 @@ isolinux_tpl = config['General'].get ('isolinux_template') lvl = config['General'].get ('logging_level') logger = _logging() -type_client = sys.argv[1] if len(sys.argv)>1 else 'host' +type_client = sys.argv[1] if len (sys.argv)>1 else 'host' if os.getuid(): logger.error ('ERROR: this program must run under root privileges!!') @@ -168,45 +143,44 @@ if os.getuid(): #os.chdir ('/tmp') +## this is convenient in case the previous run failed and we want to run this program again +try: utils.umount (btrootfsmnt) +except: pass + logger.info ('OpenGnsys CLIENT installation begins') + +logger.info ('STAGE 1 - clone opengnsys repo') ogrepo_url = 'https://ognproject.evlt.uma.es/gitea/opengnsys/opengnsys.git' ogrepo_branch = 'main' ogrepo_dir = '/tmp/ogrepo' clone_client_dirs (ogrepo_url, ogrepo_branch, ogrepo_dir) -logger.info ('FASE 1 - Asignación de variables') -osdistrib, oscodename, osrelease, osarch, oshttp = btog.GetOsInfo1(type_client) -btdir, bttargetdir, btrootfsimg, btrootfsmnt, btrootfsimglabel, log_file, versionboottools, btvirtualdisksize = btog.GetVar(osarch) -gitrelease, nameisoclient, namehostclient = btog.GetOsInfo2(ogrepo_dir, versionboottools, oscodename, osrelease, osarch) - +osdistrib, oscodename, osrelease, osarch, oshttp = btog.GetOsInfo1 (type_client) +btdir, bttargetdir, btrootfsimg, btrootfsmnt, btrootfsimglabel, log_file, versionboottools, btvirtualdisksize = btog.GetVar (osarch) +gitrelease, nameisoclient, namehostclient = btog.GetOsInfo2 (ogrepo_dir, versionboottools, oscodename, osrelease, osarch) logger.info (':'.join ([osdistrib, oscodename, osrelease, osarch, oshttp])) -## this is convenient in case the previous run failed and we want to run this program again -try: utils.umount (btrootfsmnt) -except: pass - -logger.info ('FASE 2 - Instalación de software adicional.') -_apt_update_upgrade() +logger.info ('STAGE 2 - install software in the VM') +apt.update() +apt.upgrade() pxepkg = _get_pxepkg() -utils.run (['apt-get', '-y', 'install', 'jq', 'syslinux', 'syslinux-efi', 'syslinux-utils', 'debootstrap', 'subversion', 'schroot', 'squashfs-tools', 'syslinux', 'genisoimage', 'qemu-utils', 'lsof', pxepkg]) ## TODO qemu no existe, hace falta? +apt.install (['jq', 'syslinux', 'syslinux-efi', 'syslinux-utils', 'debootstrap', 'subversion', 'schroot', 'squashfs-tools', 'syslinux', 'genisoimage', 'qemu-utils', 'lsof', pxepkg]) ## TODO qemu no existe, hace falta? -logger.info ('FASE 3 - Creación del Sistema raiz RootFS (Segundo Sistema archivos (img))') +logger.info ('STAGE 3 - create and bootstrap rootfs') _mkrootfs() _schroot (btrootfsimg) _debootstrap() -logger.info ('FASE 4 - Incorporando ficheros OpenGnsys al sistema raíz rootfs') +logger.info ('STAGE 4 - copy files to the rootfs') _initramfs_version (gitrelease, osrelease, btdir) _copy_files (btrootfsmnt, osdistrib, oscodename) -logger.info ('FASE 5 - Instalar software') -_install_compile_software (curdir, osrelease, osarch) +logger.info ('STAGE 5 - perform tasks within the chroot') +_chroot_tasks (curdir, osrelease, osarch) -logger.info ('FASE 6 - Personalizar el sistema creado') -_debconf2 (curdir) _ssh_stuff() -logger.info ('FASE 7 - Generar distribucion') +logger.info ('STAGE 6 - generate distribution files') _mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsmnt, pxepkg, isolinux_tpl, nameisoclient) logger.info ('OpenGnsys installation finished')