From c79d715ccc8ba9ab4a63b775753dcb7d4ae74f99 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Tue, 20 Aug 2024 14:09:54 +0200 Subject: [PATCH] refs #596 use a main() function --- .dockerignore | 1 + boottools/btog.py | 5 +- chroot-tasks.py | 20 ++++---- mkoglive.py | 113 ++++++++++++++++++++++++---------------------- 4 files changed, 74 insertions(+), 65 deletions(-) diff --git a/.dockerignore b/.dockerignore index 9f8c34d..472c70e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,6 +6,7 @@ Dockerfile INSTALL.es.txt Jenkinsfile kernelParameterOG.es.txt +ogclient/ ogclientImg2Sqfs ogclientSqfs2Img *.swp diff --git a/boottools/btog.py b/boottools/btog.py index f42c001..3b85b76 100644 --- a/boottools/btog.py +++ b/boottools/btog.py @@ -21,7 +21,7 @@ def GetVar (osarch): btvirtualdisksize = '3G' return bttargetdir, btrootfsimg, btrootfsmnt, btvirtualdisksize -def GetOsInfo (type_client): +def GetOsInfo (type_client='host'): if 'precise' == type_client: # ogLive 1.0.4-rc2 basado en Ubuntu 12.04 LTS. osdistrib='ubuntu' oscodename='precise' @@ -324,7 +324,8 @@ def mksquashfs (bttargetdir, btrootfsmnt): now = datetime.datetime.now(datetime.timezone.utc).strftime ('%Y%m%d-%H%M%S%z') os.rename (f'{bttargetdir}/ogclient.sqfs', f'{bttargetdir}/ogclient.sqfs.{now}') - utils.run (['mksquashfs', btrootfsmnt, f'{bttargetdir}/ogclient.sqfs']) + ## uses all CPU cores available, even within docker + utils.run (['mksquashfs', btrootfsmnt, f'{bttargetdir}/ogclient.sqfs', '-e', 'var/lib/apt/lists', '-e', 'usr/share/doc']) os.chmod (f'{bttargetdir}/ogclient.sqfs', 0o744) utils.write_md5 (f'{bttargetdir}/ogclient.sqfs') diff --git a/chroot-tasks.py b/chroot-tasks.py index 056cb36..bcdd9d9 100755 --- a/chroot-tasks.py +++ b/chroot-tasks.py @@ -36,6 +36,7 @@ def boottoolsSoftwareInstall (osarch, osrelease): _oghook_deactivate() + print ('boottoolsSoftwareInstall: debconf-set-selections', file=sys.stderr) subprocess.run (['debconf-set-selections'], input=debconf_settings, text=True) utils.run (['dpkg-reconfigure', '--frontend', 'noninteractive', 'console-setup', 'locales']) ## despues de esto, debconf-get-selections devuelve los valores antiguos, no se por que... @@ -43,16 +44,16 @@ def boottoolsSoftwareInstall (osarch, osrelease): for section in config.options('Packages'): pkgs += re.split (r'[ \n]', config['Packages'].get(section).strip()) - apt.install ( - [f'linux-image-{osrelease}', f'linux-headers-{osrelease}', f'linux-modules-{osrelease}', f'linux-modules-extra-{osrelease}', 'dkms', 'shim-signed', 'openssl', 'sshfs', 'kexec-tools'] + pkgs32 + pkgs, - opts={'DPkg::Options::': '--force-confdef'} ## hace falta --force-confdef para evitar un tema interactivo del /etc/ssh/ssh_config - ) + pkgs = [f'linux-image-{osrelease}', f'linux-headers-{osrelease}', f'linux-modules-{osrelease}', f'linux-modules-extra-{osrelease}', 'dkms', 'shim-signed', 'openssl', 'sshfs', 'kexec-tools'] + pkgs32 + pkgs + print (f'boottoolsSoftwareInstall: installing packages: {str(pkgs)}', file=sys.stderr) + apt.install (pkgs, opts={'DPkg::Options::': '--force-confdef'}) ## hace falta --force-confdef para evitar un tema interactivo del /etc/ssh/ssh_config # Instalar módulos que algunos paquetes puedan tener pendientes de compilar. + print ('boottoolsSoftwareInstall: dkms', file=sys.stderr) stdout, _ = utils.run (['dkms', 'status']) for l in stdout.strip().split ('\n'): if not l: continue - print (f'l "{l}"') + #print (f'l "{l}"') mod, vers, status = l.split (',') if 'added' in status: print (f'dkms installing {mod} {vers}') @@ -69,7 +70,7 @@ def boottoolsSoftwareCompile(): os.environ['LANGUAGE'] = os.environ['LC_ALL'] = os.environ['LANG'] = 'C' os.chdir ('/tmp') - print ('ms-sys') + print ('boottoolsSoftwareCompile: ms-sys', file=sys.stderr) try: utils.run (['which', 'ms-sys']) except: utils.run (['wget', '--quiet', 'https://sourceforge.net/projects/ms-sys/files/latest/download', '-O', 'ms-sys.tar.gz']) @@ -80,7 +81,7 @@ def boottoolsSoftwareCompile(): utils.run (['make', 'install']) os.chdir ('..') - print ('spartlnx') + print ('boottoolsSoftwareCompile: spartlnx', file=sys.stderr) try: utils.run (['which', 'spartl64.run']) except: utils.run (['wget', '--quiet', 'http://damien.guibouret.free.fr/savepart.zip']) @@ -89,7 +90,7 @@ def boottoolsSoftwareCompile(): utils.run (['unzip', '-j', '-o', 'savepart.zip', '-d', '/usr/share/doc/spartlnx/', 'doc/en/*']) if not os.path.exists ('python-libfdisk'): - print ('python-libfdisk') + print ('boottoolsSoftwareCompile: python-libfdisk', file=sys.stderr) apt.install (['python3-psutil', 'python3-dev', 'libfdisk-dev', 'python3-setuptools']) utils.run (['git', 'clone', 'git://git.48k.eu/python-libfdisk']) os.chdir ('python-libfdisk') @@ -102,12 +103,13 @@ def boottoolsSoftwareCompile(): os.environ['LANG'] = env_lang def boottoolsInitrdGenerate (osrelease): + print ('boottoolsInitrdGenerate', file=sys.stderr) for f in glob.glob ('/usr/lib/initramfs-tools/bin/*'): os.unlink (f) shutil.copy ('/bin/busybox', '/usr/lib/initramfs-tools/bin') os.chdir ('/tmp') - utils.run (['mkinitramfs', '-o', f'/tmp/initrd.img-{osrelease}', '-v', osrelease]) + utils.run (['mkinitramfs', '-o', f'/tmp/initrd.img-{osrelease}', osrelease]) shutil.copy (f'/boot/vmlinuz-{osrelease}', '/tmp/') if __name__ == '__main__': diff --git a/mkoglive.py b/mkoglive.py index 71a7f37..0151645 100755 --- a/mkoglive.py +++ b/mkoglive.py @@ -30,8 +30,7 @@ def _logging (lvl='INFO', logfile='/tmp/boot-tools_installation.log'): ) return logging.getLogger ('boottools') -def _mount_rootfs(): - global btrootfsimg, btrootfsmnt +def _mount_rootfs (btrootfsimg, btrootfsmnt): try: utils.mount (btrootfsimg, btrootfsmnt, opts=['-o', 'loop,offset=32256']) except: logger.error ('mount failed') @@ -68,7 +67,7 @@ def _debootstrap (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp): cp = subprocess.run (['schroot', '--chroot', 'IMGogclient', '--', 'stat', '/etc']) if (cp.returncode): logger.debug (f'schroot returned code "{cp.returncode}", calling btog.debootstrap()') - _mount_rootfs() + _mount_rootfs (btrootfsimg, btrootfsmnt) try: btog.debootstrap (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp) except Exception as e: logger.error (str (e)) @@ -80,21 +79,21 @@ def _initramfs_version (gitrelease, osrelease, curdir): #sed -i "1 s/$/ $GITRELEASE ($OSRELEASE)/" ${BTDIR}/includes/etc/initramfs-tools/scripts/VERSION.txt utils.run (['sed', '-i', f'1 s/$/ {gitrelease} ({osrelease})/', f'{curdir}/includes/etc/initramfs-tools/scripts/VERSION.txt']) -def _copy_og_files (btrootfsmnt, osdistrib, oscodename): - _mount_rootfs() +def _copy_og_files (btrootfsimg, btrootfsmnt, osdistrib, oscodename): + _mount_rootfs (btrootfsimg, btrootfsmnt) builder = '/tmp/opengnsys/oglive_builder' og_shared = '/tmp/opengnsys/shared' og_engine = '/tmp/opengnsys/engine' btog.copy_og_files (builder, og_shared, og_engine, btrootfsmnt, osdistrib, oscodename) utils.umount (btrootfsmnt) -def _chroot_tasks (curdir, osrelease, osarch): +def _chroot_tasks (cfgfile, curdir, osrelease, osarch): logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/chroot-tasks.py --osrelease "{osrelease}" --osarch "{osarch}"\'') - stdout, _ = utils.run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/chroot-tasks.py', '--osrelease', osrelease, '--osarch', osarch, '--config', args.config or 'mkoglive.cfg']) + stdout, _ = utils.run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/chroot-tasks.py', '--osrelease', osrelease, '--osarch', osarch, '--config', cfgfile]) ## esto deja initrd.img-6.8.0-31-generic y vmlinuz-6.8.0-31-generic en /tmp -def _ssh_stuff (btrootfsmnt): - _mount_rootfs() +def _ssh_stuff (btrootfsimg, btrootfsmnt): + _mount_rootfs (btrootfsimg, btrootfsmnt) btog.ssh_server (btrootfsmnt) btog.ssh_client (btrootfsmnt) utils.umount (btrootfsmnt) @@ -103,9 +102,9 @@ def _ssh_stuff (btrootfsmnt): ## - hay otro nuevo par de claves en el rootfs /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh ## - las dos claves públicas (una de cada par) están autorizadan en el rootfs /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh/authorized_keys -def _mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsmnt, pxepkg, isolinux_tpl, nameisoclient): +def _mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsimg, btrootfsmnt, pxepkg, isolinux_tpl, nameisoclient): logger.info ('Stage 4.1 - Put initrd in place') - _mount_rootfs() + _mount_rootfs (btrootfsimg, btrootfsmnt) btog.move_initrd (bttargetdir, osrelease) logger.info ('Stage 4.2 - make squash filesystem') @@ -115,60 +114,66 @@ def _mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsmnt, pxepkg, isoli logger.info ('Stage 4.3 - make iso filesystem') btog.mkisofs (pxepkg, isolinux_tpl, bttargetdir, nameisoclient) +def _main (cfgfile, config, type_client): + isolinux_tpl = config['General'].get ('isolinux_template') + btrootfsimglabel = config['General'].get ('rootfs_image_label') -parser = argparse.ArgumentParser() -parser.add_argument ('--loglevel', help='Log level', action='store') -parser.add_argument ('--codename', help='OS codename', action='store') -parser.add_argument ('--config', help='Path to configuration file', action='store') -args = parser.parse_args() + logger.info ('OpenGnsys CLIENT installation begins') -config = utils.read_config (args.config or 'mkoglive.cfg') -if config is None: - sys.exit (1) -isolinux_tpl = config['General'].get ('isolinux_template') -cfg_loglevel = config['General'].get ('logging_level') -btrootfsimglabel = config['General'].get ('rootfs_image_label') + fd = open (f'{curdir}/gitrelease', 'r') ## per the Dockerfile + gitrelease = fd.readline().strip() + fd.close() -logger = _logging (args.loglevel or cfg_loglevel) -type_client = args.codename or 'host' + osdistrib, oscodename, osrelease, osarch, oshttp = btog.GetOsInfo (type_client) + if osdistrib is None: + logger.error ('GetOsInfo() failed') + sys.exit (1) + bttargetdir, btrootfsimg, btrootfsmnt, btvirtualdisksize = btog.GetVar (osarch) + logger.info (':'.join ([osdistrib, oscodename, osrelease, osarch, oshttp])) -if os.getuid(): - logger.error ('ERROR: this program must run under root privileges!!') - sys.exit (1) + ## this is convenient in case the previous run failed and we want to run this program again + try: utils.umount (btrootfsmnt) + except: pass -## this is convenient in case the previous run failed and we want to run this program again -try: utils.umount (btrootfsmnt) -except: pass + logger.info ('STAGE 1 - create and bootstrap rootfs') + _mkrootfs (btrootfsimg, btrootfsmnt, btrootfsimglabel, btvirtualdisksize, bttargetdir, osarch) + _debootstrap (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp) -logger.info ('OpenGnsys CLIENT installation begins') + logger.info ('STAGE 2 - copy files to the rootfs') + _initramfs_version (gitrelease, osrelease, curdir) + _copy_og_files (btrootfsimg, btrootfsmnt, osdistrib, oscodename) -fd = open (f'{curdir}/gitrelease', 'r') ## per the Dockerfile -gitrelease = fd.readline().strip() -fd.close() + logger.info ('STAGE 3 - perform tasks within the chroot') + _chroot_tasks (cfgfile, curdir, osrelease, osarch) -osdistrib, oscodename, osrelease, osarch, oshttp = btog.GetOsInfo (type_client) -if osdistrib is None: - logger.error ('GetOsInfo() failed') - sys.exit (1) -bttargetdir, btrootfsimg, btrootfsmnt, btvirtualdisksize = btog.GetVar (osarch) -logger.info (':'.join ([osdistrib, oscodename, osrelease, osarch, oshttp])) + _ssh_stuff (btrootfsimg, btrootfsmnt) -logger.info ('STAGE 1 - create and bootstrap rootfs') -_mkrootfs (btrootfsimg, btrootfsmnt, btrootfsimglabel, btvirtualdisksize, bttargetdir, osarch) -_debootstrap (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp) + logger.info ('STAGE 4 - generate distribution files') + pxepkg = _get_pxepkg() + nameisoclient = '-'.join (['ogLive', oscodename, osrelease, osarch, gitrelease]) + _mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsimg, btrootfsmnt, pxepkg, isolinux_tpl, nameisoclient) -logger.info ('STAGE 2 - copy files to the rootfs') -_initramfs_version (gitrelease, osrelease, curdir) -_copy_og_files (btrootfsmnt, osdistrib, oscodename) + logger.info ('OpenGnsys installation finished') -logger.info ('STAGE 3 - perform tasks within the chroot') -_chroot_tasks (curdir, osrelease, osarch) -_ssh_stuff (btrootfsmnt) +if __name__ == '__main__': + if os.getuid(): + print ('ERROR: this program must run under root privileges!!', file=sys.stderr) + sys.exit (1) -logger.info ('STAGE 4 - generate distribution files') -pxepkg = _get_pxepkg() -nameisoclient = '-'.join (['ogLive', oscodename, osrelease, osarch, gitrelease]) -_mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsmnt, pxepkg, isolinux_tpl, nameisoclient) + parser = argparse.ArgumentParser() + parser.add_argument ('--loglevel', help='Log level', action='store') + parser.add_argument ('--codename', help='OS codename', action='store') + parser.add_argument ('--config', help='Path to configuration file', action='store') + args = parser.parse_args() -logger.info ('OpenGnsys installation finished') + cfgfile = args.config or 'mkoglive.cfg' + config = utils.read_config (cfgfile) + if config is None: + print ('ERROR: no configuration found', file=sys.stderr) + sys.exit (1) + cfg_loglevel = config['General'].get ('logging_level') + + logger = _logging (args.loglevel or cfg_loglevel) + + _main (cfgfile, config, args.codename)