From c6cf3786e504fbb9a1f10d76f180b3f526b204fc Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Fri, 9 Aug 2024 10:12:44 +0200 Subject: [PATCH 01/45] refs #596 add initial python test --- .gitignore | 2 + boottoolsfunctions/__init__.py | 230 +++++++++++++++++++++++++++++++++ boottoolsgenerator.py | 134 +++++++++++++++++++ 3 files changed, 366 insertions(+) create mode 100644 .gitignore create mode 100644 boottoolsfunctions/__init__.py create mode 100755 boottoolsgenerator.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7243ae5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +*.swp diff --git a/boottoolsfunctions/__init__.py b/boottoolsfunctions/__init__.py new file mode 100644 index 0000000..ba421fc --- /dev/null +++ b/boottoolsfunctions/__init__.py @@ -0,0 +1,230 @@ + +import platform +import os +import re +import time +import logging +import subprocess + +logging.basicConfig (filename='/tmp/boot-tools_installation.log', filemode='a', format='%(levelname)s %(asctime)s (%(threadName)s) (%(funcName)s) %(message)s', level=logging.INFO) +logger = logging.getLogger ('boottools') + +def btogGetVar(osarch): + btdir = '/tmp/opengnsys_installer/opengnsys/client/boot-tools' + bttargetdir = '/var/lib/tftpboot/ogclient/' + btrootfsimg = os.path.join (bttargetdir, 'ogclient.img') + btrootfsmnt = os.path.join (bttargetdir, 'ogclientmount') + btrootfsimglabel = 'ogClient' + log_file = '/tmp/boot-tools_installation.log' + versionboottools = 'ogLive' + if 'i386' == osarch: + btvirtualdisksize = '2000' # tamaño maximo limitado por schroot 2GB para 32 bits + else: + btvirtualdisksize = '3G' + return btdir, bttargetdir, btrootfsimg, btrootfsmnt, btrootfsimglabel, log_file, versionboottools, btvirtualdisksize + +def btogGetOsInfo1(type_client): + ogclientcfg = '/tmp/ogclient.cfg' + if 'precise' == type_client: # ogLive 1.0.4-rc2 basado en Ubuntu 12.04 LTS. + osdistrib='ubuntu' + oscodename='precise' + osrelease='3.2.0-23-generic' + osarch='i386' + oshttp='http://es.archive.ubuntu.com/ubuntu/' + elif 'wheezy' == type_client: # ogLive basado en Debian 7.3. + osdistrib='debian' + oscodename='wheezy' + osrelease='3.2.0-4-i386' + osarch='i386' + oshttp='http://ftp.es.debian.org/debian/' + elif 'quantal' == type_client: # ogLive 1.0.5-rc2 basado en Ubuntu 12.10 con Kernel descargado. + osdistrib='ubuntu' + oscodename='quantal' + osrelease='3.7.6-030706-generic' + osarch='i386' + oshttp='http://es.archive.ubuntu.com/ubuntu/' + elif 'raring' == type_client: # ogLive 1.0.5-rc3 basado en Ubuntu 13.04. + osdistrib='ubuntu' + oscodename='raring' + osrelease='3.8.0-22-generic' + osarch='i386' + oshttp='http://es.archive.ubuntu.com/ubuntu/' + elif 'trusty' == type_client: # ogLive 1.0.5-rc4 basado en Ubuntu 14.04. + osdistrib='ubuntu' + oscodename='trusty' + osrelease='3.13.0-24-generic' + osarch='i386' + oshttp='http://es.archive.ubuntu.com/ubuntu/' + elif 'vivid' == type_client: # ogLive 1.1.0-rc2 basado en Ubuntu 15.04. + osdistrib='ubuntu' + oscodename='vivid' + osrelease='3.19.0-49-generic' + osarch='i386' + oshttp='http://es.archive.ubuntu.com/ubuntu/' + elif 'wily' == type_client: # ogLive 1.1.0-rc1 basado en Ubuntu 15.10. + osdistrib='ubuntu' + oscodename='wily' + osrelease='4.2.0-35-generic' + osarch='i386' + oshttp='http://es.archive.ubuntu.com/ubuntu/' + elif 'xenial' == type_client or 'xenial-4.4' == type_client: # ogLive 1.1.0-rc4 basado en Ubuntu 16.04 y Kernel 4.4. + osdistrib='ubuntu' + oscodename='xenial' + osrelease='4.4.0-34-generic' + osarch='i386' + oshttp='http://es.archive.ubuntu.com/ubuntu/' + elif 'xenial-4.8' == type_client: # ogLive 1.1.0-rc5 basado en Ubuntu 16.04 y Kernel 4.8. + osdistrib='ubuntu' + oscodename='xenial' + osrelease='4.8.0-39-generic' + osarch='amd64' + oshttp='http://es.archive.ubuntu.com/ubuntu/' + elif 'xenial-4.13' == type_client: # ogLive 1.1.0-rc6 basado en Ubuntu 16.04 y Kernel 4.13. + osdistrib='ubuntu' + oscodename='xenial' + osrelease='4.13.0-17-generic' + osarch='amd64' + oshttp='http://es.archive.ubuntu.com/ubuntu/' + elif 'bionic' == type_client or 'bionic-4.15' == type_client: # ogLive 1.1.1-rc1 basado en Ubuntu 18.04 y Kernel 4.15. + osdistrib='ubuntu' + oscodename='bionic' + osrelease='4.15.0-32-generic' + osarch='amd64' + oshttp='http://es.archive.ubuntu.com/ubuntu/' + elif 'bionic-4.18' == type_client: # ogLive 1.1.1-rc3 basado en Ubuntu 18.04 y Kernel 4.18. + osdistrib='ubuntu' + oscodename='bionic' + osrelease='4.18.0-20-generic' + osarch='amd64' + oshttp='http://es.archive.ubuntu.com/ubuntu/' + elif 'bionic-5.0' == type_client: # ogLive 1.1.1-rc5 basado en Ubuntu 18.04 y Kernel 5.0. + osdistrib='ubuntu' + oscodename='bionic' + osrelease='5.0.0-27-generic' + osarch='amd64' + oshttp='http://es.archive.ubuntu.com/ubuntu/' + elif 'focal' == type_client: # ogLive 1.2.0-rc1 basado en Ubuntu 20.04 y Kernel 5.4. + osdistrib='ubuntu' + oscodename='focal' + osrelease='5.4.0-42-generic' + osarch='amd64' + oshttp='http://es.archive.ubuntu.com/ubuntu/' + elif 'host' == type_client: # ogLive basado en la distribución del servidor. + osdistrib=platform.freedesktop_os_release()['NAME'] + oscodename=platform.freedesktop_os_release()['VERSION_CODENAME'] + osrelease=platform.uname()[2] + osarch=subprocess.run (['dpkg', '--print-architecture'], capture_output=True).stdout.decode ('utf-8').strip() + oshttp='http://es.archive.ubuntu.com/ubuntu/' + else: # Parámetro desconocido + print ('Parámetro no válido.') + os.exit (1) + return ogclientcfg, osdistrib, oscodename, osrelease, osarch, oshttp + +def btogGetOsInfo2(type_client, versionboottools, ogclientcfg, osdistrib, oscodename, osrelease, osarch, oshttp): + branch = 'master' + giturl = 'https://api.github.com/repos/opengnsys/OpenGnsys/commits?sha={}&path=/client'.format (branch) + gitrelease = 'r20240808' ## TODO: are we going to keep the following? $(curl -s "$GITURL" | jq -r '"r" + (.[0].commit.committer.date | split("-") | join("")[:8]) + "." + (.[0].sha[:7])') + nameisoclient ='-'.join ([versionboottools, oscodename, osrelease, osarch, gitrelease]) + namehostclient = '-'.join ([versionboottools, oscodename, gitrelease]) + + ### El fichero de configuración debe sustituir a estos 3 ficheros (borrar las 3 líneas siguientes). + #echo "$NAMEISOCLIENT" > /tmp/opengnsys_info_rootfs + #echo "$NAMEHOSTCLIENT" > /tmp/opengnsys_chroot + + # Generar fichero de configuración. + with open(ogclientcfg, 'w') as f: + print ('TYPECLIENT="{}"'.format (type_client), file=f) + print ('OSDISTRIB="{}"'.format (osdistrib), file=f) + print ('OSCODENAME="{}"'.format (oscodename), file=f) + print ('OSRELEASE="{}"'.format (osrelease), file=f) + print ('OSARCH="{}"'.format (osarch), file=f) + print ('OSHTTP="{}"'.format (oshttp), file=f) + print ('GITRELEASE="{}"'.format (gitrelease, file=f)) + print ('NAMEISOCLIENT="{}"'.format (nameisoclient), file=f) + print ('NAMEHOSTCLIENT="{}"'.format (namehostclient), file=f) + + return gitrelease, nameisoclient, namehostclient + +def _run(args): + cp = subprocess.run (args, capture_output=True) + if cp.returncode: + logger.error ('command "{}" failed with rc "{}"'.format (' '.join(args), cp.returncode)) + raise + stdout = cp.stdout.decode ('utf-8').strip() + stderr = cp.stderr.decode ('utf-8').strip() + return stdout, stderr + +def _grep (regex, file): + with open (file, 'r') as f: + for l in f: + if (re.findall (regex, l)): return 1 + return 0 + +def _is_mounted (mntpt): + return _grep (mntpt, '/proc_mounts') + +def _umount (mntpt): + if (_is_mounted (mntpt)): + subprocess.run (['umount', mntpt]) + +def btogSetFsVirtual(btrootfsimg, btrootfsmnt, btvirtualdisksize, bttargetdir, osarch): + #if not btrootfsimg: btogGetVar() + + logger.info ('Creación y formateo del disco virtual "{}" MB'.format (btvirtualdisksize)) + + try: _umount (btrootfsmnt) + except: return 1 + + if (_is_mounted (btrootfsmnt)): + logger.error ('failed to umount "{}"'.format (btrootfsmnt)) + return 1 + + if (not os.makedirs (btrootfsmnt)): + logger.error ('Creando directorio "{}" : ERROR'.format (btrootfsmnt)) + return 1 + + try: _run (['chown', '-R', 'root:opengnsys', bttargetdir]) + except: return 1 + + if 'i386' == osarch: + try: _run (['dd', 'if=/dev/zero', 'of={}'.format (btrootfsimg), 'bs=1048576', 'count={}'.format (btvirtualdisksize)]) + except: + logger.error ('Creando el disco virtual con tamaño maxima "{}" MB : ERROR'.format (btvirtualdisksize)) + return 2 + else: + try: _run (['qemu-img', 'create', btrootfsimg, btvirtualdisksize]) + except: + logger.error ('Creando el disco virtual con tamaño maxima "{}" MB : ERROR'.format (btvirtualdisksize)) + return 2 + + diskloop, _ = _run (['losetup', '-f']) + if not diskloop: + logger.error ('no diskloop') + return 2 + print (diskloop) + + try: _run (['losetup', diskloop, btrootfsimg]) + except: return 2 + + subprocess.run ("echo -e \"n\np\n1\n\n\nt\n83\nw\" |fdisk {}".format (diskloop), shell=True) + + time.sleep (10) + try: _run (['losetup', '-d', diskloop]) + except: + logger.error ('Liberando disco virtual despues del particionado: ERROR') + return 2 + + partloop, _ = _run (['losetup', '-f']) + _run (['losetup', '-o', '32256', partloop, btrootfsimg]) + try: _run (['mkfs.ext4', '-b', '4096', '-L', btrootfsimglabel, partloop]) + except: + logger.error ('Formateando la particion principal del disco virtual: ERROR') + return 2 + + time.sleep (10) + try: _run (['losetup', '-d', partloop]) + except: + logger.error ('Liberando la particion virtual despues del formateo: ERROR') + return 2 + + logger.info ('"{}" "{}" MB : OK'.format (btrootfsimg, btvirtualdisksize)) diff --git a/boottoolsgenerator.py b/boottoolsgenerator.py new file mode 100755 index 0000000..5325de9 --- /dev/null +++ b/boottoolsgenerator.py @@ -0,0 +1,134 @@ +#!/usr/bin/python3 + +import sys +import os +import logging + +sys.path.insert (0, __file__) +from boottoolsfunctions import btogGetOsInfo1, btogGetOsInfo2, btogGetVar + +logging.basicConfig (filename='/tmp/boot-tools_installation.log', filemode='a', format='%(levelname)s %(asctime)s (%(threadName)s) (%(funcName)s) %(message)s', level=logging.INFO) +logger = logging.getLogger ('boottools') + +#BRANCH = 'branches/master' +#svn export --force https://github.com/opengnsys/OpenGnsys/$BRANCH/client /tmp/opengnsys_installer/opengnsys/client + +type_client = sys.argv[1] if len(sys.argv)>1 else 'host' +#WORKDIR = '/tmp/opengnsys_installer' +#INSTALL_TARGET = '/opt/opengnsys' +#PROGRAMDIR = $(readlink -e $(dirname "$0")) + +if os.getuid(): + logger.error ('ERROR: this program must run under root privileges!!') + os._exit (1) + +os.chdir ('/tmp') + +#for i in `mount | grep IMGogclient | grep /var | cut -f3 -d" "`; do echo $i; umount $i; done +#for i in `mount | grep IMGogclient | grep /var | cut -f3 -d" "`; do echo $i; umount $i; done +#for i in `mount | grep IMGogclient | grep /var | cut -f3 -d" "`; do echo $i; umount $i; done + + + +####################################################################3 +logger.info ('FASE 1 - Asignación de variables') +ogclientcfg, osdistrib, oscodename, osrelease, osarch, oshttp = btogGetOsInfo1(type_client) +print (osarch) +btdir, bttargetdir, btrootfsimg, btrootfsmnt, btrootfsimglabel, log_file, versionboottools, btvirtualdisksize = btogGetVar(osarch) +print (btdir) +gitrelease, nameisoclient, namehostclient = btogGetOsInfo2(type_client, versionboottools, ogclientcfg, osdistrib, oscodename, osrelease, osarch, oshttp) +print (nameisoclient) +logger.info ('OpenGnsys CLIENT installation begins') + + +########################################################################## +logger.info ('FASE 2 - Instalación de software adicional.') +""" +#grep "http://free.nchc.org.tw/drbl-core" /etc/apt/sources.list || echo "deb http://free.nchc.org.tw/drbl-core drbl stable" >> /etc/apt/sources.list +apt-get update >>/tmp/fase2.out +[ -n "$(apt-cache search gpxe)" ] && PXEPKG="gpxe" +[ -n "$(apt-cache search ipxe)" ] && PXEPKG="ipxe" +apt-get -y install debootstrap subversion schroot squashfs-tools syslinux genisoimage $PXEPKG qemu qemu-utils lsof >>/tmp/fase2.out + + +###################################################################3 +logger.info ('FASE 3 - Creación del Sistema raiz RootFS (Segundo Sistema archivos (img))') +logger.info ('Fase 3.1 Generar y formatear el disco virtual. Generar el dispositivo loop.') +file $BTROOTFSIMG | grep "partition 1: ID=0x83" +if [ $? == 1 ]: + btogSetFsVirtual || exit 2 + echo ================= nati after btogSetFsVirtual; ls -la /opt/opengnsys/tftpboot/ogclient/ + +logger.info ('Fase 3.2 Generar sistema de archivos con debootstrap') +schroot -p -c IMGogclient -- touch /tmp/ogclientOK +if [ -f /tmp/ogclientOK ] +then + rm /tmp/ogclientOK +else + btogSetFsBase >>/tmp/fase3.out +fi +echo ================= nati after fase 3; ls -la /opt/opengnsys/tftpboot/ogclient/ + + +###################################################################3 +logger.info ('FASE 4 - Configurar acceso schroot al Segundo Sistema de archivos (img)') +cat /etc/schroot/schroot.conf | grep $BTROOTFSIMG || btogSetFsAccess + + +########################################################################### +logger.info ('FASE 5 - Incorporando ficheros OpenGnsys al sistema raíz rootfs') +cp -a ${BTDIR}/includes/usr/bin/* /tmp >>/tmp/fase5.out +chmod +x /tmp/boot-tools/*.sh +# Incluir revisión. +sed -i "1 s/$/ $GITRELEASE ($OSRELEASE)/" ${BTDIR}/includes/etc/initramfs-tools/scripts/VERSION.txt +# En Ubuntu 13.04+ es necesario matar proceso de "udev" antes de desmontar. +umount $BTROOTFSMNT 2>/dev/null || (kill -9 $(lsof -t $BTROOTFSMNT); umount $BTROOTFSMNT) 2>/dev/null +schroot -p -c IMGogclient -- /tmp/boot-tools/boottoolsFsOpengnsys.sh >>/tmp/fase5.out + + +############################################################################################ +logger.info ('FASE 6 - Instalar software') +logger.info ('Fase 6.1 instalar paquetes deb con apt-get') +schroot -p -c IMGogclient -- /usr/bin/boot-tools/boottoolsSoftwareInstall.sh >>/tmp/fase6.out +logger.info ('Fase 6.2 compilar software.') +cd / +schroot -p -c IMGogclient -- /usr/bin/boot-tools/boottoolsSoftwareCompile.sh >>/tmp/fase6.out +cd - + + +############################################################################################ +logger.info ('FASE 7 - Personalizar el sistema creado') +logger.info ('Fase 7.1 Incorporar la clave publica del servidor') +cd / +ssh-keygen -q -f /root/.ssh/id_rsa -N "" +cp /root/.ssh/id_rsa.pub /tmp +schroot -p -c IMGogclient -- /usr/bin/boot-tools/boottoolsSshServer.sh +cd - +logger.info ('Fase 7.2. Incorpoar la clave publica del propio cliente') +schroot -p -c IMGogclient -- /usr/bin/boot-tools/boottoolsSshClient.sh + +logger.info ('Fase 7.3. Configurando las locales') +schroot -p -c IMGogclient -- /usr/bin/boot-tools/boottoolsFsLocales.sh + +for i in `mount | grep IMGogclient | grep /var | cut -f3 -d" "`; do echo $i; umount $i; done +for i in `mount | grep IMGogclient | grep /var | cut -f3 -d" "`; do echo $i; umount $i; done +for i in `mount | grep IMGogclient | grep /var | cut -f3 -d" "`; do echo $i; umount $i; done + +echo ================= nati after fase 7; ls -la /opt/opengnsys/tftpboot/ogclient/ + + +######################################################################### +logger.info ('FASE 8 - Generar distribucion') +logger.info ('Fase 8.1 Generar el initrd') +btogFsInitrd >>/tmp/fase8.out +echo ================= nati after fase 8.1; ls -la /opt/opengnsys/tftpboot/ogclient/ +logger.info ('Fase 8.2 Generar fichero sqfs a partir del fichero img') +btogFsSqfs +echo ================= nati after fase 8.2; ls -la /opt/opengnsys/tftpboot/ogclient/ +umount $BTROOTFSMNT 2>/dev/null +logger.info ('Fase 8.3 Generar la ISO') +btogIsoGenerator >>/tmp/fase8.out +echo ================= nati after fase 8.3; ls -la /opt/opengnsys/tftpboot/ogclient/ + +echoAndLog "OpenGnsys installation finished at $(date)" +""" From acc32a29482f7bf3283080d9245e85f8bc51b09d Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Fri, 9 Aug 2024 13:38:15 +0200 Subject: [PATCH 02/45] refs #596 fix bugs, configure schroot, do debootstrap --- boottoolsfunctions/__init__.py | 159 +++++++++++++++++++++++++-------- boottoolsgenerator.py | 135 +++++++++++++++++----------- 2 files changed, 202 insertions(+), 92 deletions(-) diff --git a/boottoolsfunctions/__init__.py b/boottoolsfunctions/__init__.py index ba421fc..2368017 100644 --- a/boottoolsfunctions/__init__.py +++ b/boottoolsfunctions/__init__.py @@ -5,11 +5,12 @@ import re import time import logging import subprocess +import shutil +import glob -logging.basicConfig (filename='/tmp/boot-tools_installation.log', filemode='a', format='%(levelname)s %(asctime)s (%(threadName)s) (%(funcName)s) %(message)s', level=logging.INFO) logger = logging.getLogger ('boottools') -def btogGetVar(osarch): +def btogGetVar (osarch): btdir = '/tmp/opengnsys_installer/opengnsys/client/boot-tools' bttargetdir = '/var/lib/tftpboot/ogclient/' btrootfsimg = os.path.join (bttargetdir, 'ogclient.img') @@ -23,7 +24,7 @@ def btogGetVar(osarch): btvirtualdisksize = '3G' return btdir, bttargetdir, btrootfsimg, btrootfsmnt, btrootfsimglabel, log_file, versionboottools, btvirtualdisksize -def btogGetOsInfo1(type_client): +def btogGetOsInfo1 (type_client): ogclientcfg = '/tmp/ogclient.cfg' if 'precise' == type_client: # ogLive 1.0.4-rc2 basado en Ubuntu 12.04 LTS. osdistrib='ubuntu' @@ -120,7 +121,7 @@ def btogGetOsInfo1(type_client): os.exit (1) return ogclientcfg, osdistrib, oscodename, osrelease, osarch, oshttp -def btogGetOsInfo2(type_client, versionboottools, ogclientcfg, osdistrib, oscodename, osrelease, osarch, oshttp): +def btogGetOsInfo2 (type_client, versionboottools, ogclientcfg, osdistrib, oscodename, osrelease, osarch, oshttp): branch = 'master' giturl = 'https://api.github.com/repos/opengnsys/OpenGnsys/commits?sha={}&path=/client'.format (branch) gitrelease = 'r20240808' ## TODO: are we going to keep the following? $(curl -s "$GITURL" | jq -r '"r" + (.[0].commit.committer.date | split("-") | join("")[:8]) + "." + (.[0].sha[:7])') @@ -132,7 +133,7 @@ def btogGetOsInfo2(type_client, versionboottools, ogclientcfg, osdistrib, oscode #echo "$NAMEHOSTCLIENT" > /tmp/opengnsys_chroot # Generar fichero de configuración. - with open(ogclientcfg, 'w') as f: + with open (ogclientcfg, 'w') as f: print ('TYPECLIENT="{}"'.format (type_client), file=f) print ('OSDISTRIB="{}"'.format (osdistrib), file=f) print ('OSCODENAME="{}"'.format (oscodename), file=f) @@ -145,7 +146,7 @@ def btogGetOsInfo2(type_client, versionboottools, ogclientcfg, osdistrib, oscode return gitrelease, nameisoclient, namehostclient -def _run(args): +def _run (args): cp = subprocess.run (args, capture_output=True) if cp.returncode: logger.error ('command "{}" failed with rc "{}"'.format (' '.join(args), cp.returncode)) @@ -161,70 +162,152 @@ def _grep (regex, file): return 0 def _is_mounted (mntpt): - return _grep (mntpt, '/proc_mounts') + return _grep (mntpt, '/proc/mounts') def _umount (mntpt): if (_is_mounted (mntpt)): subprocess.run (['umount', mntpt]) -def btogSetFsVirtual(btrootfsimg, btrootfsmnt, btvirtualdisksize, bttargetdir, osarch): +def btogSetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, bttargetdir, osarch): #if not btrootfsimg: btogGetVar() - logger.info ('Creación y formateo del disco virtual "{}" MB'.format (btvirtualdisksize)) + logger.info ('Creación y formateo del disco virtual "{}" "{}" MB'.format (btrootfsimg, btvirtualdisksize)) try: _umount (btrootfsmnt) - except: return 1 + except: pass if (_is_mounted (btrootfsmnt)): - logger.error ('failed to umount "{}"'.format (btrootfsmnt)) - return 1 + raise Exception ('failed to umount "{}"'.format (btrootfsmnt)) - if (not os.makedirs (btrootfsmnt)): - logger.error ('Creando directorio "{}" : ERROR'.format (btrootfsmnt)) - return 1 + try: os.makedirs (btrootfsmnt, exist_ok=True) + except: + raise Exception ('Creando directorio "{}" : ERROR'.format (btrootfsmnt)) try: _run (['chown', '-R', 'root:opengnsys', bttargetdir]) - except: return 1 + except: + raise Exception ('Failed to chown root:opengnsys "{}"'.format (btrootfsmnt)) + logger.info ('Creating disk image "{}"'.format (btrootfsimg)) if 'i386' == osarch: try: _run (['dd', 'if=/dev/zero', 'of={}'.format (btrootfsimg), 'bs=1048576', 'count={}'.format (btvirtualdisksize)]) except: - logger.error ('Creando el disco virtual con tamaño maxima "{}" MB : ERROR'.format (btvirtualdisksize)) - return 2 + raise Exception ('Creando el disco virtual "{}" con tamaño maxima "{}" MB : ERROR'.format (btrootfsimg, btvirtualdisksize)) else: try: _run (['qemu-img', 'create', btrootfsimg, btvirtualdisksize]) except: - logger.error ('Creando el disco virtual con tamaño maxima "{}" MB : ERROR'.format (btvirtualdisksize)) - return 2 + raise Exception ('Creando el disco virtual "{}" con tamaño maxima "{}" MB : ERROR'.format (btrootfsimg, btvirtualdisksize)) - diskloop, _ = _run (['losetup', '-f']) + logger.debug ('losetup --find') + diskloop, _ = _run (['losetup', '--find']) if not diskloop: - logger.error ('no diskloop') - return 2 - print (diskloop) + raise Exception ('no diskloop') + logger.debug ('losetup attach') try: _run (['losetup', diskloop, btrootfsimg]) - except: return 2 + except: raise Exception ('losetup failed') - subprocess.run ("echo -e \"n\np\n1\n\n\nt\n83\nw\" |fdisk {}".format (diskloop), shell=True) + logger.info ('Partitioning disk image') + cp = subprocess.run(['bash', '-c', "echo $'n\np\n1\n\n\nt\n83\nw' |fdisk {}".format (diskloop)], capture_output=True) + logger.debug ('fdisk output follows: {}'.format (cp.stdout.decode('utf-8'))) - time.sleep (10) - try: _run (['losetup', '-d', diskloop]) + time.sleep (5) + logger.debug ('losetup --detach') + try: _run (['losetup', '--detache', diskloop]) except: - logger.error ('Liberando disco virtual despues del particionado: ERROR') - return 2 + raise Exception ('Liberando disco virtual despues del particionado: ERROR') - partloop, _ = _run (['losetup', '-f']) - _run (['losetup', '-o', '32256', partloop, btrootfsimg]) + logger.debug ('losetup --find') + partloop, _ = _run (['losetup', '--find']) + if not partloop: + raise Exception ('no partloop') + + logger.debug ('losetup --offset attach') + _run (['losetup', '--offset', '32256', partloop, btrootfsimg]) + + logger.info ('Creating ext4 filesystem') try: _run (['mkfs.ext4', '-b', '4096', '-L', btrootfsimglabel, partloop]) except: - logger.error ('Formateando la particion principal del disco virtual: ERROR') - return 2 + raise Exception ('Formateando la particion principal del disco virtual: ERROR') - time.sleep (10) - try: _run (['losetup', '-d', partloop]) + time.sleep (5) + logger.debug ('losetup --detach') + try: _run (['losetup', '--detach', partloop]) except: - logger.error ('Liberando la particion virtual despues del formateo: ERROR') - return 2 + raise Exception ('Liberando la particion virtual despues del formateo: ERROR') logger.info ('"{}" "{}" MB : OK'.format (btrootfsimg, btvirtualdisksize)) + +#btogSetFsAcces: habilita el acceso al sistema root del cliente con schroot +def btogSetFsAccess (btrootfsimg): + if (_grep (btrootfsimg, '/etc/schroot/schroot.conf')): + return + logger.info ('Iniciando la configuración del schroot') + + #cp /etc/schroot/schroot.conf /etc/schroot/schroot.conf.`getDateTime` + if (os.path.exists ('/etc/schroot/schroot.conf')): + shutil.copy ('/etc/schroot/schroot.conf', '/etc/schroot/schroot.conf.bak') + + with open ('/etc/schroot/schroot.conf', 'w') as f: + print ('[IMGogclient]', file=f) + print ('type=loopback', file=f) + print ('file=/var/lib/tftpboot/ogclient/ogclient.img', file=f) + print ('description=ogclient Ubuntu image', file=f) + print ('#priority=1', file=f) + print ('users=root', file=f) + print ('groups=root', file=f) + print ('root-groups=root', file=f) + print ('mount-options=-o offset=32256', file=f) + print ('root-users=root', file=f) + + #cp /etc/schroot/mount-defaults /etc/schroot/mount-defaults.`getDateTime` + if (os.path.exists ('/etc/schroot/mount-defaults')): + shutil.copy ('/etc/schroot/mount-defaults', '/etc/schroot/mount-defaults.bak') + + with open ('/etc/schroot/mount-defaults', 'w') as f: + print ('# mount.defaults: static file system information for chroots.', file=f) + print ('# Note that the mount point will be prefixed by the chroot path', file=f) + print ('# (CHROOT_PATH)', file=f) + print ('#', file=f) + print ('# ', file=f) + print ('proc /proc proc defaults 0 0', file=f) + print ('#procbususb /proc/bus/usb usbfs defaults 0 0', file=f) + print ('#/dev /dev none rw,bind 0 0', file=f) + print ('/dev/pts /dev/pts none rw,bind 0 0', file=f) + print ('/dev/shm /dev/shm none rw,bind 0 0', file=f) + print ('#/home /home none rw,bind 0 0', file=f) + print ('/tmp /tmp none rw,bind 0 0', file=f) + + map (os.unlink, glob.glob('/etc/schroot/setup.d/*chrootname')) + #for i in glob.glob ('/etc/schroot/setup.d/*chrootname'): + # os.unlink (i) + + logger.info ('Finalizado: OK') + return 0 + +# btogSetfsBase: Genera el sistema root base con debootstrap +def btogSetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp): + logger.info ('Iniciando la generación del sistema de archivos') + + #mount | grep $BTROOTFSMNT || mount $BTROOTFSIMG $BTROOTFSMNT -o loop,offset=32256 + if (not _is_mounted (btrootfsmnt)): + try: _run (['mount', btrootfsimg, btrootfsmnt, '-o', 'loop,offset=32256']) + except: + raise Exception ('mount failed') + + #mount | grep $BTROOTFSMNT && echoAndLog "$FUNCNAME: mount $BTROOTFSIMG $BTROOTFSMNT -o loop,offset=32256 OK " || errorAndLog "$FUNCNAME: mount $BTROOTFSIMG $BTROOTFSMNT -o loop,offset=32256 : FAILURE " + if (_is_mounted (btrootfsmnt)): + logger.info ('mount "{}" "{}" -o loop,offset=32256 OK'.format (btrootfsimg, btrootfsmnt)) + else: + raise Exception ('mount "{}" "{}" -o loop,offset=32256 : FAILURE'.format (btrootfsimg, btrootfsmnt)) + + logger.info ('debootstrapping system') + try: _run (['debootstrap', '--arch="{}"'.format(osarch), '--components=main,universe', oscodename, btrootfsmnt, oshttp]) + except: + if (_is_mounted (btrootfsmnt)): + subprocess.run (['umount', btrootfsmnt]) + raise Exception ('debootstrap --arch={} --components=main,universe "{}" "{}" "{}" : ha fallado!'.format (osarch, oscodename, btrootfsmnt, oshttp)) + + if (_is_mounted (btrootfsmnt)): + subprocess.run (['umount', btrootfsmnt]) + logger.info ('debootstrap --arch="{}" --components=main,universe "{}" "{}" "{}" : ok'.format (osarch, oscodename, btrootfsmnt, oshttp)) + return 0 diff --git a/boottoolsgenerator.py b/boottoolsgenerator.py index 5325de9..d238da5 100755 --- a/boottoolsgenerator.py +++ b/boottoolsgenerator.py @@ -3,15 +3,22 @@ import sys import os import logging +import subprocess +import glob +import stat sys.path.insert (0, __file__) -from boottoolsfunctions import btogGetOsInfo1, btogGetOsInfo2, btogGetVar +from boottoolsfunctions import btogGetOsInfo1, btogGetOsInfo2, btogGetVar, btogSetFsVirtual, btogSetFsAccess, btogSetFsBase -logging.basicConfig (filename='/tmp/boot-tools_installation.log', filemode='a', format='%(levelname)s %(asctime)s (%(threadName)s) (%(funcName)s) %(message)s', level=logging.INFO) +logging.basicConfig (filename='/tmp/boot-tools_installation.log', filemode='a', format='%(levelname)s %(asctime)s (%(funcName)s) %(message)s', level=logging.INFO) logger = logging.getLogger ('boottools') -#BRANCH = 'branches/master' -#svn export --force https://github.com/opengnsys/OpenGnsys/$BRANCH/client /tmp/opengnsys_installer/opengnsys/client +if not os.path.exists ('/tmp/opengnsys_installer/opengnsys/client/engine'): + branch = 'main' + tmpdir = subprocess.run (['mktemp', '--tmpdir', '--directory', 'oggit.XXXXXX'], capture_output=True).stdout.decode ('utf-8').strip() + subprocess.run (['git', 'clone', '-c', 'http.sslVerify=false', '--branch', branch, 'https://ognproject.evlt.uma.es/gitea/opengnsys/opengnsys.git', tmpdir]) + subprocess.run (['rsync', '-aHv', '{}/client/engine'.format(tmpdir), '{}/client/shared'.format(tmpdir), '/tmp/opengnsys_installer/opengnsys/client/']) + subprocess.run (['rm', '-rf', tmpdir]) type_client = sys.argv[1] if len(sys.argv)>1 else 'host' #WORKDIR = '/tmp/opengnsys_installer' @@ -33,101 +40,121 @@ os.chdir ('/tmp') ####################################################################3 logger.info ('FASE 1 - Asignación de variables') ogclientcfg, osdistrib, oscodename, osrelease, osarch, oshttp = btogGetOsInfo1(type_client) -print (osarch) btdir, bttargetdir, btrootfsimg, btrootfsmnt, btrootfsimglabel, log_file, versionboottools, btvirtualdisksize = btogGetVar(osarch) -print (btdir) gitrelease, nameisoclient, namehostclient = btogGetOsInfo2(type_client, versionboottools, ogclientcfg, osdistrib, oscodename, osrelease, osarch, oshttp) -print (nameisoclient) logger.info ('OpenGnsys CLIENT installation begins') ########################################################################## logger.info ('FASE 2 - Instalación de software adicional.') -""" #grep "http://free.nchc.org.tw/drbl-core" /etc/apt/sources.list || echo "deb http://free.nchc.org.tw/drbl-core drbl stable" >> /etc/apt/sources.list -apt-get update >>/tmp/fase2.out -[ -n "$(apt-cache search gpxe)" ] && PXEPKG="gpxe" -[ -n "$(apt-cache search ipxe)" ] && PXEPKG="ipxe" -apt-get -y install debootstrap subversion schroot squashfs-tools syslinux genisoimage $PXEPKG qemu qemu-utils lsof >>/tmp/fase2.out +subprocess.run (['apt-get', 'update']) #>>/tmp/fase2.out +acse_gpxe = subprocess.run (['apt-cache', 'search', 'gpxe'], capture_output=True).stdout.decode ('utf-8').strip() +acse_ipxe = subprocess.run (['apt-cache', 'search', 'ipxe'], capture_output=True).stdout.decode ('utf-8').strip() +if acse_ipxe: + pxepkg = 'ipxe' +elif acse_gpxe: + pxepkg = 'gpxe' +else: + logger.error ('neither gpxe nor ipxe found in apt-cache') + os._exit (1) +logger.info ('PXE package is "{}"'.format (pxepkg)) +## TODO qemu no existe, hace falta? +subprocess.run (['apt-get', '-y', 'install', 'debootstrap', 'subversion', 'schroot', 'squashfs-tools', 'syslinux', 'genisoimage', 'qemu-utils', 'lsof', pxepkg]) #>>/tmp/fase2.out ###################################################################3 logger.info ('FASE 3 - Creación del Sistema raiz RootFS (Segundo Sistema archivos (img))') logger.info ('Fase 3.1 Generar y formatear el disco virtual. Generar el dispositivo loop.') -file $BTROOTFSIMG | grep "partition 1: ID=0x83" -if [ $? == 1 ]: - btogSetFsVirtual || exit 2 - echo ================= nati after btogSetFsVirtual; ls -la /opt/opengnsys/tftpboot/ogclient/ +rc = subprocess.run ('file {} |grep -q "partition 1 *: ID=0x83"'.format (btrootfsimg), shell=True).returncode +print (rc) +if (rc): ## 'file|grep' failed + try: btogSetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, bttargetdir, osarch) + except Exception as e: + logger.error (str (e)) + os._exit (2) + #echo ================= nati after btogSetFsVirtual; ls -la /opt/opengnsys/tftpboot/ogclient/ -logger.info ('Fase 3.2 Generar sistema de archivos con debootstrap') -schroot -p -c IMGogclient -- touch /tmp/ogclientOK -if [ -f /tmp/ogclientOK ] -then - rm /tmp/ogclientOK -else - btogSetFsBase >>/tmp/fase3.out -fi -echo ================= nati after fase 3; ls -la /opt/opengnsys/tftpboot/ogclient/ +logger.info ('Fase 3.2 - Configurar acceso schroot al Segundo Sistema de archivos (img)') +#cat /etc/schroot/schroot.conf | grep $BTROOTFSIMG || btogSetFsAccess +btogSetFsAccess (btrootfsimg) - -###################################################################3 -logger.info ('FASE 4 - Configurar acceso schroot al Segundo Sistema de archivos (img)') -cat /etc/schroot/schroot.conf | grep $BTROOTFSIMG || btogSetFsAccess +logger.info ('Fase 3.3 Generar sistema de archivos con debootstrap') +logger.debug ('Try creation of a file within chroot (this operation may fail--that is ok)') +cp = subprocess.run (['schroot', '--preserve-environment', '--chroot', 'IMGogclient', '--', 'touch', '/tmp/ogclientOK']) +if (cp.returncode): + try: btogSetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp) #>>/tmp/fase3.out + except Exception as e: + logger.error (str (e)) + os._exit (3) +else: + os.unlink ('/tmp/ogclientOK') +#echo ================= nati after fase 3; ls -la /opt/opengnsys/tftpboot/ogclient/ ########################################################################### -logger.info ('FASE 5 - Incorporando ficheros OpenGnsys al sistema raíz rootfs') -cp -a ${BTDIR}/includes/usr/bin/* /tmp >>/tmp/fase5.out -chmod +x /tmp/boot-tools/*.sh +logger.info ('FASE 4 - Incorporando ficheros OpenGnsys al sistema raíz rootfs') + +#cp -a ${BTDIR}/includes/usr/bin/* /tmp >>/tmp/fase5.out +subprocess.run (['cp', '-a'] + glob.glob ('{}/includes/usr/bin/*'.format (btdir)) + ['/tmp']) + +#chmod +x /tmp/boot-tools/*.sh +for i in glob.glob ('/tmp/boot-tools/*.sh'): + st = os.stat (i) + os.chmod (i, st.st_mode|stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH) + # Incluir revisión. -sed -i "1 s/$/ $GITRELEASE ($OSRELEASE)/" ${BTDIR}/includes/etc/initramfs-tools/scripts/VERSION.txt +#sed -i "1 s/$/ $GITRELEASE ($OSRELEASE)/" ${BTDIR}/includes/etc/initramfs-tools/scripts/VERSION.txt +subprocess.run (['sed', '-i', '1 s/$/ {} ({})/'.format (gitrelease, osrelease), '{}/includes/etc/initramfs-tools/scripts/VERSION.txt'.format (btdir)]) + # En Ubuntu 13.04+ es necesario matar proceso de "udev" antes de desmontar. -umount $BTROOTFSMNT 2>/dev/null || (kill -9 $(lsof -t $BTROOTFSMNT); umount $BTROOTFSMNT) 2>/dev/null -schroot -p -c IMGogclient -- /tmp/boot-tools/boottoolsFsOpengnsys.sh >>/tmp/fase5.out +#umount $BTROOTFSMNT 2>/dev/null || (kill -9 $(lsof -t $BTROOTFSMNT); umount $BTROOTFSMNT) 2>/dev/null +""" +schroot --preserve-environment --chroot IMGogclient -- /tmp/boot-tools/boottoolsFsOpengnsys.sh >>/tmp/fase4.out ############################################################################################ -logger.info ('FASE 6 - Instalar software') -logger.info ('Fase 6.1 instalar paquetes deb con apt-get') -schroot -p -c IMGogclient -- /usr/bin/boot-tools/boottoolsSoftwareInstall.sh >>/tmp/fase6.out -logger.info ('Fase 6.2 compilar software.') +logger.info ('FASE 5 - Instalar software') +logger.info ('Fase 5.1 instalar paquetes deb con apt-get') +schroot --preserve-environment --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSoftwareInstall.sh >>/tmp/fase5.out +logger.info ('Fase 5.2 compilar software.') cd / -schroot -p -c IMGogclient -- /usr/bin/boot-tools/boottoolsSoftwareCompile.sh >>/tmp/fase6.out +schroot --preserve-environment --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSoftwareCompile.sh >>/tmp/fase5.out cd - ############################################################################################ -logger.info ('FASE 7 - Personalizar el sistema creado') -logger.info ('Fase 7.1 Incorporar la clave publica del servidor') +logger.info ('FASE 6 - Personalizar el sistema creado') +logger.info ('Fase 6.1 Incorporar la clave publica del servidor') cd / ssh-keygen -q -f /root/.ssh/id_rsa -N "" cp /root/.ssh/id_rsa.pub /tmp -schroot -p -c IMGogclient -- /usr/bin/boot-tools/boottoolsSshServer.sh +schroot --preserve-environment --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSshServer.sh cd - -logger.info ('Fase 7.2. Incorpoar la clave publica del propio cliente') -schroot -p -c IMGogclient -- /usr/bin/boot-tools/boottoolsSshClient.sh +logger.info ('Fase 6.2. Incorpoar la clave publica del propio cliente') +schroot --preserve-environment --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSshClient.sh -logger.info ('Fase 7.3. Configurando las locales') -schroot -p -c IMGogclient -- /usr/bin/boot-tools/boottoolsFsLocales.sh +logger.info ('Fase 6.3. Configurando las locales') +schroot --preserve-environment --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsFsLocales.sh for i in `mount | grep IMGogclient | grep /var | cut -f3 -d" "`; do echo $i; umount $i; done for i in `mount | grep IMGogclient | grep /var | cut -f3 -d" "`; do echo $i; umount $i; done for i in `mount | grep IMGogclient | grep /var | cut -f3 -d" "`; do echo $i; umount $i; done -echo ================= nati after fase 7; ls -la /opt/opengnsys/tftpboot/ogclient/ +echo ================= nati after fase 6; ls -la /opt/opengnsys/tftpboot/ogclient/ ######################################################################### -logger.info ('FASE 8 - Generar distribucion') -logger.info ('Fase 8.1 Generar el initrd') -btogFsInitrd >>/tmp/fase8.out +logger.info ('FASE 7 - Generar distribucion') +logger.info ('Fase 7.1 Generar el initrd') +btogFsInitrd >>/tmp/fase7.out echo ================= nati after fase 8.1; ls -la /opt/opengnsys/tftpboot/ogclient/ -logger.info ('Fase 8.2 Generar fichero sqfs a partir del fichero img') +logger.info ('Fase 7.2 Generar fichero sqfs a partir del fichero img') btogFsSqfs echo ================= nati after fase 8.2; ls -la /opt/opengnsys/tftpboot/ogclient/ umount $BTROOTFSMNT 2>/dev/null -logger.info ('Fase 8.3 Generar la ISO') -btogIsoGenerator >>/tmp/fase8.out +logger.info ('Fase 7.3 Generar la ISO') +btogIsoGenerator >>/tmp/fase7.out echo ================= nati after fase 8.3; ls -la /opt/opengnsys/tftpboot/ogclient/ echoAndLog "OpenGnsys installation finished at $(date)" From 31e360725659358c88919580a3067c4038a53331 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Fri, 9 Aug 2024 18:42:47 +0200 Subject: [PATCH 03/45] refs #596 add 02-boottoolsFsOpengnsys.py to copy some OG files into the rootfs --- 02-boottoolsFsOpengnsys.py | 80 ++++++++++++++++++++++++++++++++++ boottoolsfunctions/__init__.py | 58 ++++++++++-------------- boottoolsgenerator.py | 35 ++++++++++----- 3 files changed, 129 insertions(+), 44 deletions(-) create mode 100755 02-boottoolsFsOpengnsys.py diff --git a/02-boottoolsFsOpengnsys.py b/02-boottoolsFsOpengnsys.py new file mode 100755 index 0000000..98c6b01 --- /dev/null +++ b/02-boottoolsFsOpengnsys.py @@ -0,0 +1,80 @@ +#!/usr/bin/python3 + +import os +import logging +import subprocess +import shutil +import argparse + +#VERSIONBOOTTOOLS="ogLive" +nameisoclientfile = '/tmp/opengnsys_info_rootfs' +namehostclientfile = '/tmp/opengnsys_chroot' + +svnclientdir = '/tmp/opengnsys_installer/opengnsys/client/boot-tools' +svnclientstructure = '/tmp/opengnsys_installer/opengnsys/client/shared' +svnclientengine = '/tmp/opengnsys_installer/opengnsys/client/engine' + +ogclientmount = '' + +def boottoolsFsOpengnsys (osdistrib, oscodename, osrelease, osarch, oshttp): + print (':'.join ([osdistrib, oscodename, osrelease, osarch, oshttp])) + + print ('Iniciando la personalización con datos del repositorio') + + ## TODO mount fs, para lo cual necesito helpers + + #sed -e "s/OSCODENAME/$OSCODENAME/g" ${SVNCLIENTDIR}/includes/etc/apt/sources.list.${OSDISTRIB,,} > ${SVNCLIENTDIR}/includes/etc/apt/sources.list + sources_list_in = '{}/includes/etc/apt/sources.list.{}'.format (svnclientdir, osdistrib.lower()) + sources_list_out = '{}/includes/etc/apt/sources.list'.format (svnclientdir) + fdin = open (sources_list_in, 'r') + fdout = open (sources_list_out, 'w') + while True: + l = fdin.readline() + if not l: break + fdout.write (l.replace ('OSCODENAME', oscodename)) + fdin.close() + fdout.close() + + subprocess.run ('chmod -R 775 {}/includes/usr/bin/*'.format (svnclientdir), shell=True) + + os.makedirs ('{}/opt/opengnsys/lib/engine/bin/'.format (ogclientmount), exist_ok=True) + os.makedirs ('{}/usr/local/etc' .format (ogclientmount), exist_ok=True) + os.makedirs ('{}/usr/local/lib' .format (ogclientmount), exist_ok=True) + os.makedirs ('{}/usr/local/plugins' .format (ogclientmount), exist_ok=True) + + subprocess.run ('rsync -aH {}/includes/* {}/' .format (svnclientdir, ogclientmount), shell=True) + subprocess.run ('rsync -aH {}/* {}/opt/opengnsys/' .format (svnclientstructure, ogclientmount), shell=True) + subprocess.run ('rsync -aH {}/* {}/opt/opengnsys/lib/engine/bin/'.format (svnclientengine, ogclientmount), shell=True) + + # Si no existe, copiar pci.ids. + if not os.path.exists ('{}/etc/pci.ids'.format (ogclientmount)): + shutil.copy ('{}/lib/pci.ids'.format (svnclientstructure), '{}/etc/'.format (ogclientmount)) + + # Dependencias Qt para el Browser. + subprocess.run ('rsync -aH {}/etc/*.qmap {}/usr/local/etc' .format (svnclientstructure, ogclientmount), shell=True) + subprocess.run ('rsync -aH {}/lib/qtlib/* {}/usr/local/lib' .format (svnclientstructure, ogclientmount), shell=True) + subprocess.run ('rsync -aH {}/lib/fonts {}/usr/local/lib' .format (svnclientstructure, ogclientmount), shell=True) + subprocess.run ('rsync -aH {}/lib/qtplugins/* {}/usr/local/plugins'.format (svnclientstructure, ogclientmount), shell=True) + + # Browser y ogAdmClient. + if os.path.exists ('{}/bin/browser'.format (svnclientstructure)): + shutil.copy ('{}/bin/browser'.format (svnclientstructure), '{}/bin/'.format (ogclientmount)) + if os.path.exists ('{}/bin/ogAdmClient'.format (svnclientstructure)): + shutil.copy ('{}/bin/ogAdmClient'.format (svnclientstructure), '{}/bin/'.format (ogclientmount)) + + # El fichero de configuración debe sustituir a los 2 ficheros (borrar las 2 líneas). + #echo "${VERSIONBOOTTOOLS}-${OSCODENAME}-${OSRELEASE}-${GITRELEASE}" > $nameisoclientfile + #echo "${VERSIONBOOTTOOLS}-${OSCODENAME}-${GITRELEASE}" > $namehostclientfile + + ## desmontar de nuevo, ejem... + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument ('--mntpt', help='rootfs mount point', action='store', required=True) + parser.add_argument ('--osdistrib', help='OS distribution', action='store', required=True) + parser.add_argument ('--oscodename', help='OS codename', action='store', required=True) + parser.add_argument ('--osrelease', help='OS release', action='store', required=True) + parser.add_argument ('--osarch', help='OS architecture', action='store', required=True) + parser.add_argument ('--oshttp', help='OS HTTP source', action='store', required=True) + args = parser.parse_args() + boottoolsFsOpengnsys (args.mntpt, args.osdistrib, args.oscodename, args.osrelease, args.osarch, args.oshttp) diff --git a/boottoolsfunctions/__init__.py b/boottoolsfunctions/__init__.py index 2368017..7eb4cd4 100644 --- a/boottoolsfunctions/__init__.py +++ b/boottoolsfunctions/__init__.py @@ -1,4 +1,3 @@ - import platform import os import re @@ -10,6 +9,28 @@ import glob logger = logging.getLogger ('boottools') +def _run (args): + cp = subprocess.run (args, capture_output=True) + if cp.returncode: + logger.error ('command "{}" failed with rc "{}"'.format (' '.join(args), cp.returncode)) + raise + stdout = cp.stdout.decode ('utf-8').strip() + stderr = cp.stderr.decode ('utf-8').strip() + return stdout, stderr + +def _grep (regex, file): + with open (file, 'r') as f: + for l in f: + if (re.findall (regex, l)): return 1 + return 0 + +def _is_mounted (mntpt): + return _grep (mntpt, '/proc/mounts') + +def _umount (mntpt): + if (_is_mounted (mntpt)): + subprocess.run (['umount', mntpt]) + def btogGetVar (osarch): btdir = '/tmp/opengnsys_installer/opengnsys/client/boot-tools' bttargetdir = '/var/lib/tftpboot/ogclient/' @@ -146,28 +167,6 @@ def btogGetOsInfo2 (type_client, versionboottools, ogclientcfg, osdistrib, oscod return gitrelease, nameisoclient, namehostclient -def _run (args): - cp = subprocess.run (args, capture_output=True) - if cp.returncode: - logger.error ('command "{}" failed with rc "{}"'.format (' '.join(args), cp.returncode)) - raise - stdout = cp.stdout.decode ('utf-8').strip() - stderr = cp.stderr.decode ('utf-8').strip() - return stdout, stderr - -def _grep (regex, file): - with open (file, 'r') as f: - for l in f: - if (re.findall (regex, l)): return 1 - return 0 - -def _is_mounted (mntpt): - return _grep (mntpt, '/proc/mounts') - -def _umount (mntpt): - if (_is_mounted (mntpt)): - subprocess.run (['umount', mntpt]) - def btogSetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, bttargetdir, osarch): #if not btrootfsimg: btogGetVar() @@ -285,20 +284,13 @@ def btogSetFsAccess (btrootfsimg): return 0 # btogSetfsBase: Genera el sistema root base con debootstrap +# trabaja sobre un rootfs ya montado def btogSetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp): logger.info ('Iniciando la generación del sistema de archivos') #mount | grep $BTROOTFSMNT || mount $BTROOTFSIMG $BTROOTFSMNT -o loop,offset=32256 if (not _is_mounted (btrootfsmnt)): - try: _run (['mount', btrootfsimg, btrootfsmnt, '-o', 'loop,offset=32256']) - except: - raise Exception ('mount failed') - - #mount | grep $BTROOTFSMNT && echoAndLog "$FUNCNAME: mount $BTROOTFSIMG $BTROOTFSMNT -o loop,offset=32256 OK " || errorAndLog "$FUNCNAME: mount $BTROOTFSIMG $BTROOTFSMNT -o loop,offset=32256 : FAILURE " - if (_is_mounted (btrootfsmnt)): - logger.info ('mount "{}" "{}" -o loop,offset=32256 OK'.format (btrootfsimg, btrootfsmnt)) - else: - raise Exception ('mount "{}" "{}" -o loop,offset=32256 : FAILURE'.format (btrootfsimg, btrootfsmnt)) + raise Exception ('rootfs should be mounted') logger.info ('debootstrapping system') try: _run (['debootstrap', '--arch="{}"'.format(osarch), '--components=main,universe', oscodename, btrootfsmnt, oshttp]) @@ -307,7 +299,5 @@ def btogSetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp): subprocess.run (['umount', btrootfsmnt]) raise Exception ('debootstrap --arch={} --components=main,universe "{}" "{}" "{}" : ha fallado!'.format (osarch, oscodename, btrootfsmnt, oshttp)) - if (_is_mounted (btrootfsmnt)): - subprocess.run (['umount', btrootfsmnt]) logger.info ('debootstrap --arch="{}" --components=main,universe "{}" "{}" "{}" : ok'.format (osarch, oscodename, btrootfsmnt, oshttp)) return 0 diff --git a/boottoolsgenerator.py b/boottoolsgenerator.py index d238da5..6ba3706 100755 --- a/boottoolsgenerator.py +++ b/boottoolsgenerator.py @@ -7,8 +7,9 @@ import subprocess import glob import stat -sys.path.insert (0, __file__) -from boottoolsfunctions import btogGetOsInfo1, btogGetOsInfo2, btogGetVar, btogSetFsVirtual, btogSetFsAccess, btogSetFsBase +curdir = os.path.dirname (__file__) +sys.path.insert (0, curdir) +from boottoolsfunctions import _run, btogGetOsInfo1, btogGetOsInfo2, btogGetVar, btogSetFsVirtual, btogSetFsAccess, btogSetFsBase logging.basicConfig (filename='/tmp/boot-tools_installation.log', filemode='a', format='%(levelname)s %(asctime)s (%(funcName)s) %(message)s', level=logging.INFO) logger = logging.getLogger ('boottools') @@ -79,10 +80,17 @@ logger.info ('Fase 3.2 - Configurar acceso schroot al Segundo Sistema de archivo #cat /etc/schroot/schroot.conf | grep $BTROOTFSIMG || btogSetFsAccess btogSetFsAccess (btrootfsimg) +try: _run (['mount', btrootfsimg, btrootfsmnt, '-o', 'loop,offset=32256']) +except: + logger.error ('mount failed') + os._exit (3) + logger.info ('Fase 3.3 Generar sistema de archivos con debootstrap') logger.debug ('Try creation of a file within chroot (this operation may fail--that is ok)') -cp = subprocess.run (['schroot', '--preserve-environment', '--chroot', 'IMGogclient', '--', 'touch', '/tmp/ogclientOK']) +logger.debug ('Running \'schroot --chroot IMGogclient -- touch /tmp/ogclientOK\'') +cp = subprocess.run (['schroot', '--chroot', 'IMGogclient', '--', 'touch', '/tmp/ogclientOK']) if (cp.returncode): + logger.debug ('schroot returned code "{}", calling btogSetFsBase()'.format (cp.returncode)) try: btogSetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp) #>>/tmp/fase3.out except Exception as e: logger.error (str (e)) @@ -95,6 +103,8 @@ else: ########################################################################### logger.info ('FASE 4 - Incorporando ficheros OpenGnsys al sistema raíz rootfs') +## por qué esta copia??? + #cp -a ${BTDIR}/includes/usr/bin/* /tmp >>/tmp/fase5.out subprocess.run (['cp', '-a'] + glob.glob ('{}/includes/usr/bin/*'.format (btdir)) + ['/tmp']) @@ -109,17 +119,22 @@ subprocess.run (['sed', '-i', '1 s/$/ {} ({})/'.format (gitrelease, osrelease), # En Ubuntu 13.04+ es necesario matar proceso de "udev" antes de desmontar. #umount $BTROOTFSMNT 2>/dev/null || (kill -9 $(lsof -t $BTROOTFSMNT); umount $BTROOTFSMNT) 2>/dev/null -""" -schroot --preserve-environment --chroot IMGogclient -- /tmp/boot-tools/boottoolsFsOpengnsys.sh >>/tmp/fase4.out + +logger.info ('running \'{}/02-boottoolsFsOpengnsys.py --mntpt "{}" --osdistrib "{}" --oscodename "{}" --osrelease "{}" --osarch "{}" --oshttp "{}"\''.format (curdir, btrootfsmnt, osdistrib, oscodename, osrelease, osarch, oshttp)) +cp = subprocess.run (['{}/02-boottoolsFsOpengnsys.py'.format (curdir), '--mntpt', btrootfsmnt, '--osdistrib', osdistrib, '--oscodename', oscodename, '--osrelease', osrelease, '--osarch', osarch, '--oshttp', oshttp], capture_output=True) +logger.info ('02-boottoolsFsOpengnsys stdout follows: {}'.format (cp.stdout.decode ('utf-8'))) +## /tmp/02.py --osdistrib ubuntu --oscodename focal --osrelease 5.4.0-42-generic --osarch amd64 --oshttp http://es.archive.ubuntu.com/ubuntu/ +## /tmp/opengnsys_installer/opengnsys/client/./boot-tools/02-boottoolsFsOpengnsys.py --mntpt "/var/lib/tftpboot/ogclient/ogclientmount" --osdistrib "Ubuntu" --oscodename "noble" --osrelease "6.8.0-39-generic" --osarch "amd64" --oshttp "http://es.archive.ubuntu.com/ubuntu/" ############################################################################################ logger.info ('FASE 5 - Instalar software') logger.info ('Fase 5.1 instalar paquetes deb con apt-get') -schroot --preserve-environment --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSoftwareInstall.sh >>/tmp/fase5.out +""" +schroot --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSoftwareInstall.sh >>/tmp/fase5.out logger.info ('Fase 5.2 compilar software.') cd / -schroot --preserve-environment --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSoftwareCompile.sh >>/tmp/fase5.out +schroot --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSoftwareCompile.sh >>/tmp/fase5.out cd - @@ -129,13 +144,13 @@ logger.info ('Fase 6.1 Incorporar la clave publica del servidor') cd / ssh-keygen -q -f /root/.ssh/id_rsa -N "" cp /root/.ssh/id_rsa.pub /tmp -schroot --preserve-environment --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSshServer.sh +schroot --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSshServer.sh cd - logger.info ('Fase 6.2. Incorpoar la clave publica del propio cliente') -schroot --preserve-environment --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSshClient.sh +schroot --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSshClient.sh logger.info ('Fase 6.3. Configurando las locales') -schroot --preserve-environment --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsFsLocales.sh +schroot --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsFsLocales.sh for i in `mount | grep IMGogclient | grep /var | cut -f3 -d" "`; do echo $i; umount $i; done for i in `mount | grep IMGogclient | grep /var | cut -f3 -d" "`; do echo $i; umount $i; done From cc5701c870e042cdbbff67a4a6162998ba2a09f0 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Mon, 12 Aug 2024 11:37:11 +0200 Subject: [PATCH 04/45] refs #596 fix bugs, add 03-boottoolsSoftwareInstall.py to install packages --- 02-boottoolsFsOpengnsys.py | 6 - 03-boottoolsSoftwareInstall.py | 133 ++++++++++++++++++ boottoolsfunctions/__init__.py | 7 +- boottoolsgenerator.py | 44 +++--- .../lib/plymouth/themes/opengnsys/box.png | Bin .../lib/plymouth/themes/opengnsys/bullet.png | Bin .../lib/plymouth/themes/opengnsys/entry.png | Bin .../lib/plymouth/themes/opengnsys/lock.png | Bin .../themes/opengnsys/logoopengnsys.png | Bin .../themes/opengnsys/opengnsys.plymouth | 0 .../plymouth/themes/opengnsys/opengnsys.png | Bin .../themes/opengnsys/opengnsys.script | 0 .../themes/opengnsys/progress_bar.png | Bin .../themes/opengnsys/progress_box.png | Bin 14 files changed, 163 insertions(+), 27 deletions(-) create mode 100755 03-boottoolsSoftwareInstall.py rename includes/{ => usr}/lib/plymouth/themes/opengnsys/box.png (100%) rename includes/{ => usr}/lib/plymouth/themes/opengnsys/bullet.png (100%) rename includes/{ => usr}/lib/plymouth/themes/opengnsys/entry.png (100%) rename includes/{ => usr}/lib/plymouth/themes/opengnsys/lock.png (100%) rename includes/{ => usr}/lib/plymouth/themes/opengnsys/logoopengnsys.png (100%) rename includes/{ => usr}/lib/plymouth/themes/opengnsys/opengnsys.plymouth (100%) rename includes/{ => usr}/lib/plymouth/themes/opengnsys/opengnsys.png (100%) rename includes/{ => usr}/lib/plymouth/themes/opengnsys/opengnsys.script (100%) rename includes/{ => usr}/lib/plymouth/themes/opengnsys/progress_bar.png (100%) rename includes/{ => usr}/lib/plymouth/themes/opengnsys/progress_box.png (100%) diff --git a/02-boottoolsFsOpengnsys.py b/02-boottoolsFsOpengnsys.py index 98c6b01..a2b9854 100755 --- a/02-boottoolsFsOpengnsys.py +++ b/02-boottoolsFsOpengnsys.py @@ -13,16 +13,12 @@ namehostclientfile = '/tmp/opengnsys_chroot' svnclientdir = '/tmp/opengnsys_installer/opengnsys/client/boot-tools' svnclientstructure = '/tmp/opengnsys_installer/opengnsys/client/shared' svnclientengine = '/tmp/opengnsys_installer/opengnsys/client/engine' - -ogclientmount = '' def boottoolsFsOpengnsys (osdistrib, oscodename, osrelease, osarch, oshttp): print (':'.join ([osdistrib, oscodename, osrelease, osarch, oshttp])) print ('Iniciando la personalización con datos del repositorio') - ## TODO mount fs, para lo cual necesito helpers - #sed -e "s/OSCODENAME/$OSCODENAME/g" ${SVNCLIENTDIR}/includes/etc/apt/sources.list.${OSDISTRIB,,} > ${SVNCLIENTDIR}/includes/etc/apt/sources.list sources_list_in = '{}/includes/etc/apt/sources.list.{}'.format (svnclientdir, osdistrib.lower()) sources_list_out = '{}/includes/etc/apt/sources.list'.format (svnclientdir) @@ -66,8 +62,6 @@ def boottoolsFsOpengnsys (osdistrib, oscodename, osrelease, osarch, oshttp): #echo "${VERSIONBOOTTOOLS}-${OSCODENAME}-${OSRELEASE}-${GITRELEASE}" > $nameisoclientfile #echo "${VERSIONBOOTTOOLS}-${OSCODENAME}-${GITRELEASE}" > $namehostclientfile - ## desmontar de nuevo, ejem... - if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument ('--mntpt', help='rootfs mount point', action='store', required=True) diff --git a/03-boottoolsSoftwareInstall.py b/03-boottoolsSoftwareInstall.py new file mode 100755 index 0000000..f912d99 --- /dev/null +++ b/03-boottoolsSoftwareInstall.py @@ -0,0 +1,133 @@ +#!/usr/bin/python3 + +import argparse +import os +import subprocess + +def _aptget_clean(): + subprocess.run (['apt-get', 'clean']) + +def _aptget_autoclean(): + subprocess.run (['apt-get', 'autoclean']) + +def _aptget_autoremove(): + subprocess.run (['apt-get', 'autoremove']) + +def _aptget_update(): + subprocess.run (['apt-get', 'update']) + +def _aptget_upgrade(): + subprocess.run (['apt-get', 'upgrade', '--yes']) + +def _aptget_install (pkgs, opts=[]): + subprocess.run (['apt-get', '--yes'] + opts + ['install'] + pkgs) + +def _oghook_deactivate(): + #Desactivamos el hook del oginitrd.img para evitar problemas, al final de este escripts se activará + #mv /etc/initramfs-tools/hooks/oghooks /etc/initramfs-tools/ + pass + +def _oghook_activate(): + #Activamos el hook del oginitrd.img + #mv /etc/initramfs-tools/oghooks /etc/initramfs-tools/hooks/ + pass + +def _mock_mtab(): + # Preparamos el mtab necesario para la instalacion correcta de paquetes. + #echo "/dev/sda1 / ext4 rw,errors=remount-ro 0 0" > /etc/mtab ## nati: falla porque es un symlink a ../proc/self/mounts + pass + +def _restore_mtab(): + # Dejamos el mtab como al principio + #echo " " > /etc/mtab + pass + +def boottoolsSoftwareInstall (osarch, osrelease): + os.environ['LANGUAGE'] = 'C' + os.environ['LC_ALL'] = 'C' + os.environ['LANG'] = 'C' + os.environ['DEBIAN_FRONTEND'] = 'noninteractive' + + subprocess.run (['dpkg-divert', '--local', '--rename', '--add', '/sbin/initctl']) + os.symlink ('/bin/true', '/sbin/initctl') + + #TEST desactivamos upstart + #apt-get update; apt-get install --no-install-recommends dbus; dbus-uuidgen > /var/lib/dbus/machine-id; dpkg-divert --local --rename --add /sbin/initctl; ln -s /bin/true /sbin/initctl + + _aptget_clean() + pkgs32 = [] + if 'i386' != osarch: + subprocess.run (['dpkg', '--add-architecture', 'i386']) + pkgs32 = 'lib32gcc1 lib32stdc++6 lib32z1 libc6-i386'.split (' ') + + _aptget_update() +#Ign:13 http://ppa.launchpad.net/zfs-native/stable/ubuntu noble InRelease +#Err:15 http://ppa.launchpad.net/zfs-native/stable/ubuntu noble Release +# 404 Not Found [IP: 185.125.190.80 80] +# +#E: The repository 'http://ppa.launchpad.net/zfs-native/stable/ubuntu noble Release' does not have a Release file. +#N: Updating from such a repository can't be done securely, and is therefore disabled by default. +#N: See apt-secure(8) manpage for repository creation and user configuration details. + _aptget_upgrade() + _oghook_deactivate() + _mock_mtab() + return + + + """ + # Instalamos el kernel. + _aptget_install (['linux-image-{}'.format (osrelease), 'linux-headers-{}'.format (osrelease), 'dkms', 'shim-signed', 'openssl']) + _aptget_install (['linux-modules-{}'.format (osrelease), 'linux-modules-extra-{}'.format (osrelease)]) + + # Valores para paquetes interactivos. + cat << EOT | debconf-set-selections -- +kexec-tools kexec-tools/load_kexec boolean true +openssh-server openssh-server/permit-root-login boolean true +refind refind/install_to_esp boolean false +EOT + ## nati: hace falta --force-confdef para evitar un tema interactivo del /etc/ssh/ssh_config + _aptget_install (['sshfs', 'kexec-tools'] + pkgs32, opts=['-o', 'DPkg::Options::=--force-confdef']) + + #comenzamos con la instalación de los paquetes a instalar. + to_install = {} + for group in `find /usr/bin/boot-tools/listpackages/ -name sw.*`; do + echo "Instalando el grupo de paquetes almacenados en $group" + for package in ` awk /^install/'{print $2}' $group `; do + echo -n $package' ' >&2 + to_install[package] = 1 + done + echo >&2 + done + del to_install['refind'] ## nati: este paquete no existe y toda la invocación de apt se muere (no sé cómo hacer que apt ignore los paquetes que no existen) + echo =========================================== nati pkgs to install ${!TO_INSTALL[*]} + _aptget_install (list (to_install.keys())) + + # Instalar módulos que algunos paquetes puedan tener pendientes de compilar. + echo "Instalando módulos adicionales con DKMS" + while read -e mod vers; do + echo -n "Intalando módulo $mod v$vers" + dkms install -m $mod -v $vers &>/dev/null + RETVAL=$? + if [ $RETVAL == 0 ]; then + echo " : OK - Módulo instalado correctamente (codigo interno de dkms $RETVAL)" + else + echo " : Error módulo $mod (codigo interno de dkms $RETVAL) " + echo "Pulse [Intro] para continuar" + read + fi + done < <(dkms status 2>/dev/null | awk -F, '$3~/added/ {print $1,$2}') + """ + + _oghook_activate() + _restore_mtab() + _aptget_clean() + _aptget_autoclean() + _aptget_autoremove() + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument ('--osarch', help='OS architecture', action='store', required=True) + parser.add_argument ('--osrelease', help='OS release', action='store', required=True) + args = parser.parse_args() + boottoolsSoftwareInstall (args.osarch, args.osrelease) + diff --git a/boottoolsfunctions/__init__.py b/boottoolsfunctions/__init__.py index 7eb4cd4..3674c6d 100644 --- a/boottoolsfunctions/__init__.py +++ b/boottoolsfunctions/__init__.py @@ -211,7 +211,7 @@ def btogSetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisks time.sleep (5) logger.debug ('losetup --detach') - try: _run (['losetup', '--detache', diskloop]) + try: _run (['losetup', '--detach', diskloop]) except: raise Exception ('Liberando disco virtual despues del particionado: ERROR') @@ -293,11 +293,12 @@ def btogSetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp): raise Exception ('rootfs should be mounted') logger.info ('debootstrapping system') - try: _run (['debootstrap', '--arch="{}"'.format(osarch), '--components=main,universe', oscodename, btrootfsmnt, oshttp]) + logger.debug ('debootstrap --arch="{}" --components=main,universe "{}" "{}" "{}"'.format (osarch, oscodename, btrootfsmnt, oshttp) + try: _run (['debootstrap', '--arch={}'.format(osarch), '--components=main,universe', oscodename, btrootfsmnt, oshttp]) except: if (_is_mounted (btrootfsmnt)): subprocess.run (['umount', btrootfsmnt]) - raise Exception ('debootstrap --arch={} --components=main,universe "{}" "{}" "{}" : ha fallado!'.format (osarch, oscodename, btrootfsmnt, oshttp)) + raise Exception ('debootstrap --arch="{}" --components=main,universe "{}" "{}" "{}" : ha fallado!'.format (osarch, oscodename, btrootfsmnt, oshttp)) logger.info ('debootstrap --arch="{}" --components=main,universe "{}" "{}" "{}" : ok'.format (osarch, oscodename, btrootfsmnt, oshttp)) return 0 diff --git a/boottoolsgenerator.py b/boottoolsgenerator.py index 6ba3706..1e799a7 100755 --- a/boottoolsgenerator.py +++ b/boottoolsgenerator.py @@ -9,7 +9,7 @@ import stat curdir = os.path.dirname (__file__) sys.path.insert (0, curdir) -from boottoolsfunctions import _run, btogGetOsInfo1, btogGetOsInfo2, btogGetVar, btogSetFsVirtual, btogSetFsAccess, btogSetFsBase +from boottoolsfunctions import _run, _is_mounted, btogGetOsInfo1, btogGetOsInfo2, btogGetVar, btogSetFsVirtual, btogSetFsAccess, btogSetFsBase logging.basicConfig (filename='/tmp/boot-tools_installation.log', filemode='a', format='%(levelname)s %(asctime)s (%(funcName)s) %(message)s', level=logging.INFO) logger = logging.getLogger ('boottools') @@ -18,7 +18,7 @@ if not os.path.exists ('/tmp/opengnsys_installer/opengnsys/client/engine'): branch = 'main' tmpdir = subprocess.run (['mktemp', '--tmpdir', '--directory', 'oggit.XXXXXX'], capture_output=True).stdout.decode ('utf-8').strip() subprocess.run (['git', 'clone', '-c', 'http.sslVerify=false', '--branch', branch, 'https://ognproject.evlt.uma.es/gitea/opengnsys/opengnsys.git', tmpdir]) - subprocess.run (['rsync', '-aHv', '{}/client/engine'.format(tmpdir), '{}/client/shared'.format(tmpdir), '/tmp/opengnsys_installer/opengnsys/client/']) + subprocess.run (['rsync', '-aH', '{}/client/engine'.format(tmpdir), '{}/client/shared'.format(tmpdir), '/tmp/opengnsys_installer/opengnsys/client/']) subprocess.run (['rm', '-rf', tmpdir]) type_client = sys.argv[1] if len(sys.argv)>1 else 'host' @@ -61,7 +61,8 @@ else: os._exit (1) logger.info ('PXE package is "{}"'.format (pxepkg)) ## TODO qemu no existe, hace falta? -subprocess.run (['apt-get', '-y', 'install', 'debootstrap', 'subversion', 'schroot', 'squashfs-tools', 'syslinux', 'genisoimage', 'qemu-utils', 'lsof', pxepkg]) #>>/tmp/fase2.out +subprocess.run (['apt-get', '-y', 'update']) +subprocess.run (['apt-get', '-y', 'install', 'jq', 'syslinux', 'syslinux-efi', 'syslinux-utils', 'debootstrap', 'subversion', 'schroot', 'squashfs-tools', 'syslinux', 'genisoimage', 'qemu-utils', 'lsof', pxepkg]) #>>/tmp/fase2.out ###################################################################3 @@ -80,25 +81,27 @@ logger.info ('Fase 3.2 - Configurar acceso schroot al Segundo Sistema de archivo #cat /etc/schroot/schroot.conf | grep $BTROOTFSIMG || btogSetFsAccess btogSetFsAccess (btrootfsimg) -try: _run (['mount', btrootfsimg, btrootfsmnt, '-o', 'loop,offset=32256']) -except: - logger.error ('mount failed') - os._exit (3) +## para hacer schroot --cosas, el mntpt tiene que estar desmontado +## si está montado da un pete tal que 'E: 10mount: mount: /run/schroot/mount/IMGogclient-7fbf51a2-e37e-48e5-8e5d-83f8901fc7ed: wrong fs type, bad option, bad superblock on /dev/loop1, missing codepage or helper program, or other error.' logger.info ('Fase 3.3 Generar sistema de archivos con debootstrap') -logger.debug ('Try creation of a file within chroot (this operation may fail--that is ok)') -logger.debug ('Running \'schroot --chroot IMGogclient -- touch /tmp/ogclientOK\'') -cp = subprocess.run (['schroot', '--chroot', 'IMGogclient', '--', 'touch', '/tmp/ogclientOK']) +logger.debug ('Try creation of a file within chroot (this operation may fail with "... etc/resolv.conf: No such file or directory"--that is ok)') +logger.debug ('Running \'schroot --chroot IMGogclient -- stat /etc\'') +cp = subprocess.run (['schroot', '--chroot', 'IMGogclient', '--', 'stat', '/etc']) if (cp.returncode): logger.debug ('schroot returned code "{}", calling btogSetFsBase()'.format (cp.returncode)) + try: _run (['mount', btrootfsimg, btrootfsmnt, '-o', 'loop,offset=32256']) + except: + logger.error ('mount failed') + os._exit (3) try: btogSetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp) #>>/tmp/fase3.out except Exception as e: logger.error (str (e)) os._exit (3) -else: - os.unlink ('/tmp/ogclientOK') #echo ================= nati after fase 3; ls -la /opt/opengnsys/tftpboot/ogclient/ +if (_is_mounted (btrootfsmnt)): + subprocess.run (['umount', btrootfsmnt]) ########################################################################### logger.info ('FASE 4 - Incorporando ficheros OpenGnsys al sistema raíz rootfs') @@ -106,23 +109,24 @@ logger.info ('FASE 4 - Incorporando ficheros OpenGnsys al sistema raíz rootfs') ## por qué esta copia??? #cp -a ${BTDIR}/includes/usr/bin/* /tmp >>/tmp/fase5.out -subprocess.run (['cp', '-a'] + glob.glob ('{}/includes/usr/bin/*'.format (btdir)) + ['/tmp']) +#subprocess.run (['cp', '-a'] + glob.glob ('{}/includes/usr/bin/*'.format (btdir)) + ['/tmp']) #chmod +x /tmp/boot-tools/*.sh -for i in glob.glob ('/tmp/boot-tools/*.sh'): - st = os.stat (i) - os.chmod (i, st.st_mode|stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH) +#for i in glob.glob ('/tmp/boot-tools/*.sh'): +# st = os.stat (i) +# os.chmod (i, st.st_mode|stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH) # Incluir revisión. +## FIXME esto la incluye incondicionalmente, y luego terminamos con "OpenGnsys Client 1.2.0-rc1 gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) ..." #sed -i "1 s/$/ $GITRELEASE ($OSRELEASE)/" ${BTDIR}/includes/etc/initramfs-tools/scripts/VERSION.txt subprocess.run (['sed', '-i', '1 s/$/ {} ({})/'.format (gitrelease, osrelease), '{}/includes/etc/initramfs-tools/scripts/VERSION.txt'.format (btdir)]) # En Ubuntu 13.04+ es necesario matar proceso de "udev" antes de desmontar. #umount $BTROOTFSMNT 2>/dev/null || (kill -9 $(lsof -t $BTROOTFSMNT); umount $BTROOTFSMNT) 2>/dev/null -logger.info ('running \'{}/02-boottoolsFsOpengnsys.py --mntpt "{}" --osdistrib "{}" --oscodename "{}" --osrelease "{}" --osarch "{}" --oshttp "{}"\''.format (curdir, btrootfsmnt, osdistrib, oscodename, osrelease, osarch, oshttp)) +logger.debug ('running \'{}/02-boottoolsFsOpengnsys.py --mntpt "{}" --osdistrib "{}" --oscodename "{}" --osrelease "{}" --osarch "{}" --oshttp "{}"\''.format (curdir, btrootfsmnt, osdistrib, oscodename, osrelease, osarch, oshttp)) cp = subprocess.run (['{}/02-boottoolsFsOpengnsys.py'.format (curdir), '--mntpt', btrootfsmnt, '--osdistrib', osdistrib, '--oscodename', oscodename, '--osrelease', osrelease, '--osarch', osarch, '--oshttp', oshttp], capture_output=True) -logger.info ('02-boottoolsFsOpengnsys stdout follows: {}'.format (cp.stdout.decode ('utf-8'))) +logger.debug ('02-boottoolsFsOpengnsys stdout follows: {}'.format (cp.stdout.decode ('utf-8'))) ## /tmp/02.py --osdistrib ubuntu --oscodename focal --osrelease 5.4.0-42-generic --osarch amd64 --oshttp http://es.archive.ubuntu.com/ubuntu/ ## /tmp/opengnsys_installer/opengnsys/client/./boot-tools/02-boottoolsFsOpengnsys.py --mntpt "/var/lib/tftpboot/ogclient/ogclientmount" --osdistrib "Ubuntu" --oscodename "noble" --osrelease "6.8.0-39-generic" --osarch "amd64" --oshttp "http://es.archive.ubuntu.com/ubuntu/" @@ -130,6 +134,10 @@ logger.info ('02-boottoolsFsOpengnsys stdout follows: {}'.format (cp.stdout.deco ############################################################################################ logger.info ('FASE 5 - Instalar software') logger.info ('Fase 5.1 instalar paquetes deb con apt-get') +logger.debug ('running \'schroot --chroot IMGogclient -- {}/03-boottoolsSoftwareInstall.py --osrelease "{}" --osarch "{}"\''.format (curdir, osrelease, osarch)) +cp = subprocess.run (['schroot', '--chroot', 'IMGogclient', '--', '{}/03-boottoolsSoftwareInstall.py'.format (curdir), '--osrelease', osrelease, '--osarch', osarch]) +logger.debug ('03-boottoolsSoftwareInstall stdout follows: {}'.format (cp.stdout.decode ('utf-8'))) + """ schroot --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSoftwareInstall.sh >>/tmp/fase5.out logger.info ('Fase 5.2 compilar software.') diff --git a/includes/lib/plymouth/themes/opengnsys/box.png b/includes/usr/lib/plymouth/themes/opengnsys/box.png similarity index 100% rename from includes/lib/plymouth/themes/opengnsys/box.png rename to includes/usr/lib/plymouth/themes/opengnsys/box.png diff --git a/includes/lib/plymouth/themes/opengnsys/bullet.png b/includes/usr/lib/plymouth/themes/opengnsys/bullet.png similarity index 100% rename from includes/lib/plymouth/themes/opengnsys/bullet.png rename to includes/usr/lib/plymouth/themes/opengnsys/bullet.png diff --git a/includes/lib/plymouth/themes/opengnsys/entry.png b/includes/usr/lib/plymouth/themes/opengnsys/entry.png similarity index 100% rename from includes/lib/plymouth/themes/opengnsys/entry.png rename to includes/usr/lib/plymouth/themes/opengnsys/entry.png diff --git a/includes/lib/plymouth/themes/opengnsys/lock.png b/includes/usr/lib/plymouth/themes/opengnsys/lock.png similarity index 100% rename from includes/lib/plymouth/themes/opengnsys/lock.png rename to includes/usr/lib/plymouth/themes/opengnsys/lock.png diff --git a/includes/lib/plymouth/themes/opengnsys/logoopengnsys.png b/includes/usr/lib/plymouth/themes/opengnsys/logoopengnsys.png similarity index 100% rename from includes/lib/plymouth/themes/opengnsys/logoopengnsys.png rename to includes/usr/lib/plymouth/themes/opengnsys/logoopengnsys.png diff --git a/includes/lib/plymouth/themes/opengnsys/opengnsys.plymouth b/includes/usr/lib/plymouth/themes/opengnsys/opengnsys.plymouth similarity index 100% rename from includes/lib/plymouth/themes/opengnsys/opengnsys.plymouth rename to includes/usr/lib/plymouth/themes/opengnsys/opengnsys.plymouth diff --git a/includes/lib/plymouth/themes/opengnsys/opengnsys.png b/includes/usr/lib/plymouth/themes/opengnsys/opengnsys.png similarity index 100% rename from includes/lib/plymouth/themes/opengnsys/opengnsys.png rename to includes/usr/lib/plymouth/themes/opengnsys/opengnsys.png diff --git a/includes/lib/plymouth/themes/opengnsys/opengnsys.script b/includes/usr/lib/plymouth/themes/opengnsys/opengnsys.script similarity index 100% rename from includes/lib/plymouth/themes/opengnsys/opengnsys.script rename to includes/usr/lib/plymouth/themes/opengnsys/opengnsys.script diff --git a/includes/lib/plymouth/themes/opengnsys/progress_bar.png b/includes/usr/lib/plymouth/themes/opengnsys/progress_bar.png similarity index 100% rename from includes/lib/plymouth/themes/opengnsys/progress_bar.png rename to includes/usr/lib/plymouth/themes/opengnsys/progress_bar.png diff --git a/includes/lib/plymouth/themes/opengnsys/progress_box.png b/includes/usr/lib/plymouth/themes/opengnsys/progress_box.png similarity index 100% rename from includes/lib/plymouth/themes/opengnsys/progress_box.png rename to includes/usr/lib/plymouth/themes/opengnsys/progress_box.png From ec2ab82eb10781f94003ca0dc693a3f473d4f4d9 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Mon, 12 Aug 2024 12:58:08 +0200 Subject: [PATCH 05/45] refs #596 fix more bugs, set debconf selections, add support for ubuntu noble --- 02-boottoolsFsOpengnsys.py | 2 +- 03-boottoolsSoftwareInstall.py | 42 ++++++++++++++++++---------------- boottoolsfunctions/__init__.py | 27 ++++++++++++++-------- 3 files changed, 40 insertions(+), 31 deletions(-) diff --git a/02-boottoolsFsOpengnsys.py b/02-boottoolsFsOpengnsys.py index a2b9854..f7c18d3 100755 --- a/02-boottoolsFsOpengnsys.py +++ b/02-boottoolsFsOpengnsys.py @@ -14,7 +14,7 @@ svnclientdir = '/tmp/opengnsys_installer/opengnsys/client/boot-tools' svnclientstructure = '/tmp/opengnsys_installer/opengnsys/client/shared' svnclientengine = '/tmp/opengnsys_installer/opengnsys/client/engine' -def boottoolsFsOpengnsys (osdistrib, oscodename, osrelease, osarch, oshttp): +def boottoolsFsOpengnsys (ogclientmount, osdistrib, oscodename, osrelease, osarch, oshttp): print (':'.join ([osdistrib, oscodename, osrelease, osarch, oshttp])) print ('Iniciando la personalización con datos del repositorio') diff --git a/03-boottoolsSoftwareInstall.py b/03-boottoolsSoftwareInstall.py index f912d99..ecc417f 100755 --- a/03-boottoolsSoftwareInstall.py +++ b/03-boottoolsSoftwareInstall.py @@ -2,8 +2,17 @@ import argparse import os +import re import subprocess +from boottoolsfunctions import _run + +debconf_settings = """ +kexec-tools kexec-tools/load_kexec boolean true +openssh-server openssh-server/permit-root-login boolean true +refind refind/install_to_esp boolean false +""".strip() + def _aptget_clean(): subprocess.run (['apt-get', 'clean']) @@ -24,13 +33,11 @@ def _aptget_install (pkgs, opts=[]): def _oghook_deactivate(): #Desactivamos el hook del oginitrd.img para evitar problemas, al final de este escripts se activará - #mv /etc/initramfs-tools/hooks/oghooks /etc/initramfs-tools/ - pass + os.rename ('/etc/initramfs-tools/hooks/oghooks', '/etc/initramfs-tools/oghooks') def _oghook_activate(): #Activamos el hook del oginitrd.img - #mv /etc/initramfs-tools/oghooks /etc/initramfs-tools/hooks/ - pass + os.rename ('/etc/initramfs-tools/oghooks', '/etc/initramfs-tools/hooks/oghooks') def _mock_mtab(): # Preparamos el mtab necesario para la instalacion correcta de paquetes. @@ -48,8 +55,10 @@ def boottoolsSoftwareInstall (osarch, osrelease): os.environ['LANG'] = 'C' os.environ['DEBIAN_FRONTEND'] = 'noninteractive' - subprocess.run (['dpkg-divert', '--local', '--rename', '--add', '/sbin/initctl']) - os.symlink ('/bin/true', '/sbin/initctl') + stdout, _ = _run (['dpkg-divert', '--list']) + if not re.findall (r'local diversion of /sbin/initctl to /sbin/initctl.distrib', stdout): + subprocess.run (['dpkg-divert', '--local', '--rename', '--add', '/sbin/initctl']) + os.symlink ('/bin/true', '/sbin/initctl') #TEST desactivamos upstart #apt-get update; apt-get install --no-install-recommends dbus; dbus-uuidgen > /var/lib/dbus/machine-id; dpkg-divert --local --rename --add /sbin/initctl; ln -s /bin/true /sbin/initctl @@ -58,9 +67,9 @@ def boottoolsSoftwareInstall (osarch, osrelease): pkgs32 = [] if 'i386' != osarch: subprocess.run (['dpkg', '--add-architecture', 'i386']) - pkgs32 = 'lib32gcc1 lib32stdc++6 lib32z1 libc6-i386'.split (' ') + pkgs32 = 'lib32gcc-s1 lib32stdc++6 lib32z1 libc6-i386'.split (' ') ## he cambiado lib32gcc1 por lib32gcc-s1 pero como queramos crear un oglive viejo, esto va a petar - _aptget_update() + _aptget_update() ## esto ya esta hecho... #Ign:13 http://ppa.launchpad.net/zfs-native/stable/ubuntu noble InRelease #Err:15 http://ppa.launchpad.net/zfs-native/stable/ubuntu noble Release # 404 Not Found [IP: 185.125.190.80 80] @@ -68,26 +77,19 @@ def boottoolsSoftwareInstall (osarch, osrelease): #E: The repository 'http://ppa.launchpad.net/zfs-native/stable/ubuntu noble Release' does not have a Release file. #N: Updating from such a repository can't be done securely, and is therefore disabled by default. #N: See apt-secure(8) manpage for repository creation and user configuration details. - _aptget_upgrade() _oghook_deactivate() + _aptget_upgrade() _mock_mtab() - return - - """ # Instalamos el kernel. - _aptget_install (['linux-image-{}'.format (osrelease), 'linux-headers-{}'.format (osrelease), 'dkms', 'shim-signed', 'openssl']) - _aptget_install (['linux-modules-{}'.format (osrelease), 'linux-modules-extra-{}'.format (osrelease)]) + _aptget_install (['linux-image-{}'.format (osrelease), 'linux-headers-{}'.format (osrelease), 'linux-modules-{}'.format (osrelease), 'linux-modules-extra-{}'.format (osrelease), 'dkms', 'shim-signed', 'openssl']) + + subprocess.run (['debconf-set-selections'], input=debconf_settings, text=True) - # Valores para paquetes interactivos. - cat << EOT | debconf-set-selections -- -kexec-tools kexec-tools/load_kexec boolean true -openssh-server openssh-server/permit-root-login boolean true -refind refind/install_to_esp boolean false -EOT ## nati: hace falta --force-confdef para evitar un tema interactivo del /etc/ssh/ssh_config _aptget_install (['sshfs', 'kexec-tools'] + pkgs32, opts=['-o', 'DPkg::Options::=--force-confdef']) + """ #comenzamos con la instalación de los paquetes a instalar. to_install = {} for group in `find /usr/bin/boot-tools/listpackages/ -name sw.*`; do diff --git a/boottoolsfunctions/__init__.py b/boottoolsfunctions/__init__.py index 3674c6d..b66426d 100644 --- a/boottoolsfunctions/__init__.py +++ b/boottoolsfunctions/__init__.py @@ -10,12 +10,13 @@ import glob logger = logging.getLogger ('boottools') def _run (args): - cp = subprocess.run (args, capture_output=True) + cp = subprocess.run (args, text=True, capture_output=True) if cp.returncode: + logger.error ('stdout "{}" stderr "{}"'.format (cp.stdout, cp.stderr)) logger.error ('command "{}" failed with rc "{}"'.format (' '.join(args), cp.returncode)) - raise - stdout = cp.stdout.decode ('utf-8').strip() - stderr = cp.stderr.decode ('utf-8').strip() + raise Exception ('command "{}" failed with rc "{}"'.format (' '.join(args), cp.returncode)) + stdout = cp.stdout.strip() + stderr = cp.stderr.strip() return stdout, stderr def _grep (regex, file): @@ -131,11 +132,17 @@ def btogGetOsInfo1 (type_client): osrelease='5.4.0-42-generic' osarch='amd64' oshttp='http://es.archive.ubuntu.com/ubuntu/' + elif 'noble' == type_client: # ogLive 1.2.0-rc1 basado en Ubuntu 24.04 y Kernel 6.8. + osdistrib='ubuntu' + oscodename='noble' + osrelease='6.8.0-31-generic' + osarch='amd64' + oshttp='http://es.archive.ubuntu.com/ubuntu/' elif 'host' == type_client: # ogLive basado en la distribución del servidor. osdistrib=platform.freedesktop_os_release()['NAME'] oscodename=platform.freedesktop_os_release()['VERSION_CODENAME'] osrelease=platform.uname()[2] - osarch=subprocess.run (['dpkg', '--print-architecture'], capture_output=True).stdout.decode ('utf-8').strip() + osarch, _=_run (['dpkg', '--print-architecture']) oshttp='http://es.archive.ubuntu.com/ubuntu/' else: # Parámetro desconocido print ('Parámetro no válido.') @@ -206,10 +213,10 @@ def btogSetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisks except: raise Exception ('losetup failed') logger.info ('Partitioning disk image') - cp = subprocess.run(['bash', '-c', "echo $'n\np\n1\n\n\nt\n83\nw' |fdisk {}".format (diskloop)], capture_output=True) - logger.debug ('fdisk output follows: {}'.format (cp.stdout.decode('utf-8'))) + stdout, _ = _run (['bash', '-c', "echo $'n\np\n1\n\n\nt\n83\nw' |fdisk {}".format (diskloop)]) + logger.debug ('fdisk output follows: {}'.format (stdout)) - time.sleep (5) + time.sleep (3) logger.debug ('losetup --detach') try: _run (['losetup', '--detach', diskloop]) except: @@ -228,7 +235,7 @@ def btogSetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisks except: raise Exception ('Formateando la particion principal del disco virtual: ERROR') - time.sleep (5) + time.sleep (3) logger.debug ('losetup --detach') try: _run (['losetup', '--detach', partloop]) except: @@ -293,7 +300,7 @@ def btogSetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp): raise Exception ('rootfs should be mounted') logger.info ('debootstrapping system') - logger.debug ('debootstrap --arch="{}" --components=main,universe "{}" "{}" "{}"'.format (osarch, oscodename, btrootfsmnt, oshttp) + logger.debug ('debootstrap --arch="{}" --components=main,universe "{}" "{}" "{}"'.format (osarch, oscodename, btrootfsmnt, oshttp)) try: _run (['debootstrap', '--arch={}'.format(osarch), '--components=main,universe', oscodename, btrootfsmnt, oshttp]) except: if (_is_mounted (btrootfsmnt)): From 2729431ad1f03622c169654cc5b6d6037a488b08 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Mon, 12 Aug 2024 13:03:33 +0200 Subject: [PATCH 06/45] refs #596 add some forgotten changes --- boottoolsgenerator.py | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/boottoolsgenerator.py b/boottoolsgenerator.py index 1e799a7..6740463 100755 --- a/boottoolsgenerator.py +++ b/boottoolsgenerator.py @@ -16,10 +16,10 @@ logger = logging.getLogger ('boottools') if not os.path.exists ('/tmp/opengnsys_installer/opengnsys/client/engine'): branch = 'main' - tmpdir = subprocess.run (['mktemp', '--tmpdir', '--directory', 'oggit.XXXXXX'], capture_output=True).stdout.decode ('utf-8').strip() - subprocess.run (['git', 'clone', '-c', 'http.sslVerify=false', '--branch', branch, 'https://ognproject.evlt.uma.es/gitea/opengnsys/opengnsys.git', tmpdir]) - subprocess.run (['rsync', '-aH', '{}/client/engine'.format(tmpdir), '{}/client/shared'.format(tmpdir), '/tmp/opengnsys_installer/opengnsys/client/']) - subprocess.run (['rm', '-rf', tmpdir]) + tmpdir = subprocess.run (['mktemp', '--tmpdir', '--directory', 'oggit.XXXXXX']).stdout.decode ('utf-8').strip() + _run (['git', 'clone', '-c', 'http.sslVerify=false', '--branch', branch, 'https://ognproject.evlt.uma.es/gitea/opengnsys/opengnsys.git', tmpdir]) + _run (['rsync', '-aH', '{}/client/engine'.format(tmpdir), '{}/client/shared'.format(tmpdir), '/tmp/opengnsys_installer/opengnsys/client/']) + _run (['rm', '-rf', tmpdir]) type_client = sys.argv[1] if len(sys.argv)>1 else 'host' #WORKDIR = '/tmp/opengnsys_installer' @@ -49,9 +49,9 @@ logger.info ('OpenGnsys CLIENT installation begins') ########################################################################## logger.info ('FASE 2 - Instalación de software adicional.') #grep "http://free.nchc.org.tw/drbl-core" /etc/apt/sources.list || echo "deb http://free.nchc.org.tw/drbl-core drbl stable" >> /etc/apt/sources.list -subprocess.run (['apt-get', 'update']) #>>/tmp/fase2.out -acse_gpxe = subprocess.run (['apt-cache', 'search', 'gpxe'], capture_output=True).stdout.decode ('utf-8').strip() -acse_ipxe = subprocess.run (['apt-cache', 'search', 'ipxe'], capture_output=True).stdout.decode ('utf-8').strip() +_run (['apt-get', 'update']) #>>/tmp/fase2.out +acse_gpxe, _ = _run (['apt-cache', 'search', 'gpxe']) +acse_ipxe, _ = _run (['apt-cache', 'search', 'ipxe']) if acse_ipxe: pxepkg = 'ipxe' elif acse_gpxe: @@ -61,8 +61,7 @@ else: os._exit (1) logger.info ('PXE package is "{}"'.format (pxepkg)) ## TODO qemu no existe, hace falta? -subprocess.run (['apt-get', '-y', 'update']) -subprocess.run (['apt-get', '-y', 'install', 'jq', 'syslinux', 'syslinux-efi', 'syslinux-utils', 'debootstrap', 'subversion', 'schroot', 'squashfs-tools', 'syslinux', 'genisoimage', 'qemu-utils', 'lsof', pxepkg]) #>>/tmp/fase2.out +_run (['apt-get', '-y', 'install', 'jq', 'syslinux', 'syslinux-efi', 'syslinux-utils', 'debootstrap', 'subversion', 'schroot', 'squashfs-tools', 'syslinux', 'genisoimage', 'qemu-utils', 'lsof', pxepkg]) #>>/tmp/fase2.out ###################################################################3 @@ -100,16 +99,13 @@ if (cp.returncode): os._exit (3) #echo ================= nati after fase 3; ls -la /opt/opengnsys/tftpboot/ogclient/ -if (_is_mounted (btrootfsmnt)): - subprocess.run (['umount', btrootfsmnt]) - ########################################################################### logger.info ('FASE 4 - Incorporando ficheros OpenGnsys al sistema raíz rootfs') ## por qué esta copia??? #cp -a ${BTDIR}/includes/usr/bin/* /tmp >>/tmp/fase5.out -#subprocess.run (['cp', '-a'] + glob.glob ('{}/includes/usr/bin/*'.format (btdir)) + ['/tmp']) +#_run (['cp', '-a'] + glob.glob ('{}/includes/usr/bin/*'.format (btdir)) + ['/tmp']) #chmod +x /tmp/boot-tools/*.sh #for i in glob.glob ('/tmp/boot-tools/*.sh'): @@ -119,24 +115,32 @@ logger.info ('FASE 4 - Incorporando ficheros OpenGnsys al sistema raíz rootfs') # Incluir revisión. ## FIXME esto la incluye incondicionalmente, y luego terminamos con "OpenGnsys Client 1.2.0-rc1 gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) ..." #sed -i "1 s/$/ $GITRELEASE ($OSRELEASE)/" ${BTDIR}/includes/etc/initramfs-tools/scripts/VERSION.txt -subprocess.run (['sed', '-i', '1 s/$/ {} ({})/'.format (gitrelease, osrelease), '{}/includes/etc/initramfs-tools/scripts/VERSION.txt'.format (btdir)]) +_run (['sed', '-i', '1 s/$/ {} ({})/'.format (gitrelease, osrelease), '{}/includes/etc/initramfs-tools/scripts/VERSION.txt'.format (btdir)]) # En Ubuntu 13.04+ es necesario matar proceso de "udev" antes de desmontar. #umount $BTROOTFSMNT 2>/dev/null || (kill -9 $(lsof -t $BTROOTFSMNT); umount $BTROOTFSMNT) 2>/dev/null +if (not _is_mounted (btrootfsmnt)): + try: _run (['mount', btrootfsimg, btrootfsmnt, '-o', 'loop,offset=32256']) + except: + logger.error ('mount failed') + os._exit (3) + logger.debug ('running \'{}/02-boottoolsFsOpengnsys.py --mntpt "{}" --osdistrib "{}" --oscodename "{}" --osrelease "{}" --osarch "{}" --oshttp "{}"\''.format (curdir, btrootfsmnt, osdistrib, oscodename, osrelease, osarch, oshttp)) -cp = subprocess.run (['{}/02-boottoolsFsOpengnsys.py'.format (curdir), '--mntpt', btrootfsmnt, '--osdistrib', osdistrib, '--oscodename', oscodename, '--osrelease', osrelease, '--osarch', osarch, '--oshttp', oshttp], capture_output=True) -logger.debug ('02-boottoolsFsOpengnsys stdout follows: {}'.format (cp.stdout.decode ('utf-8'))) +stdout, _ = _run (['{}/02-boottoolsFsOpengnsys.py'.format (curdir), '--mntpt', btrootfsmnt, '--osdistrib', osdistrib, '--oscodename', oscodename, '--osrelease', osrelease, '--osarch', osarch, '--oshttp', oshttp]) +logger.debug ('02-boottoolsFsOpengnsys stdout follows: {}'.format (stdout)) ## /tmp/02.py --osdistrib ubuntu --oscodename focal --osrelease 5.4.0-42-generic --osarch amd64 --oshttp http://es.archive.ubuntu.com/ubuntu/ ## /tmp/opengnsys_installer/opengnsys/client/./boot-tools/02-boottoolsFsOpengnsys.py --mntpt "/var/lib/tftpboot/ogclient/ogclientmount" --osdistrib "Ubuntu" --oscodename "noble" --osrelease "6.8.0-39-generic" --osarch "amd64" --oshttp "http://es.archive.ubuntu.com/ubuntu/" +_run (['umount', btrootfsmnt]) + ############################################################################################ logger.info ('FASE 5 - Instalar software') logger.info ('Fase 5.1 instalar paquetes deb con apt-get') logger.debug ('running \'schroot --chroot IMGogclient -- {}/03-boottoolsSoftwareInstall.py --osrelease "{}" --osarch "{}"\''.format (curdir, osrelease, osarch)) -cp = subprocess.run (['schroot', '--chroot', 'IMGogclient', '--', '{}/03-boottoolsSoftwareInstall.py'.format (curdir), '--osrelease', osrelease, '--osarch', osarch]) -logger.debug ('03-boottoolsSoftwareInstall stdout follows: {}'.format (cp.stdout.decode ('utf-8'))) +stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', '{}/03-boottoolsSoftwareInstall.py'.format (curdir), '--osrelease', osrelease, '--osarch', osarch]) +logger.debug ('03-boottoolsSoftwareInstall stdout follows: {}'.format (stdout)) """ schroot --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSoftwareInstall.sh >>/tmp/fase5.out From 3fffd212da725ff6fff9ce66d96167e9dc6889f8 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Mon, 12 Aug 2024 14:58:08 +0200 Subject: [PATCH 07/45] refs #596 add config file, install more packages per the config --- 03-boottoolsSoftwareInstall.py | 47 +++++++++++---------------- boottoolsfunctions/__init__.py | 6 ++-- boottoolsgenerator.cfg | 58 ++++++++++++++++++++++++++++++++++ boottoolsgenerator.py | 7 +++- 4 files changed, 85 insertions(+), 33 deletions(-) create mode 100644 boottoolsgenerator.cfg diff --git a/03-boottoolsSoftwareInstall.py b/03-boottoolsSoftwareInstall.py index ecc417f..dc56ffb 100755 --- a/03-boottoolsSoftwareInstall.py +++ b/03-boottoolsSoftwareInstall.py @@ -4,9 +4,13 @@ import argparse import os import re import subprocess +import configparser from boottoolsfunctions import _run +config = configparser.ConfigParser (inline_comment_prefixes='#') +config.read ('boottoolsgenerator.cfg') + debconf_settings = """ kexec-tools kexec-tools/load_kexec boolean true openssh-server openssh-server/permit-root-login boolean true @@ -14,22 +18,22 @@ refind refind/install_to_esp boolean false """.strip() def _aptget_clean(): - subprocess.run (['apt-get', 'clean']) + _run (['apt-get', 'clean']) def _aptget_autoclean(): - subprocess.run (['apt-get', 'autoclean']) + _run (['apt-get', 'autoclean']) def _aptget_autoremove(): - subprocess.run (['apt-get', 'autoremove']) + _run (['apt-get', 'autoremove']) def _aptget_update(): - subprocess.run (['apt-get', 'update']) + _run (['apt-get', 'update']) def _aptget_upgrade(): - subprocess.run (['apt-get', 'upgrade', '--yes']) + _run (['apt-get', 'upgrade', '--yes']) def _aptget_install (pkgs, opts=[]): - subprocess.run (['apt-get', '--yes'] + opts + ['install'] + pkgs) + _run (['apt-get', '--yes'] + opts + ['install'] + pkgs) def _oghook_deactivate(): #Desactivamos el hook del oginitrd.img para evitar problemas, al final de este escripts se activará @@ -57,7 +61,7 @@ def boottoolsSoftwareInstall (osarch, osrelease): stdout, _ = _run (['dpkg-divert', '--list']) if not re.findall (r'local diversion of /sbin/initctl to /sbin/initctl.distrib', stdout): - subprocess.run (['dpkg-divert', '--local', '--rename', '--add', '/sbin/initctl']) + _run (['dpkg-divert', '--local', '--rename', '--add', '/sbin/initctl']) os.symlink ('/bin/true', '/sbin/initctl') #TEST desactivamos upstart @@ -66,17 +70,10 @@ def boottoolsSoftwareInstall (osarch, osrelease): _aptget_clean() pkgs32 = [] if 'i386' != osarch: - subprocess.run (['dpkg', '--add-architecture', 'i386']) + _run (['dpkg', '--add-architecture', 'i386']) pkgs32 = 'lib32gcc-s1 lib32stdc++6 lib32z1 libc6-i386'.split (' ') ## he cambiado lib32gcc1 por lib32gcc-s1 pero como queramos crear un oglive viejo, esto va a petar _aptget_update() ## esto ya esta hecho... -#Ign:13 http://ppa.launchpad.net/zfs-native/stable/ubuntu noble InRelease -#Err:15 http://ppa.launchpad.net/zfs-native/stable/ubuntu noble Release -# 404 Not Found [IP: 185.125.190.80 80] -# -#E: The repository 'http://ppa.launchpad.net/zfs-native/stable/ubuntu noble Release' does not have a Release file. -#N: Updating from such a repository can't be done securely, and is therefore disabled by default. -#N: See apt-secure(8) manpage for repository creation and user configuration details. _oghook_deactivate() _aptget_upgrade() _mock_mtab() @@ -89,21 +86,13 @@ def boottoolsSoftwareInstall (osarch, osrelease): ## nati: hace falta --force-confdef para evitar un tema interactivo del /etc/ssh/ssh_config _aptget_install (['sshfs', 'kexec-tools'] + pkgs32, opts=['-o', 'DPkg::Options::=--force-confdef']) - """ - #comenzamos con la instalación de los paquetes a instalar. - to_install = {} - for group in `find /usr/bin/boot-tools/listpackages/ -name sw.*`; do - echo "Instalando el grupo de paquetes almacenados en $group" - for package in ` awk /^install/'{print $2}' $group `; do - echo -n $package' ' >&2 - to_install[package] = 1 - done - echo >&2 - done - del to_install['refind'] ## nati: este paquete no existe y toda la invocación de apt se muere (no sé cómo hacer que apt ignore los paquetes que no existen) - echo =========================================== nati pkgs to install ${!TO_INSTALL[*]} - _aptget_install (list (to_install.keys())) + pkgs = [] + for section in config.options('Packages'): + pkgs += re.split (r'[ \n]', config['Packages'].get(section).strip()) + print ('about to install these packages: "{}"'.format (' '.join (pkgs))) + _aptget_install (pkgs) + """ # Instalar módulos que algunos paquetes puedan tener pendientes de compilar. echo "Instalando módulos adicionales con DKMS" while read -e mod vers; do diff --git a/boottoolsfunctions/__init__.py b/boottoolsfunctions/__init__.py index b66426d..a9b54cd 100644 --- a/boottoolsfunctions/__init__.py +++ b/boottoolsfunctions/__init__.py @@ -30,7 +30,7 @@ def _is_mounted (mntpt): def _umount (mntpt): if (_is_mounted (mntpt)): - subprocess.run (['umount', mntpt]) + _run (['umount', mntpt]) def btogGetVar (osarch): btdir = '/tmp/opengnsys_installer/opengnsys/client/boot-tools' @@ -209,7 +209,7 @@ def btogSetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisks raise Exception ('no diskloop') logger.debug ('losetup attach') - try: _run (['losetup', diskloop, btrootfsimg]) + try: _run (['losetup', '--partscan', diskloop, btrootfsimg]) except: raise Exception ('losetup failed') logger.info ('Partitioning disk image') @@ -304,7 +304,7 @@ def btogSetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp): try: _run (['debootstrap', '--arch={}'.format(osarch), '--components=main,universe', oscodename, btrootfsmnt, oshttp]) except: if (_is_mounted (btrootfsmnt)): - subprocess.run (['umount', btrootfsmnt]) + _run (['umount', btrootfsmnt]) raise Exception ('debootstrap --arch="{}" --components=main,universe "{}" "{}" "{}" : ha fallado!'.format (osarch, oscodename, btrootfsmnt, oshttp)) logger.info ('debootstrap --arch="{}" --components=main,universe "{}" "{}" "{}" : ok'.format (osarch, oscodename, btrootfsmnt, oshttp)) diff --git a/boottoolsgenerator.cfg b/boottoolsgenerator.cfg new file mode 100644 index 0000000..46e28ce --- /dev/null +++ b/boottoolsgenerator.cfg @@ -0,0 +1,58 @@ +[Packages] + +basic = + dialog man-db fbset gdebi-core bash-static busybox-static locales lshw gawk git python-openssl python php-cli dmidecode rpm sqlite3 os-prober moreutils jq + #console-data # ogLive anterior a Ubuntu 20.04 + #subversion # ogLive anterior a Ubuntu 18.04 + #php5-cli # ogLive anterior a Ubuntu 16.04 + #realpath # ogLive hasta Ubuntu 16.04 + #zypper # PROBAR + +cloning = fsarchiver pv pxe-kexec mbuffer parted gdisk chntpw clonezilla registry-tools reglookup libparse-win32registry-perl libwin-hivex-perl grokevt libhivex0 libhivex-bin rsync + +compile = + build-essential libattr1 libattr1-dev uuid-dev attr make m4 gettext libmhash-dev + #libattr* # Satura el tamaño del archivo en Ubuntu 15.04 + +compressor = lzma zip unzip gzip lzop pigz pbzip2 lbzip2 rzip p7zip-full + +filesystem_local = + squashfs-tools unionfs-fuse ntfs-3g dosfstools fatresize exfat-utils dmraid dmsetup lvm2 e2fsprogs + jfsutils reiserfsprogs reiser4progs xfsprogs mhddfs hfsplus hfsprogs hfsutils nilfs-tools vmfs-tools + btrfs-progs # ogLive a partir de Ubuntu 20.04 + zfsutils-linux # ogLive a partir de Ubuntu 16.04 + android-tools-adb # ogLive a partir de Ubuntu 16.04 + android-tools-fastboot # ogLive a partir de Ubuntu 16.04 + f2fs-tools + #drbl-ntfsprogs # Eliminado + #ntfs-config # ogLive anterior a Ubuntu 20.04 + #btrfs-tools # ogLive anterior a Ubuntu 20.04 + #ufsutils # ogLive anterior a Ubuntu 14.04 + #zfsutils # ogLive anterior a Ubuntu 16.04 + #android-tools-fsutils # ogLive entre Ubuntu 16.04 y Ubuntu 20.04 + +filesystem_remote = + sshfs cifs-utils smbclient open-iscsi openssh-server + #nfs-common # Provoca error de instalación en Ubuntu 15.04 + #bittornado # ogLive anterior a Ubuntu 20.04 + +monitoring = htop ncdu bwbar bmon iftop ifstat dstat hdparm sdparm blktool testdisk + +networking = netpipes curl wget tftp dnsutils trickle lighttpd ethtool ssmping tcpdump nmap arping ntpdate ctorrent udpcast #iptraf + +testing = + screen schroot xmlstarlet + efibootmgr efitools refind grub-efi-amd64-bin grub-efi-ia32-bin grub-pc-bin ## UEFI + #libxss1 python-prctl python-six python-requests ## OGAgent + ############## + fusioninventory-agent nvme-cli disktype laptop-detect + #discover casper lupin-casper xinit obconf xserver-xorg x11-xserver-utils xterm network-manager-gnome plymouth-x11 plymouth-label plymouth-theme-ubuntu-logo pcmanfm chromium-browser gtk-theme-switch gtk2-engines murrine-themes + ####################################3 + #gnome-icon-theme gnome-brave-icon-theme dmz-cursor-theme python-wnck python-xlib python-pyinotify python-alsaaudio python-vte maximus gpicview leafpad lxappearance lxmenu-data lxrandr lxterminal nitrogen ttf-ubuntu-font-family time synaptic libglib-perl libgtk2-perl libxml-simple-perl smartmontools gnome-disk-utility policykit-1-gnome policykit-desktop-privileges baobab lshw-gtk usb-creator-gtk wodim curlftpfs libnotify-bin cryptsetup system-config-lvm + +xwindows = + v86d # VESA Kernel 3.7+ + #xorg-dev xorg lxde #+300M + #roxterm gparted #+80M + #openbox midori #xvesa en compilacion + diff --git a/boottoolsgenerator.py b/boottoolsgenerator.py index 6740463..ed5a9b4 100755 --- a/boottoolsgenerator.py +++ b/boottoolsgenerator.py @@ -6,6 +6,7 @@ import logging import subprocess import glob import stat +import configparser curdir = os.path.dirname (__file__) sys.path.insert (0, curdir) @@ -14,9 +15,13 @@ from boottoolsfunctions import _run, _is_mounted, btogGetOsInfo1, btogGetOsInfo2 logging.basicConfig (filename='/tmp/boot-tools_installation.log', filemode='a', format='%(levelname)s %(asctime)s (%(funcName)s) %(message)s', level=logging.INFO) logger = logging.getLogger ('boottools') +#config = configparser.ConfigParser() +#config.read ('boottoolsgenerator.cfg') + if not os.path.exists ('/tmp/opengnsys_installer/opengnsys/client/engine'): branch = 'main' - tmpdir = subprocess.run (['mktemp', '--tmpdir', '--directory', 'oggit.XXXXXX']).stdout.decode ('utf-8').strip() + tmpdir, _ = _run (['mktemp', '--tmpdir', '--directory', 'oggit.XXXXXX']) + logger.debug ('tmpdir "{}"'.format(tmpdir)) _run (['git', 'clone', '-c', 'http.sslVerify=false', '--branch', branch, 'https://ognproject.evlt.uma.es/gitea/opengnsys/opengnsys.git', tmpdir]) _run (['rsync', '-aH', '{}/client/engine'.format(tmpdir), '{}/client/shared'.format(tmpdir), '/tmp/opengnsys_installer/opengnsys/client/']) _run (['rm', '-rf', tmpdir]) From 2a2da12966fe448036d59347b5a98d9f4d08f867 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Mon, 12 Aug 2024 15:29:01 +0200 Subject: [PATCH 08/45] refs #596 install newly added dkms modules --- 03-boottoolsSoftwareInstall.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/03-boottoolsSoftwareInstall.py b/03-boottoolsSoftwareInstall.py index dc56ffb..8ac6c65 100755 --- a/03-boottoolsSoftwareInstall.py +++ b/03-boottoolsSoftwareInstall.py @@ -92,22 +92,15 @@ def boottoolsSoftwareInstall (osarch, osrelease): print ('about to install these packages: "{}"'.format (' '.join (pkgs))) _aptget_install (pkgs) - """ # Instalar módulos que algunos paquetes puedan tener pendientes de compilar. - echo "Instalando módulos adicionales con DKMS" - while read -e mod vers; do - echo -n "Intalando módulo $mod v$vers" - dkms install -m $mod -v $vers &>/dev/null - RETVAL=$? - if [ $RETVAL == 0 ]; then - echo " : OK - Módulo instalado correctamente (codigo interno de dkms $RETVAL)" - else - echo " : Error módulo $mod (codigo interno de dkms $RETVAL) " - echo "Pulse [Intro] para continuar" - read - fi - done < <(dkms status 2>/dev/null | awk -F, '$3~/added/ {print $1,$2}') - """ + stdout, _ = _run (['dkms', 'status']) + for l in stdout.strip().split ('\n'): + if not l: continue + print ('l "{}"'.format (l)) + mod, vers, status = l.split (',') + if 'added' in status: + print ('dkms installing {} {}'.format (mod, vers)) + _run (['dkms', 'install', '-m', mod.strip(), '-v', vers.strip()]) _oghook_activate() _restore_mtab() From 7fc700a6ab3752eb733458b0b39a69a3721ba1a3 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Mon, 12 Aug 2024 15:30:57 +0200 Subject: [PATCH 09/45] refs #596 remove unused files, remove non-existing apt repo --- includes/etc/apt/sources.list.ubuntu | 4 +- .../bin/boot-tools/listpackages/sw.Xwindow | 12 ---- .../usr/bin/boot-tools/listpackages/sw.basic | 25 ------- .../bin/boot-tools/listpackages/sw.cloning | 17 ----- .../bin/boot-tools/listpackages/sw.compile | 11 --- .../bin/boot-tools/listpackages/sw.compresor | 11 --- .../listpackages/sw.filesystem.local | 32 --------- .../listpackages/sw.filesystem.remote | 8 --- .../bin/boot-tools/listpackages/sw.monitoring | 12 ---- .../bin/boot-tools/listpackages/sw.networking | 17 ----- .../bin/boot-tools/listpackages/sw.testing | 71 ------------------- 11 files changed, 2 insertions(+), 218 deletions(-) delete mode 100644 includes/usr/bin/boot-tools/listpackages/sw.Xwindow delete mode 100644 includes/usr/bin/boot-tools/listpackages/sw.basic delete mode 100644 includes/usr/bin/boot-tools/listpackages/sw.cloning delete mode 100644 includes/usr/bin/boot-tools/listpackages/sw.compile delete mode 100644 includes/usr/bin/boot-tools/listpackages/sw.compresor delete mode 100644 includes/usr/bin/boot-tools/listpackages/sw.filesystem.local delete mode 100644 includes/usr/bin/boot-tools/listpackages/sw.filesystem.remote delete mode 100644 includes/usr/bin/boot-tools/listpackages/sw.monitoring delete mode 100644 includes/usr/bin/boot-tools/listpackages/sw.networking delete mode 100644 includes/usr/bin/boot-tools/listpackages/sw.testing diff --git a/includes/etc/apt/sources.list.ubuntu b/includes/etc/apt/sources.list.ubuntu index 51b19a4..1243416 100644 --- a/includes/etc/apt/sources.list.ubuntu +++ b/includes/etc/apt/sources.list.ubuntu @@ -55,6 +55,6 @@ deb http://free.nchc.org.tw/ubuntu OSCODENAME-security main restricted universe #deb http://ppa.launchpad.net/randomaction/ppa/ubuntu OSCODENAME main #deb-src http://ppa.launchpad.net/randomaction/ppa/ubuntu OSCODENAME main # Ubuntu-ZFS -deb http://ppa.launchpad.net/zfs-native/stable/ubuntu OSCODENAME main -deb-src http://ppa.launchpad.net/zfs-native/stable/ubuntu OSCODENAME main +#deb http://ppa.launchpad.net/zfs-native/stable/ubuntu OSCODENAME main +#deb-src http://ppa.launchpad.net/zfs-native/stable/ubuntu OSCODENAME main diff --git a/includes/usr/bin/boot-tools/listpackages/sw.Xwindow b/includes/usr/bin/boot-tools/listpackages/sw.Xwindow deleted file mode 100644 index 2f383b9..0000000 --- a/includes/usr/bin/boot-tools/listpackages/sw.Xwindow +++ /dev/null @@ -1,12 +0,0 @@ -#sw.X -#+300M -#install xorg-dev -#install xorg -#install lxde -#+80M roxterm y gparted -#install roxterm -#install gparted -#xvesa en compilacion -#install openbox -#install midori -install v86d # VESA Kernel 3.7+ diff --git a/includes/usr/bin/boot-tools/listpackages/sw.basic b/includes/usr/bin/boot-tools/listpackages/sw.basic deleted file mode 100644 index a1b89f5..0000000 --- a/includes/usr/bin/boot-tools/listpackages/sw.basic +++ /dev/null @@ -1,25 +0,0 @@ -#Format install paquete -install dialog -install man-db -install fbset -install gdebi-core -install bash-static -install busybox-static -#install console-data # ogLive anterior a Ubuntu 20.04 -install locales -install lshw -install gawk -#install subversion # ogLive anterior a Ubuntu 18.04 -install git # ogLive a partir de Ubuntu 18.04 -install python-openssl -install python -#install php5-cli # ogLive anterior a Ubuntu 16.04 -install php-cli # ogLive a partir de Ubuntu 16.04 -install dmidecode -#install realpath # ogLive hasta Ubuntu 16.04 -install rpm -#install zypper # PROBAR -install sqlite3 -install os-prober -install moreutils -install jq diff --git a/includes/usr/bin/boot-tools/listpackages/sw.cloning b/includes/usr/bin/boot-tools/listpackages/sw.cloning deleted file mode 100644 index 3e6a766..0000000 --- a/includes/usr/bin/boot-tools/listpackages/sw.cloning +++ /dev/null @@ -1,17 +0,0 @@ -#sw.cloning -install fsarchiver -install pv -install pxe-kexec -install mbuffer -install parted -install gdisk -install chntpw -install clonezilla -install registry-tools -install reglookup -install libparse-win32registry-perl -install libwin-hivex-perl -install grokevt -install libhivex0 -install libhivex-bin -install rsync diff --git a/includes/usr/bin/boot-tools/listpackages/sw.compile b/includes/usr/bin/boot-tools/listpackages/sw.compile deleted file mode 100644 index cc54747..0000000 --- a/includes/usr/bin/boot-tools/listpackages/sw.compile +++ /dev/null @@ -1,11 +0,0 @@ -install build-essential -#install libattr* # Satura el tamaño del archivo en Ubuntu 15.04 -install libattr1 -install libattr1-dev -install uuid-dev -install attr -install make -install m4 -install gettext -install libmhash-dev - diff --git a/includes/usr/bin/boot-tools/listpackages/sw.compresor b/includes/usr/bin/boot-tools/listpackages/sw.compresor deleted file mode 100644 index 196cbce..0000000 --- a/includes/usr/bin/boot-tools/listpackages/sw.compresor +++ /dev/null @@ -1,11 +0,0 @@ -#sw.compressor -install lzma -install zip -install unzip -install gzip -install lzop -install pigz -install pbzip2 -install lbzip2 -install rzip -install p7zip-full diff --git a/includes/usr/bin/boot-tools/listpackages/sw.filesystem.local b/includes/usr/bin/boot-tools/listpackages/sw.filesystem.local deleted file mode 100644 index 7bb08c4..0000000 --- a/includes/usr/bin/boot-tools/listpackages/sw.filesystem.local +++ /dev/null @@ -1,32 +0,0 @@ -#sw.filesystem.local -install squashfs-tools -install unionfs-fuse -#install drbl-ntfsprogs # Eliminado -install ntfs-3g -#install ntfs-config # ogLive anterior a Ubuntu 20.04 -install dosfstools -install fatresize -install exfat-utils -install dmraid -install dmsetup -install lvm2 -install e2fsprogs -#install btrfs-tools # ogLive anterior a Ubuntu 20.04 -install btrfs-progs # ogLive a partir de Ubuntu 20.04 -install jfsutils -install reiserfsprogs -install reiser4progs -install xfsprogs -install mhddfs -install hfsplus -install hfsprogs -install hfsutils -install nilfs-tools -install vmfs-tools -#install ufsutils # ogLive anterior a Ubuntu 14.04 -#install zfsutils # ogLive anterior a Ubuntu 16.04 -install zfsutils-linux # ogLive a partir de Ubuntu 16.04 -install android-tools-adb # ogLive a partir de Ubuntu 16.04 -install android-tools-fastboot # ogLive a partir de Ubuntu 16.04 -#install android-tools-fsutils # ogLive entre Ubuntu 16.04 y Ubuntu 20.04 -install f2fs-tools diff --git a/includes/usr/bin/boot-tools/listpackages/sw.filesystem.remote b/includes/usr/bin/boot-tools/listpackages/sw.filesystem.remote deleted file mode 100644 index 1050690..0000000 --- a/includes/usr/bin/boot-tools/listpackages/sw.filesystem.remote +++ /dev/null @@ -1,8 +0,0 @@ -#sw.filesystem.remote -#install nfs-common # Provoca error de instalación en Ubuntu 15.04 -install sshfs -install cifs-utils -install smbclient -install open-iscsi -install openssh-server -#install bittornado # ogLive anterior a Ubuntu 20.04 diff --git a/includes/usr/bin/boot-tools/listpackages/sw.monitoring b/includes/usr/bin/boot-tools/listpackages/sw.monitoring deleted file mode 100644 index 49adc6f..0000000 --- a/includes/usr/bin/boot-tools/listpackages/sw.monitoring +++ /dev/null @@ -1,12 +0,0 @@ -#sw.monitoring -install htop -install ncdu -install bwbar -install bmon -install iftop -install ifstat -install dstat -install hdparm -install sdparm -install blktool -install testdisk \ No newline at end of file diff --git a/includes/usr/bin/boot-tools/listpackages/sw.networking b/includes/usr/bin/boot-tools/listpackages/sw.networking deleted file mode 100644 index 42d44e8..0000000 --- a/includes/usr/bin/boot-tools/listpackages/sw.networking +++ /dev/null @@ -1,17 +0,0 @@ -#sw.networking -install netpipes -install curl -install wget -install tftp -install dnsutils -install trickle -install lighttpd -install ethtool -install ssmping -install tcpdump -install nmap -install arping -#install iptraf -install ntpdate -install ctorrent -install udpcast diff --git a/includes/usr/bin/boot-tools/listpackages/sw.testing b/includes/usr/bin/boot-tools/listpackages/sw.testing deleted file mode 100644 index 41d8a30..0000000 --- a/includes/usr/bin/boot-tools/listpackages/sw.testing +++ /dev/null @@ -1,71 +0,0 @@ -#sw.other -install screen -install schroot -install xmlstarlet -############# UEFI -install efibootmgr -install efitools -install refind -install grub-efi-amd64-bin -install grub-efi-ia32-bin -install grub-pc-bin -############# OGAgent -#install libxss1 -#install python-prctl -#install python-six -#install python-requests -############## -install fusioninventory-agent -install nvme-cli -install disktype -install laptop-detect -#install discover -#install casper -#install lupin-casper -#install xinit -#install obconf -#install xserver-xorg -#install x11-xserver-utils -#install xterm network-manager-gnome -#install plymouth-x11 -#install plymouth-label -#install plymouth-theme-ubuntu-logo -#install pcmanfm -#install chromium-browser -#install gtk-theme-switch -#install gtk2-engines -#install murrine-themes -####################################3 -#install gnome-icon-theme -#install gnome-brave-icon-theme -#install dmz-cursor-theme -#install python-wnck -#install python-xlib -#install python-pyinotify -#install python-alsaaudio -#install python-vte maximus -#install gpicview -#install leafpad -#install lxappearance -#install lxmenu-data -#install lxrandr -#install lxterminal -#install nitrogen -#install ttf-ubuntu-font-family -#install time -#install synaptic -#install libglib-perl -#install libgtk2-perl -#install libxml-simple-perl -#install smartmontools -#install gnome-disk-utility -#install policykit-1-gnome -#install policykit-desktop-privileges -#install baobab -#install lshw-gtk -#install usb-creator-gtk -#install wodim -#install curlftpfs -#install libnotify-bin -#install cryptsetup -#install system-config-lvm From 8419065f0bb04fce0c04284482e997192f9f533a Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Mon, 12 Aug 2024 16:26:23 +0200 Subject: [PATCH 10/45] refs #596 add 04-boottoolsSoftwareCompile --- 04-boottoolsSoftwareCompile.py | 40 ++++++++++++++++++++++++++++++++++ boottoolsgenerator.py | 6 +++-- 2 files changed, 44 insertions(+), 2 deletions(-) create mode 100755 04-boottoolsSoftwareCompile.py diff --git a/04-boottoolsSoftwareCompile.py b/04-boottoolsSoftwareCompile.py new file mode 100755 index 0000000..cf85567 --- /dev/null +++ b/04-boottoolsSoftwareCompile.py @@ -0,0 +1,40 @@ +#!/usr/bin/python3 + +import os +import subprocess +from boottoolsfunctions import _run + +os.environ['LANGUAGE'] = os.environ['LC_ALL'] = os.environ['LANG'] = 'C' + +#source /opt/opengnsys/lib/engine/bin/ToolsGNU.c &>/dev/null + +os.chdir ('/tmp') + +## apt-get install wget gettext unzip git + +print ('ms-sys') +try: _run (['which', 'ms-sys']) +except: + _run (['wget', 'https://sourceforge.net/projects/ms-sys/files/latest/download', '-O', 'ms-sys.tar.gz']) + _run (['tar', '-xpzf', 'ms-sys.tar.gz']) + mssys_dir = subprocess.run (['tar tzf ms-sys.tar.gz |head -n 1'], shell=True, capture_output=True, text=True).stdout.strip() + print ('mssys_dir "{}"'.format (mssys_dir)) + os.chdir (mssys_dir) + _run (['make', 'install']) + os.chdir ('..') + +print ('spartlnx') +try: _run (['which', 'spartl64.run']) +except: + _run (['wget', 'http://damien.guibouret.free.fr/savepart.zip']) + _run (['unzip', '-o', 'savepart.zip', '-d', '/sbin/', 'spartl64.run']) + _run (['mkdir', '/usr/share/doc/spartlnx']) + _run (['unzip', '-j', '-o', 'savepart.zip', '-d', '/usr/share/doc/spartlnx/', 'doc/en/*']) + +print ('python-libfdisk') +_run (['apt-get', '-y', 'install', 'python3-psutil', 'python3-dev', 'libfdisk-dev', 'python3-setuptools']) +_run (['git', 'clone', 'git://git.48k.eu/python-libfdisk']) +os.chdir ('python-libfdisk') +_run (['python3', 'setup.py', 'install']) +os.chdir ('..') +_run (['apt-get', '-y', 'remove', 'python3-dev', 'python3-setuptools']) diff --git a/boottoolsgenerator.py b/boottoolsgenerator.py index ed5a9b4..897e488 100755 --- a/boottoolsgenerator.py +++ b/boottoolsgenerator.py @@ -147,9 +147,11 @@ logger.debug ('running \'schroot --chroot IMGogclient -- {}/03-boottoolsSoftware stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', '{}/03-boottoolsSoftwareInstall.py'.format (curdir), '--osrelease', osrelease, '--osarch', osarch]) logger.debug ('03-boottoolsSoftwareInstall stdout follows: {}'.format (stdout)) -""" -schroot --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSoftwareInstall.sh >>/tmp/fase5.out logger.info ('Fase 5.2 compilar software.') +logger.debug ('running \'schroot --chroot IMGogclient -- {}/04-boottoolsSoftwareCompile.py\'') +stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', '{}/04-boottoolsSoftwareCompile.py'.format (curdir)]) +logger.debug ('04-boottoolsSoftwareCompile stdout follows: {}'.format (stdout)) +""" cd / schroot --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSoftwareCompile.sh >>/tmp/fase5.out cd - From a193b678c052794f398182819e0709fc3eead7ef Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Tue, 13 Aug 2024 09:42:36 +0200 Subject: [PATCH 11/45] refs #596 use sys.exit(), improve logging, remove uninstallable packages --- 03-boottoolsSoftwareInstall.py | 4 ++++ boottoolsfunctions/__init__.py | 8 +++++++- boottoolsgenerator.cfg | 18 ++++++++++++------ boottoolsgenerator.py | 24 ++++++++++++++++-------- 4 files changed, 39 insertions(+), 15 deletions(-) diff --git a/03-boottoolsSoftwareInstall.py b/03-boottoolsSoftwareInstall.py index 8ac6c65..d7f06a1 100755 --- a/03-boottoolsSoftwareInstall.py +++ b/03-boottoolsSoftwareInstall.py @@ -2,6 +2,7 @@ import argparse import os +import sys import re import subprocess import configparser @@ -9,6 +10,9 @@ import configparser from boottoolsfunctions import _run config = configparser.ConfigParser (inline_comment_prefixes='#') +if not os.path.exists ('boottoolsgenerator.cfg'): ## si pongo uno que no existe, + print ('configuration file "boottoolsgenerator.cfg" not found') ## este error no llega al caller... + sys.exit (1) config.read ('boottoolsgenerator.cfg') debconf_settings = """ diff --git a/boottoolsfunctions/__init__.py b/boottoolsfunctions/__init__.py index a9b54cd..9649fe2 100644 --- a/boottoolsfunctions/__init__.py +++ b/boottoolsfunctions/__init__.py @@ -12,8 +12,14 @@ logger = logging.getLogger ('boottools') def _run (args): cp = subprocess.run (args, text=True, capture_output=True) if cp.returncode: - logger.error ('stdout "{}" stderr "{}"'.format (cp.stdout, cp.stderr)) logger.error ('command "{}" failed with rc "{}"'.format (' '.join(args), cp.returncode)) + + logger.error ('stdout follows:') + for i in cp.stdout.strip().split('\n'): logger.error (' ' + i) + + logger.error ('stderr follows:') + for i in cp.stderr.strip().split('\n'): logger.error (' ' + i) + raise Exception ('command "{}" failed with rc "{}"'.format (' '.join(args), cp.returncode)) stdout = cp.stdout.strip() stderr = cp.stderr.strip() diff --git a/boottoolsgenerator.cfg b/boottoolsgenerator.cfg index 46e28ce..f05027c 100644 --- a/boottoolsgenerator.cfg +++ b/boottoolsgenerator.cfg @@ -1,23 +1,27 @@ [Packages] basic = - dialog man-db fbset gdebi-core bash-static busybox-static locales lshw gawk git python-openssl python php-cli dmidecode rpm sqlite3 os-prober moreutils jq + dialog man-db fbset gdebi-core bash-static busybox-static locales lshw gawk git python3-openssl python3 php-cli dmidecode rpm sqlite3 os-prober moreutils jq #console-data # ogLive anterior a Ubuntu 20.04 #subversion # ogLive anterior a Ubuntu 18.04 #php5-cli # ogLive anterior a Ubuntu 16.04 #realpath # ogLive hasta Ubuntu 16.04 #zypper # PROBAR -cloning = fsarchiver pv pxe-kexec mbuffer parted gdisk chntpw clonezilla registry-tools reglookup libparse-win32registry-perl libwin-hivex-perl grokevt libhivex0 libhivex-bin rsync +cloning = fsarchiver pv + #pxe-kexec # ubuntu noble: Unable to locate package + mbuffer parted gdisk chntpw clonezilla registry-tools reglookup libparse-win32registry-perl libwin-hivex-perl grokevt libhivex0 libhivex-bin rsync compile = build-essential libattr1 libattr1-dev uuid-dev attr make m4 gettext libmhash-dev - #libattr* # Satura el tamaño del archivo en Ubuntu 15.04 + #libattr* # Satura el tamaño del archivo en Ubuntu 15.04 compressor = lzma zip unzip gzip lzop pigz pbzip2 lbzip2 rzip p7zip-full filesystem_local = - squashfs-tools unionfs-fuse ntfs-3g dosfstools fatresize exfat-utils dmraid dmsetup lvm2 e2fsprogs + squashfs-tools unionfs-fuse ntfs-3g dosfstools fatresize + #exfat-utils # ubuntu noble: has no installation candidate + dmraid dmsetup lvm2 e2fsprogs jfsutils reiserfsprogs reiser4progs xfsprogs mhddfs hfsplus hfsprogs hfsutils nilfs-tools vmfs-tools btrfs-progs # ogLive a partir de Ubuntu 20.04 zfsutils-linux # ogLive a partir de Ubuntu 16.04 @@ -34,11 +38,13 @@ filesystem_local = filesystem_remote = sshfs cifs-utils smbclient open-iscsi openssh-server #nfs-common # Provoca error de instalación en Ubuntu 15.04 - #bittornado # ogLive anterior a Ubuntu 20.04 + #bittornado # ogLive anterior a Ubuntu 20.04 monitoring = htop ncdu bwbar bmon iftop ifstat dstat hdparm sdparm blktool testdisk -networking = netpipes curl wget tftp dnsutils trickle lighttpd ethtool ssmping tcpdump nmap arping ntpdate ctorrent udpcast #iptraf +networking = netpipes curl wget tftp-hpa dnsutils + #trickle # ubuntu noble: has no installation candidate + lighttpd ethtool ssmping tcpdump nmap arping ntpdate ctorrent udpcast #iptraf testing = screen schroot xmlstarlet diff --git a/boottoolsgenerator.py b/boottoolsgenerator.py index 897e488..576cc9c 100755 --- a/boottoolsgenerator.py +++ b/boottoolsgenerator.py @@ -12,7 +12,15 @@ curdir = os.path.dirname (__file__) sys.path.insert (0, curdir) from boottoolsfunctions import _run, _is_mounted, btogGetOsInfo1, btogGetOsInfo2, btogGetVar, btogSetFsVirtual, btogSetFsAccess, btogSetFsBase -logging.basicConfig (filename='/tmp/boot-tools_installation.log', filemode='a', format='%(levelname)s %(asctime)s (%(funcName)s) %(message)s', level=logging.INFO) +#logging.root.handlers = [] +logging.basicConfig ( + format='%(levelname)s %(asctime)s (%(funcName)s) %(message)s', + level=logging.INFO, + handlers = [ + logging.FileHandler ('/tmp/boot-tools_installation.log'), + logging.StreamHandler (sys.stdout), + ], +) logger = logging.getLogger ('boottools') #config = configparser.ConfigParser() @@ -33,9 +41,9 @@ type_client = sys.argv[1] if len(sys.argv)>1 else 'host' if os.getuid(): logger.error ('ERROR: this program must run under root privileges!!') - os._exit (1) + sys.exit (1) -os.chdir ('/tmp') +#os.chdir ('/tmp') #for i in `mount | grep IMGogclient | grep /var | cut -f3 -d" "`; do echo $i; umount $i; done #for i in `mount | grep IMGogclient | grep /var | cut -f3 -d" "`; do echo $i; umount $i; done @@ -63,7 +71,7 @@ elif acse_gpxe: pxepkg = 'gpxe' else: logger.error ('neither gpxe nor ipxe found in apt-cache') - os._exit (1) + sys.exit (1) logger.info ('PXE package is "{}"'.format (pxepkg)) ## TODO qemu no existe, hace falta? _run (['apt-get', '-y', 'install', 'jq', 'syslinux', 'syslinux-efi', 'syslinux-utils', 'debootstrap', 'subversion', 'schroot', 'squashfs-tools', 'syslinux', 'genisoimage', 'qemu-utils', 'lsof', pxepkg]) #>>/tmp/fase2.out @@ -78,7 +86,7 @@ if (rc): ## 'file|grep' failed try: btogSetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, bttargetdir, osarch) except Exception as e: logger.error (str (e)) - os._exit (2) + sys.exit (2) #echo ================= nati after btogSetFsVirtual; ls -la /opt/opengnsys/tftpboot/ogclient/ logger.info ('Fase 3.2 - Configurar acceso schroot al Segundo Sistema de archivos (img)') @@ -97,11 +105,11 @@ if (cp.returncode): try: _run (['mount', btrootfsimg, btrootfsmnt, '-o', 'loop,offset=32256']) except: logger.error ('mount failed') - os._exit (3) + sys.exit (3) try: btogSetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp) #>>/tmp/fase3.out except Exception as e: logger.error (str (e)) - os._exit (3) + sys.exit (3) #echo ================= nati after fase 3; ls -la /opt/opengnsys/tftpboot/ogclient/ ########################################################################### @@ -129,7 +137,7 @@ if (not _is_mounted (btrootfsmnt)): try: _run (['mount', btrootfsimg, btrootfsmnt, '-o', 'loop,offset=32256']) except: logger.error ('mount failed') - os._exit (3) + sys.exit (3) logger.debug ('running \'{}/02-boottoolsFsOpengnsys.py --mntpt "{}" --osdistrib "{}" --oscodename "{}" --osrelease "{}" --osarch "{}" --oshttp "{}"\''.format (curdir, btrootfsmnt, osdistrib, oscodename, osrelease, osarch, oshttp)) stdout, _ = _run (['{}/02-boottoolsFsOpengnsys.py'.format (curdir), '--mntpt', btrootfsmnt, '--osdistrib', osdistrib, '--oscodename', oscodename, '--osrelease', osrelease, '--osarch', osarch, '--oshttp', oshttp]) From 960b66ea8117b14158c281e576eb523438ba21d6 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Tue, 13 Aug 2024 12:22:02 +0200 Subject: [PATCH 12/45] refs #596 add ssh config, 05-boottoolsFsLocales.py and 06-boottoolsInitrdGenerate.py --- 05-boottoolsFsLocales.py | 16 ++++++++ 06-boottoolsInitrdGenerate.py | 21 ++++++++++ boottoolsfunctions/__init__.py | 70 ++++++++++++++++++++++++++++++++++ boottoolsgenerator.py | 51 ++++++++++++++++--------- 4 files changed, 139 insertions(+), 19 deletions(-) create mode 100755 05-boottoolsFsLocales.py create mode 100755 06-boottoolsInitrdGenerate.py diff --git a/05-boottoolsFsLocales.py b/05-boottoolsFsLocales.py new file mode 100755 index 0000000..f61b469 --- /dev/null +++ b/05-boottoolsFsLocales.py @@ -0,0 +1,16 @@ +#!/usr/bin/python3 + +import os +import subprocess + +debconf_settings = """ +console-setup console-setup/charmap47 select UTF-8 +console-setup console-setup/codeset47 select . Combined - Latin; Slavic Cyrillic; Greek +console-setup console-setup/fontface47 select VGA +console-setup console-setup/fontsize-fb47 select 8x16 +""".strip() + +subprocess.run (['debconf-set-selections'], input=debconf_settings, text=True) + +os.environ['DEBIAN_FRONTEND'] = 'noninteractive' +_run (['dpkg-reconfigure', 'console-setup', 'locales']) diff --git a/06-boottoolsInitrdGenerate.py b/06-boottoolsInitrdGenerate.py new file mode 100755 index 0000000..328bf18 --- /dev/null +++ b/06-boottoolsInitrdGenerate.py @@ -0,0 +1,21 @@ +#!/usr/bin/python3 + +import argparse +import shutil +import os +import glob +from boottoolsfunctions import _run + +def boottoolsInitrdGenerate (osrelease): + for f in glob.glob ('/usr/lib/initramfs-tools/bin/*'): + os.unlink (f) + shutil.copy ('/bin/busybox', '/usr/lib/initramfs-tools/bin') + os.chdir ('/tmp') + _run (['mkinitramfs', '-o', '/tmp/initrd.img-{}'.format (osrelease), '-v', osrelease]) + shutil.copy ('/boot/vmlinuz-{}'.format (osrelease), '/tmp/') + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument ('--osrelease', help='OS release', action='store', required=True) + args = parser.parse_args() + boottoolsInitrdGenerate (args.osrelease) diff --git a/boottoolsfunctions/__init__.py b/boottoolsfunctions/__init__.py index 9649fe2..143ee42 100644 --- a/boottoolsfunctions/__init__.py +++ b/boottoolsfunctions/__init__.py @@ -315,3 +315,73 @@ def btogSetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp): logger.info ('debootstrap --arch="{}" --components=main,universe "{}" "{}" "{}" : ok'.format (osarch, oscodename, btrootfsmnt, oshttp)) return 0 + +def boottoolsSshServer(): + if not os.path.exists ('/root/.ssh/id_rsa'): ## crea un par de claves en la VM, no en el chroot + _run (['ssh-keygen', '-q', '-f', '/root/.ssh/id_rsa', '-N', '']) + #shutil.copy ('/root/.ssh/id_rsa.pub', '/tmp/') + + logger.debug ('comprobando directorio .ssh del root') + if not os.path.exists ('{}/root/.ssh'.format (btrootfsmnt)): ## crea directorio dentro del chroot + logger.debug ('creando directorio .ssh 600') + os.mkdir ('{}/root/.ssh'.format (btrootfsmnt)) + os.chmod (0o700, '{}/root/.ssh'.format (btrootfsmnt)) + + logger.debug ('creando el fichero authorized_keys') ## crea archivo en el chroot + if not os.path.exists ('{}/root/.ssh/authorized_keys'.format (btrootfsmnt)): + open ('{}/root/.ssh/authorized_keys'.format (btrootfsmnt), 'w').close() + os.chmod (0o600, '{}/root/.ssh/authorized_keys'.format (btrootfsmnt)) + + logger.debug ('importando la clave publica del servidor OG') + #cat /tmp/id_rsa.pub + if os.path.exists ('/root/.ssh/id_rsa.pub'): ## coge la publica de la VM y la pone en el authorized_keys del chroot + #cat /tmp/id_rsa.pub >> /root/.ssh/authorized_keys + fdin = open ('/root/.ssh/id_rsa.pub', 'r') + fdout = open ('{}/root/.ssh/authorized_keys'.format (btrootfsmnt), 'a') + while True: + l = fdin.readline() + if not l: break + fdout.write (l) + fdin.close() + fdout.close() + + else: + logger.error ('no key publica og') + +def boottoolsSshClient(): + _run (['ssh-keygen', '-q', '-f', '{}/root/.ssh/id_rsa'.format (btrootfsmnt), '-N', '']) ## crea un par de claves en el chroot + + #cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys ## coge la publica y se la autoriza a sí mismo + fdin = open ('{}//root/.ssh/id_rsa.pub'.format (btrootfsmnt), 'r') + fdout = open ('{}/root/.ssh/authorized_keys'.format (btrootfsmnt), 'a') + while True: + l = fdin.readline() + if not l: break + fdout.write (l) + fdin.close() + fdout.close() + + ## TODO: exportamos la publica a los repos + #cp /root/.ssh/id_rsa.pub /tmp/rsa.ogclient.pub + +#btogFsInitrd genera un initrd. +def btogFsInitrd(): + pass + #echo "cp /tmp/*-${OSRELEASE} ${BTTARGETDIR}" ## esto copia algo?? + #cp /tmp/*-${OSRELEASE} ${BTTARGETDIR} + + ## backup de oginitrd.img, oginitrd.img.sum, ogvmlinuz y ogvmlinuz.sum + #TIMEMOD=`date +%Y%m%d-%H%M%S` + #[ -f $BTTARGETDIR/oginitrd.img ] && mv $BTTARGETDIR/oginitrd.img $BTTARGETDIR/oginitrd.img.$TIMEMOD; mv $BTTARGETDIR/oginitrd.img.sum $BTTARGETDIR/oginitrd.img.sum.$TIMEMOD + #[ -f $BTTARGETDIR/ogvmlinuz ] && mv $BTTARGETDIR/ogvmlinuz $BTTARGETDIR/ogvmlinuz.$TIMEMOD; mv $BTTARGETDIR/ogvmlinuz.sum $BTTARGETDIR/ogvmlinuz.sum.$TIMEMOD + + #cp /tmp/initrd.img-${OSRELEASE} ${BTTARGETDIR}/oginitrd.img + #cp /tmp/vmlinuz-${OSRELEASE} ${BTTARGETDIR}/ogvmlinuz + + #DATASUM=`md5sum "${BTTARGETDIR}/oginitrd.img" | cut -f1 -d" "` + #echo $DATASUM > ${BTTARGETDIR}/oginitrd.img.sum + #DATASUM=`md5sum "${BTTARGETDIR}/ogvmlinuz" | cut -f1 -d" "` + #echo $DATASUM > ${BTTARGETDIR}/ogvmlinuz.sum + + #cd - + #chmod -R 755 $BTTARGETDIR diff --git a/boottoolsgenerator.py b/boottoolsgenerator.py index 576cc9c..579a12b 100755 --- a/boottoolsgenerator.py +++ b/boottoolsgenerator.py @@ -7,10 +7,11 @@ import subprocess import glob import stat import configparser +import shutil curdir = os.path.dirname (__file__) sys.path.insert (0, curdir) -from boottoolsfunctions import _run, _is_mounted, btogGetOsInfo1, btogGetOsInfo2, btogGetVar, btogSetFsVirtual, btogSetFsAccess, btogSetFsBase +from boottoolsfunctions import _run, _is_mounted, btogGetOsInfo1, btogGetOsInfo2, btogGetVar, btogSetFsVirtual, btogSetFsAccess, btogSetFsBase, boottoolsSshServer, boottoolsSshClient, btogFsInitrd #logging.root.handlers = [] logging.basicConfig ( @@ -157,36 +158,48 @@ logger.debug ('03-boottoolsSoftwareInstall stdout follows: {}'.format (stdout)) logger.info ('Fase 5.2 compilar software.') logger.debug ('running \'schroot --chroot IMGogclient -- {}/04-boottoolsSoftwareCompile.py\'') +#cd / stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', '{}/04-boottoolsSoftwareCompile.py'.format (curdir)]) +#cd - logger.debug ('04-boottoolsSoftwareCompile stdout follows: {}'.format (stdout)) -""" -cd / -schroot --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSoftwareCompile.sh >>/tmp/fase5.out -cd - ############################################################################################ logger.info ('FASE 6 - Personalizar el sistema creado') logger.info ('Fase 6.1 Incorporar la clave publica del servidor') -cd / -ssh-keygen -q -f /root/.ssh/id_rsa -N "" -cp /root/.ssh/id_rsa.pub /tmp -schroot --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSshServer.sh -cd - +#cd / +#schroot --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSshServer.sh ## no necesita chroot +boottoolsSshServer() +#cd - + logger.info ('Fase 6.2. Incorpoar la clave publica del propio cliente') -schroot --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSshClient.sh +#schroot --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSshClient.sh ## no necesita chroot +boottoolsSshClient() + +## el resultado es: +## - hay un nuevo par de claves en la VM /root/.ssh +## - hay otro nuevo par de claves en el rootfs /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh +## - las dos claves públicas (una de cada par) están autorizadan en el rootfs /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh/authorized_keys +############################################################################################ logger.info ('Fase 6.3. Configurando las locales') -schroot --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsFsLocales.sh +logger.debug ('running \'schroot --chroot IMGogclient -- {}/05-boottoolsFsLocales.py\'') +stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', '{}/05-boottoolsFsLocales.py'.format (curdir)]) +logger.debug ('05-boottoolsFsLocales stdout follows: {}'.format (stdout)) -for i in `mount | grep IMGogclient | grep /var | cut -f3 -d" "`; do echo $i; umount $i; done -for i in `mount | grep IMGogclient | grep /var | cut -f3 -d" "`; do echo $i; umount $i; done -for i in `mount | grep IMGogclient | grep /var | cut -f3 -d" "`; do echo $i; umount $i; done +if _is_mounted (btrootfsmnt): + _run (['umount', btrootfsmnt]) -echo ================= nati after fase 6; ls -la /opt/opengnsys/tftpboot/ogclient/ +#cd / +#schroot -c IMGogclient -- /usr/bin/boot-tools/boottoolsInitrdGenerate.sh +logger.debug ('running \'schroot --chroot IMGogclient -- {}/06-boottoolsInitrdGenerate.py --osrelease "{}"\''.format (curdir, osrelease)) +stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', '{}/06-boottoolsInitrdGenerate.py'.format (curdir), '--osrelease', osrelease]) +logger.debug ('06-boottoolsInitrdGenerate stdout follows: {}'.format (stdout)) + +btogFsInitrd() -######################################################################### +""" logger.info ('FASE 7 - Generar distribucion') logger.info ('Fase 7.1 Generar el initrd') btogFsInitrd >>/tmp/fase7.out @@ -198,6 +211,6 @@ umount $BTROOTFSMNT 2>/dev/null logger.info ('Fase 7.3 Generar la ISO') btogIsoGenerator >>/tmp/fase7.out echo ================= nati after fase 8.3; ls -la /opt/opengnsys/tftpboot/ogclient/ - -echoAndLog "OpenGnsys installation finished at $(date)" """ + +logger.info ('OpenGnsys installation finished') From be59919977b8e7e9197ff643339a6df873561f1a Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Tue, 13 Aug 2024 13:24:02 +0200 Subject: [PATCH 13/45] refs #596 fix bugs, actually implement btogFsInitrd() --- 04-boottoolsSoftwareCompile.py | 15 ++++++------ 05-boottoolsFsLocales.py | 1 + 06-boottoolsInitrdGenerate.py | 1 + boottoolsfunctions/__init__.py | 44 +++++++++++++++++++++++----------- boottoolsgenerator.py | 21 +++++++++------- 5 files changed, 53 insertions(+), 29 deletions(-) diff --git a/04-boottoolsSoftwareCompile.py b/04-boottoolsSoftwareCompile.py index cf85567..696082f 100755 --- a/04-boottoolsSoftwareCompile.py +++ b/04-boottoolsSoftwareCompile.py @@ -31,10 +31,11 @@ except: _run (['mkdir', '/usr/share/doc/spartlnx']) _run (['unzip', '-j', '-o', 'savepart.zip', '-d', '/usr/share/doc/spartlnx/', 'doc/en/*']) -print ('python-libfdisk') -_run (['apt-get', '-y', 'install', 'python3-psutil', 'python3-dev', 'libfdisk-dev', 'python3-setuptools']) -_run (['git', 'clone', 'git://git.48k.eu/python-libfdisk']) -os.chdir ('python-libfdisk') -_run (['python3', 'setup.py', 'install']) -os.chdir ('..') -_run (['apt-get', '-y', 'remove', 'python3-dev', 'python3-setuptools']) +if not os.path.exists ('python-libfdisk'): + print ('python-libfdisk') + _run (['apt-get', '-y', 'install', 'python3-psutil', 'python3-dev', 'libfdisk-dev', 'python3-setuptools']) + _run (['git', 'clone', 'git://git.48k.eu/python-libfdisk']) + os.chdir ('python-libfdisk') + _run (['python3', 'setup.py', 'install']) + os.chdir ('..') + _run (['apt-get', '-y', 'remove', 'python3-dev', 'python3-setuptools']) diff --git a/05-boottoolsFsLocales.py b/05-boottoolsFsLocales.py index f61b469..a7a5387 100755 --- a/05-boottoolsFsLocales.py +++ b/05-boottoolsFsLocales.py @@ -2,6 +2,7 @@ import os import subprocess +from boottoolsfunctions import _run debconf_settings = """ console-setup console-setup/charmap47 select UTF-8 diff --git a/06-boottoolsInitrdGenerate.py b/06-boottoolsInitrdGenerate.py index 328bf18..2d50941 100755 --- a/06-boottoolsInitrdGenerate.py +++ b/06-boottoolsInitrdGenerate.py @@ -10,6 +10,7 @@ def boottoolsInitrdGenerate (osrelease): for f in glob.glob ('/usr/lib/initramfs-tools/bin/*'): os.unlink (f) shutil.copy ('/bin/busybox', '/usr/lib/initramfs-tools/bin') + os.chdir ('/tmp') _run (['mkinitramfs', '-o', '/tmp/initrd.img-{}'.format (osrelease), '-v', osrelease]) shutil.copy ('/boot/vmlinuz-{}'.format (osrelease), '/tmp/') diff --git a/boottoolsfunctions/__init__.py b/boottoolsfunctions/__init__.py index 143ee42..04498f5 100644 --- a/boottoolsfunctions/__init__.py +++ b/boottoolsfunctions/__init__.py @@ -6,6 +6,7 @@ import logging import subprocess import shutil import glob +import datetime logger = logging.getLogger ('boottools') @@ -316,7 +317,7 @@ def btogSetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp): logger.info ('debootstrap --arch="{}" --components=main,universe "{}" "{}" "{}" : ok'.format (osarch, oscodename, btrootfsmnt, oshttp)) return 0 -def boottoolsSshServer(): +def boottoolsSshServer (btrootfsmnt): if not os.path.exists ('/root/.ssh/id_rsa'): ## crea un par de claves en la VM, no en el chroot _run (['ssh-keygen', '-q', '-f', '/root/.ssh/id_rsa', '-N', '']) #shutil.copy ('/root/.ssh/id_rsa.pub', '/tmp/') @@ -325,12 +326,12 @@ def boottoolsSshServer(): if not os.path.exists ('{}/root/.ssh'.format (btrootfsmnt)): ## crea directorio dentro del chroot logger.debug ('creando directorio .ssh 600') os.mkdir ('{}/root/.ssh'.format (btrootfsmnt)) - os.chmod (0o700, '{}/root/.ssh'.format (btrootfsmnt)) + os.chmod ('{}/root/.ssh'.format (btrootfsmnt), 0o700) logger.debug ('creando el fichero authorized_keys') ## crea archivo en el chroot if not os.path.exists ('{}/root/.ssh/authorized_keys'.format (btrootfsmnt)): open ('{}/root/.ssh/authorized_keys'.format (btrootfsmnt), 'w').close() - os.chmod (0o600, '{}/root/.ssh/authorized_keys'.format (btrootfsmnt)) + os.chmod ('{}/root/.ssh/authorized_keys'.format (btrootfsmnt), 0o600) logger.debug ('importando la clave publica del servidor OG') #cat /tmp/id_rsa.pub @@ -348,8 +349,9 @@ def boottoolsSshServer(): else: logger.error ('no key publica og') -def boottoolsSshClient(): - _run (['ssh-keygen', '-q', '-f', '{}/root/.ssh/id_rsa'.format (btrootfsmnt), '-N', '']) ## crea un par de claves en el chroot +def boottoolsSshClient (btrootfsmnt): + if not os.path.exists ('{}/root/.ssh/id_rsa'.format (btrootfsmnt)): + _run (['ssh-keygen', '-q', '-f', '{}/root/.ssh/id_rsa'.format (btrootfsmnt), '-N', '']) ## crea un par de claves en el chroot #cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys ## coge la publica y se la autoriza a sí mismo fdin = open ('{}//root/.ssh/id_rsa.pub'.format (btrootfsmnt), 'r') @@ -365,23 +367,37 @@ def boottoolsSshClient(): #cp /root/.ssh/id_rsa.pub /tmp/rsa.ogclient.pub #btogFsInitrd genera un initrd. -def btogFsInitrd(): - pass - #echo "cp /tmp/*-${OSRELEASE} ${BTTARGETDIR}" ## esto copia algo?? - #cp /tmp/*-${OSRELEASE} ${BTTARGETDIR} +def btogFsInitrd (bttargetdir, osrelease): + #echo "cp /tmp/*-${OSRELEASE} ${BTTARGETDIR}" + #cp /tmp/*-${OSRELEASE} ${BTTARGETDIR} ## esto copia algo?? ## backup de oginitrd.img, oginitrd.img.sum, ogvmlinuz y ogvmlinuz.sum - #TIMEMOD=`date +%Y%m%d-%H%M%S` - #[ -f $BTTARGETDIR/oginitrd.img ] && mv $BTTARGETDIR/oginitrd.img $BTTARGETDIR/oginitrd.img.$TIMEMOD; mv $BTTARGETDIR/oginitrd.img.sum $BTTARGETDIR/oginitrd.img.sum.$TIMEMOD - #[ -f $BTTARGETDIR/ogvmlinuz ] && mv $BTTARGETDIR/ogvmlinuz $BTTARGETDIR/ogvmlinuz.$TIMEMOD; mv $BTTARGETDIR/ogvmlinuz.sum $BTTARGETDIR/ogvmlinuz.sum.$TIMEMOD + now = datetime.datetime.now(datetime.timezone.utc).strftime ('%Y%m%d-%H%M%S%z') + if os.path.exists ('{}/oginitrd.img'.format (bttargetdir)): + os.rename ('{}/oginitrd.img' .format (bttargetdir), '{}/oginitrd.img.{}' .format (bttargetdir, now)) + os.rename ('{}/oginitrd.img.sum'.format (bttargetdir), '{}/oginitrd.img.sum.{}'.format (bttargetdir, now)) + if os.path.exists ('{}/ogvmlinuz'.format (bttargetdir)): + os.rename ('{}/ogvmlinuz' .format (bttargetdir), '{}/ogvmlinuz.{}' .format (bttargetdir, now)) + os.rename ('{}/ogvmlinuz.sum'.format (bttargetdir), '{}/ogvmlinuz.sum.{}'.format (bttargetdir, now)) - #cp /tmp/initrd.img-${OSRELEASE} ${BTTARGETDIR}/oginitrd.img - #cp /tmp/vmlinuz-${OSRELEASE} ${BTTARGETDIR}/ogvmlinuz + shutil.copy ('/tmp/initrd.img-{}'.format (osrelease), '{}/oginitrd.img'.format (bttargetdir)) + shutil.copy ('/tmp/vmlinuz-{}' .format (osrelease), '{}/ogvmlinuz' .format (bttargetdir)) #DATASUM=`md5sum "${BTTARGETDIR}/oginitrd.img" | cut -f1 -d" "` + md5, _ = _run (['md5sum', '{}/oginitrd.img'.format (bttargetdir)]) + md5, rest = md5.split (' ', 1) #echo $DATASUM > ${BTTARGETDIR}/oginitrd.img.sum + with open ('{}/oginitrd.img.sum'.format (bttargetdir), 'w') as fd: + fd.write (md5) + #DATASUM=`md5sum "${BTTARGETDIR}/ogvmlinuz" | cut -f1 -d" "` + md5, _ = _run (['md5sum', '{}/ogvmlinuz'.format (bttargetdir)]) + md5, rest = md5.split (' ', 1) #echo $DATASUM > ${BTTARGETDIR}/ogvmlinuz.sum + with open ('{}/ogvmlinuz.sum'.format (bttargetdir), 'w') as fd: + fd.write (md5) #cd - #chmod -R 755 $BTTARGETDIR + for f in glob.glob ('{}/oginitrd*'.format (bttargetdir)) + glob.glob ('{}/vmlinuz*'.format (bttargetdir)): + os.chmod (f, 0o755) diff --git a/boottoolsgenerator.py b/boottoolsgenerator.py index 579a12b..187fd5e 100755 --- a/boottoolsgenerator.py +++ b/boottoolsgenerator.py @@ -163,18 +163,22 @@ stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', '{}/04-boottoolsS #cd - logger.debug ('04-boottoolsSoftwareCompile stdout follows: {}'.format (stdout)) +try: _run (['mount', btrootfsimg, btrootfsmnt, '-o', 'loop,offset=32256']) +except: + logger.error ('mount failed') + sys.exit (3) ############################################################################################ logger.info ('FASE 6 - Personalizar el sistema creado') logger.info ('Fase 6.1 Incorporar la clave publica del servidor') #cd / #schroot --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSshServer.sh ## no necesita chroot -boottoolsSshServer() +boottoolsSshServer (btrootfsmnt) #cd - -logger.info ('Fase 6.2. Incorpoar la clave publica del propio cliente') +logger.info ('Fase 6.2 Incorporar la clave publica del propio cliente') #schroot --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSshClient.sh ## no necesita chroot -boottoolsSshClient() +boottoolsSshClient (btrootfsmnt) ## el resultado es: ## - hay un nuevo par de claves en la VM /root/.ssh @@ -182,21 +186,22 @@ boottoolsSshClient() ## - las dos claves públicas (una de cada par) están autorizadan en el rootfs /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh/authorized_keys ############################################################################################ -logger.info ('Fase 6.3. Configurando las locales') +if _is_mounted (btrootfsmnt): + _run (['umount', btrootfsmnt]) + +logger.info ('Fase 6.3 Configurando las locales') logger.debug ('running \'schroot --chroot IMGogclient -- {}/05-boottoolsFsLocales.py\'') stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', '{}/05-boottoolsFsLocales.py'.format (curdir)]) logger.debug ('05-boottoolsFsLocales stdout follows: {}'.format (stdout)) -if _is_mounted (btrootfsmnt): - _run (['umount', btrootfsmnt]) - #cd / #schroot -c IMGogclient -- /usr/bin/boot-tools/boottoolsInitrdGenerate.sh logger.debug ('running \'schroot --chroot IMGogclient -- {}/06-boottoolsInitrdGenerate.py --osrelease "{}"\''.format (curdir, osrelease)) stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', '{}/06-boottoolsInitrdGenerate.py'.format (curdir), '--osrelease', osrelease]) logger.debug ('06-boottoolsInitrdGenerate stdout follows: {}'.format (stdout)) +## esto deja initrd.img-6.8.0-31-generic y vmlinuz-6.8.0-31-generic en /tmp -btogFsInitrd() +btogFsInitrd (bttargetdir, osrelease) """ From bf161038172fcba1385132198856db7d7fb19a70 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Tue, 13 Aug 2024 14:33:13 +0200 Subject: [PATCH 14/45] refs #596 build squash and iso filesystems --- boottoolsfunctions/__init__.py | 135 ++++++++++++++++++++++++++++++++- boottoolsgenerator.py | 31 ++++---- 2 files changed, 148 insertions(+), 18 deletions(-) diff --git a/boottoolsfunctions/__init__.py b/boottoolsfunctions/__init__.py index 04498f5..1f2668b 100644 --- a/boottoolsfunctions/__init__.py +++ b/boottoolsfunctions/__init__.py @@ -10,6 +10,61 @@ import datetime logger = logging.getLogger ('boottools') +isolinux_tpl = """ +DEFAULT menu.c32 +PROMPT 0 +ALLOWOPTIONS 1 + +MENU TITLE OpenGnsys __NAMEISOCLIENT__ + + +LABEL 0 +MENU LABEL ogLive vga irqpool acpi ogdebug ip:none +KERNEL /ogclient/linuxISO +APPEND initrd=/ogclient/oginitrd.img ro vga=788 irqpoll acpi=on boot=oginit og2nd=sqfs ogprotocol=local ogactiveadmin=true ogdebug=true ip=none + +LABEL 1 +MENU LABEL ogLive irqpoll acpi ip:none +KERNEL /ogclient/linuxISO +APPEND initrd=/ogclient/oginitrd.img ro irqpoll acpi=on boot=oginit og2nd=sqfs ogprotocol=local ogactiveadmin=true ogdebug=false ip=none + +LABEL 2 +MENU LABEL ogLive acpi debug ip=dhcp +KERNEL /ogclient/linuxISO +APPEND initrd=/ogclient/oginitrd.img ro acpi=on boot=oginit og2nd=sqfs ogprotocol=local ogactiveadmin=true ogdebug=true ip=dhcp + +LABEL 3 +MENU LABEL ogLive ip=dhcp +KERNEL /ogclient/linuxISO +APPEND initrd=/ogclient/oginitrd.img ro acpi=off boot=oginit og2nd=sqfs ogprotocol=local ogactiveadmin=true ogdebug=false ip=dhcp + +LABEL 4 +MENU LABEL local +localboot 0x80 +append - + +label 5 +MENU LABEL Network boot via gPXE lkrn +KERNEL __PXEPKG__.lkrn + +label 6 +MENU LABEL Network boot via gPXE usb +KERNEL __PXEPKG__.usb + +label 7 +MENU LABEL Network boot via gPXE pxe +KERNEL __PXEPKG__.pxe + +label 8 +MENU LABEL Network boot via gPXE iso +KERNEL __PXEPKG__.iso + +label 9 +MENU LABEL pxe +KERNEL /clonezilla/live/vmlinuz1 +APPEND initrd=/clonezilla/live/initrd1.img boot=live union=aufs noswap vga=788 ip=frommedia +""" + def _run (args): cp = subprocess.run (args, text=True, capture_output=True) if cp.returncode: @@ -175,7 +230,7 @@ def btogGetOsInfo2 (type_client, versionboottools, ogclientcfg, osdistrib, oscod print ('OSRELEASE="{}"'.format (osrelease), file=f) print ('OSARCH="{}"'.format (osarch), file=f) print ('OSHTTP="{}"'.format (oshttp), file=f) - print ('GITRELEASE="{}"'.format (gitrelease, file=f)) + print ('GITRELEASE="{}"'.format (gitrelease), file=f) print ('NAMEISOCLIENT="{}"'.format (nameisoclient), file=f) print ('NAMEHOSTCLIENT="{}"'.format (namehostclient), file=f) @@ -401,3 +456,81 @@ def btogFsInitrd (bttargetdir, osrelease): #chmod -R 755 $BTTARGETDIR for f in glob.glob ('{}/oginitrd*'.format (bttargetdir)) + glob.glob ('{}/vmlinuz*'.format (bttargetdir)): os.chmod (f, 0o755) + +#btogFsSqfs convierte el sistema root en sqfs +def btogFsSqfs (bttargetdir, btrootfsmnt): + logger.info ('Iniciando la creación del sistema de archivos en sqfs') + + # si ya existe un sqfs lo renombramos + if os.path.exists ('{}/ogclient.sqfs'.format (bttargetdir)): + now = datetime.datetime.now(datetime.timezone.utc).strftime ('%Y%m%d-%H%M%S%z') + os.rename ('{}/ogclient.sqfs'.format (bttargetdir), '{}/ogclient.sqfs.{}'.format (bttargetdir, now)) + + _run (['mksquashfs', btrootfsmnt, '{}/ogclient.sqfs'.format (bttargetdir)]) + os.chmod ('{}/ogclient.sqfs'.format (bttargetdir), 0o744) + + #DATASUM=`md5sum "${BTTARGETDIR}/ogclient.sqfs" | cut -f1 -d" "` + md5, _ = _run (['md5sum', '{}/ogclient.sqfs'.format (bttargetdir)]) + md5, rest = md5.split (' ', 1) + #echo $DATASUM > ${BTTARGETDIR}/ogclient.sqfs.sum + with open ('{}/ogclient.sqfs.sum'.format (bttargetdir), 'w') as fd: + fd.write (md5) + +# btogIsoGenerator genera la iso del cliente +def btogIsoGenerator (pxepkg, bttargetdir, nameisoclient): + #Preparamos los gestores de arranque + try: os.makedirs ('/tmp/iso/isolinux', exist_ok=True) + except: raise + + subprocess.run (['cp -a /usr/lib/syslinux/* /tmp/iso/isolinux/'], shell=True) + + if 'gpxe' == pxepkg: + subprocess.run (['cp -a /usr/share/gpxe/* /tmp/iso/isolinux'], shell=True) + elif 'ipxe' == pxepkg: + subprocess.run (['cp -a /usr/lib/ipxe/* /tmp/iso/isolinux'], shell=True) + else: + raise Exception ('unknown pxepkg value "{}"'.format (pxepkg)) + + # Si existe el fichero ISO, montarlo para extraer isolinux.bin. + if os.path.exists ('/tmp/iso/isolinux/{}.iso'.format (pxepkg)): + os.mkdir ('/tmp/iso/isolinux/mount') + _run (['mount', '-o', 'loop', '/tmp/iso/isolinux/{}.iso'.format (pxepkg), '/tmp/iso/isolinux/mount']) + subprocess.run (['cp -a /tmp/iso/isolinux/mount/* /tmp/iso/isolinux'], shell=True) + _run (['umount', '/tmp/iso/isolinux/mount']) + os.rmdir ('/tmp/iso/isolinux/mount') + os.unlink ('/tmp/iso/isolinux/{}.iso'.format (pxepkg)) + + ## ojo que aquí hay que interpolar $NAMEISOCLIENT y $PXEPKG + with open ('/tmp/iso/isolinux/isolinux.cfg', 'w') as fd: + fd.write (isolinux_tpl.strip().replace('__NAMEISOCLIENT__', nameisoclient).replace('__PXEPKG__', pxepkg)) + + # preparamos el directorio boot-tools. + if not os.path.exists ('/tmp/iso/ogclient'): + os.mkdir ('/tmp/iso/ogclient') + shutil.copy ('{}/ogclient.sqfs' .format (bttargetdir), '/tmp/iso/ogclient/') + shutil.copy ('{}/ogclient.sqfs.sum'.format (bttargetdir), '/tmp/iso/ogclient/') + shutil.copy ('{}/ogvmlinuz' .format (bttargetdir), '/tmp/iso/ogclient/') + shutil.copy ('{}/ogvmlinuz.sum' .format (bttargetdir), '/tmp/iso/ogclient/') + shutil.copy ('{}/ogvmlinuz' .format (bttargetdir), '/tmp/iso/ogclient/linuxISO') + shutil.copy ('{}/ogvmlinuz.sum' .format (bttargetdir), '/tmp/iso/ogclient/linuxISO.sum') + shutil.copy ('{}/oginitrd.img' .format (bttargetdir), '/tmp/iso/ogclient/') + shutil.copy ('{}/oginitrd.img.sum' .format (bttargetdir), '/tmp/iso/ogclient/') + #el ogclienteToISO debe tener una copia del ogvmlinuz como linuxISO + #cp -prv /var/lib/tftpboot/ogclientToIso/* /tmp/iso/ogclient + + oldpwd = os.getcwd() + os.chdir ('/tmp') + logger.debug ('mkisofs -V ogClient -o {}.iso -b isolinux/isolinux.bin -c isolinux/boot.cat -J -no-emul-boot -boot-load-size 4 -boot-info-table /tmp/iso'.format (nameisoclient)) + _run (['mkisofs', '-V', 'ogClient', '-o', '{}.iso'.format (nameisoclient), '-b', 'isolinux/isolinux.bin', '-c', 'isolinux/boot.cat', '-J', '-no-emul-boot', '-boot-load-size', '4', '-boot-info-table', '/tmp/iso']) + + ##Generamos el md5 + #DATASUM=`md5sum ${NAMEISOCLIENT}.iso | cut -f1 -d" "` + md5, _ = _run (['md5sum', '{}.iso'.format (nameisoclient)]) + md5, rest = md5.split (' ', 1) + #echo $DATASUM > ${NAMEISOCLIENT}.iso.sum + with open ('{}.iso.sum'.format (nameisoclient), 'w') as fd: + fd.write (md5) + os.chdir (oldpwd) + + os.rename ('/tmp/{}.iso' .format (nameisoclient), '/var/lib/tftpboot/ogclient/{}.iso' .format (nameisoclient)) + os.rename ('/tmp/{}.iso.sum'.format (nameisoclient), '/var/lib/tftpboot/ogclient/{}.iso.sum'.format (nameisoclient)) diff --git a/boottoolsgenerator.py b/boottoolsgenerator.py index 187fd5e..e166db8 100755 --- a/boottoolsgenerator.py +++ b/boottoolsgenerator.py @@ -11,7 +11,7 @@ import shutil curdir = os.path.dirname (__file__) sys.path.insert (0, curdir) -from boottoolsfunctions import _run, _is_mounted, btogGetOsInfo1, btogGetOsInfo2, btogGetVar, btogSetFsVirtual, btogSetFsAccess, btogSetFsBase, boottoolsSshServer, boottoolsSshClient, btogFsInitrd +from boottoolsfunctions import _run, _is_mounted, btogGetOsInfo1, btogGetOsInfo2, btogGetVar, btogSetFsVirtual, btogSetFsAccess, btogSetFsBase, boottoolsSshServer, boottoolsSshClient, btogFsInitrd, btogFsSqfs, btogIsoGenerator #logging.root.handlers = [] logging.basicConfig ( @@ -46,11 +46,6 @@ if os.getuid(): #os.chdir ('/tmp') -#for i in `mount | grep IMGogclient | grep /var | cut -f3 -d" "`; do echo $i; umount $i; done -#for i in `mount | grep IMGogclient | grep /var | cut -f3 -d" "`; do echo $i; umount $i; done -#for i in `mount | grep IMGogclient | grep /var | cut -f3 -d" "`; do echo $i; umount $i; done - - ####################################################################3 logger.info ('FASE 1 - Asignación de variables') @@ -59,6 +54,9 @@ btdir, bttargetdir, btrootfsimg, btrootfsmnt, btrootfsimglabel, log_file, versio gitrelease, nameisoclient, namehostclient = btogGetOsInfo2(type_client, versionboottools, ogclientcfg, osdistrib, oscodename, osrelease, osarch, oshttp) logger.info ('OpenGnsys CLIENT installation begins') +if _is_mounted (btrootfsmnt): + _run (['umount', btrootfsmnt]) + ########################################################################## logger.info ('FASE 2 - Instalación de software adicional.') @@ -201,21 +199,20 @@ stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', '{}/06-boottoolsI logger.debug ('06-boottoolsInitrdGenerate stdout follows: {}'.format (stdout)) ## esto deja initrd.img-6.8.0-31-generic y vmlinuz-6.8.0-31-generic en /tmp +logger.info ('FASE 7 - Generar distribucion') +logger.info ('Fase 7.1 Generar el initrd') btogFsInitrd (bttargetdir, osrelease) -""" -logger.info ('FASE 7 - Generar distribucion') -logger.info ('Fase 7.1 Generar el initrd') -btogFsInitrd >>/tmp/fase7.out -echo ================= nati after fase 8.1; ls -la /opt/opengnsys/tftpboot/ogclient/ logger.info ('Fase 7.2 Generar fichero sqfs a partir del fichero img') -btogFsSqfs -echo ================= nati after fase 8.2; ls -la /opt/opengnsys/tftpboot/ogclient/ -umount $BTROOTFSMNT 2>/dev/null +try: _run (['mount', btrootfsimg, btrootfsmnt, '-o', 'loop,offset=32256']) +except: + logger.error ('mount failed') + sys.exit (3) +btogFsSqfs (bttargetdir, btrootfsmnt) +_run (['umount', btrootfsmnt]) + logger.info ('Fase 7.3 Generar la ISO') -btogIsoGenerator >>/tmp/fase7.out -echo ================= nati after fase 8.3; ls -la /opt/opengnsys/tftpboot/ogclient/ -""" +btogIsoGenerator (pxepkg, bttargetdir, nameisoclient) logger.info ('OpenGnsys installation finished') From 305e6209bf00c63f2890674509162fca2d864362 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Tue, 13 Aug 2024 14:57:11 +0200 Subject: [PATCH 15/45] refs #596 move isolinux.cfg template and debconf settings to the config --- 03-boottoolsSoftwareInstall.py | 21 ++- boottoolsfunctions/__init__.py | 65 ++-------- boottoolsgenerator.cfg | 62 +++++++++ boottoolsgenerator.py | 10 +- boottoolsgenerator.sh | 121 ------------------ .../bin/boot-tools/boottoolsBootGraphics.sh | 16 --- .../usr/bin/boot-tools/boottoolsFsLocales.sh | 10 -- .../bin/boot-tools/boottoolsFsOpengnsys.sh | 74 ----------- .../boottoolsFsPostconfiguration.sh | 5 - .../bin/boot-tools/boottoolsInitrdGenerate.sh | 14 -- .../boot-tools/boottoolsSoftwareCompile.sh | 36 ------ .../boot-tools/boottoolsSoftwareInstall.sh | 104 --------------- .../bin/boot-tools/boottoolsSoftwareXwindows | 14 -- .../usr/bin/boot-tools/boottoolsSshClient.sh | 25 ---- .../usr/bin/boot-tools/boottoolsSshServer.sh | 18 --- 15 files changed, 86 insertions(+), 509 deletions(-) delete mode 100755 boottoolsgenerator.sh delete mode 100755 includes/usr/bin/boot-tools/boottoolsBootGraphics.sh delete mode 100755 includes/usr/bin/boot-tools/boottoolsFsLocales.sh delete mode 100755 includes/usr/bin/boot-tools/boottoolsFsOpengnsys.sh delete mode 100755 includes/usr/bin/boot-tools/boottoolsFsPostconfiguration.sh delete mode 100755 includes/usr/bin/boot-tools/boottoolsInitrdGenerate.sh delete mode 100755 includes/usr/bin/boot-tools/boottoolsSoftwareCompile.sh delete mode 100755 includes/usr/bin/boot-tools/boottoolsSoftwareInstall.sh delete mode 100755 includes/usr/bin/boot-tools/boottoolsSoftwareXwindows delete mode 100755 includes/usr/bin/boot-tools/boottoolsSshClient.sh delete mode 100755 includes/usr/bin/boot-tools/boottoolsSshServer.sh diff --git a/03-boottoolsSoftwareInstall.py b/03-boottoolsSoftwareInstall.py index d7f06a1..070a907 100755 --- a/03-boottoolsSoftwareInstall.py +++ b/03-boottoolsSoftwareInstall.py @@ -10,31 +10,26 @@ import configparser from boottoolsfunctions import _run config = configparser.ConfigParser (inline_comment_prefixes='#') -if not os.path.exists ('boottoolsgenerator.cfg'): ## si pongo uno que no existe, - print ('configuration file "boottoolsgenerator.cfg" not found') ## este error no llega al caller... +if not os.path.exists ('boottoolsgenerator.cfg'): + print ('configuration file "boottoolsgenerator.cfg" not found') sys.exit (1) config.read ('boottoolsgenerator.cfg') - -debconf_settings = """ -kexec-tools kexec-tools/load_kexec boolean true -openssh-server openssh-server/permit-root-login boolean true -refind refind/install_to_esp boolean false -""".strip() +debconf_settings = config['debconf'].get ('settings') def _aptget_clean(): - _run (['apt-get', 'clean']) + _run (['apt-get', '--yes', 'clean']) def _aptget_autoclean(): - _run (['apt-get', 'autoclean']) + _run (['apt-get', '--yes', 'autoclean']) def _aptget_autoremove(): - _run (['apt-get', 'autoremove']) + _run (['apt-get', '--yes', 'autoremove']) def _aptget_update(): - _run (['apt-get', 'update']) + _run (['apt-get', '--yes', 'update']) def _aptget_upgrade(): - _run (['apt-get', 'upgrade', '--yes']) + _run (['apt-get', '--yes', 'upgrade']) def _aptget_install (pkgs, opts=[]): _run (['apt-get', '--yes'] + opts + ['install'] + pkgs) diff --git a/boottoolsfunctions/__init__.py b/boottoolsfunctions/__init__.py index 1f2668b..af611f0 100644 --- a/boottoolsfunctions/__init__.py +++ b/boottoolsfunctions/__init__.py @@ -10,61 +10,6 @@ import datetime logger = logging.getLogger ('boottools') -isolinux_tpl = """ -DEFAULT menu.c32 -PROMPT 0 -ALLOWOPTIONS 1 - -MENU TITLE OpenGnsys __NAMEISOCLIENT__ - - -LABEL 0 -MENU LABEL ogLive vga irqpool acpi ogdebug ip:none -KERNEL /ogclient/linuxISO -APPEND initrd=/ogclient/oginitrd.img ro vga=788 irqpoll acpi=on boot=oginit og2nd=sqfs ogprotocol=local ogactiveadmin=true ogdebug=true ip=none - -LABEL 1 -MENU LABEL ogLive irqpoll acpi ip:none -KERNEL /ogclient/linuxISO -APPEND initrd=/ogclient/oginitrd.img ro irqpoll acpi=on boot=oginit og2nd=sqfs ogprotocol=local ogactiveadmin=true ogdebug=false ip=none - -LABEL 2 -MENU LABEL ogLive acpi debug ip=dhcp -KERNEL /ogclient/linuxISO -APPEND initrd=/ogclient/oginitrd.img ro acpi=on boot=oginit og2nd=sqfs ogprotocol=local ogactiveadmin=true ogdebug=true ip=dhcp - -LABEL 3 -MENU LABEL ogLive ip=dhcp -KERNEL /ogclient/linuxISO -APPEND initrd=/ogclient/oginitrd.img ro acpi=off boot=oginit og2nd=sqfs ogprotocol=local ogactiveadmin=true ogdebug=false ip=dhcp - -LABEL 4 -MENU LABEL local -localboot 0x80 -append - - -label 5 -MENU LABEL Network boot via gPXE lkrn -KERNEL __PXEPKG__.lkrn - -label 6 -MENU LABEL Network boot via gPXE usb -KERNEL __PXEPKG__.usb - -label 7 -MENU LABEL Network boot via gPXE pxe -KERNEL __PXEPKG__.pxe - -label 8 -MENU LABEL Network boot via gPXE iso -KERNEL __PXEPKG__.iso - -label 9 -MENU LABEL pxe -KERNEL /clonezilla/live/vmlinuz1 -APPEND initrd=/clonezilla/live/initrd1.img boot=live union=aufs noswap vga=788 ip=frommedia -""" - def _run (args): cp = subprocess.run (args, text=True, capture_output=True) if cp.returncode: @@ -477,7 +422,7 @@ def btogFsSqfs (bttargetdir, btrootfsmnt): fd.write (md5) # btogIsoGenerator genera la iso del cliente -def btogIsoGenerator (pxepkg, bttargetdir, nameisoclient): +def btogIsoGenerator (pxepkg, isolinux_tpl, bttargetdir, nameisoclient): #Preparamos los gestores de arranque try: os.makedirs ('/tmp/iso/isolinux', exist_ok=True) except: raise @@ -534,3 +479,11 @@ def btogIsoGenerator (pxepkg, bttargetdir, nameisoclient): os.rename ('/tmp/{}.iso' .format (nameisoclient), '/var/lib/tftpboot/ogclient/{}.iso' .format (nameisoclient)) os.rename ('/tmp/{}.iso.sum'.format (nameisoclient), '/var/lib/tftpboot/ogclient/{}.iso.sum'.format (nameisoclient)) + +#def __unused_boottoolsBootGraphics(): +# find /tmp/opengnsys_installer/ -name .svn -type d -exec rm -fr {} \; 2>/dev/null; +# apt-get -y install plymouth plymouth-theme-script +# update-alternatives --install /lib/plymouth/themes/default.plymouth default.plymouth /lib/plymouth/themes/opengnsys/opengnsys.plymouth 100 +# update-alternatives --set default.plymouth /lib/plymouth/themes/opengnsys/opengnsys.plymouth +# mkdir -p /etc/initramfs-tools/conf.d +# echo "FRAMEBUFFER=y" > /etc/initramfs-tools/conf.d/splash diff --git a/boottoolsgenerator.cfg b/boottoolsgenerator.cfg index f05027c..4bda2ff 100644 --- a/boottoolsgenerator.cfg +++ b/boottoolsgenerator.cfg @@ -62,3 +62,65 @@ xwindows = #roxterm gparted #+80M #openbox midori #xvesa en compilacion +[debconf] + +settings = + kexec-tools kexec-tools/load_kexec boolean true + openssh-server openssh-server/permit-root-login boolean true + refind refind/install_to_esp boolean false + +[ISOLinux] + +template = + DEFAULT menu.c32 + PROMPT 0 + ALLOWOPTIONS 1 + + MENU TITLE OpenGnsys __NAMEISOCLIENT__ + + + LABEL 0 + MENU LABEL ogLive vga irqpool acpi ogdebug ip:none + KERNEL /ogclient/linuxISO + APPEND initrd=/ogclient/oginitrd.img ro vga=788 irqpoll acpi=on boot=oginit og2nd=sqfs ogprotocol=local ogactiveadmin=true ogdebug=true ip=none + + LABEL 1 + MENU LABEL ogLive irqpoll acpi ip:none + KERNEL /ogclient/linuxISO + APPEND initrd=/ogclient/oginitrd.img ro irqpoll acpi=on boot=oginit og2nd=sqfs ogprotocol=local ogactiveadmin=true ogdebug=false ip=none + + LABEL 2 + MENU LABEL ogLive acpi debug ip=dhcp + KERNEL /ogclient/linuxISO + APPEND initrd=/ogclient/oginitrd.img ro acpi=on boot=oginit og2nd=sqfs ogprotocol=local ogactiveadmin=true ogdebug=true ip=dhcp + + LABEL 3 + MENU LABEL ogLive ip=dhcp + KERNEL /ogclient/linuxISO + APPEND initrd=/ogclient/oginitrd.img ro acpi=off boot=oginit og2nd=sqfs ogprotocol=local ogactiveadmin=true ogdebug=false ip=dhcp + + LABEL 4 + MENU LABEL local + localboot 0x80 + append - + + label 5 + MENU LABEL Network boot via gPXE lkrn + KERNEL __PXEPKG__.lkrn + + label 6 + MENU LABEL Network boot via gPXE usb + KERNEL __PXEPKG__.usb + + label 7 + MENU LABEL Network boot via gPXE pxe + KERNEL __PXEPKG__.pxe + + label 8 + MENU LABEL Network boot via gPXE iso + KERNEL __PXEPKG__.iso + + label 9 + MENU LABEL pxe + KERNEL /clonezilla/live/vmlinuz1 + APPEND initrd=/clonezilla/live/initrd1.img boot=live union=aufs noswap vga=788 ip=frommedia diff --git a/boottoolsgenerator.py b/boottoolsgenerator.py index e166db8..e91e7e2 100755 --- a/boottoolsgenerator.py +++ b/boottoolsgenerator.py @@ -24,8 +24,12 @@ logging.basicConfig ( ) logger = logging.getLogger ('boottools') -#config = configparser.ConfigParser() -#config.read ('boottoolsgenerator.cfg') +config = configparser.ConfigParser (inline_comment_prefixes='#') +if not os.path.exists ('boottoolsgenerator.cfg'): + print ('configuration file "boottoolsgenerator.cfg" not found') + sys.exit (1) +config.read ('boottoolsgenerator.cfg') +isolinux_tpl = config['ISOLinux'].get ('template') if not os.path.exists ('/tmp/opengnsys_installer/opengnsys/client/engine'): branch = 'main' @@ -213,6 +217,6 @@ btogFsSqfs (bttargetdir, btrootfsmnt) _run (['umount', btrootfsmnt]) logger.info ('Fase 7.3 Generar la ISO') -btogIsoGenerator (pxepkg, bttargetdir, nameisoclient) +btogIsoGenerator (pxepkg, isolinux_tpl, bttargetdir, nameisoclient) logger.info ('OpenGnsys installation finished') diff --git a/boottoolsgenerator.sh b/boottoolsgenerator.sh deleted file mode 100755 index d4f8436..0000000 --- a/boottoolsgenerator.sh +++ /dev/null @@ -1,121 +0,0 @@ -#!/bin/bash -#@file boottoolsgenerator.sh -#@brief Script generación del sistema opertativo cliente OpenGnsys -#@warning -#@version 0.9 - Prototipo de sistema operativo multiarranque de opengnsys. -#@author Antonio J. Doblas Viso. Universidad de Malaga. -#@date 2010/05/24 -#@version 1.0 - Compatibilidad OpengGnsys X. -#@author Antonio J. Doblas Viso. Universidad de Malaga. -#@date 2011/08/03 -#*/ - - -BRANCH="branches/master" -svn export --force https://github.com/opengnsys/OpenGnsys/$BRANCH/client /tmp/opengnsys_installer/opengnsys/client || exit 1 - -#Variables -TYPECLIENT="${1:-host}" -WORKDIR=/tmp/opengnsys_installer -INSTALL_TARGET=/opt/opengnsys -PROGRAMDIR=$(readlink -e $(dirname "$0")) - -# Solo ejecutable por usuario root -if [ "$(whoami)" != 'root' ] -then - echo "ERROR: this program must run under root privileges!!" - exit 1 -fi - -# Cambiar a directorio temporal. -cd /tmp - -for i in `mount | grep IMGogclient | grep /var | cut -f3 -d" "`; do echo $i; umount $i; done -for i in `mount | grep IMGogclient | grep /var | cut -f3 -d" "`; do echo $i; umount $i; done -for i in `mount | grep IMGogclient | grep /var | cut -f3 -d" "`; do echo $i; umount $i; done - - -#funciones especificas del cliente. -source $PROGRAMDIR/boottoolsfunctions.lib - -####################################################################3 -echo "FASE 1 - Asignación de variables" -#obtenemos las variables necesarias y la información del host. -btogGetVar -echoAndLog "OpenGnsys CLIENT installation begins at $(date)" -btogGetOsInfo $TYPECLIENT -########################################################################## -echo "FASE 2 - Instalación de software adicional." -#grep "http://free.nchc.org.tw/drbl-core" /etc/apt/sources.list || echo "deb http://free.nchc.org.tw/drbl-core drbl stable" >> /etc/apt/sources.list -apt-get update -[ -n "$(apt-cache search gpxe)" ] && PXEPKG="gpxe" -[ -n "$(apt-cache search ipxe)" ] && PXEPKG="ipxe" -apt-get -y install debootstrap subversion schroot squashfs-tools syslinux genisoimage $PXEPKG qemu qemu-utils lsof -###################################################################3 -echo "FASE 3 - Creación del Sistema raiz RootFS (Segundo Sistema archivos (img)) " -echo "Fase 3.1 Generar y formatear el disco virtual. Generar el dispositivo loop." -file $BTROOTFSIMG | grep "partition 1: ID=0x83" -if [ $? == 1 ] -then - btogSetFsVirtual || exit 2 -fi -echo "Fase 3.2 Generar sistema de archivos con debootstrap" -schroot -p -c IMGogclient -- touch /tmp/ogclientOK -if [ -f /tmp/ogclientOK ] -then - rm /tmp/ogclientOK -else - btogSetFsBase || exit 3 -fi -###################################################################3 -echo "FASE 4 - Configurar acceso schroot al Segundo Sistema de archivos (img)" -cat /etc/schroot/schroot.conf | grep $BTROOTFSIMG || btogSetFsAccess -########################################################################### -echo "FASE 5 - Incorporando ficheros OpenGnsys al sistema raíz rootfs " -cp -a ${BTDIR}/includes/usr/bin/* /tmp -chmod +x /tmp/boot-tools/*.sh -# Incluir revisión. -sed -i "1 s/$/ $GITRELEASE ($OSRELEASE)/" ${BTDIR}/includes/etc/initramfs-tools/scripts/VERSION.txt -# En Ubuntu 13.04+ es necesario matar proceso de "udev" antes de desmontar. -umount $BTROOTFSMNT 2>/dev/null || (kill -9 $(lsof -t $BTROOTFSMNT); umount $BTROOTFSMNT) 2>/dev/null -schroot -p -c IMGogclient -- /tmp/boot-tools/boottoolsFsOpengnsys.sh -############################################################################################ -echo "FASE 6 - Instalar software" -echo "Fase 6.1 instalar paquetes deb con apt-get" -schroot -p -c IMGogclient -- /usr/bin/boot-tools/boottoolsSoftwareInstall.sh -echo "Fase 6.2 compilar software." -cd / -schroot -p -c IMGogclient -- /usr/bin/boot-tools/boottoolsSoftwareCompile.sh -cd - - -echo "FASE 7 - Personalizar el sistema creado" -echo "Fase 7.1 Incorporar la clave publica del servidor" -cd / -ssh-keygen -q -f /root/.ssh/id_rsa -N "" -cp /root/.ssh/id_rsa.pub /tmp -schroot -p -c IMGogclient -- /usr/bin/boot-tools/boottoolsSshServer.sh -cd - -echo "Fase 7.2. Incorpoar la clave publica del propio cliente" -schroot -p -c IMGogclient -- /usr/bin/boot-tools/boottoolsSshClient.sh - -echo "Fase 7.3. Configurando las locales" -schroot -p -c IMGogclient -- /usr/bin/boot-tools/boottoolsFsLocales.sh - - -for i in `mount | grep IMGogclient | grep /var | cut -f3 -d" "`; do echo $i; umount $i; done -for i in `mount | grep IMGogclient | grep /var | cut -f3 -d" "`; do echo $i; umount $i; done -for i in `mount | grep IMGogclient | grep /var | cut -f3 -d" "`; do echo $i; umount $i; done - -######################################################################### -echo "FASE 8 - Generar distribucion" -echo "Fase 8.1 Generar el initrd" -btogFsInitrd -echo "Fase 8.2 Generar fichero sqfs a partir del fichero img" -btogFsSqfs -umount $BTROOTFSMNT 2>/dev/null -echo "Fase 8.3 Generar la ISO" -btogIsoGenerator -######################################################################3 -######################################################################## -echoAndLog "OpenGnsys installation finished at $(date)" - diff --git a/includes/usr/bin/boot-tools/boottoolsBootGraphics.sh b/includes/usr/bin/boot-tools/boottoolsBootGraphics.sh deleted file mode 100755 index 67d738b..0000000 --- a/includes/usr/bin/boot-tools/boottoolsBootGraphics.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -find /tmp/opengnsys_installer/ -name .svn -type d -exec rm -fr {} \; 2>/dev/null; - -#plymouth -apt-get -y install plymouth plymouth-theme-script - - -#plymoutyh -update-alternatives --install /lib/plymouth/themes/default.plymouth default.plymouth /lib/plymouth/themes/opengnsys/opengnsys.plymouth 100 -update-alternatives --set default.plymouth /lib/plymouth/themes/opengnsys/opengnsys.plymouth - -mkdir -p /etc/initramfs-tools/conf.d -echo "FRAMEBUFFER=y" > /etc/initramfs-tools/conf.d/splash - -history -c diff --git a/includes/usr/bin/boot-tools/boottoolsFsLocales.sh b/includes/usr/bin/boot-tools/boottoolsFsLocales.sh deleted file mode 100755 index 11d1226..0000000 --- a/includes/usr/bin/boot-tools/boottoolsFsLocales.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -cat << EOT | debconf-set-selections -- -console-setup console-setup/charmap47 select UTF-8 -console-setup console-setup/codeset47 select . Combined - Latin; Slavic Cyrillic; Greek -console-setup console-setup/fontface47 select VGA -console-setup console-setup/fontsize-fb47 select 8x16 -EOT -dpkg-reconfigure console-setup locales - -history -c diff --git a/includes/usr/bin/boot-tools/boottoolsFsOpengnsys.sh b/includes/usr/bin/boot-tools/boottoolsFsOpengnsys.sh deleted file mode 100755 index 6296292..0000000 --- a/includes/usr/bin/boot-tools/boottoolsFsOpengnsys.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash - -VERSIONBOOTTOOLS="ogLive" - -NAMEISOCLIENTFILE="/tmp/opengnsys_info_rootfs" -NAMEHOSTCLIENTFILE="/tmp/opengnsys_chroot" - -SVNCLIENTDIR=/tmp/opengnsys_installer/opengnsys/client/boot-tools -SVNCLIENTSTRUCTURE=/tmp/opengnsys_installer/opengnsys/client/shared -SVNCLIENTENGINE=/tmp/opengnsys_installer/opengnsys/client/engine - -OGCLIENTMOUNT="" - -OGCLIENTCFG=${OGCLIENTCFG:-/tmp/ogclient.cfg} -[ -f $OGCLIENTCFG ] && source $OGCLIENTCFG -OSDISTRIB=${OSDISTRIB:-$(lsb_release -is)} -OSCODENAME=${OSCODENAME:-$(lsb_release -cs)} -OSRELEASE=${OSRELEASE:-$(uname -r)} -OSARCH=${OSARCH:-$(dpkg --print-architecture)} -OSHTTP=${OSHTTP:-"http://es.archive.ubuntu.com/ubuntu/"} - -echo "$OSDISTRIB:$OSCODENAME:$OSRELEASE:$OSARCH:$OSHTTP" - - -LERROR=TRUE - -echo "$FUNCNAME: Iniciando la personalización con datos del repositorio" - -# parseamos el apt.source de la distribución (en minúsculas) -sed -e "s/OSCODENAME/$OSCODENAME/g" ${SVNCLIENTDIR}/includes/etc/apt/sources.list.${OSDISTRIB,,} > ${SVNCLIENTDIR}/includes/etc/apt/sources.list -if [ $? -ne 0 ] -then - echo "$FUNCNAME(): Parsing apt.sources : ERROR" - exit 1 -fi - -#damos permiso al directorio de scripts -chmod -R 775 ${SVNCLIENTDIR}/includes/usr/bin/* - -# los copiamos -cp -av ${SVNCLIENTDIR}/includes/* ${OGCLIENTMOUNT}/ -mkdir -p ${OGCLIENTMOUNT}/opt/opengnsys/ -cp -av ${SVNCLIENTSTRUCTURE}/* ${OGCLIENTMOUNT}/opt/opengnsys/ -mkdir -p ${OGCLIENTMOUNT}/opt/opengnsys/lib/engine/bin/ -cp -av ${SVNCLIENTENGINE}/* ${OGCLIENTMOUNT}/opt/opengnsys/lib/engine/bin/ - -if [ $? -ne 0 ] -then - echo "$FUNCNAME(): Copying client data : ERROR" - exit 1 -fi - - -# Si no existe, copiar pci.ids. -[ -f $OGCLIENTMOUNT/etc/pci.ids ] || cp -va ${SVNCLIENTSTRUCTURE}/lib/pci.ids $OGCLIENTMOUNT/etc - -# Dependencias Qt para el Browser. -mkdir -p $OGCLIENTMOUNT/usr/local/{etc,lib,plugins} -cp -av ${SVNCLIENTSTRUCTURE}/lib/qtlib/* $OGCLIENTMOUNT/usr/local/lib -cp -av ${SVNCLIENTSTRUCTURE}/lib/fonts $OGCLIENTMOUNT/usr/local/lib -cp -av ${SVNCLIENTSTRUCTURE}/lib/qtplugins/* $OGCLIENTMOUNT/usr/local/plugins -cp -av ${SVNCLIENTSTRUCTURE}/etc/*.qmap $OGCLIENTMOUNT/usr/local/etc - -# Browser y ogAdmClient. -[ -x ${SVNCLIENTSTRUCTURE}/bin/browser ] && cp -av ${SVNCLIENTSTRUCTURE}/bin/browser $OGCLIENTMOUNT/bin -[ -x ${SVNCLIENTSTRUCTURE}/bin/ogAdmClient ] && cp -av ${SVNCLIENTSTRUCTURE}/bin/ogAdmClient $OGCLIENTMOUNT/bin - -# El fichero de configuración debe sustituir a los 2 ficheros (borrar las 2 líneas). -echo "${VERSIONBOOTTOOLS}-${OSCODENAME}-${OSRELEASE}-${GITRELEASE}" > /$NAMEISOCLIENTFILE -echo "${VERSIONBOOTTOOLS}-${OSCODENAME}-${GITRELEASE}" > $NAMEHOSTCLIENTFILE - - -history -c - diff --git a/includes/usr/bin/boot-tools/boottoolsFsPostconfiguration.sh b/includes/usr/bin/boot-tools/boottoolsFsPostconfiguration.sh deleted file mode 100755 index 140cffb..0000000 --- a/includes/usr/bin/boot-tools/boottoolsFsPostconfiguration.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - - - -history -c \ No newline at end of file diff --git a/includes/usr/bin/boot-tools/boottoolsInitrdGenerate.sh b/includes/usr/bin/boot-tools/boottoolsInitrdGenerate.sh deleted file mode 100755 index 714fba5..0000000 --- a/includes/usr/bin/boot-tools/boottoolsInitrdGenerate.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -OGCLIENTCFG=${OGCLIENTCFG:-/tmp/ogclient.cfg} -[ -f $OGCLIENTCFG ] && source $OGCLIENTCFG -OSRELEASE=${OSRELEASE:-$(uname -r)} - -rm -f /usr/lib/initramfs-tools/bin/* -cp /bin/busybox /usr/lib/initramfs-tools/bin -cd /tmp -mkinitramfs -o /tmp/initrd.img-$OSRELEASE -v $OSRELEASE -cp -v /boot/vmlinuz-$OSRELEASE /tmp - -history -c - diff --git a/includes/usr/bin/boot-tools/boottoolsSoftwareCompile.sh b/includes/usr/bin/boot-tools/boottoolsSoftwareCompile.sh deleted file mode 100755 index 3996636..0000000 --- a/includes/usr/bin/boot-tools/boottoolsSoftwareCompile.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -################################################################ -##################### SOFTWARE ##################### -################################################################ -OLDLANGUAGE=$LANGUAGE -OLDLC_ALL=$LC_ALL -OLDLANG=$LANG -export LANGUAGE=C -export LC_ALL=C -export LANG=C - - -source /opt/opengnsys/lib/engine/bin/ToolsGNU.c &>/dev/null -pushd /tmp - -echo "ms-sys" -which ms-sys || (wget https://sourceforge.net/projects/ms-sys/files/latest/download -O ms-sys.tar.gz; tar xpvzf ms-sys.tar.gz; cd $(tar tzf ms-sys.tar.gz|head -1); make install; cd ..) - -echo "spartlnx" -which spartlnx.run || $(wget http://damien.guibouret.free.fr/savepart.zip &>/dev/null; unzip -o savepart.zip -d /sbin/) - -echo "python-libfdisk" -apt-get -y install python3-psutil python3-dev libfdisk-dev python3-setuptools -git clone git://git.48k.eu/python-libfdisk -pushd python-libfdisk -python3 setup.py install -popd -apt-get -y remove python3-dev python3-setuptools - -popd -export LANGUAGE=$OLDLANGUAGE -export LC_ALL=$OLDLC_ALL -export LANG=$OLDLANG - -history -c - diff --git a/includes/usr/bin/boot-tools/boottoolsSoftwareInstall.sh b/includes/usr/bin/boot-tools/boottoolsSoftwareInstall.sh deleted file mode 100755 index 6c7b23b..0000000 --- a/includes/usr/bin/boot-tools/boottoolsSoftwareInstall.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/bin/bash -# Script de instalación de paquetes de software. - -export LANGUAGE=C -export LC_ALL=C -export LANG=C -#LOG_FILE=/tmp/boot-tools-software.txt - -#ADV -#Desactivamos upstart -dpkg-divert --local --rename --add /sbin/initctl -ln -s /bin/true /sbin/initctl -#TEST desactivamos upstart -#apt-get update; apt-get install --no-install-recommends dbus; dbus-uuidgen > /var/lib/dbus/machine-id; dpkg-divert --local --rename --add /sbin/initctl; ln -s /bin/true /sbin/initctl - -#cp /tmp/sources.list /etc/apt/sources.list -#Limpiamos y actualizamos los repositorios apt (incluir siempre paquetes de 32 bits) -apt-get clean -OSARCH=${OSARCH:-$(dpkg --print-architecture)} -if [ "$OSARCH" != "i386" ]; then - dpkg --add-architecture i386 - PKGS32="lib32gcc1 lib32stdc++6 lib32z1 libc6-i386" -fi -apt-get update -apt-get upgrade -y - -#Desactivamos el hook del oginitrd.img para evitar problemas, al final de este escripts se activará -mv /etc/initramfs-tools/hooks/oghooks /etc/initramfs-tools/ - -# Preparamos el mtab necesario para la instalacion correcta de paquetes. -echo "/dev/sda1 / ext4 rw,errors=remount-ro 0 0" > /etc/mtab - -# Instalamos el kernel. -# Deteccion de la versión y kernel a usar -OGCLIENTCFG=${OGCLIENTCFG:-/tmp/ogclient.cfg} -[ -f $OGCLIENTCFG ] && source $OGCLIENTCFG -OSRELEASE=${OSRELEASE:-$(uname -r)} -# Instalar Kernel firmado del repositorio de paquetes. -apt-get -y install linux-image-${OSRELEASE} linux-headers-${OSRELEASE} dkms shim-signed -apt-get -y install linux-modules-${OSRELEASE} linux-modules-extra-${OSRELEASE} 2>/dev/null - -# Valores para paquetes interactivos. -cat << EOT | debconf-set-selections -- -kexec-tools kexec-tools/load_kexec boolean true -openssh-server openssh-server/permit-root-login boolean true -refind refind/install_to_esp boolean false -EOT -apt-get -y install sshfs kexec-tools $PKGS32 - -#comenzamos con la instalación de los paquetes a instalar. -for group in `find /usr/bin/boot-tools/listpackages/ -name sw.*` -do - echo "Instalando el grupo de paquetes almacenados en $group" - for package in ` awk /^install/'{print $2}' $group ` - do - echo -n $package - apt-get -y install $package &>/dev/null - RETVAL=$? - if [ $RETVAL == 0 ] - then - echo " : OK - Paquete instalado correctamente (codigo interno de apt-get $RETVAL)" - else - echo " : Error Paquete $package del grupo $group (codigo interno de apt-get $RETVAL) " - echo "Pulse [Intro] para continuar" - read - fi - done -done - -# Instalar módulos que algunos paquetes puedan tener pendientes de compilar. -echo "Instalando módulos adicionales con DKMS" -while read -e mod vers; do - echo -n "Intalando módulo $mod v$vers" - dkms install -m $mod -v $vers &>/dev/null - RETVAL=$? - if [ $RETVAL == 0 ]; then - echo " : OK - Módulo instalado correctamente (codigo interno de dkms $RETVAL)" - else - echo " : Error módulo $mod (codigo interno de dkms $RETVAL) " - echo "Pulse [Intro] para continuar" - read - fi -done < <(dkms status 2>/dev/null | awk -F, '$3~/added/ {print $1,$2}') - -#Activamos el hook del oginitrd.img -mv /etc/initramfs-tools/oghooks /etc/initramfs-tools/hooks/ - -# Dejamos el mtab como al principio -echo " " > /etc/mtab - - -#ADV -#TEST -#apt-get install localepurge -#localepurge -#rm /var/lib/dbus/machine-id; rm /sbin/initctl; dpkg-divert --rename --remove /sbin/initctl; -#FIN ADV - -apt-get clean -apt-get autoclean -apt-get autoremove - -history -c - diff --git a/includes/usr/bin/boot-tools/boottoolsSoftwareXwindows b/includes/usr/bin/boot-tools/boottoolsSoftwareXwindows deleted file mode 100755 index 41824f1..0000000 --- a/includes/usr/bin/boot-tools/boottoolsSoftwareXwindows +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -export LANGUAGE=C -export LC_ALL=C -export LANG=C - -apt-get clean -apt-get -y update - -apt-get -y --force-yes install xorg-dev xorg lxde roxterm - -apt-get clean - -history -c \ No newline at end of file diff --git a/includes/usr/bin/boot-tools/boottoolsSshClient.sh b/includes/usr/bin/boot-tools/boottoolsSshClient.sh deleted file mode 100755 index 10aa673..0000000 --- a/includes/usr/bin/boot-tools/boottoolsSshClient.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -echo "comprobando directorio .ssh del root" -if [ ! -d /root/.ssh ] -then - echo "creando directorio .ssh 600" - mkdir -p /root/.ssh - chmod 700 /root/.ssh -fi - - -echo "comprobando el fichero authorized_keys .ssh del root" -if [ ! -f /root/.ssh/authorized_keys ] -then - echo "creando el fichero authorized_keys" - touch /root/.ssh/authorized_keys - chmod 600 /root/.ssh/authorized_keys -fi - -ssh-keygen -q -f /root/.ssh/id_rsa -N "" -cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys - -## TODO: exportamos la publica a los repos -cp /root/.ssh/id_rsa.pub /tmp/rsa.ogclient.pub - -history -c \ No newline at end of file diff --git a/includes/usr/bin/boot-tools/boottoolsSshServer.sh b/includes/usr/bin/boot-tools/boottoolsSshServer.sh deleted file mode 100755 index 02ea71c..0000000 --- a/includes/usr/bin/boot-tools/boottoolsSshServer.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -echo "comprobando directorio .ssh del root" -if [ ! -d /root/.ssh ] -then - echo "creando directorio .ssh 600" - mkdir -p /root/.ssh - chmod 700 /root/.ssh -fi -echo "creando el fichero authorized_keys" -touch /root/.ssh/authorized_keys -chmod 600 /root/.ssh/authorized_keys - -echo "importando la clave publica del servidor OG" -cat /tmp/id_rsa.pub - -[ -f /tmp/id_rsa.pub ] && cat /tmp/id_rsa.pub >> /root/.ssh/authorized_keys || echo "no key publica og" - - history -c \ No newline at end of file From 4ac6b8cdd45c7c189c9212d14781e2a9d51fa065 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Tue, 13 Aug 2024 15:11:17 +0200 Subject: [PATCH 16/45] refs #596 move more debconf settings to the config --- 03-boottoolsSoftwareInstall.py | 2 +- 05-boottoolsFsLocales.py | 15 ++++++++------- boottoolsgenerator.cfg | 6 ++++++ boottoolsgenerator.py | 7 +------ 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/03-boottoolsSoftwareInstall.py b/03-boottoolsSoftwareInstall.py index 070a907..f7ab2f7 100755 --- a/03-boottoolsSoftwareInstall.py +++ b/03-boottoolsSoftwareInstall.py @@ -9,7 +9,7 @@ import configparser from boottoolsfunctions import _run -config = configparser.ConfigParser (inline_comment_prefixes='#') +config = configparser.ConfigParser (comment_prefixes='#', inline_comment_prefixes='#') if not os.path.exists ('boottoolsgenerator.cfg'): print ('configuration file "boottoolsgenerator.cfg" not found') sys.exit (1) diff --git a/05-boottoolsFsLocales.py b/05-boottoolsFsLocales.py index a7a5387..e3995b8 100755 --- a/05-boottoolsFsLocales.py +++ b/05-boottoolsFsLocales.py @@ -2,16 +2,17 @@ import os import subprocess +import configparser from boottoolsfunctions import _run -debconf_settings = """ -console-setup console-setup/charmap47 select UTF-8 -console-setup console-setup/codeset47 select . Combined - Latin; Slavic Cyrillic; Greek -console-setup console-setup/fontface47 select VGA -console-setup console-setup/fontsize-fb47 select 8x16 -""".strip() +config = configparser.ConfigParser (inline_comment_prefixes='#') +if not os.path.exists ('boottoolsgenerator.cfg'): + print ('configuration file "boottoolsgenerator.cfg" not found') + sys.exit (1) +config.read ('boottoolsgenerator.cfg') +debconf_settings2 = config['debconf'].get ('settings2') -subprocess.run (['debconf-set-selections'], input=debconf_settings, text=True) +subprocess.run (['debconf-set-selections'], input=debconf_settings2, text=True) os.environ['DEBIAN_FRONTEND'] = 'noninteractive' _run (['dpkg-reconfigure', 'console-setup', 'locales']) diff --git a/boottoolsgenerator.cfg b/boottoolsgenerator.cfg index 4bda2ff..3542537 100644 --- a/boottoolsgenerator.cfg +++ b/boottoolsgenerator.cfg @@ -69,6 +69,12 @@ settings = openssh-server openssh-server/permit-root-login boolean true refind refind/install_to_esp boolean false +settings2 = + console-setup console-setup/charmap47 select UTF-8 + console-setup console-setup/codeset47 select . Combined - Latin; Slavic Cyrillic; Greek + console-setup console-setup/fontface47 select VGA + console-setup console-setup/fontsize-fb47 select 8x16 + [ISOLinux] template = diff --git a/boottoolsgenerator.py b/boottoolsgenerator.py index e91e7e2..2b6a362 100755 --- a/boottoolsgenerator.py +++ b/boottoolsgenerator.py @@ -24,7 +24,7 @@ logging.basicConfig ( ) logger = logging.getLogger ('boottools') -config = configparser.ConfigParser (inline_comment_prefixes='#') +config = configparser.ConfigParser (comment_prefixes='#', inline_comment_prefixes='#') if not os.path.exists ('boottoolsgenerator.cfg'): print ('configuration file "boottoolsgenerator.cfg" not found') sys.exit (1) @@ -90,10 +90,8 @@ if (rc): ## 'file|grep' failed except Exception as e: logger.error (str (e)) sys.exit (2) - #echo ================= nati after btogSetFsVirtual; ls -la /opt/opengnsys/tftpboot/ogclient/ logger.info ('Fase 3.2 - Configurar acceso schroot al Segundo Sistema de archivos (img)') -#cat /etc/schroot/schroot.conf | grep $BTROOTFSIMG || btogSetFsAccess btogSetFsAccess (btrootfsimg) ## para hacer schroot --cosas, el mntpt tiene que estar desmontado @@ -113,7 +111,6 @@ if (cp.returncode): except Exception as e: logger.error (str (e)) sys.exit (3) -#echo ================= nati after fase 3; ls -la /opt/opengnsys/tftpboot/ogclient/ ########################################################################### logger.info ('FASE 4 - Incorporando ficheros OpenGnsys al sistema raíz rootfs') @@ -145,8 +142,6 @@ if (not _is_mounted (btrootfsmnt)): logger.debug ('running \'{}/02-boottoolsFsOpengnsys.py --mntpt "{}" --osdistrib "{}" --oscodename "{}" --osrelease "{}" --osarch "{}" --oshttp "{}"\''.format (curdir, btrootfsmnt, osdistrib, oscodename, osrelease, osarch, oshttp)) stdout, _ = _run (['{}/02-boottoolsFsOpengnsys.py'.format (curdir), '--mntpt', btrootfsmnt, '--osdistrib', osdistrib, '--oscodename', oscodename, '--osrelease', osrelease, '--osarch', osarch, '--oshttp', oshttp]) logger.debug ('02-boottoolsFsOpengnsys stdout follows: {}'.format (stdout)) -## /tmp/02.py --osdistrib ubuntu --oscodename focal --osrelease 5.4.0-42-generic --osarch amd64 --oshttp http://es.archive.ubuntu.com/ubuntu/ -## /tmp/opengnsys_installer/opengnsys/client/./boot-tools/02-boottoolsFsOpengnsys.py --mntpt "/var/lib/tftpboot/ogclient/ogclientmount" --osdistrib "Ubuntu" --oscodename "noble" --osrelease "6.8.0-39-generic" --osarch "amd64" --oshttp "http://es.archive.ubuntu.com/ubuntu/" _run (['umount', btrootfsmnt]) From c5e9f6858df82c82f5d2a235a7ff89f525a26ac9 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Wed, 14 Aug 2024 09:47:28 +0200 Subject: [PATCH 17/45] refs #596 use f-strings, refactor the main program a bit --- 02-boottoolsFsOpengnsys.py | 38 +++-- 03-boottoolsSoftwareInstall.py | 6 +- 04-boottoolsSoftwareCompile.py | 2 +- 06-boottoolsInitrdGenerate.py | 4 +- boottoolsfunctions/__init__.py | 146 +++++++++---------- boottoolsgenerator.py | 250 +++++++++++++++++++-------------- 6 files changed, 245 insertions(+), 201 deletions(-) diff --git a/02-boottoolsFsOpengnsys.py b/02-boottoolsFsOpengnsys.py index f7c18d3..a26d018 100755 --- a/02-boottoolsFsOpengnsys.py +++ b/02-boottoolsFsOpengnsys.py @@ -20,8 +20,8 @@ def boottoolsFsOpengnsys (ogclientmount, osdistrib, oscodename, osrelease, osarc print ('Iniciando la personalización con datos del repositorio') #sed -e "s/OSCODENAME/$OSCODENAME/g" ${SVNCLIENTDIR}/includes/etc/apt/sources.list.${OSDISTRIB,,} > ${SVNCLIENTDIR}/includes/etc/apt/sources.list - sources_list_in = '{}/includes/etc/apt/sources.list.{}'.format (svnclientdir, osdistrib.lower()) - sources_list_out = '{}/includes/etc/apt/sources.list'.format (svnclientdir) + sources_list_in = f'{svnclientdir}/includes/etc/apt/sources.list.{osdistrib.lower()}' + sources_list_out = f'{svnclientdir}/includes/etc/apt/sources.list' fdin = open (sources_list_in, 'r') fdout = open (sources_list_out, 'w') while True: @@ -31,32 +31,30 @@ def boottoolsFsOpengnsys (ogclientmount, osdistrib, oscodename, osrelease, osarc fdin.close() fdout.close() - subprocess.run ('chmod -R 775 {}/includes/usr/bin/*'.format (svnclientdir), shell=True) + subprocess.run (f'chmod -R 775 {svnclientdir}/includes/usr/bin/*', shell=True) - os.makedirs ('{}/opt/opengnsys/lib/engine/bin/'.format (ogclientmount), exist_ok=True) - os.makedirs ('{}/usr/local/etc' .format (ogclientmount), exist_ok=True) - os.makedirs ('{}/usr/local/lib' .format (ogclientmount), exist_ok=True) - os.makedirs ('{}/usr/local/plugins' .format (ogclientmount), exist_ok=True) + os.makedirs (f'{ogclientmount}/opt/opengnsys/lib/engine/bin/', exist_ok=True) + os.makedirs (f'{ogclientmount}/usr/local/etc', exist_ok=True) + os.makedirs (f'{ogclientmount}/usr/local/lib', exist_ok=True) + os.makedirs (f'{ogclientmount}/usr/local/plugins', exist_ok=True) - subprocess.run ('rsync -aH {}/includes/* {}/' .format (svnclientdir, ogclientmount), shell=True) - subprocess.run ('rsync -aH {}/* {}/opt/opengnsys/' .format (svnclientstructure, ogclientmount), shell=True) - subprocess.run ('rsync -aH {}/* {}/opt/opengnsys/lib/engine/bin/'.format (svnclientengine, ogclientmount), shell=True) + subprocess.run (f'rsync -aH {svnclientdir}/includes/* {ogclientmount}/' , shell=True) + subprocess.run (f'rsync -aH {svnclientstructure}/* {ogclientmount}/opt/opengnsys/' , shell=True) + subprocess.run (f'rsync -aH {svnclientengine}/* {ogclientmount}/opt/opengnsys/lib/engine/bin/', shell=True) # Si no existe, copiar pci.ids. - if not os.path.exists ('{}/etc/pci.ids'.format (ogclientmount)): - shutil.copy ('{}/lib/pci.ids'.format (svnclientstructure), '{}/etc/'.format (ogclientmount)) + if not os.path.exists (f'{ogclientmount}/etc/pci.ids'): + shutil.copy (f'{svnclientstructure}/lib/pci.ids', f'{ogclientmount}/etc/') # Dependencias Qt para el Browser. - subprocess.run ('rsync -aH {}/etc/*.qmap {}/usr/local/etc' .format (svnclientstructure, ogclientmount), shell=True) - subprocess.run ('rsync -aH {}/lib/qtlib/* {}/usr/local/lib' .format (svnclientstructure, ogclientmount), shell=True) - subprocess.run ('rsync -aH {}/lib/fonts {}/usr/local/lib' .format (svnclientstructure, ogclientmount), shell=True) - subprocess.run ('rsync -aH {}/lib/qtplugins/* {}/usr/local/plugins'.format (svnclientstructure, ogclientmount), shell=True) + subprocess.run (f'rsync -aH {svnclientstructure}/etc/*.qmap {ogclientmount}/usr/local/etc', shell=True) + subprocess.run (f'rsync -aH {svnclientstructure}/lib/qtlib/* {ogclientmount}/usr/local/lib', shell=True) + subprocess.run (f'rsync -aH {svnclientstructure}/lib/fonts {ogclientmount}/usr/local/lib', shell=True) + subprocess.run (f'rsync -aH {svnclientstructure}/lib/qtplugins/* {ogclientmount}/usr/local/plugins', shell=True) # Browser y ogAdmClient. - if os.path.exists ('{}/bin/browser'.format (svnclientstructure)): - shutil.copy ('{}/bin/browser'.format (svnclientstructure), '{}/bin/'.format (ogclientmount)) - if os.path.exists ('{}/bin/ogAdmClient'.format (svnclientstructure)): - shutil.copy ('{}/bin/ogAdmClient'.format (svnclientstructure), '{}/bin/'.format (ogclientmount)) + if os.path.exists (f'{svnclientstructure}/bin/browser'): shutil.copy (f'{svnclientstructure}/bin/browser', f'{ogclientmount}/bin/') + if os.path.exists (f'{svnclientstructure}/bin/ogAdmClient'): shutil.copy (f'{svnclientstructure}/bin/ogAdmClient', f'{ogclientmount}/bin/') # El fichero de configuración debe sustituir a los 2 ficheros (borrar las 2 líneas). #echo "${VERSIONBOOTTOOLS}-${OSCODENAME}-${OSRELEASE}-${GITRELEASE}" > $nameisoclientfile diff --git a/03-boottoolsSoftwareInstall.py b/03-boottoolsSoftwareInstall.py index f7ab2f7..53f1f2e 100755 --- a/03-boottoolsSoftwareInstall.py +++ b/03-boottoolsSoftwareInstall.py @@ -78,7 +78,7 @@ def boottoolsSoftwareInstall (osarch, osrelease): _mock_mtab() # Instalamos el kernel. - _aptget_install (['linux-image-{}'.format (osrelease), 'linux-headers-{}'.format (osrelease), 'linux-modules-{}'.format (osrelease), 'linux-modules-extra-{}'.format (osrelease), 'dkms', 'shim-signed', 'openssl']) + _aptget_install ([f'linux-image-{osrelease}', f'linux-headers-{osrelease}', f'linux-modules-{osrelease}', f'linux-modules-extra-{osrelease}', 'dkms', 'shim-signed', 'openssl']) subprocess.run (['debconf-set-selections'], input=debconf_settings, text=True) @@ -95,10 +95,10 @@ def boottoolsSoftwareInstall (osarch, osrelease): stdout, _ = _run (['dkms', 'status']) for l in stdout.strip().split ('\n'): if not l: continue - print ('l "{}"'.format (l)) + print (f'l "{l}"') mod, vers, status = l.split (',') if 'added' in status: - print ('dkms installing {} {}'.format (mod, vers)) + print (f'dkms installing {mod} {vers}') _run (['dkms', 'install', '-m', mod.strip(), '-v', vers.strip()]) _oghook_activate() diff --git a/04-boottoolsSoftwareCompile.py b/04-boottoolsSoftwareCompile.py index 696082f..46adbcd 100755 --- a/04-boottoolsSoftwareCompile.py +++ b/04-boottoolsSoftwareCompile.py @@ -18,7 +18,7 @@ except: _run (['wget', 'https://sourceforge.net/projects/ms-sys/files/latest/download', '-O', 'ms-sys.tar.gz']) _run (['tar', '-xpzf', 'ms-sys.tar.gz']) mssys_dir = subprocess.run (['tar tzf ms-sys.tar.gz |head -n 1'], shell=True, capture_output=True, text=True).stdout.strip() - print ('mssys_dir "{}"'.format (mssys_dir)) + print (f'mssys_dir "{mssys_dir}"') os.chdir (mssys_dir) _run (['make', 'install']) os.chdir ('..') diff --git a/06-boottoolsInitrdGenerate.py b/06-boottoolsInitrdGenerate.py index 2d50941..254743e 100755 --- a/06-boottoolsInitrdGenerate.py +++ b/06-boottoolsInitrdGenerate.py @@ -12,8 +12,8 @@ def boottoolsInitrdGenerate (osrelease): shutil.copy ('/bin/busybox', '/usr/lib/initramfs-tools/bin') os.chdir ('/tmp') - _run (['mkinitramfs', '-o', '/tmp/initrd.img-{}'.format (osrelease), '-v', osrelease]) - shutil.copy ('/boot/vmlinuz-{}'.format (osrelease), '/tmp/') + _run (['mkinitramfs', '-o', f'/tmp/initrd.img-{osrelease}', '-v', osrelease]) + shutil.copy (f'/boot/vmlinuz-{osrelease}', '/tmp/') if __name__ == '__main__': parser = argparse.ArgumentParser() diff --git a/boottoolsfunctions/__init__.py b/boottoolsfunctions/__init__.py index af611f0..d5b9a15 100644 --- a/boottoolsfunctions/__init__.py +++ b/boottoolsfunctions/__init__.py @@ -158,7 +158,7 @@ def btogGetOsInfo1 (type_client): def btogGetOsInfo2 (type_client, versionboottools, ogclientcfg, osdistrib, oscodename, osrelease, osarch, oshttp): branch = 'master' - giturl = 'https://api.github.com/repos/opengnsys/OpenGnsys/commits?sha={}&path=/client'.format (branch) + giturl = f'https://api.github.com/repos/opengnsys/OpenGnsys/commits?sha={branch}&path=/client' gitrelease = 'r20240808' ## TODO: are we going to keep the following? $(curl -s "$GITURL" | jq -r '"r" + (.[0].commit.committer.date | split("-") | join("")[:8]) + "." + (.[0].sha[:7])') nameisoclient ='-'.join ([versionboottools, oscodename, osrelease, osarch, gitrelease]) namehostclient = '-'.join ([versionboottools, oscodename, gitrelease]) @@ -169,46 +169,46 @@ def btogGetOsInfo2 (type_client, versionboottools, ogclientcfg, osdistrib, oscod # Generar fichero de configuración. with open (ogclientcfg, 'w') as f: - print ('TYPECLIENT="{}"'.format (type_client), file=f) - print ('OSDISTRIB="{}"'.format (osdistrib), file=f) - print ('OSCODENAME="{}"'.format (oscodename), file=f) - print ('OSRELEASE="{}"'.format (osrelease), file=f) - print ('OSARCH="{}"'.format (osarch), file=f) - print ('OSHTTP="{}"'.format (oshttp), file=f) - print ('GITRELEASE="{}"'.format (gitrelease), file=f) - print ('NAMEISOCLIENT="{}"'.format (nameisoclient), file=f) - print ('NAMEHOSTCLIENT="{}"'.format (namehostclient), file=f) + print (f'TYPECLIENT="{type_client}"', file=f) + print (f'OSDISTRIB="{osdistrib}"', file=f) + print (f'OSCODENAME="{oscodename}"', file=f) + print (f'OSRELEASE="{osrelease}"', file=f) + print (f'OSARCH="{osarch}"', file=f) + print (f'OSHTTP="{oshttp}"', file=f) + print (f'GITRELEASE="{gitrelease}"', file=f) + print (f'NAMEISOCLIENT="{nameisoclient}"', file=f) + print (f'NAMEHOSTCLIENT="{namehostclient}"', file=f) return gitrelease, nameisoclient, namehostclient def btogSetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, bttargetdir, osarch): #if not btrootfsimg: btogGetVar() - logger.info ('Creación y formateo del disco virtual "{}" "{}" MB'.format (btrootfsimg, btvirtualdisksize)) + logger.info (f'Creación y formateo del disco virtual "{btrootfsimg}" "{btvirtualdisksize}" MB') try: _umount (btrootfsmnt) except: pass if (_is_mounted (btrootfsmnt)): - raise Exception ('failed to umount "{}"'.format (btrootfsmnt)) + raise Exception (f'failed to umount "{btrootfsmnt}"') try: os.makedirs (btrootfsmnt, exist_ok=True) except: - raise Exception ('Creando directorio "{}" : ERROR'.format (btrootfsmnt)) + raise Exception (f'Creando directorio "{btrootfsmnt}" : ERROR') try: _run (['chown', '-R', 'root:opengnsys', bttargetdir]) except: - raise Exception ('Failed to chown root:opengnsys "{}"'.format (btrootfsmnt)) + raise Exception (f'Failed to chown root:opengnsys "{btrootfsmnt}"') - logger.info ('Creating disk image "{}"'.format (btrootfsimg)) + logger.info (f'Creating disk image "{btrootfsimg}"') if 'i386' == osarch: - try: _run (['dd', 'if=/dev/zero', 'of={}'.format (btrootfsimg), 'bs=1048576', 'count={}'.format (btvirtualdisksize)]) + try: _run (['dd', 'if=/dev/zero', f'of={btrootfsimg}', 'bs=1048576', f'count={btvirtualdisksize}']) except: - raise Exception ('Creando el disco virtual "{}" con tamaño maxima "{}" MB : ERROR'.format (btrootfsimg, btvirtualdisksize)) + raise Exception (f'Creando el disco virtual "{btrootfsimg}" con tamaño maxima "{btvirtualdisksize}" MB : ERROR') else: try: _run (['qemu-img', 'create', btrootfsimg, btvirtualdisksize]) except: - raise Exception ('Creando el disco virtual "{}" con tamaño maxima "{}" MB : ERROR'.format (btrootfsimg, btvirtualdisksize)) + raise Exception (f'Creando el disco virtual "{btrootfsimg}" con tamaño maxima "{btvirtualdisksize}" MB : ERROR') logger.debug ('losetup --find') diskloop, _ = _run (['losetup', '--find']) @@ -220,8 +220,8 @@ def btogSetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisks except: raise Exception ('losetup failed') logger.info ('Partitioning disk image') - stdout, _ = _run (['bash', '-c', "echo $'n\np\n1\n\n\nt\n83\nw' |fdisk {}".format (diskloop)]) - logger.debug ('fdisk output follows: {}'.format (stdout)) + stdout, _ = _run (['bash', '-c', f"echo $'n\np\n1\n\n\nt\n83\nw' |fdisk {diskloop}"]) + logger.debug (f'fdisk output follows: {stdout}') time.sleep (3) logger.debug ('losetup --detach') @@ -248,7 +248,7 @@ def btogSetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisks except: raise Exception ('Liberando la particion virtual despues del formateo: ERROR') - logger.info ('"{}" "{}" MB : OK'.format (btrootfsimg, btvirtualdisksize)) + logger.info (f'"{btrootfsimg}" "{btvirtualdisksize}" MB : OK') #btogSetFsAcces: habilita el acceso al sistema root del cliente con schroot def btogSetFsAccess (btrootfsimg): @@ -307,14 +307,14 @@ def btogSetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp): raise Exception ('rootfs should be mounted') logger.info ('debootstrapping system') - logger.debug ('debootstrap --arch="{}" --components=main,universe "{}" "{}" "{}"'.format (osarch, oscodename, btrootfsmnt, oshttp)) - try: _run (['debootstrap', '--arch={}'.format(osarch), '--components=main,universe', oscodename, btrootfsmnt, oshttp]) + logger.debug (f'debootstrap --arch="{osarch}" --components=main,universe "{oscodename}" "{btrootfsmnt}" "{oshttp}"') + try: _run (['debootstrap', f'--arch={osarch}', '--components=main,universe', oscodename, btrootfsmnt, oshttp]) except: if (_is_mounted (btrootfsmnt)): _run (['umount', btrootfsmnt]) - raise Exception ('debootstrap --arch="{}" --components=main,universe "{}" "{}" "{}" : ha fallado!'.format (osarch, oscodename, btrootfsmnt, oshttp)) + raise Exception (f'debootstrap --arch="{osarch}" --components=main,universe "{oscodename}" "{btrootfsmnt}" "{oshttp}" : ha fallado!') - logger.info ('debootstrap --arch="{}" --components=main,universe "{}" "{}" "{}" : ok'.format (osarch, oscodename, btrootfsmnt, oshttp)) + logger.info (f'debootstrap --arch="{osarch}" --components=main,universe "{oscodename}" "{btrootfsmnt}" "{oshttp}" : ok') return 0 def boottoolsSshServer (btrootfsmnt): @@ -323,22 +323,22 @@ def boottoolsSshServer (btrootfsmnt): #shutil.copy ('/root/.ssh/id_rsa.pub', '/tmp/') logger.debug ('comprobando directorio .ssh del root') - if not os.path.exists ('{}/root/.ssh'.format (btrootfsmnt)): ## crea directorio dentro del chroot + if not os.path.exists (f'{btrootfsmnt}/root/.ssh'): ## crea directorio dentro del chroot logger.debug ('creando directorio .ssh 600') - os.mkdir ('{}/root/.ssh'.format (btrootfsmnt)) - os.chmod ('{}/root/.ssh'.format (btrootfsmnt), 0o700) + os.mkdir (f'{btrootfsmnt}/root/.ssh') + os.chmod (f'{btrootfsmnt}/root/.ssh', 0o700) logger.debug ('creando el fichero authorized_keys') ## crea archivo en el chroot - if not os.path.exists ('{}/root/.ssh/authorized_keys'.format (btrootfsmnt)): - open ('{}/root/.ssh/authorized_keys'.format (btrootfsmnt), 'w').close() - os.chmod ('{}/root/.ssh/authorized_keys'.format (btrootfsmnt), 0o600) + if not os.path.exists (f'{btrootfsmnt}/root/.ssh/authorized_keys'): + open (f'{btrootfsmnt}/root/.ssh/authorized_keys', 'w').close() + os.chmod (f'{btrootfsmnt}/root/.ssh/authorized_keys', 0o600) logger.debug ('importando la clave publica del servidor OG') #cat /tmp/id_rsa.pub if os.path.exists ('/root/.ssh/id_rsa.pub'): ## coge la publica de la VM y la pone en el authorized_keys del chroot #cat /tmp/id_rsa.pub >> /root/.ssh/authorized_keys fdin = open ('/root/.ssh/id_rsa.pub', 'r') - fdout = open ('{}/root/.ssh/authorized_keys'.format (btrootfsmnt), 'a') + fdout = open (f'{btrootfsmnt}/root/.ssh/authorized_keys', 'a') while True: l = fdin.readline() if not l: break @@ -350,12 +350,12 @@ def boottoolsSshServer (btrootfsmnt): logger.error ('no key publica og') def boottoolsSshClient (btrootfsmnt): - if not os.path.exists ('{}/root/.ssh/id_rsa'.format (btrootfsmnt)): - _run (['ssh-keygen', '-q', '-f', '{}/root/.ssh/id_rsa'.format (btrootfsmnt), '-N', '']) ## crea un par de claves en el chroot + if not os.path.exists (f'{btrootfsmnt}/root/.ssh/id_rsa'): + _run (['ssh-keygen', '-q', '-f', f'{btrootfsmnt}/root/.ssh/id_rsa', '-N', '']) ## crea un par de claves en el chroot #cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys ## coge la publica y se la autoriza a sí mismo - fdin = open ('{}//root/.ssh/id_rsa.pub'.format (btrootfsmnt), 'r') - fdout = open ('{}/root/.ssh/authorized_keys'.format (btrootfsmnt), 'a') + fdin = open (f'{btrootfsmnt}//root/.ssh/id_rsa.pub', 'r') + fdout = open (f'{btrootfsmnt}/root/.ssh/authorized_keys', 'a') while True: l = fdin.readline() if not l: break @@ -373,33 +373,33 @@ def btogFsInitrd (bttargetdir, osrelease): ## backup de oginitrd.img, oginitrd.img.sum, ogvmlinuz y ogvmlinuz.sum now = datetime.datetime.now(datetime.timezone.utc).strftime ('%Y%m%d-%H%M%S%z') - if os.path.exists ('{}/oginitrd.img'.format (bttargetdir)): - os.rename ('{}/oginitrd.img' .format (bttargetdir), '{}/oginitrd.img.{}' .format (bttargetdir, now)) - os.rename ('{}/oginitrd.img.sum'.format (bttargetdir), '{}/oginitrd.img.sum.{}'.format (bttargetdir, now)) - if os.path.exists ('{}/ogvmlinuz'.format (bttargetdir)): - os.rename ('{}/ogvmlinuz' .format (bttargetdir), '{}/ogvmlinuz.{}' .format (bttargetdir, now)) - os.rename ('{}/ogvmlinuz.sum'.format (bttargetdir), '{}/ogvmlinuz.sum.{}'.format (bttargetdir, now)) + if os.path.exists (f'{bttargetdir}/oginitrd.img'): + os.rename (f'{bttargetdir}/oginitrd.img' , f'{bttargetdir}/oginitrd.img.{now}') + os.rename (f'{bttargetdir}/oginitrd.img.sum', f'{bttargetdir}/oginitrd.img.sum.{now}') + if os.path.exists (f'{bttargetdir}/ogvmlinuz'): + os.rename (f'{bttargetdir}/ogvmlinuz' , f'{bttargetdir}/ogvmlinuz.{now}') + os.rename (f'{bttargetdir}/ogvmlinuz.sum', f'{bttargetdir}/ogvmlinuz.sum.{now}') - shutil.copy ('/tmp/initrd.img-{}'.format (osrelease), '{}/oginitrd.img'.format (bttargetdir)) - shutil.copy ('/tmp/vmlinuz-{}' .format (osrelease), '{}/ogvmlinuz' .format (bttargetdir)) + shutil.copy (f'/tmp/initrd.img-{osrelease}', f'{bttargetdir}/oginitrd.img') + shutil.copy (f'/tmp/vmlinuz-{osrelease}', f'{bttargetdir}/ogvmlinuz') #DATASUM=`md5sum "${BTTARGETDIR}/oginitrd.img" | cut -f1 -d" "` - md5, _ = _run (['md5sum', '{}/oginitrd.img'.format (bttargetdir)]) + md5, _ = _run (['md5sum', f'{bttargetdir}/oginitrd.img']) md5, rest = md5.split (' ', 1) #echo $DATASUM > ${BTTARGETDIR}/oginitrd.img.sum - with open ('{}/oginitrd.img.sum'.format (bttargetdir), 'w') as fd: + with open (f'{bttargetdir}/oginitrd.img.sum', 'w') as fd: fd.write (md5) #DATASUM=`md5sum "${BTTARGETDIR}/ogvmlinuz" | cut -f1 -d" "` - md5, _ = _run (['md5sum', '{}/ogvmlinuz'.format (bttargetdir)]) + md5, _ = _run (['md5sum', f'{bttargetdir}/ogvmlinuz']) md5, rest = md5.split (' ', 1) #echo $DATASUM > ${BTTARGETDIR}/ogvmlinuz.sum - with open ('{}/ogvmlinuz.sum'.format (bttargetdir), 'w') as fd: + with open (f'{bttargetdir}/ogvmlinuz.sum', 'w') as fd: fd.write (md5) #cd - #chmod -R 755 $BTTARGETDIR - for f in glob.glob ('{}/oginitrd*'.format (bttargetdir)) + glob.glob ('{}/vmlinuz*'.format (bttargetdir)): + for f in glob.glob (f'{bttargetdir}/oginitrd*') + glob.glob (f'{bttargetdir}/vmlinuz*'): os.chmod (f, 0o755) #btogFsSqfs convierte el sistema root en sqfs @@ -407,18 +407,18 @@ def btogFsSqfs (bttargetdir, btrootfsmnt): logger.info ('Iniciando la creación del sistema de archivos en sqfs') # si ya existe un sqfs lo renombramos - if os.path.exists ('{}/ogclient.sqfs'.format (bttargetdir)): + if os.path.exists (f'{bttargetdir}/ogclient.sqfs'): now = datetime.datetime.now(datetime.timezone.utc).strftime ('%Y%m%d-%H%M%S%z') - os.rename ('{}/ogclient.sqfs'.format (bttargetdir), '{}/ogclient.sqfs.{}'.format (bttargetdir, now)) + os.rename (f'{bttargetdir}/ogclient.sqfs', f'{bttargetdir}/ogclient.sqfs.{now}') - _run (['mksquashfs', btrootfsmnt, '{}/ogclient.sqfs'.format (bttargetdir)]) - os.chmod ('{}/ogclient.sqfs'.format (bttargetdir), 0o744) + _run (['mksquashfs', btrootfsmnt, f'{bttargetdir}/ogclient.sqfs']) + os.chmod (f'{bttargetdir}/ogclient.sqfs', 0o744) #DATASUM=`md5sum "${BTTARGETDIR}/ogclient.sqfs" | cut -f1 -d" "` - md5, _ = _run (['md5sum', '{}/ogclient.sqfs'.format (bttargetdir)]) + md5, _ = _run (['md5sum', f'{bttargetdir}/ogclient.sqfs']) md5, rest = md5.split (' ', 1) #echo $DATASUM > ${BTTARGETDIR}/ogclient.sqfs.sum - with open ('{}/ogclient.sqfs.sum'.format (bttargetdir), 'w') as fd: + with open (f'{bttargetdir}/ogclient.sqfs.sum', 'w') as fd: fd.write (md5) # btogIsoGenerator genera la iso del cliente @@ -434,16 +434,16 @@ def btogIsoGenerator (pxepkg, isolinux_tpl, bttargetdir, nameisoclient): elif 'ipxe' == pxepkg: subprocess.run (['cp -a /usr/lib/ipxe/* /tmp/iso/isolinux'], shell=True) else: - raise Exception ('unknown pxepkg value "{}"'.format (pxepkg)) + raise Exception (f'unknown pxepkg value "{pxepkg}"') # Si existe el fichero ISO, montarlo para extraer isolinux.bin. - if os.path.exists ('/tmp/iso/isolinux/{}.iso'.format (pxepkg)): + if os.path.exists (f'/tmp/iso/isolinux/{pxepkg}.iso'): os.mkdir ('/tmp/iso/isolinux/mount') - _run (['mount', '-o', 'loop', '/tmp/iso/isolinux/{}.iso'.format (pxepkg), '/tmp/iso/isolinux/mount']) + _run (['mount', '-o', 'loop', f'/tmp/iso/isolinux/{pxepkg}.iso', '/tmp/iso/isolinux/mount']) subprocess.run (['cp -a /tmp/iso/isolinux/mount/* /tmp/iso/isolinux'], shell=True) _run (['umount', '/tmp/iso/isolinux/mount']) os.rmdir ('/tmp/iso/isolinux/mount') - os.unlink ('/tmp/iso/isolinux/{}.iso'.format (pxepkg)) + os.unlink (f'/tmp/iso/isolinux/{pxepkg}.iso') ## ojo que aquí hay que interpolar $NAMEISOCLIENT y $PXEPKG with open ('/tmp/iso/isolinux/isolinux.cfg', 'w') as fd: @@ -452,33 +452,33 @@ def btogIsoGenerator (pxepkg, isolinux_tpl, bttargetdir, nameisoclient): # preparamos el directorio boot-tools. if not os.path.exists ('/tmp/iso/ogclient'): os.mkdir ('/tmp/iso/ogclient') - shutil.copy ('{}/ogclient.sqfs' .format (bttargetdir), '/tmp/iso/ogclient/') - shutil.copy ('{}/ogclient.sqfs.sum'.format (bttargetdir), '/tmp/iso/ogclient/') - shutil.copy ('{}/ogvmlinuz' .format (bttargetdir), '/tmp/iso/ogclient/') - shutil.copy ('{}/ogvmlinuz.sum' .format (bttargetdir), '/tmp/iso/ogclient/') - shutil.copy ('{}/ogvmlinuz' .format (bttargetdir), '/tmp/iso/ogclient/linuxISO') - shutil.copy ('{}/ogvmlinuz.sum' .format (bttargetdir), '/tmp/iso/ogclient/linuxISO.sum') - shutil.copy ('{}/oginitrd.img' .format (bttargetdir), '/tmp/iso/ogclient/') - shutil.copy ('{}/oginitrd.img.sum' .format (bttargetdir), '/tmp/iso/ogclient/') + shutil.copy (f'{bttargetdir}/ogclient.sqfs', '/tmp/iso/ogclient/') + shutil.copy (f'{bttargetdir}/ogclient.sqfs.sum', '/tmp/iso/ogclient/') + shutil.copy (f'{bttargetdir}/ogvmlinuz', '/tmp/iso/ogclient/') + shutil.copy (f'{bttargetdir}/ogvmlinuz.sum', '/tmp/iso/ogclient/') + shutil.copy (f'{bttargetdir}/ogvmlinuz', '/tmp/iso/ogclient/linuxISO') + shutil.copy (f'{bttargetdir}/ogvmlinuz.sum', '/tmp/iso/ogclient/linuxISO.sum') + shutil.copy (f'{bttargetdir}/oginitrd.img', '/tmp/iso/ogclient/') + shutil.copy (f'{bttargetdir}/oginitrd.img.sum', '/tmp/iso/ogclient/') #el ogclienteToISO debe tener una copia del ogvmlinuz como linuxISO #cp -prv /var/lib/tftpboot/ogclientToIso/* /tmp/iso/ogclient oldpwd = os.getcwd() os.chdir ('/tmp') - logger.debug ('mkisofs -V ogClient -o {}.iso -b isolinux/isolinux.bin -c isolinux/boot.cat -J -no-emul-boot -boot-load-size 4 -boot-info-table /tmp/iso'.format (nameisoclient)) - _run (['mkisofs', '-V', 'ogClient', '-o', '{}.iso'.format (nameisoclient), '-b', 'isolinux/isolinux.bin', '-c', 'isolinux/boot.cat', '-J', '-no-emul-boot', '-boot-load-size', '4', '-boot-info-table', '/tmp/iso']) + logger.debug (f'mkisofs -V ogClient -o {nameisoclient}.iso -b isolinux/isolinux.bin -c isolinux/boot.cat -J -no-emul-boot -boot-load-size 4 -boot-info-table /tmp/iso') + _run (['mkisofs', '-V', 'ogClient', '-o', f'{nameisoclient}.iso', '-b', 'isolinux/isolinux.bin', '-c', 'isolinux/boot.cat', '-J', '-no-emul-boot', '-boot-load-size', '4', '-boot-info-table', '/tmp/iso']) ##Generamos el md5 #DATASUM=`md5sum ${NAMEISOCLIENT}.iso | cut -f1 -d" "` - md5, _ = _run (['md5sum', '{}.iso'.format (nameisoclient)]) + md5, _ = _run (['md5sum', f'{nameisoclient}.iso']) md5, rest = md5.split (' ', 1) #echo $DATASUM > ${NAMEISOCLIENT}.iso.sum - with open ('{}.iso.sum'.format (nameisoclient), 'w') as fd: + with open (f'{nameisoclient}.iso.sum', 'w') as fd: fd.write (md5) os.chdir (oldpwd) - os.rename ('/tmp/{}.iso' .format (nameisoclient), '/var/lib/tftpboot/ogclient/{}.iso' .format (nameisoclient)) - os.rename ('/tmp/{}.iso.sum'.format (nameisoclient), '/var/lib/tftpboot/ogclient/{}.iso.sum'.format (nameisoclient)) + os.rename (f'/tmp/{nameisoclient}.iso' , f'/var/lib/tftpboot/ogclient/{nameisoclient}.iso') + os.rename (f'/tmp/{nameisoclient}.iso.sum', f'/var/lib/tftpboot/ogclient/{nameisoclient}.iso.sum') #def __unused_boottoolsBootGraphics(): # find /tmp/opengnsys_installer/ -name .svn -type d -exec rm -fr {} \; 2>/dev/null; diff --git a/boottoolsgenerator.py b/boottoolsgenerator.py index 2b6a362..285f1cf 100755 --- a/boottoolsgenerator.py +++ b/boottoolsgenerator.py @@ -13,16 +13,19 @@ curdir = os.path.dirname (__file__) sys.path.insert (0, curdir) from boottoolsfunctions import _run, _is_mounted, btogGetOsInfo1, btogGetOsInfo2, btogGetVar, btogSetFsVirtual, btogSetFsAccess, btogSetFsBase, boottoolsSshServer, boottoolsSshClient, btogFsInitrd, btogFsSqfs, btogIsoGenerator -#logging.root.handlers = [] -logging.basicConfig ( - format='%(levelname)s %(asctime)s (%(funcName)s) %(message)s', - level=logging.INFO, - handlers = [ - logging.FileHandler ('/tmp/boot-tools_installation.log'), - logging.StreamHandler (sys.stdout), - ], -) -logger = logging.getLogger ('boottools') +def _logging(): + #logging.root.handlers = [] + logging.basicConfig ( + format='%(levelname)s %(asctime)s (%(funcName)s) %(message)s', + level=logging.INFO, + handlers = [ + logging.FileHandler ('/tmp/boot-tools_installation.log'), + logging.StreamHandler (sys.stdout), + ], + ) + return = logging.getLogger ('boottools') + +logger = _logging() config = configparser.ConfigParser (comment_prefixes='#', inline_comment_prefixes='#') if not os.path.exists ('boottoolsgenerator.cfg'): @@ -31,13 +34,14 @@ if not os.path.exists ('boottoolsgenerator.cfg'): config.read ('boottoolsgenerator.cfg') isolinux_tpl = config['ISOLinux'].get ('template') -if not os.path.exists ('/tmp/opengnsys_installer/opengnsys/client/engine'): - branch = 'main' - tmpdir, _ = _run (['mktemp', '--tmpdir', '--directory', 'oggit.XXXXXX']) - logger.debug ('tmpdir "{}"'.format(tmpdir)) - _run (['git', 'clone', '-c', 'http.sslVerify=false', '--branch', branch, 'https://ognproject.evlt.uma.es/gitea/opengnsys/opengnsys.git', tmpdir]) - _run (['rsync', '-aH', '{}/client/engine'.format(tmpdir), '{}/client/shared'.format(tmpdir), '/tmp/opengnsys_installer/opengnsys/client/']) - _run (['rm', '-rf', tmpdir]) +def clone_client_dirs(): + if not os.path.exists ('/tmp/opengnsys_installer/opengnsys/client/engine'): + branch = 'main' + tmpdir, _ = _run (['mktemp', '--tmpdir', '--directory', 'oggit.XXXXXX']) + logger.debug (f'tmpdir "{tmpdir}"') + _run (['git', 'clone', '-c', 'http.sslVerify=false', '--branch', branch, 'https://ognproject.evlt.uma.es/gitea/opengnsys/opengnsys.git', tmpdir]) + _run (['rsync', '-aH', f'{tmpdir}/client/engine', f'{tmpdir}/client/shared', '/tmp/opengnsys_installer/opengnsys/client/']) + _run (['rm', '-rf', tmpdir]) type_client = sys.argv[1] if len(sys.argv)>1 else 'host' #WORKDIR = '/tmp/opengnsys_installer' @@ -50,46 +54,55 @@ if os.getuid(): #os.chdir ('/tmp') +logger.info ('OpenGnsys CLIENT installation begins') +clone_client_dirs() ####################################################################3 logger.info ('FASE 1 - Asignación de variables') ogclientcfg, osdistrib, oscodename, osrelease, osarch, oshttp = btogGetOsInfo1(type_client) btdir, bttargetdir, btrootfsimg, btrootfsmnt, btrootfsimglabel, log_file, versionboottools, btvirtualdisksize = btogGetVar(osarch) gitrelease, nameisoclient, namehostclient = btogGetOsInfo2(type_client, versionboottools, ogclientcfg, osdistrib, oscodename, osrelease, osarch, oshttp) -logger.info ('OpenGnsys CLIENT installation begins') +## this is convenient in case the previous run failed and we want to run this program again if _is_mounted (btrootfsmnt): _run (['umount', btrootfsmnt]) ########################################################################## logger.info ('FASE 2 - Instalación de software adicional.') -#grep "http://free.nchc.org.tw/drbl-core" /etc/apt/sources.list || echo "deb http://free.nchc.org.tw/drbl-core drbl stable" >> /etc/apt/sources.list -_run (['apt-get', 'update']) #>>/tmp/fase2.out -acse_gpxe, _ = _run (['apt-cache', 'search', 'gpxe']) -acse_ipxe, _ = _run (['apt-cache', 'search', 'ipxe']) -if acse_ipxe: - pxepkg = 'ipxe' -elif acse_gpxe: - pxepkg = 'gpxe' -else: - logger.error ('neither gpxe nor ipxe found in apt-cache') - sys.exit (1) -logger.info ('PXE package is "{}"'.format (pxepkg)) + +def _get_pxepkg(): + #grep "http://free.nchc.org.tw/drbl-core" /etc/apt/sources.list || echo "deb http://free.nchc.org.tw/drbl-core drbl stable" >> /etc/apt/sources.list + _run (['apt-get', 'update']) #>>/tmp/fase2.out + acse_gpxe, _ = _run (['apt-cache', 'search', 'gpxe']) + acse_ipxe, _ = _run (['apt-cache', 'search', 'ipxe']) + if acse_ipxe: + pxepkg = 'ipxe' + elif acse_gpxe: + pxepkg = 'gpxe' + else: + logger.error ('neither gpxe nor ipxe found in apt-cache') + sys.exit (1) + logger.info (f'PXE package is "{pxepkg}"') + return pxepkg + +pxepkg = _get_pxepkg() ## TODO qemu no existe, hace falta? _run (['apt-get', '-y', 'install', 'jq', 'syslinux', 'syslinux-efi', 'syslinux-utils', 'debootstrap', 'subversion', 'schroot', 'squashfs-tools', 'syslinux', 'genisoimage', 'qemu-utils', 'lsof', pxepkg]) #>>/tmp/fase2.out -###################################################################3 +def _mkrootfs(): + rc = subprocess.run (f'file "{btrootfsimg}" |grep -q "partition 1 *: ID=0x83"', shell=True).returncode + print (rc) + if (rc): ## 'file|grep' failed + try: btogSetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, bttargetdir, osarch) + except Exception as e: + logger.error (str (e)) + sys.exit (2) + logger.info ('FASE 3 - Creación del Sistema raiz RootFS (Segundo Sistema archivos (img))') logger.info ('Fase 3.1 Generar y formatear el disco virtual. Generar el dispositivo loop.') -rc = subprocess.run ('file {} |grep -q "partition 1 *: ID=0x83"'.format (btrootfsimg), shell=True).returncode -print (rc) -if (rc): ## 'file|grep' failed - try: btogSetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, bttargetdir, osarch) - except Exception as e: - logger.error (str (e)) - sys.exit (2) +_mkrootfs() logger.info ('Fase 3.2 - Configurar acceso schroot al Segundo Sistema de archivos (img)') btogSetFsAccess (btrootfsimg) @@ -97,28 +110,30 @@ btogSetFsAccess (btrootfsimg) ## para hacer schroot --cosas, el mntpt tiene que estar desmontado ## si está montado da un pete tal que 'E: 10mount: mount: /run/schroot/mount/IMGogclient-7fbf51a2-e37e-48e5-8e5d-83f8901fc7ed: wrong fs type, bad option, bad superblock on /dev/loop1, missing codepage or helper program, or other error.' -logger.info ('Fase 3.3 Generar sistema de archivos con debootstrap') -logger.debug ('Try creation of a file within chroot (this operation may fail with "... etc/resolv.conf: No such file or directory"--that is ok)') -logger.debug ('Running \'schroot --chroot IMGogclient -- stat /etc\'') -cp = subprocess.run (['schroot', '--chroot', 'IMGogclient', '--', 'stat', '/etc']) -if (cp.returncode): - logger.debug ('schroot returned code "{}", calling btogSetFsBase()'.format (cp.returncode)) - try: _run (['mount', btrootfsimg, btrootfsmnt, '-o', 'loop,offset=32256']) - except: - logger.error ('mount failed') - sys.exit (3) - try: btogSetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp) #>>/tmp/fase3.out - except Exception as e: - logger.error (str (e)) - sys.exit (3) +def _debootstrap(): + logger.debug ('Try creation of a file within chroot (this operation may fail with "... etc/resolv.conf: No such file or directory"--that is ok)') + logger.debug ('Running \'schroot --chroot IMGogclient -- stat /etc\'') + cp = subprocess.run (['schroot', '--chroot', 'IMGogclient', '--', 'stat', '/etc']) + if (cp.returncode): + logger.debug (f'schroot returned code "{cp.returncode}", calling btogSetFsBase()') + try: _run (['mount', btrootfsimg, btrootfsmnt, '-o', 'loop,offset=32256']) + except: + logger.error ('mount failed') + sys.exit (3) + try: btogSetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp) #>>/tmp/fase3.out + except Exception as e: + logger.error (str (e)) + sys.exit (3) + +logger.info ('Fase 3.3 Generar sistema de archivos con debootstrap') +_debootstrap() -########################################################################### logger.info ('FASE 4 - Incorporando ficheros OpenGnsys al sistema raíz rootfs') ## por qué esta copia??? #cp -a ${BTDIR}/includes/usr/bin/* /tmp >>/tmp/fase5.out -#_run (['cp', '-a'] + glob.glob ('{}/includes/usr/bin/*'.format (btdir)) + ['/tmp']) +#_run (['cp', '-a'] + glob.glob (f'{btdir}/includes/usr/bin/*') + ['/tmp']) #chmod +x /tmp/boot-tools/*.sh #for i in glob.glob ('/tmp/boot-tools/*.sh'): @@ -128,75 +143,106 @@ logger.info ('FASE 4 - Incorporando ficheros OpenGnsys al sistema raíz rootfs') # Incluir revisión. ## FIXME esto la incluye incondicionalmente, y luego terminamos con "OpenGnsys Client 1.2.0-rc1 gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) ..." #sed -i "1 s/$/ $GITRELEASE ($OSRELEASE)/" ${BTDIR}/includes/etc/initramfs-tools/scripts/VERSION.txt -_run (['sed', '-i', '1 s/$/ {} ({})/'.format (gitrelease, osrelease), '{}/includes/etc/initramfs-tools/scripts/VERSION.txt'.format (btdir)]) +_run (['sed', '-i', f'1 s/$/ {gitrelease} ({osrelease})/', f'{btdir}/includes/etc/initramfs-tools/scripts/VERSION.txt']) # En Ubuntu 13.04+ es necesario matar proceso de "udev" antes de desmontar. #umount $BTROOTFSMNT 2>/dev/null || (kill -9 $(lsof -t $BTROOTFSMNT); umount $BTROOTFSMNT) 2>/dev/null -if (not _is_mounted (btrootfsmnt)): +def _cerodos(): + if (not _is_mounted (btrootfsmnt)): + try: _run (['mount', btrootfsimg, btrootfsmnt, '-o', 'loop,offset=32256']) + except: + logger.error ('mount failed') + sys.exit (3) + + logger.debug (f'running \'{curdir}/02-boottoolsFsOpengnsys.py --mntpt "{btrootfsmnt}" --osdistrib "{osdistrib}" --oscodename "{oscodename}" --osrelease "{osrelease}" --osarch "{osarch}" --oshttp "{oshttp))}"\'') + stdout, _ = _run ([f'{curdir}/02-boottoolsFsOpengnsys.py', '--mntpt', btrootfsmnt, '--osdistrib', osdistrib, '--oscodename', oscodename, '--osrelease', osrelease, '--osarch', osarch, '--oshttp', oshttp]) + logger.debug (f'02-boottoolsFsOpengnsys stdout follows: {stdout}') + + _run (['umount', btrootfsmnt]) + +_cerodos() + + +## aqui empiezan las cosas de chroot + +def _cerotres(): + logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/03-boottoolsSoftwareInstall.py --osrelease "{osrelease}" --osarch "{osarch))}"\'') + stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/03-boottoolsSoftwareInstall.py', '--osrelease', osrelease, '--osarch', osarch]) + logger.debug (f'03-boottoolsSoftwareInstall stdout follows: {stdout}') + +logger.info ('FASE 5 - Instalar software') +logger.info ('Fase 5.1 instalar paquetes deb con apt-get') +_cerotres() + +def _cerocuatro(): + logger.debug ('running \'schroot --chroot IMGogclient -- {}/04-boottoolsSoftwareCompile.py\'') + #cd / + stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/04-boottoolsSoftwareCompile.py']) + #cd - + logger.debug (f'04-boottoolsSoftwareCompile stdout follows: {stdout}') + +logger.info ('Fase 5.2 compilar software.') +_cerocuatro(): + +## ya no chroot + + + + +def _ssh_stuff(): try: _run (['mount', btrootfsimg, btrootfsmnt, '-o', 'loop,offset=32256']) except: logger.error ('mount failed') sys.exit (3) -logger.debug ('running \'{}/02-boottoolsFsOpengnsys.py --mntpt "{}" --osdistrib "{}" --oscodename "{}" --osrelease "{}" --osarch "{}" --oshttp "{}"\''.format (curdir, btrootfsmnt, osdistrib, oscodename, osrelease, osarch, oshttp)) -stdout, _ = _run (['{}/02-boottoolsFsOpengnsys.py'.format (curdir), '--mntpt', btrootfsmnt, '--osdistrib', osdistrib, '--oscodename', oscodename, '--osrelease', osrelease, '--osarch', osarch, '--oshttp', oshttp]) -logger.debug ('02-boottoolsFsOpengnsys stdout follows: {}'.format (stdout)) + #cd / + #schroot --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSshServer.sh ## no necesita chroot + boottoolsSshServer (btrootfsmnt) + #cd - -_run (['umount', btrootfsmnt]) + #schroot --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSshClient.sh ## no necesita chroot + boottoolsSshClient (btrootfsmnt) + if _is_mounted (btrootfsmnt): + _run (['umount', btrootfsmnt]) -############################################################################################ -logger.info ('FASE 5 - Instalar software') -logger.info ('Fase 5.1 instalar paquetes deb con apt-get') -logger.debug ('running \'schroot --chroot IMGogclient -- {}/03-boottoolsSoftwareInstall.py --osrelease "{}" --osarch "{}"\''.format (curdir, osrelease, osarch)) -stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', '{}/03-boottoolsSoftwareInstall.py'.format (curdir), '--osrelease', osrelease, '--osarch', osarch]) -logger.debug ('03-boottoolsSoftwareInstall stdout follows: {}'.format (stdout)) + ## el resultado es: + ## - hay un nuevo par de claves en la VM /root/.ssh + ## - hay otro nuevo par de claves en el rootfs /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh + ## - las dos claves públicas (una de cada par) están autorizadan en el rootfs /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh/authorized_keys -logger.info ('Fase 5.2 compilar software.') -logger.debug ('running \'schroot --chroot IMGogclient -- {}/04-boottoolsSoftwareCompile.py\'') -#cd / -stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', '{}/04-boottoolsSoftwareCompile.py'.format (curdir)]) -#cd - -logger.debug ('04-boottoolsSoftwareCompile stdout follows: {}'.format (stdout)) - -try: _run (['mount', btrootfsimg, btrootfsmnt, '-o', 'loop,offset=32256']) -except: - logger.error ('mount failed') - sys.exit (3) - -############################################################################################ logger.info ('FASE 6 - Personalizar el sistema creado') logger.info ('Fase 6.1 Incorporar la clave publica del servidor') -#cd / -#schroot --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSshServer.sh ## no necesita chroot -boottoolsSshServer (btrootfsmnt) -#cd - - logger.info ('Fase 6.2 Incorporar la clave publica del propio cliente') -#schroot --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSshClient.sh ## no necesita chroot -boottoolsSshClient (btrootfsmnt) +_ssh_stuff() -## el resultado es: -## - hay un nuevo par de claves en la VM /root/.ssh -## - hay otro nuevo par de claves en el rootfs /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh -## - las dos claves públicas (una de cada par) están autorizadan en el rootfs /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh/authorized_keys -############################################################################################ -if _is_mounted (btrootfsmnt): - _run (['umount', btrootfsmnt]) + +## chroot de nuevo + +def _cerocinco(): + logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/05-boottoolsFsLocales.py\'') + stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/05-boottoolsFsLocales.py']) + logger.debug (f'05-boottoolsFsLocales stdout follows: {stdout}') logger.info ('Fase 6.3 Configurando las locales') -logger.debug ('running \'schroot --chroot IMGogclient -- {}/05-boottoolsFsLocales.py\'') -stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', '{}/05-boottoolsFsLocales.py'.format (curdir)]) -logger.debug ('05-boottoolsFsLocales stdout follows: {}'.format (stdout)) +_cerocinco() + +def _ceroseis(): + #cd / + #schroot -c IMGogclient -- /usr/bin/boot-tools/boottoolsInitrdGenerate.sh + logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/06-boottoolsInitrdGenerate.py --osrelease "{osrelease))}"\'') + stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/06-boottoolsInitrdGenerate.py', '--osrelease', osrelease]) + logger.debug (f'06-boottoolsInitrdGenerate stdout follows: {stdout}') + ## esto deja initrd.img-6.8.0-31-generic y vmlinuz-6.8.0-31-generic en /tmp + +_ceroseis() + +## ya no chroot + + -#cd / -#schroot -c IMGogclient -- /usr/bin/boot-tools/boottoolsInitrdGenerate.sh -logger.debug ('running \'schroot --chroot IMGogclient -- {}/06-boottoolsInitrdGenerate.py --osrelease "{}"\''.format (curdir, osrelease)) -stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', '{}/06-boottoolsInitrdGenerate.py'.format (curdir), '--osrelease', osrelease]) -logger.debug ('06-boottoolsInitrdGenerate stdout follows: {}'.format (stdout)) -## esto deja initrd.img-6.8.0-31-generic y vmlinuz-6.8.0-31-generic en /tmp logger.info ('FASE 7 - Generar distribucion') logger.info ('Fase 7.1 Generar el initrd') From c67b7f6e91e1f80d73b2b70f8b76945d5d2cd701 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Wed, 14 Aug 2024 10:59:56 +0200 Subject: [PATCH 18/45] refs #596 refactor (u)mounts --- boottoolsfunctions/__init__.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/boottoolsfunctions/__init__.py b/boottoolsfunctions/__init__.py index d5b9a15..7693f4f 100644 --- a/boottoolsfunctions/__init__.py +++ b/boottoolsfunctions/__init__.py @@ -35,6 +35,10 @@ def _grep (regex, file): def _is_mounted (mntpt): return _grep (mntpt, '/proc/mounts') +def _mount (dev, mntpt, opts=[]): + if not _is_mounted (mntpt): + _run (['mount', dev, mntpt] + opts) + def _umount (mntpt): if (_is_mounted (mntpt)): _run (['umount', mntpt]) @@ -302,17 +306,10 @@ def btogSetFsAccess (btrootfsimg): def btogSetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp): logger.info ('Iniciando la generación del sistema de archivos') - #mount | grep $BTROOTFSMNT || mount $BTROOTFSIMG $BTROOTFSMNT -o loop,offset=32256 - if (not _is_mounted (btrootfsmnt)): - raise Exception ('rootfs should be mounted') - logger.info ('debootstrapping system') logger.debug (f'debootstrap --arch="{osarch}" --components=main,universe "{oscodename}" "{btrootfsmnt}" "{oshttp}"') try: _run (['debootstrap', f'--arch={osarch}', '--components=main,universe', oscodename, btrootfsmnt, oshttp]) - except: - if (_is_mounted (btrootfsmnt)): - _run (['umount', btrootfsmnt]) - raise Exception (f'debootstrap --arch="{osarch}" --components=main,universe "{oscodename}" "{btrootfsmnt}" "{oshttp}" : ha fallado!') + except: raise Exception (f'debootstrap --arch="{osarch}" --components=main,universe "{oscodename}" "{btrootfsmnt}" "{oshttp}" : ha fallado!') logger.info (f'debootstrap --arch="{osarch}" --components=main,universe "{oscodename}" "{btrootfsmnt}" "{oshttp}" : ok') return 0 @@ -441,7 +438,7 @@ def btogIsoGenerator (pxepkg, isolinux_tpl, bttargetdir, nameisoclient): os.mkdir ('/tmp/iso/isolinux/mount') _run (['mount', '-o', 'loop', f'/tmp/iso/isolinux/{pxepkg}.iso', '/tmp/iso/isolinux/mount']) subprocess.run (['cp -a /tmp/iso/isolinux/mount/* /tmp/iso/isolinux'], shell=True) - _run (['umount', '/tmp/iso/isolinux/mount']) + _umount ('/tmp/iso/isolinux/mount') os.rmdir ('/tmp/iso/isolinux/mount') os.unlink (f'/tmp/iso/isolinux/{pxepkg}.iso') From 45f533aca9384d9ea2c1788288c5c4f71864e3f7 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Wed, 14 Aug 2024 11:01:35 +0200 Subject: [PATCH 19/45] refs #596 refactor (u)mounts --- boottoolsgenerator.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/boottoolsgenerator.py b/boottoolsgenerator.py index 285f1cf..6891796 100755 --- a/boottoolsgenerator.py +++ b/boottoolsgenerator.py @@ -11,7 +11,7 @@ import shutil curdir = os.path.dirname (__file__) sys.path.insert (0, curdir) -from boottoolsfunctions import _run, _is_mounted, btogGetOsInfo1, btogGetOsInfo2, btogGetVar, btogSetFsVirtual, btogSetFsAccess, btogSetFsBase, boottoolsSshServer, boottoolsSshClient, btogFsInitrd, btogFsSqfs, btogIsoGenerator +from boottoolsfunctions import _run, _mount, _umount, btogGetOsInfo1, btogGetOsInfo2, btogGetVar, btogSetFsVirtual, btogSetFsAccess, btogSetFsBase, boottoolsSshServer, boottoolsSshClient, btogFsInitrd, btogFsSqfs, btogIsoGenerator def _logging(): #logging.root.handlers = [] @@ -63,9 +63,13 @@ ogclientcfg, osdistrib, oscodename, osrelease, osarch, oshttp = btogGetOsInfo1(t btdir, bttargetdir, btrootfsimg, btrootfsmnt, btrootfsimglabel, log_file, versionboottools, btvirtualdisksize = btogGetVar(osarch) gitrelease, nameisoclient, namehostclient = btogGetOsInfo2(type_client, versionboottools, ogclientcfg, osdistrib, oscodename, osrelease, osarch, oshttp) +def _mount_rootfs(): + global btrootfsimg, btrootfsmnt + _mount (btrootfsimg, btrootfsmnt, opts=['-o', 'loop,offset=32256']) + ## this is convenient in case the previous run failed and we want to run this program again -if _is_mounted (btrootfsmnt): - _run (['umount', btrootfsmnt]) +try: _umount (btrootfsmnt) +except: pass ########################################################################## @@ -116,7 +120,7 @@ def _debootstrap(): cp = subprocess.run (['schroot', '--chroot', 'IMGogclient', '--', 'stat', '/etc']) if (cp.returncode): logger.debug (f'schroot returned code "{cp.returncode}", calling btogSetFsBase()') - try: _run (['mount', btrootfsimg, btrootfsmnt, '-o', 'loop,offset=32256']) + try: _mount_rootfs() except: logger.error ('mount failed') sys.exit (3) @@ -149,17 +153,16 @@ _run (['sed', '-i', f'1 s/$/ {gitrelease} ({osrelease})/', f'{btdir}/includes/et #umount $BTROOTFSMNT 2>/dev/null || (kill -9 $(lsof -t $BTROOTFSMNT); umount $BTROOTFSMNT) 2>/dev/null def _cerodos(): - if (not _is_mounted (btrootfsmnt)): - try: _run (['mount', btrootfsimg, btrootfsmnt, '-o', 'loop,offset=32256']) - except: - logger.error ('mount failed') - sys.exit (3) + try: _mount_rootfs() + except: + logger.error ('mount failed') + sys.exit (3) logger.debug (f'running \'{curdir}/02-boottoolsFsOpengnsys.py --mntpt "{btrootfsmnt}" --osdistrib "{osdistrib}" --oscodename "{oscodename}" --osrelease "{osrelease}" --osarch "{osarch}" --oshttp "{oshttp))}"\'') stdout, _ = _run ([f'{curdir}/02-boottoolsFsOpengnsys.py', '--mntpt', btrootfsmnt, '--osdistrib', osdistrib, '--oscodename', oscodename, '--osrelease', osrelease, '--osarch', osarch, '--oshttp', oshttp]) logger.debug (f'02-boottoolsFsOpengnsys stdout follows: {stdout}') - _run (['umount', btrootfsmnt]) + _umount (btrootfsmnt) _cerodos() @@ -191,7 +194,7 @@ _cerocuatro(): def _ssh_stuff(): - try: _run (['mount', btrootfsimg, btrootfsmnt, '-o', 'loop,offset=32256']) + try: _mount_rootfs() except: logger.error ('mount failed') sys.exit (3) @@ -204,8 +207,7 @@ def _ssh_stuff(): #schroot --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSshClient.sh ## no necesita chroot boottoolsSshClient (btrootfsmnt) - if _is_mounted (btrootfsmnt): - _run (['umount', btrootfsmnt]) + _umount (btrootfsmnt) ## el resultado es: ## - hay un nuevo par de claves en la VM /root/.ssh @@ -250,12 +252,12 @@ btogFsInitrd (bttargetdir, osrelease) logger.info ('Fase 7.2 Generar fichero sqfs a partir del fichero img') -try: _run (['mount', btrootfsimg, btrootfsmnt, '-o', 'loop,offset=32256']) +try: _mount_rootfs() except: logger.error ('mount failed') sys.exit (3) btogFsSqfs (bttargetdir, btrootfsmnt) -_run (['umount', btrootfsmnt]) +_umount (btrootfsmnt) logger.info ('Fase 7.3 Generar la ISO') btogIsoGenerator (pxepkg, isolinux_tpl, bttargetdir, nameisoclient) From 0371870fa0802729fb9cffcf17c691247e619295 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Wed, 14 Aug 2024 11:32:24 +0200 Subject: [PATCH 20/45] refs #596 refactor read_config, remove unused ogclient.cfg, use templates, make logging level configurable, fix bugs --- 03-boottoolsSoftwareInstall.py | 8 +--- 05-boottoolsFsLocales.py | 8 +--- boottoolsfunctions/__init__.py | 84 +++++++++++++++++----------------- boottoolsgenerator.cfg | 4 ++ boottoolsgenerator.py | 47 ++++++++++--------- 5 files changed, 76 insertions(+), 75 deletions(-) diff --git a/03-boottoolsSoftwareInstall.py b/03-boottoolsSoftwareInstall.py index 53f1f2e..6f4667d 100755 --- a/03-boottoolsSoftwareInstall.py +++ b/03-boottoolsSoftwareInstall.py @@ -7,13 +7,9 @@ import re import subprocess import configparser -from boottoolsfunctions import _run +from boottoolsfunctions import _run, _read_config -config = configparser.ConfigParser (comment_prefixes='#', inline_comment_prefixes='#') -if not os.path.exists ('boottoolsgenerator.cfg'): - print ('configuration file "boottoolsgenerator.cfg" not found') - sys.exit (1) -config.read ('boottoolsgenerator.cfg') +config = _read_config ('boottoolsgenerator.cfg') debconf_settings = config['debconf'].get ('settings') def _aptget_clean(): diff --git a/05-boottoolsFsLocales.py b/05-boottoolsFsLocales.py index e3995b8..486e21f 100755 --- a/05-boottoolsFsLocales.py +++ b/05-boottoolsFsLocales.py @@ -3,13 +3,9 @@ import os import subprocess import configparser -from boottoolsfunctions import _run +from boottoolsfunctions import _run, _read_config -config = configparser.ConfigParser (inline_comment_prefixes='#') -if not os.path.exists ('boottoolsgenerator.cfg'): - print ('configuration file "boottoolsgenerator.cfg" not found') - sys.exit (1) -config.read ('boottoolsgenerator.cfg') +config = _read_config ('boottoolsgenerator.cfg') debconf_settings2 = config['debconf'].get ('settings2') subprocess.run (['debconf-set-selections'], input=debconf_settings2, text=True) diff --git a/boottoolsfunctions/__init__.py b/boottoolsfunctions/__init__.py index 7693f4f..e7acbae 100644 --- a/boottoolsfunctions/__init__.py +++ b/boottoolsfunctions/__init__.py @@ -43,6 +43,14 @@ def _umount (mntpt): if (_is_mounted (mntpt)): _run (['umount', mntpt]) +def _read_config (fn): + if not os.path.exists (fn): + print (f'configuration file "{fn}" not found') + sys.exit (1) + config = configparser.ConfigParser (comment_prefixes='#', inline_comment_prefixes='#') + config.read (fn) + return config + def btogGetVar (osarch): btdir = '/tmp/opengnsys_installer/opengnsys/client/boot-tools' bttargetdir = '/var/lib/tftpboot/ogclient/' @@ -58,7 +66,6 @@ def btogGetVar (osarch): return btdir, bttargetdir, btrootfsimg, btrootfsmnt, btrootfsimglabel, log_file, versionboottools, btvirtualdisksize def btogGetOsInfo1 (type_client): - ogclientcfg = '/tmp/ogclient.cfg' if 'precise' == type_client: # ogLive 1.0.4-rc2 basado en Ubuntu 12.04 LTS. osdistrib='ubuntu' oscodename='precise' @@ -158,31 +165,15 @@ def btogGetOsInfo1 (type_client): else: # Parámetro desconocido print ('Parámetro no válido.') os.exit (1) - return ogclientcfg, osdistrib, oscodename, osrelease, osarch, oshttp + return osdistrib, oscodename, osrelease, osarch, oshttp -def btogGetOsInfo2 (type_client, versionboottools, ogclientcfg, osdistrib, oscodename, osrelease, osarch, oshttp): +def btogGetOsInfo2 (type_client, versionboottools, osdistrib, oscodename, osrelease, osarch, oshttp): branch = 'master' giturl = f'https://api.github.com/repos/opengnsys/OpenGnsys/commits?sha={branch}&path=/client' gitrelease = 'r20240808' ## TODO: are we going to keep the following? $(curl -s "$GITURL" | jq -r '"r" + (.[0].commit.committer.date | split("-") | join("")[:8]) + "." + (.[0].sha[:7])') nameisoclient ='-'.join ([versionboottools, oscodename, osrelease, osarch, gitrelease]) namehostclient = '-'.join ([versionboottools, oscodename, gitrelease]) - ### El fichero de configuración debe sustituir a estos 3 ficheros (borrar las 3 líneas siguientes). - #echo "$NAMEISOCLIENT" > /tmp/opengnsys_info_rootfs - #echo "$NAMEHOSTCLIENT" > /tmp/opengnsys_chroot - - # Generar fichero de configuración. - with open (ogclientcfg, 'w') as f: - print (f'TYPECLIENT="{type_client}"', file=f) - print (f'OSDISTRIB="{osdistrib}"', file=f) - print (f'OSCODENAME="{oscodename}"', file=f) - print (f'OSRELEASE="{osrelease}"', file=f) - print (f'OSARCH="{osarch}"', file=f) - print (f'OSHTTP="{oshttp}"', file=f) - print (f'GITRELEASE="{gitrelease}"', file=f) - print (f'NAMEISOCLIENT="{nameisoclient}"', file=f) - print (f'NAMEHOSTCLIENT="{namehostclient}"', file=f) - return gitrelease, nameisoclient, namehostclient def btogSetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, bttargetdir, osarch): @@ -225,7 +216,8 @@ def btogSetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisks logger.info ('Partitioning disk image') stdout, _ = _run (['bash', '-c', f"echo $'n\np\n1\n\n\nt\n83\nw' |fdisk {diskloop}"]) - logger.debug (f'fdisk output follows: {stdout}') + logger.debug (f'fdisk stdout follows:') + for i in stdout.strip().split('\n'): logger.debug (' ' + i) time.sleep (3) logger.debug ('losetup --detach') @@ -254,6 +246,34 @@ def btogSetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisks logger.info (f'"{btrootfsimg}" "{btvirtualdisksize}" MB : OK') +schroot_conf_tpl = """ +[IMGogclient] +type=loopback +file=/var/lib/tftpboot/ogclient/ogclient.img +description=ogclient Ubuntu image +#priority=1 +users=root +groups=root +root-groups=root +mount-options=-o offset=32256 +root-users=root +""".strip() + +mount_defaults_tpl = """ +# mount.defaults: static file system information for chroots. +# Note that the mount point will be prefixed by the chroot path +# (CHROOT_PATH) +# +# +proc /proc proc defaults 0 0 +#procbususb /proc/bus/usb usbfs defaults 0 0 +#/dev /dev none rw,bind 0 0 +/dev/pts /dev/pts none rw,bind 0 0 +/dev/shm /dev/shm none rw,bind 0 0 +#/home /home none rw,bind 0 0 +/tmp /tmp none rw,bind 0 0 +""".strip() + #btogSetFsAcces: habilita el acceso al sistema root del cliente con schroot def btogSetFsAccess (btrootfsimg): if (_grep (btrootfsimg, '/etc/schroot/schroot.conf')): @@ -265,34 +285,14 @@ def btogSetFsAccess (btrootfsimg): shutil.copy ('/etc/schroot/schroot.conf', '/etc/schroot/schroot.conf.bak') with open ('/etc/schroot/schroot.conf', 'w') as f: - print ('[IMGogclient]', file=f) - print ('type=loopback', file=f) - print ('file=/var/lib/tftpboot/ogclient/ogclient.img', file=f) - print ('description=ogclient Ubuntu image', file=f) - print ('#priority=1', file=f) - print ('users=root', file=f) - print ('groups=root', file=f) - print ('root-groups=root', file=f) - print ('mount-options=-o offset=32256', file=f) - print ('root-users=root', file=f) + f.write (schroot_conf_tpl + '\n') #cp /etc/schroot/mount-defaults /etc/schroot/mount-defaults.`getDateTime` if (os.path.exists ('/etc/schroot/mount-defaults')): shutil.copy ('/etc/schroot/mount-defaults', '/etc/schroot/mount-defaults.bak') with open ('/etc/schroot/mount-defaults', 'w') as f: - print ('# mount.defaults: static file system information for chroots.', file=f) - print ('# Note that the mount point will be prefixed by the chroot path', file=f) - print ('# (CHROOT_PATH)', file=f) - print ('#', file=f) - print ('# ', file=f) - print ('proc /proc proc defaults 0 0', file=f) - print ('#procbususb /proc/bus/usb usbfs defaults 0 0', file=f) - print ('#/dev /dev none rw,bind 0 0', file=f) - print ('/dev/pts /dev/pts none rw,bind 0 0', file=f) - print ('/dev/shm /dev/shm none rw,bind 0 0', file=f) - print ('#/home /home none rw,bind 0 0', file=f) - print ('/tmp /tmp none rw,bind 0 0', file=f) + f.write (mount_defaults_tpl + '\n') map (os.unlink, glob.glob('/etc/schroot/setup.d/*chrootname')) #for i in glob.glob ('/etc/schroot/setup.d/*chrootname'): diff --git a/boottoolsgenerator.cfg b/boottoolsgenerator.cfg index 3542537..2797d4b 100644 --- a/boottoolsgenerator.cfg +++ b/boottoolsgenerator.cfg @@ -62,6 +62,10 @@ xwindows = #roxterm gparted #+80M #openbox midori #xvesa en compilacion +[General] + +logging_level = INFO + [debconf] settings = diff --git a/boottoolsgenerator.py b/boottoolsgenerator.py index 6891796..4b57dae 100755 --- a/boottoolsgenerator.py +++ b/boottoolsgenerator.py @@ -11,29 +11,29 @@ import shutil curdir = os.path.dirname (__file__) sys.path.insert (0, curdir) -from boottoolsfunctions import _run, _mount, _umount, btogGetOsInfo1, btogGetOsInfo2, btogGetVar, btogSetFsVirtual, btogSetFsAccess, btogSetFsBase, boottoolsSshServer, boottoolsSshClient, btogFsInitrd, btogFsSqfs, btogIsoGenerator +from boottoolsfunctions import _run, _mount, _umount, _read_config, btogGetOsInfo1, btogGetOsInfo2, btogGetVar, btogSetFsVirtual, btogSetFsAccess, btogSetFsBase, boottoolsSshServer, boottoolsSshClient, btogFsInitrd, btogFsSqfs, btogIsoGenerator +config = _read_config ('boottoolsgenerator.cfg') +isolinux_tpl = config['ISOLinux'].get ('template') +lvl = config['General'].get ('logging_level') + +numeric_level = getattr (logging, lvl.upper(), None) +if numeric_level is None: + numeric_level = getattr (logging, 'INFO') def _logging(): #logging.root.handlers = [] logging.basicConfig ( format='%(levelname)s %(asctime)s (%(funcName)s) %(message)s', - level=logging.INFO, + level=numeric_level, handlers = [ logging.FileHandler ('/tmp/boot-tools_installation.log'), logging.StreamHandler (sys.stdout), ], ) - return = logging.getLogger ('boottools') + return logging.getLogger ('boottools') logger = _logging() -config = configparser.ConfigParser (comment_prefixes='#', inline_comment_prefixes='#') -if not os.path.exists ('boottoolsgenerator.cfg'): - print ('configuration file "boottoolsgenerator.cfg" not found') - sys.exit (1) -config.read ('boottoolsgenerator.cfg') -isolinux_tpl = config['ISOLinux'].get ('template') - def clone_client_dirs(): if not os.path.exists ('/tmp/opengnsys_installer/opengnsys/client/engine'): branch = 'main' @@ -59,9 +59,9 @@ clone_client_dirs() ####################################################################3 logger.info ('FASE 1 - Asignación de variables') -ogclientcfg, osdistrib, oscodename, osrelease, osarch, oshttp = btogGetOsInfo1(type_client) +osdistrib, oscodename, osrelease, osarch, oshttp = btogGetOsInfo1(type_client) btdir, bttargetdir, btrootfsimg, btrootfsmnt, btrootfsimglabel, log_file, versionboottools, btvirtualdisksize = btogGetVar(osarch) -gitrelease, nameisoclient, namehostclient = btogGetOsInfo2(type_client, versionboottools, ogclientcfg, osdistrib, oscodename, osrelease, osarch, oshttp) +gitrelease, nameisoclient, namehostclient = btogGetOsInfo2(type_client, versionboottools, osdistrib, oscodename, osrelease, osarch, oshttp) def _mount_rootfs(): global btrootfsimg, btrootfsmnt @@ -158,9 +158,10 @@ def _cerodos(): logger.error ('mount failed') sys.exit (3) - logger.debug (f'running \'{curdir}/02-boottoolsFsOpengnsys.py --mntpt "{btrootfsmnt}" --osdistrib "{osdistrib}" --oscodename "{oscodename}" --osrelease "{osrelease}" --osarch "{osarch}" --oshttp "{oshttp))}"\'') + logger.debug (f'running \'{curdir}/02-boottoolsFsOpengnsys.py --mntpt "{btrootfsmnt}" --osdistrib "{osdistrib}" --oscodename "{oscodename}" --osrelease "{osrelease}" --osarch "{osarch}" --oshttp "{oshttp}"\'') stdout, _ = _run ([f'{curdir}/02-boottoolsFsOpengnsys.py', '--mntpt', btrootfsmnt, '--osdistrib', osdistrib, '--oscodename', oscodename, '--osrelease', osrelease, '--osarch', osarch, '--oshttp', oshttp]) - logger.debug (f'02-boottoolsFsOpengnsys stdout follows: {stdout}') + logger.debug (f'02-boottoolsFsOpengnsys stdout follows:') + for i in stdout.strip().split('\n'): logger.debug (' ' + i) _umount (btrootfsmnt) @@ -170,9 +171,10 @@ _cerodos() ## aqui empiezan las cosas de chroot def _cerotres(): - logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/03-boottoolsSoftwareInstall.py --osrelease "{osrelease}" --osarch "{osarch))}"\'') + logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/03-boottoolsSoftwareInstall.py --osrelease "{osrelease}" --osarch "{osarch}"\'') stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/03-boottoolsSoftwareInstall.py', '--osrelease', osrelease, '--osarch', osarch]) - logger.debug (f'03-boottoolsSoftwareInstall stdout follows: {stdout}') + logger.debug (f'03-boottoolsSoftwareInstall stdout follows:') + for i in stdout.strip().split('\n'): logger.debug (' ' + i) logger.info ('FASE 5 - Instalar software') logger.info ('Fase 5.1 instalar paquetes deb con apt-get') @@ -183,10 +185,11 @@ def _cerocuatro(): #cd / stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/04-boottoolsSoftwareCompile.py']) #cd - - logger.debug (f'04-boottoolsSoftwareCompile stdout follows: {stdout}') + logger.debug (f'04-boottoolsSoftwareCompile stdout follows:') + for i in stdout.strip().split('\n'): logger.debug (' ' + i) logger.info ('Fase 5.2 compilar software.') -_cerocuatro(): +_cerocuatro() ## ya no chroot @@ -226,7 +229,8 @@ _ssh_stuff() def _cerocinco(): logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/05-boottoolsFsLocales.py\'') stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/05-boottoolsFsLocales.py']) - logger.debug (f'05-boottoolsFsLocales stdout follows: {stdout}') + logger.debug (f'05-boottoolsFsLocales stdout follows:') + for i in stdout.strip().split('\n'): logger.debug (' ' + i) logger.info ('Fase 6.3 Configurando las locales') _cerocinco() @@ -234,9 +238,10 @@ _cerocinco() def _ceroseis(): #cd / #schroot -c IMGogclient -- /usr/bin/boot-tools/boottoolsInitrdGenerate.sh - logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/06-boottoolsInitrdGenerate.py --osrelease "{osrelease))}"\'') + logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/06-boottoolsInitrdGenerate.py --osrelease "{osrelease}"\'') stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/06-boottoolsInitrdGenerate.py', '--osrelease', osrelease]) - logger.debug (f'06-boottoolsInitrdGenerate stdout follows: {stdout}') + logger.debug (f'06-boottoolsInitrdGenerate stdout follows:') + for i in stdout.strip().split('\n'): logger.debug (' ' + i) ## esto deja initrd.img-6.8.0-31-generic y vmlinuz-6.8.0-31-generic en /tmp _ceroseis() From ef212bc84e24c65271a201dd61dac1837b95f7c0 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Wed, 14 Aug 2024 12:04:23 +0200 Subject: [PATCH 21/45] refs #596 reorganise configuration, refactor md5 hashing, move installation files --- 02-boottoolsFsOpengnsys.py | 10 ++++----- 03-boottoolsSoftwareInstall.py | 3 +-- 05-boottoolsFsLocales.py | 3 +-- boottoolsfunctions/__init__.py | 39 +++++++++++----------------------- boottoolsgenerator.cfg | 10 +++------ boottoolsgenerator.py | 7 +++--- 6 files changed, 25 insertions(+), 47 deletions(-) diff --git a/02-boottoolsFsOpengnsys.py b/02-boottoolsFsOpengnsys.py index a26d018..7ae3f85 100755 --- a/02-boottoolsFsOpengnsys.py +++ b/02-boottoolsFsOpengnsys.py @@ -7,12 +7,12 @@ import shutil import argparse #VERSIONBOOTTOOLS="ogLive" -nameisoclientfile = '/tmp/opengnsys_info_rootfs' -namehostclientfile = '/tmp/opengnsys_chroot' +#nameisoclientfile = '/tmp/opengnsys_info_rootfs' +#namehostclientfile = '/tmp/opengnsys_chroot' -svnclientdir = '/tmp/opengnsys_installer/opengnsys/client/boot-tools' -svnclientstructure = '/tmp/opengnsys_installer/opengnsys/client/shared' -svnclientengine = '/tmp/opengnsys_installer/opengnsys/client/engine' +svnclientdir = '/tmp/opengnsys/oglive_builder' +svnclientstructure = '/tmp/opengnsys/shared' +svnclientengine = '/tmp/opengnsys/engine' def boottoolsFsOpengnsys (ogclientmount, osdistrib, oscodename, osrelease, osarch, oshttp): print (':'.join ([osdistrib, oscodename, osrelease, osarch, oshttp])) diff --git a/03-boottoolsSoftwareInstall.py b/03-boottoolsSoftwareInstall.py index 6f4667d..eb0bb4c 100755 --- a/03-boottoolsSoftwareInstall.py +++ b/03-boottoolsSoftwareInstall.py @@ -5,12 +5,11 @@ import os import sys import re import subprocess -import configparser from boottoolsfunctions import _run, _read_config config = _read_config ('boottoolsgenerator.cfg') -debconf_settings = config['debconf'].get ('settings') +debconf_settings = config['General'].get ('debconf_settings') def _aptget_clean(): _run (['apt-get', '--yes', 'clean']) diff --git a/05-boottoolsFsLocales.py b/05-boottoolsFsLocales.py index 486e21f..91ae033 100755 --- a/05-boottoolsFsLocales.py +++ b/05-boottoolsFsLocales.py @@ -2,11 +2,10 @@ import os import subprocess -import configparser from boottoolsfunctions import _run, _read_config config = _read_config ('boottoolsgenerator.cfg') -debconf_settings2 = config['debconf'].get ('settings2') +debconf_settings2 = config['General'].get ('debconf_settings2') subprocess.run (['debconf-set-selections'], input=debconf_settings2, text=True) diff --git a/boottoolsfunctions/__init__.py b/boottoolsfunctions/__init__.py index e7acbae..b47fc63 100644 --- a/boottoolsfunctions/__init__.py +++ b/boottoolsfunctions/__init__.py @@ -7,6 +7,7 @@ import subprocess import shutil import glob import datetime +import configparser logger = logging.getLogger ('boottools') @@ -51,8 +52,14 @@ def _read_config (fn): config.read (fn) return config +def _write_md5 (fn): + md5, _ = _run (['md5sum', fn]) + md5, rest = md5.split (' ', 1) + with open (f'{fn}.sum', 'w') as fd: + fd.write (md5 + '\n') + def btogGetVar (osarch): - btdir = '/tmp/opengnsys_installer/opengnsys/client/boot-tools' + btdir = '/tmp/opengnsys/oglive_builder' bttargetdir = '/var/lib/tftpboot/ogclient/' btrootfsimg = os.path.join (bttargetdir, 'ogclient.img') btrootfsmnt = os.path.join (bttargetdir, 'ogclientmount') @@ -380,19 +387,8 @@ def btogFsInitrd (bttargetdir, osrelease): shutil.copy (f'/tmp/initrd.img-{osrelease}', f'{bttargetdir}/oginitrd.img') shutil.copy (f'/tmp/vmlinuz-{osrelease}', f'{bttargetdir}/ogvmlinuz') - #DATASUM=`md5sum "${BTTARGETDIR}/oginitrd.img" | cut -f1 -d" "` - md5, _ = _run (['md5sum', f'{bttargetdir}/oginitrd.img']) - md5, rest = md5.split (' ', 1) - #echo $DATASUM > ${BTTARGETDIR}/oginitrd.img.sum - with open (f'{bttargetdir}/oginitrd.img.sum', 'w') as fd: - fd.write (md5) - - #DATASUM=`md5sum "${BTTARGETDIR}/ogvmlinuz" | cut -f1 -d" "` - md5, _ = _run (['md5sum', f'{bttargetdir}/ogvmlinuz']) - md5, rest = md5.split (' ', 1) - #echo $DATASUM > ${BTTARGETDIR}/ogvmlinuz.sum - with open (f'{bttargetdir}/ogvmlinuz.sum', 'w') as fd: - fd.write (md5) + _write_md5 (f'{bttargetdir}/oginitrd.img') + _write_md5 (f'{bttargetdir}/ogvmlinuz') #cd - #chmod -R 755 $BTTARGETDIR @@ -411,12 +407,7 @@ def btogFsSqfs (bttargetdir, btrootfsmnt): _run (['mksquashfs', btrootfsmnt, f'{bttargetdir}/ogclient.sqfs']) os.chmod (f'{bttargetdir}/ogclient.sqfs', 0o744) - #DATASUM=`md5sum "${BTTARGETDIR}/ogclient.sqfs" | cut -f1 -d" "` - md5, _ = _run (['md5sum', f'{bttargetdir}/ogclient.sqfs']) - md5, rest = md5.split (' ', 1) - #echo $DATASUM > ${BTTARGETDIR}/ogclient.sqfs.sum - with open (f'{bttargetdir}/ogclient.sqfs.sum', 'w') as fd: - fd.write (md5) + _write_md5 (f'{bttargetdir}/ogclient.sqfs') # btogIsoGenerator genera la iso del cliente def btogIsoGenerator (pxepkg, isolinux_tpl, bttargetdir, nameisoclient): @@ -465,13 +456,7 @@ def btogIsoGenerator (pxepkg, isolinux_tpl, bttargetdir, nameisoclient): logger.debug (f'mkisofs -V ogClient -o {nameisoclient}.iso -b isolinux/isolinux.bin -c isolinux/boot.cat -J -no-emul-boot -boot-load-size 4 -boot-info-table /tmp/iso') _run (['mkisofs', '-V', 'ogClient', '-o', f'{nameisoclient}.iso', '-b', 'isolinux/isolinux.bin', '-c', 'isolinux/boot.cat', '-J', '-no-emul-boot', '-boot-load-size', '4', '-boot-info-table', '/tmp/iso']) - ##Generamos el md5 - #DATASUM=`md5sum ${NAMEISOCLIENT}.iso | cut -f1 -d" "` - md5, _ = _run (['md5sum', f'{nameisoclient}.iso']) - md5, rest = md5.split (' ', 1) - #echo $DATASUM > ${NAMEISOCLIENT}.iso.sum - with open (f'{nameisoclient}.iso.sum', 'w') as fd: - fd.write (md5) + _write_md5 (f'{nameisoclient}.iso') os.chdir (oldpwd) os.rename (f'/tmp/{nameisoclient}.iso' , f'/var/lib/tftpboot/ogclient/{nameisoclient}.iso') diff --git a/boottoolsgenerator.cfg b/boottoolsgenerator.cfg index 2797d4b..c6eeda8 100644 --- a/boottoolsgenerator.cfg +++ b/boottoolsgenerator.cfg @@ -66,22 +66,18 @@ xwindows = logging_level = INFO -[debconf] - -settings = +debconf_settings = kexec-tools kexec-tools/load_kexec boolean true openssh-server openssh-server/permit-root-login boolean true refind refind/install_to_esp boolean false -settings2 = +debconf_settings2 = console-setup console-setup/charmap47 select UTF-8 console-setup console-setup/codeset47 select . Combined - Latin; Slavic Cyrillic; Greek console-setup console-setup/fontface47 select VGA console-setup console-setup/fontsize-fb47 select 8x16 -[ISOLinux] - -template = +isolinux_template = DEFAULT menu.c32 PROMPT 0 ALLOWOPTIONS 1 diff --git a/boottoolsgenerator.py b/boottoolsgenerator.py index 4b57dae..515ebf7 100755 --- a/boottoolsgenerator.py +++ b/boottoolsgenerator.py @@ -6,7 +6,6 @@ import logging import subprocess import glob import stat -import configparser import shutil curdir = os.path.dirname (__file__) @@ -14,7 +13,7 @@ sys.path.insert (0, curdir) from boottoolsfunctions import _run, _mount, _umount, _read_config, btogGetOsInfo1, btogGetOsInfo2, btogGetVar, btogSetFsVirtual, btogSetFsAccess, btogSetFsBase, boottoolsSshServer, boottoolsSshClient, btogFsInitrd, btogFsSqfs, btogIsoGenerator config = _read_config ('boottoolsgenerator.cfg') -isolinux_tpl = config['ISOLinux'].get ('template') +isolinux_tpl = config['General'].get ('isolinux_template') lvl = config['General'].get ('logging_level') numeric_level = getattr (logging, lvl.upper(), None) @@ -35,12 +34,12 @@ def _logging(): logger = _logging() def clone_client_dirs(): - if not os.path.exists ('/tmp/opengnsys_installer/opengnsys/client/engine'): + if not os.path.exists ('/tmp/opengnsys/engine'): branch = 'main' tmpdir, _ = _run (['mktemp', '--tmpdir', '--directory', 'oggit.XXXXXX']) logger.debug (f'tmpdir "{tmpdir}"') _run (['git', 'clone', '-c', 'http.sslVerify=false', '--branch', branch, 'https://ognproject.evlt.uma.es/gitea/opengnsys/opengnsys.git', tmpdir]) - _run (['rsync', '-aH', f'{tmpdir}/client/engine', f'{tmpdir}/client/shared', '/tmp/opengnsys_installer/opengnsys/client/']) + _run (['rsync', '-aH', f'{tmpdir}/client/engine', f'{tmpdir}/client/shared', '/tmp/opengnsys/']) _run (['rm', '-rf', tmpdir]) type_client = sys.argv[1] if len(sys.argv)>1 else 'host' From 727cd1a666f4577f12bb981639c6b7014b09b614 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Wed, 14 Aug 2024 12:50:17 +0200 Subject: [PATCH 22/45] refs #596 unhardcode gitrelease --- boottoolsfunctions/__init__.py | 7 ++----- boottoolsgenerator.py | 17 ++++++++--------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/boottoolsfunctions/__init__.py b/boottoolsfunctions/__init__.py index b47fc63..1e8cc09 100644 --- a/boottoolsfunctions/__init__.py +++ b/boottoolsfunctions/__init__.py @@ -174,13 +174,10 @@ def btogGetOsInfo1 (type_client): os.exit (1) return osdistrib, oscodename, osrelease, osarch, oshttp -def btogGetOsInfo2 (type_client, versionboottools, osdistrib, oscodename, osrelease, osarch, oshttp): - branch = 'master' - giturl = f'https://api.github.com/repos/opengnsys/OpenGnsys/commits?sha={branch}&path=/client' - gitrelease = 'r20240808' ## TODO: are we going to keep the following? $(curl -s "$GITURL" | jq -r '"r" + (.[0].commit.committer.date | split("-") | join("")[:8]) + "." + (.[0].sha[:7])') +def btogGetOsInfo2 (ogrepo_dir, versionboottools, oscodename, osrelease, osarch): + gitrelease = subprocess.run (['git', '--git-dir', f'{ogrepo_dir}/.git', 'log', '--date', 'format:r%Y%m%d', '--format=%ad', '-1'], capture_output=True, text=True).stdout.strip() nameisoclient ='-'.join ([versionboottools, oscodename, osrelease, osarch, gitrelease]) namehostclient = '-'.join ([versionboottools, oscodename, gitrelease]) - return gitrelease, nameisoclient, namehostclient def btogSetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, bttargetdir, osarch): diff --git a/boottoolsgenerator.py b/boottoolsgenerator.py index 515ebf7..23cacf6 100755 --- a/boottoolsgenerator.py +++ b/boottoolsgenerator.py @@ -33,14 +33,10 @@ def _logging(): logger = _logging() -def clone_client_dirs(): +def clone_client_dirs (ogrepo_url, ogrepo_branch, ogrepo_dir): if not os.path.exists ('/tmp/opengnsys/engine'): - branch = 'main' - tmpdir, _ = _run (['mktemp', '--tmpdir', '--directory', 'oggit.XXXXXX']) - logger.debug (f'tmpdir "{tmpdir}"') - _run (['git', 'clone', '-c', 'http.sslVerify=false', '--branch', branch, 'https://ognproject.evlt.uma.es/gitea/opengnsys/opengnsys.git', tmpdir]) - _run (['rsync', '-aH', f'{tmpdir}/client/engine', f'{tmpdir}/client/shared', '/tmp/opengnsys/']) - _run (['rm', '-rf', tmpdir]) + _run (['git', 'clone', '-c', 'http.sslVerify=false', '--branch', ogrepo_branch, ogrepo_url, ogrepo_dir]) + _run (['rsync', '-aH', f'{ogrepo_dir}/client/engine', f'{ogrepo_dir}/client/shared', '/tmp/opengnsys/']) type_client = sys.argv[1] if len(sys.argv)>1 else 'host' #WORKDIR = '/tmp/opengnsys_installer' @@ -54,13 +50,16 @@ if os.getuid(): #os.chdir ('/tmp') logger.info ('OpenGnsys CLIENT installation begins') -clone_client_dirs() +ogrepo_url = 'https://ognproject.evlt.uma.es/gitea/opengnsys/opengnsys.git' +ogrepo_branch = 'main' +ogrepo_dir = '/tmp/ogrepo' +clone_client_dirs (ogrepo_url, ogrepo_branch, ogrepo_dir) ####################################################################3 logger.info ('FASE 1 - Asignación de variables') osdistrib, oscodename, osrelease, osarch, oshttp = btogGetOsInfo1(type_client) btdir, bttargetdir, btrootfsimg, btrootfsmnt, btrootfsimglabel, log_file, versionboottools, btvirtualdisksize = btogGetVar(osarch) -gitrelease, nameisoclient, namehostclient = btogGetOsInfo2(type_client, versionboottools, osdistrib, oscodename, osrelease, osarch, oshttp) +gitrelease, nameisoclient, namehostclient = btogGetOsInfo2(ogrepo_dir, versionboottools, oscodename, osrelease, osarch) def _mount_rootfs(): global btrootfsimg, btrootfsmnt From f04ace2b3ce19bbfbaf3b5b208de0b84d8e05c1a Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Wed, 14 Aug 2024 13:21:18 +0200 Subject: [PATCH 23/45] refs #596 some cleanup, move code around --- 02-boottoolsFsOpengnsys.py | 11 -- 03-boottoolsSoftwareInstall.py | 28 +++--- 04-boottoolsSoftwareCompile.py | 5 - boottoolsfunctions/__init__.py | 71 ++++++------- boottoolsgenerator.py | 178 +++++++++++++-------------------- 5 files changed, 114 insertions(+), 179 deletions(-) diff --git a/02-boottoolsFsOpengnsys.py b/02-boottoolsFsOpengnsys.py index 7ae3f85..0063f29 100755 --- a/02-boottoolsFsOpengnsys.py +++ b/02-boottoolsFsOpengnsys.py @@ -6,20 +6,14 @@ import subprocess import shutil import argparse -#VERSIONBOOTTOOLS="ogLive" -#nameisoclientfile = '/tmp/opengnsys_info_rootfs' -#namehostclientfile = '/tmp/opengnsys_chroot' - svnclientdir = '/tmp/opengnsys/oglive_builder' svnclientstructure = '/tmp/opengnsys/shared' svnclientengine = '/tmp/opengnsys/engine' def boottoolsFsOpengnsys (ogclientmount, osdistrib, oscodename, osrelease, osarch, oshttp): print (':'.join ([osdistrib, oscodename, osrelease, osarch, oshttp])) - print ('Iniciando la personalización con datos del repositorio') - #sed -e "s/OSCODENAME/$OSCODENAME/g" ${SVNCLIENTDIR}/includes/etc/apt/sources.list.${OSDISTRIB,,} > ${SVNCLIENTDIR}/includes/etc/apt/sources.list sources_list_in = f'{svnclientdir}/includes/etc/apt/sources.list.{osdistrib.lower()}' sources_list_out = f'{svnclientdir}/includes/etc/apt/sources.list' fdin = open (sources_list_in, 'r') @@ -42,7 +36,6 @@ def boottoolsFsOpengnsys (ogclientmount, osdistrib, oscodename, osrelease, osarc subprocess.run (f'rsync -aH {svnclientstructure}/* {ogclientmount}/opt/opengnsys/' , shell=True) subprocess.run (f'rsync -aH {svnclientengine}/* {ogclientmount}/opt/opengnsys/lib/engine/bin/', shell=True) - # Si no existe, copiar pci.ids. if not os.path.exists (f'{ogclientmount}/etc/pci.ids'): shutil.copy (f'{svnclientstructure}/lib/pci.ids', f'{ogclientmount}/etc/') @@ -56,10 +49,6 @@ def boottoolsFsOpengnsys (ogclientmount, osdistrib, oscodename, osrelease, osarc if os.path.exists (f'{svnclientstructure}/bin/browser'): shutil.copy (f'{svnclientstructure}/bin/browser', f'{ogclientmount}/bin/') if os.path.exists (f'{svnclientstructure}/bin/ogAdmClient'): shutil.copy (f'{svnclientstructure}/bin/ogAdmClient', f'{ogclientmount}/bin/') - # El fichero de configuración debe sustituir a los 2 ficheros (borrar las 2 líneas). - #echo "${VERSIONBOOTTOOLS}-${OSCODENAME}-${OSRELEASE}-${GITRELEASE}" > $nameisoclientfile - #echo "${VERSIONBOOTTOOLS}-${OSCODENAME}-${GITRELEASE}" > $namehostclientfile - if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument ('--mntpt', help='rootfs mount point', action='store', required=True) diff --git a/03-boottoolsSoftwareInstall.py b/03-boottoolsSoftwareInstall.py index eb0bb4c..1ede4f8 100755 --- a/03-boottoolsSoftwareInstall.py +++ b/03-boottoolsSoftwareInstall.py @@ -37,15 +37,15 @@ def _oghook_activate(): #Activamos el hook del oginitrd.img os.rename ('/etc/initramfs-tools/oghooks', '/etc/initramfs-tools/hooks/oghooks') -def _mock_mtab(): - # Preparamos el mtab necesario para la instalacion correcta de paquetes. - #echo "/dev/sda1 / ext4 rw,errors=remount-ro 0 0" > /etc/mtab ## nati: falla porque es un symlink a ../proc/self/mounts - pass +#def _mock_mtab(): +# # Preparamos el mtab necesario para la instalacion correcta de paquetes. +# #echo "/dev/sda1 / ext4 rw,errors=remount-ro 0 0" > /etc/mtab ## nati: falla porque es un symlink a ../proc/self/mounts +# pass -def _restore_mtab(): - # Dejamos el mtab como al principio - #echo " " > /etc/mtab - pass +#def _restore_mtab(): +# # Dejamos el mtab como al principio +# #echo " " > /etc/mtab +# pass def boottoolsSoftwareInstall (osarch, osrelease): os.environ['LANGUAGE'] = 'C' @@ -58,9 +58,6 @@ def boottoolsSoftwareInstall (osarch, osrelease): _run (['dpkg-divert', '--local', '--rename', '--add', '/sbin/initctl']) os.symlink ('/bin/true', '/sbin/initctl') - #TEST desactivamos upstart - #apt-get update; apt-get install --no-install-recommends dbus; dbus-uuidgen > /var/lib/dbus/machine-id; dpkg-divert --local --rename --add /sbin/initctl; ln -s /bin/true /sbin/initctl - _aptget_clean() pkgs32 = [] if 'i386' != osarch: @@ -70,20 +67,19 @@ def boottoolsSoftwareInstall (osarch, osrelease): _aptget_update() ## esto ya esta hecho... _oghook_deactivate() _aptget_upgrade() - _mock_mtab() + #_mock_mtab() - # Instalamos el kernel. _aptget_install ([f'linux-image-{osrelease}', f'linux-headers-{osrelease}', f'linux-modules-{osrelease}', f'linux-modules-extra-{osrelease}', 'dkms', 'shim-signed', 'openssl']) subprocess.run (['debconf-set-selections'], input=debconf_settings, text=True) - ## nati: hace falta --force-confdef para evitar un tema interactivo del /etc/ssh/ssh_config - _aptget_install (['sshfs', 'kexec-tools'] + pkgs32, opts=['-o', 'DPkg::Options::=--force-confdef']) + _aptget_install (['sshfs', 'kexec-tools'] + pkgs32, opts=['-o', 'DPkg::Options::=--force-confdef']) ## hace falta --force-confdef para evitar un tema interactivo del /etc/ssh/ssh_config pkgs = [] for section in config.options('Packages'): pkgs += re.split (r'[ \n]', config['Packages'].get(section).strip()) print ('about to install these packages: "{}"'.format (' '.join (pkgs))) + ## TODO don't fail when there are unknown/uninstallable packages _aptget_install (pkgs) # Instalar módulos que algunos paquetes puedan tener pendientes de compilar. @@ -97,7 +93,7 @@ def boottoolsSoftwareInstall (osarch, osrelease): _run (['dkms', 'install', '-m', mod.strip(), '-v', vers.strip()]) _oghook_activate() - _restore_mtab() + #_restore_mtab() _aptget_clean() _aptget_autoclean() _aptget_autoremove() diff --git a/04-boottoolsSoftwareCompile.py b/04-boottoolsSoftwareCompile.py index 46adbcd..912008c 100755 --- a/04-boottoolsSoftwareCompile.py +++ b/04-boottoolsSoftwareCompile.py @@ -5,13 +5,8 @@ import subprocess from boottoolsfunctions import _run os.environ['LANGUAGE'] = os.environ['LC_ALL'] = os.environ['LANG'] = 'C' - -#source /opt/opengnsys/lib/engine/bin/ToolsGNU.c &>/dev/null - os.chdir ('/tmp') -## apt-get install wget gettext unzip git - print ('ms-sys') try: _run (['which', 'ms-sys']) except: diff --git a/boottoolsfunctions/__init__.py b/boottoolsfunctions/__init__.py index 1e8cc09..f9eeb79 100644 --- a/boottoolsfunctions/__init__.py +++ b/boottoolsfunctions/__init__.py @@ -11,6 +11,34 @@ import configparser logger = logging.getLogger ('boottools') +schroot_conf_tpl = """ +[IMGogclient] +type=loopback +file=/var/lib/tftpboot/ogclient/ogclient.img +description=ogclient Ubuntu image +#priority=1 +users=root +groups=root +root-groups=root +mount-options=-o offset=32256 +root-users=root +""".strip() + +mount_defaults_tpl = """ +# mount.defaults: static file system information for chroots. +# Note that the mount point will be prefixed by the chroot path +# (CHROOT_PATH) +# +# +proc /proc proc defaults 0 0 +#procbususb /proc/bus/usb usbfs defaults 0 0 +#/dev /dev none rw,bind 0 0 +/dev/pts /dev/pts none rw,bind 0 0 +/dev/shm /dev/shm none rw,bind 0 0 +#/home /home none rw,bind 0 0 +/tmp /tmp none rw,bind 0 0 +""".strip() + def _run (args): cp = subprocess.run (args, text=True, capture_output=True) if cp.returncode: @@ -181,8 +209,6 @@ def btogGetOsInfo2 (ogrepo_dir, versionboottools, oscodename, osrelease, osarch) return gitrelease, nameisoclient, namehostclient def btogSetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, bttargetdir, osarch): - #if not btrootfsimg: btogGetVar() - logger.info (f'Creación y formateo del disco virtual "{btrootfsimg}" "{btvirtualdisksize}" MB') try: _umount (btrootfsmnt) @@ -250,57 +276,24 @@ def btogSetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisks logger.info (f'"{btrootfsimg}" "{btvirtualdisksize}" MB : OK') -schroot_conf_tpl = """ -[IMGogclient] -type=loopback -file=/var/lib/tftpboot/ogclient/ogclient.img -description=ogclient Ubuntu image -#priority=1 -users=root -groups=root -root-groups=root -mount-options=-o offset=32256 -root-users=root -""".strip() - -mount_defaults_tpl = """ -# mount.defaults: static file system information for chroots. -# Note that the mount point will be prefixed by the chroot path -# (CHROOT_PATH) -# -# -proc /proc proc defaults 0 0 -#procbususb /proc/bus/usb usbfs defaults 0 0 -#/dev /dev none rw,bind 0 0 -/dev/pts /dev/pts none rw,bind 0 0 -/dev/shm /dev/shm none rw,bind 0 0 -#/home /home none rw,bind 0 0 -/tmp /tmp none rw,bind 0 0 -""".strip() - #btogSetFsAcces: habilita el acceso al sistema root del cliente con schroot def btogSetFsAccess (btrootfsimg): if (_grep (btrootfsimg, '/etc/schroot/schroot.conf')): return logger.info ('Iniciando la configuración del schroot') - #cp /etc/schroot/schroot.conf /etc/schroot/schroot.conf.`getDateTime` if (os.path.exists ('/etc/schroot/schroot.conf')): shutil.copy ('/etc/schroot/schroot.conf', '/etc/schroot/schroot.conf.bak') - with open ('/etc/schroot/schroot.conf', 'w') as f: f.write (schroot_conf_tpl + '\n') - #cp /etc/schroot/mount-defaults /etc/schroot/mount-defaults.`getDateTime` if (os.path.exists ('/etc/schroot/mount-defaults')): shutil.copy ('/etc/schroot/mount-defaults', '/etc/schroot/mount-defaults.bak') - with open ('/etc/schroot/mount-defaults', 'w') as f: f.write (mount_defaults_tpl + '\n') - map (os.unlink, glob.glob('/etc/schroot/setup.d/*chrootname')) - #for i in glob.glob ('/etc/schroot/setup.d/*chrootname'): - # os.unlink (i) + for i in glob.glob ('/etc/schroot/setup.d/*chrootname'): + os.unlink (i) logger.info ('Finalizado: OK') return 0 @@ -321,7 +314,6 @@ def btogSetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp): def boottoolsSshServer (btrootfsmnt): if not os.path.exists ('/root/.ssh/id_rsa'): ## crea un par de claves en la VM, no en el chroot _run (['ssh-keygen', '-q', '-f', '/root/.ssh/id_rsa', '-N', '']) - #shutil.copy ('/root/.ssh/id_rsa.pub', '/tmp/') logger.debug ('comprobando directorio .ssh del root') if not os.path.exists (f'{btrootfsmnt}/root/.ssh'): ## crea directorio dentro del chroot @@ -337,7 +329,6 @@ def boottoolsSshServer (btrootfsmnt): logger.debug ('importando la clave publica del servidor OG') #cat /tmp/id_rsa.pub if os.path.exists ('/root/.ssh/id_rsa.pub'): ## coge la publica de la VM y la pone en el authorized_keys del chroot - #cat /tmp/id_rsa.pub >> /root/.ssh/authorized_keys fdin = open ('/root/.ssh/id_rsa.pub', 'r') fdout = open (f'{btrootfsmnt}/root/.ssh/authorized_keys', 'a') while True: @@ -387,7 +378,6 @@ def btogFsInitrd (bttargetdir, osrelease): _write_md5 (f'{bttargetdir}/oginitrd.img') _write_md5 (f'{bttargetdir}/ogvmlinuz') - #cd - #chmod -R 755 $BTTARGETDIR for f in glob.glob (f'{bttargetdir}/oginitrd*') + glob.glob (f'{bttargetdir}/vmlinuz*'): os.chmod (f, 0o755) @@ -430,7 +420,6 @@ def btogIsoGenerator (pxepkg, isolinux_tpl, bttargetdir, nameisoclient): os.rmdir ('/tmp/iso/isolinux/mount') os.unlink (f'/tmp/iso/isolinux/{pxepkg}.iso') - ## ojo que aquí hay que interpolar $NAMEISOCLIENT y $PXEPKG with open ('/tmp/iso/isolinux/isolinux.cfg', 'w') as fd: fd.write (isolinux_tpl.strip().replace('__NAMEISOCLIENT__', nameisoclient).replace('__PXEPKG__', pxepkg)) diff --git a/boottoolsgenerator.py b/boottoolsgenerator.py index 23cacf6..7964a6b 100755 --- a/boottoolsgenerator.py +++ b/boottoolsgenerator.py @@ -16,11 +16,13 @@ config = _read_config ('boottoolsgenerator.cfg') isolinux_tpl = config['General'].get ('isolinux_template') lvl = config['General'].get ('logging_level') -numeric_level = getattr (logging, lvl.upper(), None) -if numeric_level is None: - numeric_level = getattr (logging, 'INFO') def _logging(): #logging.root.handlers = [] + + numeric_level = getattr (logging, lvl.upper(), None) + if numeric_level is None: + numeric_level = getattr (logging, 'INFO') + logging.basicConfig ( format='%(levelname)s %(asctime)s (%(funcName)s) %(message)s', level=numeric_level, @@ -31,48 +33,15 @@ def _logging(): ) return logging.getLogger ('boottools') -logger = _logging() - def clone_client_dirs (ogrepo_url, ogrepo_branch, ogrepo_dir): if not os.path.exists ('/tmp/opengnsys/engine'): _run (['git', 'clone', '-c', 'http.sslVerify=false', '--branch', ogrepo_branch, ogrepo_url, ogrepo_dir]) _run (['rsync', '-aH', f'{ogrepo_dir}/client/engine', f'{ogrepo_dir}/client/shared', '/tmp/opengnsys/']) -type_client = sys.argv[1] if len(sys.argv)>1 else 'host' -#WORKDIR = '/tmp/opengnsys_installer' -#INSTALL_TARGET = '/opt/opengnsys' -#PROGRAMDIR = $(readlink -e $(dirname "$0")) - -if os.getuid(): - logger.error ('ERROR: this program must run under root privileges!!') - sys.exit (1) - -#os.chdir ('/tmp') - -logger.info ('OpenGnsys CLIENT installation begins') -ogrepo_url = 'https://ognproject.evlt.uma.es/gitea/opengnsys/opengnsys.git' -ogrepo_branch = 'main' -ogrepo_dir = '/tmp/ogrepo' -clone_client_dirs (ogrepo_url, ogrepo_branch, ogrepo_dir) - -####################################################################3 -logger.info ('FASE 1 - Asignación de variables') -osdistrib, oscodename, osrelease, osarch, oshttp = btogGetOsInfo1(type_client) -btdir, bttargetdir, btrootfsimg, btrootfsmnt, btrootfsimglabel, log_file, versionboottools, btvirtualdisksize = btogGetVar(osarch) -gitrelease, nameisoclient, namehostclient = btogGetOsInfo2(ogrepo_dir, versionboottools, oscodename, osrelease, osarch) - def _mount_rootfs(): global btrootfsimg, btrootfsmnt _mount (btrootfsimg, btrootfsmnt, opts=['-o', 'loop,offset=32256']) -## this is convenient in case the previous run failed and we want to run this program again -try: _umount (btrootfsmnt) -except: pass - - -########################################################################## -logger.info ('FASE 2 - Instalación de software adicional.') - def _get_pxepkg(): #grep "http://free.nchc.org.tw/drbl-core" /etc/apt/sources.list || echo "deb http://free.nchc.org.tw/drbl-core drbl stable" >> /etc/apt/sources.list _run (['apt-get', 'update']) #>>/tmp/fase2.out @@ -88,11 +57,6 @@ def _get_pxepkg(): logger.info (f'PXE package is "{pxepkg}"') return pxepkg -pxepkg = _get_pxepkg() -## TODO qemu no existe, hace falta? -_run (['apt-get', '-y', 'install', 'jq', 'syslinux', 'syslinux-efi', 'syslinux-utils', 'debootstrap', 'subversion', 'schroot', 'squashfs-tools', 'syslinux', 'genisoimage', 'qemu-utils', 'lsof', pxepkg]) #>>/tmp/fase2.out - - def _mkrootfs(): rc = subprocess.run (f'file "{btrootfsimg}" |grep -q "partition 1 *: ID=0x83"', shell=True).returncode print (rc) @@ -102,13 +66,6 @@ def _mkrootfs(): logger.error (str (e)) sys.exit (2) -logger.info ('FASE 3 - Creación del Sistema raiz RootFS (Segundo Sistema archivos (img))') -logger.info ('Fase 3.1 Generar y formatear el disco virtual. Generar el dispositivo loop.') -_mkrootfs() - -logger.info ('Fase 3.2 - Configurar acceso schroot al Segundo Sistema de archivos (img)') -btogSetFsAccess (btrootfsimg) - ## para hacer schroot --cosas, el mntpt tiene que estar desmontado ## si está montado da un pete tal que 'E: 10mount: mount: /run/schroot/mount/IMGogclient-7fbf51a2-e37e-48e5-8e5d-83f8901fc7ed: wrong fs type, bad option, bad superblock on /dev/loop1, missing codepage or helper program, or other error.' @@ -127,29 +84,6 @@ def _debootstrap(): logger.error (str (e)) sys.exit (3) -logger.info ('Fase 3.3 Generar sistema de archivos con debootstrap') -_debootstrap() - -logger.info ('FASE 4 - Incorporando ficheros OpenGnsys al sistema raíz rootfs') - -## por qué esta copia??? - -#cp -a ${BTDIR}/includes/usr/bin/* /tmp >>/tmp/fase5.out -#_run (['cp', '-a'] + glob.glob (f'{btdir}/includes/usr/bin/*') + ['/tmp']) - -#chmod +x /tmp/boot-tools/*.sh -#for i in glob.glob ('/tmp/boot-tools/*.sh'): -# st = os.stat (i) -# os.chmod (i, st.st_mode|stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH) - -# Incluir revisión. -## FIXME esto la incluye incondicionalmente, y luego terminamos con "OpenGnsys Client 1.2.0-rc1 gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) ..." -#sed -i "1 s/$/ $GITRELEASE ($OSRELEASE)/" ${BTDIR}/includes/etc/initramfs-tools/scripts/VERSION.txt -_run (['sed', '-i', f'1 s/$/ {gitrelease} ({osrelease})/', f'{btdir}/includes/etc/initramfs-tools/scripts/VERSION.txt']) - -# En Ubuntu 13.04+ es necesario matar proceso de "udev" antes de desmontar. -#umount $BTROOTFSMNT 2>/dev/null || (kill -9 $(lsof -t $BTROOTFSMNT); umount $BTROOTFSMNT) 2>/dev/null - def _cerodos(): try: _mount_rootfs() except: @@ -163,22 +97,13 @@ def _cerodos(): _umount (btrootfsmnt) -_cerodos() - - -## aqui empiezan las cosas de chroot - -def _cerotres(): +def _chroot_cerotres(): logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/03-boottoolsSoftwareInstall.py --osrelease "{osrelease}" --osarch "{osarch}"\'') stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/03-boottoolsSoftwareInstall.py', '--osrelease', osrelease, '--osarch', osarch]) logger.debug (f'03-boottoolsSoftwareInstall stdout follows:') for i in stdout.strip().split('\n'): logger.debug (' ' + i) -logger.info ('FASE 5 - Instalar software') -logger.info ('Fase 5.1 instalar paquetes deb con apt-get') -_cerotres() - -def _cerocuatro(): +def _chroot_cerocuatro(): logger.debug ('running \'schroot --chroot IMGogclient -- {}/04-boottoolsSoftwareCompile.py\'') #cd / stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/04-boottoolsSoftwareCompile.py']) @@ -186,14 +111,6 @@ def _cerocuatro(): logger.debug (f'04-boottoolsSoftwareCompile stdout follows:') for i in stdout.strip().split('\n'): logger.debug (' ' + i) -logger.info ('Fase 5.2 compilar software.') -_cerocuatro() - -## ya no chroot - - - - def _ssh_stuff(): try: _mount_rootfs() except: @@ -215,25 +132,13 @@ def _ssh_stuff(): ## - hay otro nuevo par de claves en el rootfs /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh ## - las dos claves públicas (una de cada par) están autorizadan en el rootfs /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh/authorized_keys -logger.info ('FASE 6 - Personalizar el sistema creado') -logger.info ('Fase 6.1 Incorporar la clave publica del servidor') -logger.info ('Fase 6.2 Incorporar la clave publica del propio cliente') -_ssh_stuff() - - - -## chroot de nuevo - -def _cerocinco(): +def _chroot_cerocinco(): logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/05-boottoolsFsLocales.py\'') stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/05-boottoolsFsLocales.py']) logger.debug (f'05-boottoolsFsLocales stdout follows:') for i in stdout.strip().split('\n'): logger.debug (' ' + i) -logger.info ('Fase 6.3 Configurando las locales') -_cerocinco() - -def _ceroseis(): +def _chroot_ceroseis(): #cd / #schroot -c IMGogclient -- /usr/bin/boot-tools/boottoolsInitrdGenerate.sh logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/06-boottoolsInitrdGenerate.py --osrelease "{osrelease}"\'') @@ -242,12 +147,73 @@ def _ceroseis(): for i in stdout.strip().split('\n'): logger.debug (' ' + i) ## esto deja initrd.img-6.8.0-31-generic y vmlinuz-6.8.0-31-generic en /tmp -_ceroseis() -## ya no chroot +logger = _logging() +type_client = sys.argv[1] if len(sys.argv)>1 else 'host' + +if os.getuid(): + logger.error ('ERROR: this program must run under root privileges!!') + sys.exit (1) + +#os.chdir ('/tmp') + +logger.info ('OpenGnsys CLIENT installation begins') +ogrepo_url = 'https://ognproject.evlt.uma.es/gitea/opengnsys/opengnsys.git' +ogrepo_branch = 'main' +ogrepo_dir = '/tmp/ogrepo' +clone_client_dirs (ogrepo_url, ogrepo_branch, ogrepo_dir) + +logger.info ('FASE 1 - Asignación de variables') +osdistrib, oscodename, osrelease, osarch, oshttp = btogGetOsInfo1(type_client) +btdir, bttargetdir, btrootfsimg, btrootfsmnt, btrootfsimglabel, log_file, versionboottools, btvirtualdisksize = btogGetVar(osarch) +gitrelease, nameisoclient, namehostclient = btogGetOsInfo2(ogrepo_dir, versionboottools, oscodename, osrelease, osarch) + +## this is convenient in case the previous run failed and we want to run this program again +try: _umount (btrootfsmnt) +except: pass + +logger.info ('FASE 2 - Instalación de software adicional.') + +pxepkg = _get_pxepkg() +_run (['apt-get', '-y', 'install', 'jq', 'syslinux', 'syslinux-efi', 'syslinux-utils', 'debootstrap', 'subversion', 'schroot', 'squashfs-tools', 'syslinux', 'genisoimage', 'qemu-utils', 'lsof', pxepkg]) ## TODO qemu no existe, hace falta? +logger.info ('FASE 3 - Creación del Sistema raiz RootFS (Segundo Sistema archivos (img))') +logger.info ('Fase 3.1 Generar y formatear el disco virtual. Generar el dispositivo loop.') +_mkrootfs() +logger.info ('Fase 3.2 - Configurar acceso schroot al Segundo Sistema de archivos (img)') +btogSetFsAccess (btrootfsimg) + +logger.info ('Fase 3.3 Generar sistema de archivos con debootstrap') +_debootstrap() + +logger.info ('FASE 4 - Incorporando ficheros OpenGnsys al sistema raíz rootfs') + +# Incluir revisión. +## FIXME esto la incluye incondicionalmente, y luego terminamos con "OpenGnsys Client 1.2.0-rc1 gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) ..." +#sed -i "1 s/$/ $GITRELEASE ($OSRELEASE)/" ${BTDIR}/includes/etc/initramfs-tools/scripts/VERSION.txt +_run (['sed', '-i', f'1 s/$/ {gitrelease} ({osrelease})/', f'{btdir}/includes/etc/initramfs-tools/scripts/VERSION.txt']) + +_cerodos() + +logger.info ('FASE 5 - Instalar software') +logger.info ('Fase 5.1 instalar paquetes deb con apt-get') +_chroot_cerotres() + +logger.info ('Fase 5.2 compilar software.') +_chroot_cerocuatro() + + +logger.info ('FASE 6 - Personalizar el sistema creado') +logger.info ('Fase 6.1 Incorporar la clave publica del servidor') +logger.info ('Fase 6.2 Incorporar la clave publica del propio cliente') +_ssh_stuff() + +logger.info ('Fase 6.3 Configurando las locales') +_chroot_cerocinco() + +_chroot_ceroseis() logger.info ('FASE 7 - Generar distribucion') logger.info ('Fase 7.1 Generar el initrd') From e6aa11e5b33e083f30d980d60512bd7f8a265c3b Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Wed, 14 Aug 2024 13:37:59 +0200 Subject: [PATCH 24/45] refs #596 sort phases out --- 03-boottoolsSoftwareInstall.py | 2 + 05-boottoolsFsLocales.py | 2 + boottoolsfunctions/__init__.py | 2 +- boottoolsgenerator.py | 93 ++++++++++++++++++---------------- 4 files changed, 54 insertions(+), 45 deletions(-) diff --git a/03-boottoolsSoftwareInstall.py b/03-boottoolsSoftwareInstall.py index 1ede4f8..e45bdf3 100755 --- a/03-boottoolsSoftwareInstall.py +++ b/03-boottoolsSoftwareInstall.py @@ -9,6 +9,8 @@ import subprocess from boottoolsfunctions import _run, _read_config config = _read_config ('boottoolsgenerator.cfg') +if config is None: + sys.exit (1) debconf_settings = config['General'].get ('debconf_settings') def _aptget_clean(): diff --git a/05-boottoolsFsLocales.py b/05-boottoolsFsLocales.py index 91ae033..bb4e500 100755 --- a/05-boottoolsFsLocales.py +++ b/05-boottoolsFsLocales.py @@ -5,6 +5,8 @@ import subprocess from boottoolsfunctions import _run, _read_config config = _read_config ('boottoolsgenerator.cfg') +if config is None: + sys.exit (1) debconf_settings2 = config['General'].get ('debconf_settings2') subprocess.run (['debconf-set-selections'], input=debconf_settings2, text=True) diff --git a/boottoolsfunctions/__init__.py b/boottoolsfunctions/__init__.py index f9eeb79..c3aad00 100644 --- a/boottoolsfunctions/__init__.py +++ b/boottoolsfunctions/__init__.py @@ -75,7 +75,7 @@ def _umount (mntpt): def _read_config (fn): if not os.path.exists (fn): print (f'configuration file "{fn}" not found') - sys.exit (1) + return config = configparser.ConfigParser (comment_prefixes='#', inline_comment_prefixes='#') config.read (fn) return config diff --git a/boottoolsgenerator.py b/boottoolsgenerator.py index 7964a6b..fc2faf6 100755 --- a/boottoolsgenerator.py +++ b/boottoolsgenerator.py @@ -12,10 +12,6 @@ curdir = os.path.dirname (__file__) sys.path.insert (0, curdir) from boottoolsfunctions import _run, _mount, _umount, _read_config, btogGetOsInfo1, btogGetOsInfo2, btogGetVar, btogSetFsVirtual, btogSetFsAccess, btogSetFsBase, boottoolsSshServer, boottoolsSshClient, btogFsInitrd, btogFsSqfs, btogIsoGenerator -config = _read_config ('boottoolsgenerator.cfg') -isolinux_tpl = config['General'].get ('isolinux_template') -lvl = config['General'].get ('logging_level') - def _logging(): #logging.root.handlers = [] @@ -44,7 +40,7 @@ def _mount_rootfs(): def _get_pxepkg(): #grep "http://free.nchc.org.tw/drbl-core" /etc/apt/sources.list || echo "deb http://free.nchc.org.tw/drbl-core drbl stable" >> /etc/apt/sources.list - _run (['apt-get', 'update']) #>>/tmp/fase2.out + _run (['apt-get', 'update']) acse_gpxe, _ = _run (['apt-cache', 'search', 'gpxe']) acse_ipxe, _ = _run (['apt-cache', 'search', 'ipxe']) if acse_ipxe: @@ -58,18 +54,24 @@ def _get_pxepkg(): return pxepkg def _mkrootfs(): + logger.info ('Fase 3.1 Generar y formatear el disco virtual. Generar el dispositivo loop.') rc = subprocess.run (f'file "{btrootfsimg}" |grep -q "partition 1 *: ID=0x83"', shell=True).returncode print (rc) if (rc): ## 'file|grep' failed try: btogSetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, bttargetdir, osarch) except Exception as e: logger.error (str (e)) - sys.exit (2) + sys.exit (1) + +def _schroot (btrootfsimg): + logger.info ('Fase 3.2 - Configurar acceso schroot al Segundo Sistema de archivos (img)') + btogSetFsAccess (btrootfsimg) ## para hacer schroot --cosas, el mntpt tiene que estar desmontado ## si está montado da un pete tal que 'E: 10mount: mount: /run/schroot/mount/IMGogclient-7fbf51a2-e37e-48e5-8e5d-83f8901fc7ed: wrong fs type, bad option, bad superblock on /dev/loop1, missing codepage or helper program, or other error.' def _debootstrap(): + logger.info ('Fase 3.3 Generar sistema de archivos con debootstrap') logger.debug ('Try creation of a file within chroot (this operation may fail with "... etc/resolv.conf: No such file or directory"--that is ok)') logger.debug ('Running \'schroot --chroot IMGogclient -- stat /etc\'') cp = subprocess.run (['schroot', '--chroot', 'IMGogclient', '--', 'stat', '/etc']) @@ -78,17 +80,23 @@ def _debootstrap(): try: _mount_rootfs() except: logger.error ('mount failed') - sys.exit (3) - try: btogSetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp) #>>/tmp/fase3.out + sys.exit (1) + try: btogSetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp) except Exception as e: logger.error (str (e)) - sys.exit (3) + sys.exit (1) + +# Incluir revisión. +def _initramfs_version (gitrelease, osrelease, btdir): + ## FIXME esto la incluye incondicionalmente, y luego terminamos con "OpenGnsys Client 1.2.0-rc1 gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) ..." + #sed -i "1 s/$/ $GITRELEASE ($OSRELEASE)/" ${BTDIR}/includes/etc/initramfs-tools/scripts/VERSION.txt + _run (['sed', '-i', f'1 s/$/ {gitrelease} ({osrelease})/', f'{btdir}/includes/etc/initramfs-tools/scripts/VERSION.txt']) def _cerodos(): try: _mount_rootfs() except: logger.error ('mount failed') - sys.exit (3) + sys.exit (1) logger.debug (f'running \'{curdir}/02-boottoolsFsOpengnsys.py --mntpt "{btrootfsmnt}" --osdistrib "{osdistrib}" --oscodename "{oscodename}" --osrelease "{osrelease}" --osarch "{osarch}" --oshttp "{oshttp}"\'') stdout, _ = _run ([f'{curdir}/02-boottoolsFsOpengnsys.py', '--mntpt', btrootfsmnt, '--osdistrib', osdistrib, '--oscodename', oscodename, '--osrelease', osrelease, '--osarch', osarch, '--oshttp', oshttp]) @@ -98,12 +106,14 @@ def _cerodos(): _umount (btrootfsmnt) def _chroot_cerotres(): + logger.info ('Fase 5.1 instalar paquetes deb con apt-get') logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/03-boottoolsSoftwareInstall.py --osrelease "{osrelease}" --osarch "{osarch}"\'') stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/03-boottoolsSoftwareInstall.py', '--osrelease', osrelease, '--osarch', osarch]) logger.debug (f'03-boottoolsSoftwareInstall stdout follows:') for i in stdout.strip().split('\n'): logger.debug (' ' + i) def _chroot_cerocuatro(): + logger.info ('Fase 5.2 compilar software.') logger.debug ('running \'schroot --chroot IMGogclient -- {}/04-boottoolsSoftwareCompile.py\'') #cd / stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/04-boottoolsSoftwareCompile.py']) @@ -112,10 +122,11 @@ def _chroot_cerocuatro(): for i in stdout.strip().split('\n'): logger.debug (' ' + i) def _ssh_stuff(): + logger.info ('Fase 6.1 Configurar ssh') try: _mount_rootfs() except: logger.error ('mount failed') - sys.exit (3) + sys.exit (1) #cd / #schroot --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSshServer.sh ## no necesita chroot @@ -133,12 +144,14 @@ def _ssh_stuff(): ## - las dos claves públicas (una de cada par) están autorizadan en el rootfs /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh/authorized_keys def _chroot_cerocinco(): + logger.info ('Fase 6.2 Configurar las locales') logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/05-boottoolsFsLocales.py\'') stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/05-boottoolsFsLocales.py']) logger.debug (f'05-boottoolsFsLocales stdout follows:') for i in stdout.strip().split('\n'): logger.debug (' ' + i) def _chroot_ceroseis(): + logger.info ('Fase 6.3 Crear initrd') ## what?? #cd / #schroot -c IMGogclient -- /usr/bin/boot-tools/boottoolsInitrdGenerate.sh logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/06-boottoolsInitrdGenerate.py --osrelease "{osrelease}"\'') @@ -147,6 +160,25 @@ def _chroot_ceroseis(): for i in stdout.strip().split('\n'): logger.debug (' ' + i) ## esto deja initrd.img-6.8.0-31-generic y vmlinuz-6.8.0-31-generic en /tmp +def _mkinitrd (bttargetdir, osrelease): + logger.info ('Fase 7.1 Generar el initrd') ## what?? + btogFsInitrd (bttargetdir, osrelease) + +def _mksquashfs (bttargetdir, btrootfsmnt): + logger.info ('Fase 7.2 Generar fichero sqfs a partir del fichero img') + btogFsSqfs (bttargetdir, btrootfsmnt) + +def _mkisofs (pxepkg, isolinux_tpl, bttargetdir, nameisoclient): + logger.info ('Fase 7.3 Generar la ISO') + btogIsoGenerator (pxepkg, isolinux_tpl, bttargetdir, nameisoclient) + + + +config = _read_config ('boottoolsgenerator.cfg') +if config is None: + sys.exit (1) +isolinux_tpl = config['General'].get ('isolinux_template') +lvl = config['General'].get ('logging_level') logger = _logging() type_client = sys.argv[1] if len(sys.argv)>1 else 'host' @@ -173,62 +205,35 @@ try: _umount (btrootfsmnt) except: pass logger.info ('FASE 2 - Instalación de software adicional.') - pxepkg = _get_pxepkg() _run (['apt-get', '-y', 'install', 'jq', 'syslinux', 'syslinux-efi', 'syslinux-utils', 'debootstrap', 'subversion', 'schroot', 'squashfs-tools', 'syslinux', 'genisoimage', 'qemu-utils', 'lsof', pxepkg]) ## TODO qemu no existe, hace falta? - logger.info ('FASE 3 - Creación del Sistema raiz RootFS (Segundo Sistema archivos (img))') -logger.info ('Fase 3.1 Generar y formatear el disco virtual. Generar el dispositivo loop.') _mkrootfs() - -logger.info ('Fase 3.2 - Configurar acceso schroot al Segundo Sistema de archivos (img)') -btogSetFsAccess (btrootfsimg) - -logger.info ('Fase 3.3 Generar sistema de archivos con debootstrap') +_schroot (btrootfsimg) _debootstrap() logger.info ('FASE 4 - Incorporando ficheros OpenGnsys al sistema raíz rootfs') - -# Incluir revisión. -## FIXME esto la incluye incondicionalmente, y luego terminamos con "OpenGnsys Client 1.2.0-rc1 gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) ..." -#sed -i "1 s/$/ $GITRELEASE ($OSRELEASE)/" ${BTDIR}/includes/etc/initramfs-tools/scripts/VERSION.txt -_run (['sed', '-i', f'1 s/$/ {gitrelease} ({osrelease})/', f'{btdir}/includes/etc/initramfs-tools/scripts/VERSION.txt']) - +_initramfs_version (gitrelease, osrelease, btdir) _cerodos() logger.info ('FASE 5 - Instalar software') -logger.info ('Fase 5.1 instalar paquetes deb con apt-get') _chroot_cerotres() - -logger.info ('Fase 5.2 compilar software.') _chroot_cerocuatro() - logger.info ('FASE 6 - Personalizar el sistema creado') -logger.info ('Fase 6.1 Incorporar la clave publica del servidor') -logger.info ('Fase 6.2 Incorporar la clave publica del propio cliente') _ssh_stuff() - -logger.info ('Fase 6.3 Configurando las locales') _chroot_cerocinco() - _chroot_ceroseis() logger.info ('FASE 7 - Generar distribucion') -logger.info ('Fase 7.1 Generar el initrd') -btogFsInitrd (bttargetdir, osrelease) - - -logger.info ('Fase 7.2 Generar fichero sqfs a partir del fichero img') +_mkinitrd (bttargetdir, osrelease) try: _mount_rootfs() except: logger.error ('mount failed') - sys.exit (3) -btogFsSqfs (bttargetdir, btrootfsmnt) + sys.exit (1) +_mksquashfs (bttargetdir, btrootfsmnt) _umount (btrootfsmnt) - -logger.info ('Fase 7.3 Generar la ISO') -btogIsoGenerator (pxepkg, isolinux_tpl, bttargetdir, nameisoclient) +_mkisofs (pxepkg, isolinux_tpl, bttargetdir, nameisoclient) logger.info ('OpenGnsys installation finished') From bd56977510baa5dc38b65d4c83a43c779467f4a2 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Wed, 14 Aug 2024 14:57:25 +0200 Subject: [PATCH 25/45] refs #596 move 02-boottoolsFsOpengnsys to a function, rename functions, reorganise code and config --- 02-boottoolsFsOpengnsys.py | 61 ---------------- 05-boottoolsFsLocales.py | 4 +- boottoolsfunctions/__init__.py | 38 ++++++++++ boottoolsgenerator.cfg | 128 ++++++++++++++++----------------- boottoolsgenerator.py | 105 ++++++++++----------------- 5 files changed, 142 insertions(+), 194 deletions(-) delete mode 100755 02-boottoolsFsOpengnsys.py diff --git a/02-boottoolsFsOpengnsys.py b/02-boottoolsFsOpengnsys.py deleted file mode 100755 index 0063f29..0000000 --- a/02-boottoolsFsOpengnsys.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/python3 - -import os -import logging -import subprocess -import shutil -import argparse - -svnclientdir = '/tmp/opengnsys/oglive_builder' -svnclientstructure = '/tmp/opengnsys/shared' -svnclientengine = '/tmp/opengnsys/engine' - -def boottoolsFsOpengnsys (ogclientmount, osdistrib, oscodename, osrelease, osarch, oshttp): - print (':'.join ([osdistrib, oscodename, osrelease, osarch, oshttp])) - print ('Iniciando la personalización con datos del repositorio') - - sources_list_in = f'{svnclientdir}/includes/etc/apt/sources.list.{osdistrib.lower()}' - sources_list_out = f'{svnclientdir}/includes/etc/apt/sources.list' - fdin = open (sources_list_in, 'r') - fdout = open (sources_list_out, 'w') - while True: - l = fdin.readline() - if not l: break - fdout.write (l.replace ('OSCODENAME', oscodename)) - fdin.close() - fdout.close() - - subprocess.run (f'chmod -R 775 {svnclientdir}/includes/usr/bin/*', shell=True) - - os.makedirs (f'{ogclientmount}/opt/opengnsys/lib/engine/bin/', exist_ok=True) - os.makedirs (f'{ogclientmount}/usr/local/etc', exist_ok=True) - os.makedirs (f'{ogclientmount}/usr/local/lib', exist_ok=True) - os.makedirs (f'{ogclientmount}/usr/local/plugins', exist_ok=True) - - subprocess.run (f'rsync -aH {svnclientdir}/includes/* {ogclientmount}/' , shell=True) - subprocess.run (f'rsync -aH {svnclientstructure}/* {ogclientmount}/opt/opengnsys/' , shell=True) - subprocess.run (f'rsync -aH {svnclientengine}/* {ogclientmount}/opt/opengnsys/lib/engine/bin/', shell=True) - - if not os.path.exists (f'{ogclientmount}/etc/pci.ids'): - shutil.copy (f'{svnclientstructure}/lib/pci.ids', f'{ogclientmount}/etc/') - - # Dependencias Qt para el Browser. - subprocess.run (f'rsync -aH {svnclientstructure}/etc/*.qmap {ogclientmount}/usr/local/etc', shell=True) - subprocess.run (f'rsync -aH {svnclientstructure}/lib/qtlib/* {ogclientmount}/usr/local/lib', shell=True) - subprocess.run (f'rsync -aH {svnclientstructure}/lib/fonts {ogclientmount}/usr/local/lib', shell=True) - subprocess.run (f'rsync -aH {svnclientstructure}/lib/qtplugins/* {ogclientmount}/usr/local/plugins', shell=True) - - # Browser y ogAdmClient. - if os.path.exists (f'{svnclientstructure}/bin/browser'): shutil.copy (f'{svnclientstructure}/bin/browser', f'{ogclientmount}/bin/') - if os.path.exists (f'{svnclientstructure}/bin/ogAdmClient'): shutil.copy (f'{svnclientstructure}/bin/ogAdmClient', f'{ogclientmount}/bin/') - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument ('--mntpt', help='rootfs mount point', action='store', required=True) - parser.add_argument ('--osdistrib', help='OS distribution', action='store', required=True) - parser.add_argument ('--oscodename', help='OS codename', action='store', required=True) - parser.add_argument ('--osrelease', help='OS release', action='store', required=True) - parser.add_argument ('--osarch', help='OS architecture', action='store', required=True) - parser.add_argument ('--oshttp', help='OS HTTP source', action='store', required=True) - args = parser.parse_args() - boottoolsFsOpengnsys (args.mntpt, args.osdistrib, args.oscodename, args.osrelease, args.osarch, args.oshttp) diff --git a/05-boottoolsFsLocales.py b/05-boottoolsFsLocales.py index bb4e500..2d04c60 100755 --- a/05-boottoolsFsLocales.py +++ b/05-boottoolsFsLocales.py @@ -11,5 +11,5 @@ debconf_settings2 = config['General'].get ('debconf_settings2') subprocess.run (['debconf-set-selections'], input=debconf_settings2, text=True) -os.environ['DEBIAN_FRONTEND'] = 'noninteractive' -_run (['dpkg-reconfigure', 'console-setup', 'locales']) +## despues de esto, debconf-get-selections devuelve los valores antiguos, no se por que... +_run (['dpkg-reconfigure', '--frontend', 'noninteractive', 'console-setup', 'locales']) diff --git a/boottoolsfunctions/__init__.py b/boottoolsfunctions/__init__.py index c3aad00..592daff 100644 --- a/boottoolsfunctions/__init__.py +++ b/boottoolsfunctions/__init__.py @@ -311,6 +311,44 @@ def btogSetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp): logger.info (f'debootstrap --arch="{osarch}" --components=main,universe "{oscodename}" "{btrootfsmnt}" "{oshttp}" : ok') return 0 +def boottoolsFsOpengnsys (builder, og_shared, og_engine, ogclientmount, osdistrib, oscodename): + print ('Iniciando la personalización con datos del repositorio') + + sources_list_in = f'{builder}/includes/etc/apt/sources.list.{osdistrib.lower()}' + sources_list_out = f'{builder}/includes/etc/apt/sources.list' + fdin = open (sources_list_in, 'r') + fdout = open (sources_list_out, 'w') + while True: + l = fdin.readline() + if not l: break + fdout.write (l.replace ('OSCODENAME', oscodename)) + fdin.close() + fdout.close() + + subprocess.run (f'chmod -R 775 {builder}/includes/usr/bin/*', shell=True) + + os.makedirs (f'{ogclientmount}/opt/opengnsys/lib/engine/bin/', exist_ok=True) + os.makedirs (f'{ogclientmount}/usr/local/etc', exist_ok=True) + os.makedirs (f'{ogclientmount}/usr/local/lib', exist_ok=True) + os.makedirs (f'{ogclientmount}/usr/local/plugins', exist_ok=True) + + subprocess.run (f'rsync -aH {builder}/includes/* {ogclientmount}/' , shell=True) + subprocess.run (f'rsync -aH {og_shared}/* {ogclientmount}/opt/opengnsys/' , shell=True) + subprocess.run (f'rsync -aH {og_engine}/* {ogclientmount}/opt/opengnsys/lib/engine/bin/', shell=True) + + if not os.path.exists (f'{ogclientmount}/etc/pci.ids'): + shutil.copy (f'{og_shared}/lib/pci.ids', f'{ogclientmount}/etc/') + + # Dependencias Qt para el Browser. + subprocess.run (f'rsync -aH {og_shared}/etc/*.qmap {ogclientmount}/usr/local/etc', shell=True) + subprocess.run (f'rsync -aH {og_shared}/lib/qtlib/* {ogclientmount}/usr/local/lib', shell=True) + subprocess.run (f'rsync -aH {og_shared}/lib/fonts {ogclientmount}/usr/local/lib', shell=True) + subprocess.run (f'rsync -aH {og_shared}/lib/qtplugins/* {ogclientmount}/usr/local/plugins', shell=True) + + # Browser y ogAdmClient. + if os.path.exists (f'{og_shared}/bin/browser'): shutil.copy (f'{og_shared}/bin/browser', f'{ogclientmount}/bin/') + if os.path.exists (f'{og_shared}/bin/ogAdmClient'): shutil.copy (f'{og_shared}/bin/ogAdmClient', f'{ogclientmount}/bin/') + def boottoolsSshServer (btrootfsmnt): if not os.path.exists ('/root/.ssh/id_rsa'): ## crea un par de claves en la VM, no en el chroot _run (['ssh-keygen', '-q', '-f', '/root/.ssh/id_rsa', '-N', '']) diff --git a/boottoolsgenerator.cfg b/boottoolsgenerator.cfg index c6eeda8..018996a 100644 --- a/boottoolsgenerator.cfg +++ b/boottoolsgenerator.cfg @@ -1,67 +1,3 @@ -[Packages] - -basic = - dialog man-db fbset gdebi-core bash-static busybox-static locales lshw gawk git python3-openssl python3 php-cli dmidecode rpm sqlite3 os-prober moreutils jq - #console-data # ogLive anterior a Ubuntu 20.04 - #subversion # ogLive anterior a Ubuntu 18.04 - #php5-cli # ogLive anterior a Ubuntu 16.04 - #realpath # ogLive hasta Ubuntu 16.04 - #zypper # PROBAR - -cloning = fsarchiver pv - #pxe-kexec # ubuntu noble: Unable to locate package - mbuffer parted gdisk chntpw clonezilla registry-tools reglookup libparse-win32registry-perl libwin-hivex-perl grokevt libhivex0 libhivex-bin rsync - -compile = - build-essential libattr1 libattr1-dev uuid-dev attr make m4 gettext libmhash-dev - #libattr* # Satura el tamaño del archivo en Ubuntu 15.04 - -compressor = lzma zip unzip gzip lzop pigz pbzip2 lbzip2 rzip p7zip-full - -filesystem_local = - squashfs-tools unionfs-fuse ntfs-3g dosfstools fatresize - #exfat-utils # ubuntu noble: has no installation candidate - dmraid dmsetup lvm2 e2fsprogs - jfsutils reiserfsprogs reiser4progs xfsprogs mhddfs hfsplus hfsprogs hfsutils nilfs-tools vmfs-tools - btrfs-progs # ogLive a partir de Ubuntu 20.04 - zfsutils-linux # ogLive a partir de Ubuntu 16.04 - android-tools-adb # ogLive a partir de Ubuntu 16.04 - android-tools-fastboot # ogLive a partir de Ubuntu 16.04 - f2fs-tools - #drbl-ntfsprogs # Eliminado - #ntfs-config # ogLive anterior a Ubuntu 20.04 - #btrfs-tools # ogLive anterior a Ubuntu 20.04 - #ufsutils # ogLive anterior a Ubuntu 14.04 - #zfsutils # ogLive anterior a Ubuntu 16.04 - #android-tools-fsutils # ogLive entre Ubuntu 16.04 y Ubuntu 20.04 - -filesystem_remote = - sshfs cifs-utils smbclient open-iscsi openssh-server - #nfs-common # Provoca error de instalación en Ubuntu 15.04 - #bittornado # ogLive anterior a Ubuntu 20.04 - -monitoring = htop ncdu bwbar bmon iftop ifstat dstat hdparm sdparm blktool testdisk - -networking = netpipes curl wget tftp-hpa dnsutils - #trickle # ubuntu noble: has no installation candidate - lighttpd ethtool ssmping tcpdump nmap arping ntpdate ctorrent udpcast #iptraf - -testing = - screen schroot xmlstarlet - efibootmgr efitools refind grub-efi-amd64-bin grub-efi-ia32-bin grub-pc-bin ## UEFI - #libxss1 python-prctl python-six python-requests ## OGAgent - ############## - fusioninventory-agent nvme-cli disktype laptop-detect - #discover casper lupin-casper xinit obconf xserver-xorg x11-xserver-utils xterm network-manager-gnome plymouth-x11 plymouth-label plymouth-theme-ubuntu-logo pcmanfm chromium-browser gtk-theme-switch gtk2-engines murrine-themes - ####################################3 - #gnome-icon-theme gnome-brave-icon-theme dmz-cursor-theme python-wnck python-xlib python-pyinotify python-alsaaudio python-vte maximus gpicview leafpad lxappearance lxmenu-data lxrandr lxterminal nitrogen ttf-ubuntu-font-family time synaptic libglib-perl libgtk2-perl libxml-simple-perl smartmontools gnome-disk-utility policykit-1-gnome policykit-desktop-privileges baobab lshw-gtk usb-creator-gtk wodim curlftpfs libnotify-bin cryptsetup system-config-lvm - -xwindows = - v86d # VESA Kernel 3.7+ - #xorg-dev xorg lxde #+300M - #roxterm gparted #+80M - #openbox midori #xvesa en compilacion - [General] logging_level = INFO @@ -130,3 +66,67 @@ isolinux_template = MENU LABEL pxe KERNEL /clonezilla/live/vmlinuz1 APPEND initrd=/clonezilla/live/initrd1.img boot=live union=aufs noswap vga=788 ip=frommedia + +[Packages] + +basic = + dialog man-db fbset gdebi-core bash-static busybox-static locales lshw gawk git python3-openssl python3 php-cli dmidecode rpm sqlite3 os-prober moreutils jq + #console-data # ogLive anterior a Ubuntu 20.04 + #subversion # ogLive anterior a Ubuntu 18.04 + #php5-cli # ogLive anterior a Ubuntu 16.04 + #realpath # ogLive hasta Ubuntu 16.04 + #zypper # PROBAR + +cloning = fsarchiver pv + #pxe-kexec # ubuntu noble: Unable to locate package + mbuffer parted gdisk chntpw clonezilla registry-tools reglookup libparse-win32registry-perl libwin-hivex-perl grokevt libhivex0 libhivex-bin rsync + +compile = + build-essential libattr1 libattr1-dev uuid-dev attr make m4 gettext libmhash-dev + #libattr* # Satura el tamaño del archivo en Ubuntu 15.04 + +compressor = lzma zip unzip gzip lzop pigz pbzip2 lbzip2 rzip p7zip-full + +filesystem_local = + squashfs-tools unionfs-fuse ntfs-3g dosfstools fatresize + #exfat-utils # ubuntu noble: has no installation candidate + dmraid dmsetup lvm2 e2fsprogs + jfsutils reiserfsprogs reiser4progs xfsprogs mhddfs hfsplus hfsprogs hfsutils nilfs-tools vmfs-tools + btrfs-progs # ogLive a partir de Ubuntu 20.04 + zfsutils-linux # ogLive a partir de Ubuntu 16.04 + android-tools-adb # ogLive a partir de Ubuntu 16.04 + android-tools-fastboot # ogLive a partir de Ubuntu 16.04 + f2fs-tools + #drbl-ntfsprogs # Eliminado + #ntfs-config # ogLive anterior a Ubuntu 20.04 + #btrfs-tools # ogLive anterior a Ubuntu 20.04 + #ufsutils # ogLive anterior a Ubuntu 14.04 + #zfsutils # ogLive anterior a Ubuntu 16.04 + #android-tools-fsutils # ogLive entre Ubuntu 16.04 y Ubuntu 20.04 + +filesystem_remote = + sshfs cifs-utils smbclient open-iscsi openssh-server + #nfs-common # Provoca error de instalación en Ubuntu 15.04 + #bittornado # ogLive anterior a Ubuntu 20.04 + +monitoring = htop ncdu bwbar bmon iftop ifstat dstat hdparm sdparm blktool testdisk + +networking = netpipes curl wget tftp-hpa dnsutils + #trickle # ubuntu noble: has no installation candidate + lighttpd ethtool ssmping tcpdump nmap arping ntpdate ctorrent udpcast #iptraf + +testing = + screen schroot xmlstarlet + efibootmgr efitools refind grub-efi-amd64-bin grub-efi-ia32-bin grub-pc-bin ## UEFI + #libxss1 python-prctl python-six python-requests ## OGAgent + ############## + fusioninventory-agent nvme-cli disktype laptop-detect + #discover casper lupin-casper xinit obconf xserver-xorg x11-xserver-utils xterm network-manager-gnome plymouth-x11 plymouth-label plymouth-theme-ubuntu-logo pcmanfm chromium-browser gtk-theme-switch gtk2-engines murrine-themes + ####################################3 + #gnome-icon-theme gnome-brave-icon-theme dmz-cursor-theme python-wnck python-xlib python-pyinotify python-alsaaudio python-vte maximus gpicview leafpad lxappearance lxmenu-data lxrandr lxterminal nitrogen ttf-ubuntu-font-family time synaptic libglib-perl libgtk2-perl libxml-simple-perl smartmontools gnome-disk-utility policykit-1-gnome policykit-desktop-privileges baobab lshw-gtk usb-creator-gtk wodim curlftpfs libnotify-bin cryptsetup system-config-lvm + +xwindows = + v86d # VESA Kernel 3.7+ + #xorg-dev xorg lxde #+300M + #roxterm gparted #+80M + #openbox midori #xvesa en compilacion diff --git a/boottoolsgenerator.py b/boottoolsgenerator.py index fc2faf6..2dc47ca 100755 --- a/boottoolsgenerator.py +++ b/boottoolsgenerator.py @@ -10,7 +10,7 @@ import shutil curdir = os.path.dirname (__file__) sys.path.insert (0, curdir) -from boottoolsfunctions import _run, _mount, _umount, _read_config, btogGetOsInfo1, btogGetOsInfo2, btogGetVar, btogSetFsVirtual, btogSetFsAccess, btogSetFsBase, boottoolsSshServer, boottoolsSshClient, btogFsInitrd, btogFsSqfs, btogIsoGenerator +from boottoolsfunctions import _run, _mount, _umount, _read_config, btogGetOsInfo1, btogGetOsInfo2, btogGetVar, btogSetFsVirtual, btogSetFsAccess, btogSetFsBase, boottoolsFsOpengnsys, boottoolsSshServer, boottoolsSshClient, btogFsInitrd, btogFsSqfs, btogIsoGenerator def _logging(): #logging.root.handlers = [] @@ -36,7 +36,10 @@ def clone_client_dirs (ogrepo_url, ogrepo_branch, ogrepo_dir): def _mount_rootfs(): global btrootfsimg, btrootfsmnt - _mount (btrootfsimg, btrootfsmnt, opts=['-o', 'loop,offset=32256']) + try: _mount (btrootfsimg, btrootfsmnt, opts=['-o', 'loop,offset=32256']) + except: + logger.error ('mount failed') + sys.exit (1) def _get_pxepkg(): #grep "http://free.nchc.org.tw/drbl-core" /etc/apt/sources.list || echo "deb http://free.nchc.org.tw/drbl-core drbl stable" >> /etc/apt/sources.list @@ -77,10 +80,7 @@ def _debootstrap(): cp = subprocess.run (['schroot', '--chroot', 'IMGogclient', '--', 'stat', '/etc']) if (cp.returncode): logger.debug (f'schroot returned code "{cp.returncode}", calling btogSetFsBase()') - try: _mount_rootfs() - except: - logger.error ('mount failed') - sys.exit (1) + _mount_rootfs() try: btogSetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp) except Exception as e: logger.error (str (e)) @@ -92,83 +92,61 @@ def _initramfs_version (gitrelease, osrelease, btdir): #sed -i "1 s/$/ $GITRELEASE ($OSRELEASE)/" ${BTDIR}/includes/etc/initramfs-tools/scripts/VERSION.txt _run (['sed', '-i', f'1 s/$/ {gitrelease} ({osrelease})/', f'{btdir}/includes/etc/initramfs-tools/scripts/VERSION.txt']) -def _cerodos(): - try: _mount_rootfs() - except: - logger.error ('mount failed') - sys.exit (1) - - logger.debug (f'running \'{curdir}/02-boottoolsFsOpengnsys.py --mntpt "{btrootfsmnt}" --osdistrib "{osdistrib}" --oscodename "{oscodename}" --osrelease "{osrelease}" --osarch "{osarch}" --oshttp "{oshttp}"\'') - stdout, _ = _run ([f'{curdir}/02-boottoolsFsOpengnsys.py', '--mntpt', btrootfsmnt, '--osdistrib', osdistrib, '--oscodename', oscodename, '--osrelease', osrelease, '--osarch', osarch, '--oshttp', oshttp]) - logger.debug (f'02-boottoolsFsOpengnsys stdout follows:') - for i in stdout.strip().split('\n'): logger.debug (' ' + i) - +def _copy_files (btrootfsmnt, osdistrib, oscodename): + _mount_rootfs() + builder = '/tmp/opengnsys/oglive_builder' + og_shared = '/tmp/opengnsys/shared' + og_engine = '/tmp/opengnsys/engine' + boottoolsFsOpengnsys (builder, og_shared, og_engine, btrootfsmnt, osdistrib, oscodename) _umount (btrootfsmnt) -def _chroot_cerotres(): +def _install_compile_software (curdir, osrelease, osarch): logger.info ('Fase 5.1 instalar paquetes deb con apt-get') logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/03-boottoolsSoftwareInstall.py --osrelease "{osrelease}" --osarch "{osarch}"\'') stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/03-boottoolsSoftwareInstall.py', '--osrelease', osrelease, '--osarch', osarch]) logger.debug (f'03-boottoolsSoftwareInstall stdout follows:') for i in stdout.strip().split('\n'): logger.debug (' ' + i) -def _chroot_cerocuatro(): logger.info ('Fase 5.2 compilar software.') - logger.debug ('running \'schroot --chroot IMGogclient -- {}/04-boottoolsSoftwareCompile.py\'') - #cd / + logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/04-boottoolsSoftwareCompile.py\'') stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/04-boottoolsSoftwareCompile.py']) - #cd - logger.debug (f'04-boottoolsSoftwareCompile stdout follows:') for i in stdout.strip().split('\n'): logger.debug (' ' + i) -def _ssh_stuff(): - logger.info ('Fase 6.1 Configurar ssh') - try: _mount_rootfs() - except: - logger.error ('mount failed') - sys.exit (1) - - #cd / - #schroot --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSshServer.sh ## no necesita chroot - boottoolsSshServer (btrootfsmnt) - #cd - - - #schroot --chroot IMGogclient -- /usr/bin/boot-tools/boottoolsSshClient.sh ## no necesita chroot - boottoolsSshClient (btrootfsmnt) - - _umount (btrootfsmnt) - - ## el resultado es: - ## - hay un nuevo par de claves en la VM /root/.ssh - ## - hay otro nuevo par de claves en el rootfs /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh - ## - las dos claves públicas (una de cada par) están autorizadan en el rootfs /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh/authorized_keys - -def _chroot_cerocinco(): - logger.info ('Fase 6.2 Configurar las locales') +def _debconf2 (curdir): + logger.info ('Fase 6.1 Configurar las locales') logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/05-boottoolsFsLocales.py\'') stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/05-boottoolsFsLocales.py']) logger.debug (f'05-boottoolsFsLocales stdout follows:') for i in stdout.strip().split('\n'): logger.debug (' ' + i) -def _chroot_ceroseis(): - logger.info ('Fase 6.3 Crear initrd') ## what?? - #cd / - #schroot -c IMGogclient -- /usr/bin/boot-tools/boottoolsInitrdGenerate.sh + logger.info ('Fase 6.3 Crear initrd') logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/06-boottoolsInitrdGenerate.py --osrelease "{osrelease}"\'') stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/06-boottoolsInitrdGenerate.py', '--osrelease', osrelease]) logger.debug (f'06-boottoolsInitrdGenerate stdout follows:') for i in stdout.strip().split('\n'): logger.debug (' ' + i) ## esto deja initrd.img-6.8.0-31-generic y vmlinuz-6.8.0-31-generic en /tmp -def _mkinitrd (bttargetdir, osrelease): - logger.info ('Fase 7.1 Generar el initrd') ## what?? +def _ssh_stuff(): + logger.info ('Fase 6.2 Configurar ssh') + _mount_rootfs() + boottoolsSshServer (btrootfsmnt) + boottoolsSshClient (btrootfsmnt) + _umount (btrootfsmnt) + ## el resultado es: + ## - hay un nuevo par de claves en la VM /root/.ssh + ## - hay otro nuevo par de claves en el rootfs /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh + ## - las dos claves públicas (una de cada par) están autorizadan en el rootfs /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh/authorized_keys + +def _mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsmnt, pxepkg, isolinux_tpl nameisoclient): + logger.info ('Fase 7.1 Copiar el initrd a su sitio') + _mount_rootfs() btogFsInitrd (bttargetdir, osrelease) -def _mksquashfs (bttargetdir, btrootfsmnt): logger.info ('Fase 7.2 Generar fichero sqfs a partir del fichero img') btogFsSqfs (bttargetdir, btrootfsmnt) + _umount (btrootfsmnt) -def _mkisofs (pxepkg, isolinux_tpl, bttargetdir, nameisoclient): logger.info ('Fase 7.3 Generar la ISO') btogIsoGenerator (pxepkg, isolinux_tpl, bttargetdir, nameisoclient) @@ -200,6 +178,8 @@ osdistrib, oscodename, osrelease, osarch, oshttp = btogGetOsInfo1(type_client) btdir, bttargetdir, btrootfsimg, btrootfsmnt, btrootfsimglabel, log_file, versionboottools, btvirtualdisksize = btogGetVar(osarch) gitrelease, nameisoclient, namehostclient = btogGetOsInfo2(ogrepo_dir, versionboottools, oscodename, osrelease, osarch) +logger.info (':'.join ([osdistrib, oscodename, osrelease, osarch, oshttp])) + ## this is convenient in case the previous run failed and we want to run this program again try: _umount (btrootfsmnt) except: pass @@ -215,25 +195,16 @@ _debootstrap() logger.info ('FASE 4 - Incorporando ficheros OpenGnsys al sistema raíz rootfs') _initramfs_version (gitrelease, osrelease, btdir) -_cerodos() +_copy_files (btrootfsmnt, osdistrib, oscodename, osrelease, osarch, oshttp) logger.info ('FASE 5 - Instalar software') -_chroot_cerotres() -_chroot_cerocuatro() +_install_compile_software (curdir, osrelease, osarch) logger.info ('FASE 6 - Personalizar el sistema creado') +_debconf2 (curdir) _ssh_stuff() -_chroot_cerocinco() -_chroot_ceroseis() logger.info ('FASE 7 - Generar distribucion') -_mkinitrd (bttargetdir, osrelease) -try: _mount_rootfs() -except: - logger.error ('mount failed') - sys.exit (1) -_mksquashfs (bttargetdir, btrootfsmnt) -_umount (btrootfsmnt) -_mkisofs (pxepkg, isolinux_tpl, bttargetdir, nameisoclient) +_mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsmnt, pxepkg, isolinux_tpl nameisoclient) logger.info ('OpenGnsys installation finished') From 95a24ac6a6cb964a3ea98153f8f1edbc80cd74b6 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Wed, 14 Aug 2024 15:57:54 +0200 Subject: [PATCH 26/45] refs #596 separate some functions to a new "utils" module --- 03-boottoolsSoftwareInstall.py | 26 ++++----- 04-boottoolsSoftwareCompile.py | 28 ++++----- 05-boottoolsFsLocales.py | 6 +- 06-boottoolsInitrdGenerate.py | 4 +- boottoolsfunctions/__init__.py | 102 +++++++++------------------------ boottoolsgenerator.py | 42 +++++++------- 6 files changed, 80 insertions(+), 128 deletions(-) diff --git a/03-boottoolsSoftwareInstall.py b/03-boottoolsSoftwareInstall.py index e45bdf3..6d0cefe 100755 --- a/03-boottoolsSoftwareInstall.py +++ b/03-boottoolsSoftwareInstall.py @@ -6,30 +6,30 @@ import sys import re import subprocess -from boottoolsfunctions import _run, _read_config +from boottoolsfunctions import utils -config = _read_config ('boottoolsgenerator.cfg') +config = utils.read_config ('boottoolsgenerator.cfg') if config is None: sys.exit (1) debconf_settings = config['General'].get ('debconf_settings') def _aptget_clean(): - _run (['apt-get', '--yes', 'clean']) + utils.run (['apt-get', '--yes', 'clean']) def _aptget_autoclean(): - _run (['apt-get', '--yes', 'autoclean']) + utils.run (['apt-get', '--yes', 'autoclean']) def _aptget_autoremove(): - _run (['apt-get', '--yes', 'autoremove']) + utils.run (['apt-get', '--yes', 'autoremove']) def _aptget_update(): - _run (['apt-get', '--yes', 'update']) + utils.run (['apt-get', '--yes', 'update']) def _aptget_upgrade(): - _run (['apt-get', '--yes', 'upgrade']) + utils.run (['apt-get', '--yes', 'upgrade']) def _aptget_install (pkgs, opts=[]): - _run (['apt-get', '--yes'] + opts + ['install'] + pkgs) + utils.run (['apt-get', '--yes'] + opts + ['install'] + pkgs) def _oghook_deactivate(): #Desactivamos el hook del oginitrd.img para evitar problemas, al final de este escripts se activará @@ -55,15 +55,15 @@ def boottoolsSoftwareInstall (osarch, osrelease): os.environ['LANG'] = 'C' os.environ['DEBIAN_FRONTEND'] = 'noninteractive' - stdout, _ = _run (['dpkg-divert', '--list']) + stdout, _ = utils.run (['dpkg-divert', '--list']) if not re.findall (r'local diversion of /sbin/initctl to /sbin/initctl.distrib', stdout): - _run (['dpkg-divert', '--local', '--rename', '--add', '/sbin/initctl']) + utils.run (['dpkg-divert', '--local', '--rename', '--add', '/sbin/initctl']) os.symlink ('/bin/true', '/sbin/initctl') _aptget_clean() pkgs32 = [] if 'i386' != osarch: - _run (['dpkg', '--add-architecture', 'i386']) + utils.run (['dpkg', '--add-architecture', 'i386']) pkgs32 = 'lib32gcc-s1 lib32stdc++6 lib32z1 libc6-i386'.split (' ') ## he cambiado lib32gcc1 por lib32gcc-s1 pero como queramos crear un oglive viejo, esto va a petar _aptget_update() ## esto ya esta hecho... @@ -85,14 +85,14 @@ def boottoolsSoftwareInstall (osarch, osrelease): _aptget_install (pkgs) # Instalar módulos que algunos paquetes puedan tener pendientes de compilar. - stdout, _ = _run (['dkms', 'status']) + stdout, _ = utils.run (['dkms', 'status']) for l in stdout.strip().split ('\n'): if not l: continue print (f'l "{l}"') mod, vers, status = l.split (',') if 'added' in status: print (f'dkms installing {mod} {vers}') - _run (['dkms', 'install', '-m', mod.strip(), '-v', vers.strip()]) + utils.run (['dkms', 'install', '-m', mod.strip(), '-v', vers.strip()]) _oghook_activate() #_restore_mtab() diff --git a/04-boottoolsSoftwareCompile.py b/04-boottoolsSoftwareCompile.py index 912008c..484cde5 100755 --- a/04-boottoolsSoftwareCompile.py +++ b/04-boottoolsSoftwareCompile.py @@ -2,35 +2,35 @@ import os import subprocess -from boottoolsfunctions import _run +from boottoolsfunctions import utils os.environ['LANGUAGE'] = os.environ['LC_ALL'] = os.environ['LANG'] = 'C' os.chdir ('/tmp') print ('ms-sys') -try: _run (['which', 'ms-sys']) +try: utils.run (['which', 'ms-sys']) except: - _run (['wget', 'https://sourceforge.net/projects/ms-sys/files/latest/download', '-O', 'ms-sys.tar.gz']) - _run (['tar', '-xpzf', 'ms-sys.tar.gz']) + utils.run (['wget', 'https://sourceforge.net/projects/ms-sys/files/latest/download', '-O', 'ms-sys.tar.gz']) + utils.run (['tar', '-xpzf', 'ms-sys.tar.gz']) mssys_dir = subprocess.run (['tar tzf ms-sys.tar.gz |head -n 1'], shell=True, capture_output=True, text=True).stdout.strip() print (f'mssys_dir "{mssys_dir}"') os.chdir (mssys_dir) - _run (['make', 'install']) + utils.run (['make', 'install']) os.chdir ('..') print ('spartlnx') -try: _run (['which', 'spartl64.run']) +try: utils.run (['which', 'spartl64.run']) except: - _run (['wget', 'http://damien.guibouret.free.fr/savepart.zip']) - _run (['unzip', '-o', 'savepart.zip', '-d', '/sbin/', 'spartl64.run']) - _run (['mkdir', '/usr/share/doc/spartlnx']) - _run (['unzip', '-j', '-o', 'savepart.zip', '-d', '/usr/share/doc/spartlnx/', 'doc/en/*']) + utils.run (['wget', 'http://damien.guibouret.free.fr/savepart.zip']) + utils.run (['unzip', '-o', 'savepart.zip', '-d', '/sbin/', 'spartl64.run']) + utils.run (['mkdir', '/usr/share/doc/spartlnx']) + utils.run (['unzip', '-j', '-o', 'savepart.zip', '-d', '/usr/share/doc/spartlnx/', 'doc/en/*']) if not os.path.exists ('python-libfdisk'): print ('python-libfdisk') - _run (['apt-get', '-y', 'install', 'python3-psutil', 'python3-dev', 'libfdisk-dev', 'python3-setuptools']) - _run (['git', 'clone', 'git://git.48k.eu/python-libfdisk']) + utils.run (['apt-get', '-y', 'install', 'python3-psutil', 'python3-dev', 'libfdisk-dev', 'python3-setuptools']) + utils.run (['git', 'clone', 'git://git.48k.eu/python-libfdisk']) os.chdir ('python-libfdisk') - _run (['python3', 'setup.py', 'install']) + utils.run (['python3', 'setup.py', 'install']) os.chdir ('..') - _run (['apt-get', '-y', 'remove', 'python3-dev', 'python3-setuptools']) + utils.run (['apt-get', '-y', 'remove', 'python3-dev', 'python3-setuptools']) diff --git a/05-boottoolsFsLocales.py b/05-boottoolsFsLocales.py index 2d04c60..02684ca 100755 --- a/05-boottoolsFsLocales.py +++ b/05-boottoolsFsLocales.py @@ -2,9 +2,9 @@ import os import subprocess -from boottoolsfunctions import _run, _read_config +from boottoolsfunctions import utils -config = _read_config ('boottoolsgenerator.cfg') +config = utils.read_config ('boottoolsgenerator.cfg') if config is None: sys.exit (1) debconf_settings2 = config['General'].get ('debconf_settings2') @@ -12,4 +12,4 @@ debconf_settings2 = config['General'].get ('debconf_settings2') subprocess.run (['debconf-set-selections'], input=debconf_settings2, text=True) ## despues de esto, debconf-get-selections devuelve los valores antiguos, no se por que... -_run (['dpkg-reconfigure', '--frontend', 'noninteractive', 'console-setup', 'locales']) +utils.run (['dpkg-reconfigure', '--frontend', 'noninteractive', 'console-setup', 'locales']) diff --git a/06-boottoolsInitrdGenerate.py b/06-boottoolsInitrdGenerate.py index 254743e..02c94b1 100755 --- a/06-boottoolsInitrdGenerate.py +++ b/06-boottoolsInitrdGenerate.py @@ -4,7 +4,7 @@ import argparse import shutil import os import glob -from boottoolsfunctions import _run +from boottoolsfunctions import utils def boottoolsInitrdGenerate (osrelease): for f in glob.glob ('/usr/lib/initramfs-tools/bin/*'): @@ -12,7 +12,7 @@ def boottoolsInitrdGenerate (osrelease): shutil.copy ('/bin/busybox', '/usr/lib/initramfs-tools/bin') os.chdir ('/tmp') - _run (['mkinitramfs', '-o', f'/tmp/initrd.img-{osrelease}', '-v', osrelease]) + utils.run (['mkinitramfs', '-o', f'/tmp/initrd.img-{osrelease}', '-v', osrelease]) shutil.copy (f'/boot/vmlinuz-{osrelease}', '/tmp/') if __name__ == '__main__': diff --git a/boottoolsfunctions/__init__.py b/boottoolsfunctions/__init__.py index 592daff..d605174 100644 --- a/boottoolsfunctions/__init__.py +++ b/boottoolsfunctions/__init__.py @@ -1,15 +1,14 @@ import platform import os -import re import time import logging import subprocess import shutil import glob import datetime -import configparser logger = logging.getLogger ('boottools') +from . import utils schroot_conf_tpl = """ [IMGogclient] @@ -39,53 +38,6 @@ proc /proc proc defaults 0 0 /tmp /tmp none rw,bind 0 0 """.strip() -def _run (args): - cp = subprocess.run (args, text=True, capture_output=True) - if cp.returncode: - logger.error ('command "{}" failed with rc "{}"'.format (' '.join(args), cp.returncode)) - - logger.error ('stdout follows:') - for i in cp.stdout.strip().split('\n'): logger.error (' ' + i) - - logger.error ('stderr follows:') - for i in cp.stderr.strip().split('\n'): logger.error (' ' + i) - - raise Exception ('command "{}" failed with rc "{}"'.format (' '.join(args), cp.returncode)) - stdout = cp.stdout.strip() - stderr = cp.stderr.strip() - return stdout, stderr - -def _grep (regex, file): - with open (file, 'r') as f: - for l in f: - if (re.findall (regex, l)): return 1 - return 0 - -def _is_mounted (mntpt): - return _grep (mntpt, '/proc/mounts') - -def _mount (dev, mntpt, opts=[]): - if not _is_mounted (mntpt): - _run (['mount', dev, mntpt] + opts) - -def _umount (mntpt): - if (_is_mounted (mntpt)): - _run (['umount', mntpt]) - -def _read_config (fn): - if not os.path.exists (fn): - print (f'configuration file "{fn}" not found') - return - config = configparser.ConfigParser (comment_prefixes='#', inline_comment_prefixes='#') - config.read (fn) - return config - -def _write_md5 (fn): - md5, _ = _run (['md5sum', fn]) - md5, rest = md5.split (' ', 1) - with open (f'{fn}.sum', 'w') as fd: - fd.write (md5 + '\n') - def btogGetVar (osarch): btdir = '/tmp/opengnsys/oglive_builder' bttargetdir = '/var/lib/tftpboot/ogclient/' @@ -195,7 +147,7 @@ def btogGetOsInfo1 (type_client): osdistrib=platform.freedesktop_os_release()['NAME'] oscodename=platform.freedesktop_os_release()['VERSION_CODENAME'] osrelease=platform.uname()[2] - osarch, _=_run (['dpkg', '--print-architecture']) + osarch, _=utils.run (['dpkg', '--print-architecture']) oshttp='http://es.archive.ubuntu.com/ubuntu/' else: # Parámetro desconocido print ('Parámetro no válido.') @@ -211,66 +163,66 @@ def btogGetOsInfo2 (ogrepo_dir, versionboottools, oscodename, osrelease, osarch) def btogSetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, bttargetdir, osarch): logger.info (f'Creación y formateo del disco virtual "{btrootfsimg}" "{btvirtualdisksize}" MB') - try: _umount (btrootfsmnt) + try: utils.umount (btrootfsmnt) except: pass - if (_is_mounted (btrootfsmnt)): + if (utils.is_mounted (btrootfsmnt)): raise Exception (f'failed to umount "{btrootfsmnt}"') try: os.makedirs (btrootfsmnt, exist_ok=True) except: raise Exception (f'Creando directorio "{btrootfsmnt}" : ERROR') - try: _run (['chown', '-R', 'root:opengnsys', bttargetdir]) + try: utils.run (['chown', '-R', 'root:opengnsys', bttargetdir]) except: raise Exception (f'Failed to chown root:opengnsys "{btrootfsmnt}"') logger.info (f'Creating disk image "{btrootfsimg}"') if 'i386' == osarch: - try: _run (['dd', 'if=/dev/zero', f'of={btrootfsimg}', 'bs=1048576', f'count={btvirtualdisksize}']) + try: utils.run (['dd', 'if=/dev/zero', f'of={btrootfsimg}', 'bs=1048576', f'count={btvirtualdisksize}']) except: raise Exception (f'Creando el disco virtual "{btrootfsimg}" con tamaño maxima "{btvirtualdisksize}" MB : ERROR') else: - try: _run (['qemu-img', 'create', btrootfsimg, btvirtualdisksize]) + try: utils.run (['qemu-img', 'create', btrootfsimg, btvirtualdisksize]) except: raise Exception (f'Creando el disco virtual "{btrootfsimg}" con tamaño maxima "{btvirtualdisksize}" MB : ERROR') logger.debug ('losetup --find') - diskloop, _ = _run (['losetup', '--find']) + diskloop, _ = utils.run (['losetup', '--find']) if not diskloop: raise Exception ('no diskloop') logger.debug ('losetup attach') - try: _run (['losetup', '--partscan', diskloop, btrootfsimg]) + try: utils.run (['losetup', '--partscan', diskloop, btrootfsimg]) except: raise Exception ('losetup failed') logger.info ('Partitioning disk image') - stdout, _ = _run (['bash', '-c', f"echo $'n\np\n1\n\n\nt\n83\nw' |fdisk {diskloop}"]) + stdout, _ = utils.run (['bash', '-c', f"echo $'n\np\n1\n\n\nt\n83\nw' |fdisk {diskloop}"]) logger.debug (f'fdisk stdout follows:') for i in stdout.strip().split('\n'): logger.debug (' ' + i) time.sleep (3) logger.debug ('losetup --detach') - try: _run (['losetup', '--detach', diskloop]) + try: utils.run (['losetup', '--detach', diskloop]) except: raise Exception ('Liberando disco virtual despues del particionado: ERROR') logger.debug ('losetup --find') - partloop, _ = _run (['losetup', '--find']) + partloop, _ = utils.run (['losetup', '--find']) if not partloop: raise Exception ('no partloop') logger.debug ('losetup --offset attach') - _run (['losetup', '--offset', '32256', partloop, btrootfsimg]) + utils.run (['losetup', '--offset', '32256', partloop, btrootfsimg]) logger.info ('Creating ext4 filesystem') - try: _run (['mkfs.ext4', '-b', '4096', '-L', btrootfsimglabel, partloop]) + try: utils.run (['mkfs.ext4', '-b', '4096', '-L', btrootfsimglabel, partloop]) except: raise Exception ('Formateando la particion principal del disco virtual: ERROR') time.sleep (3) logger.debug ('losetup --detach') - try: _run (['losetup', '--detach', partloop]) + try: utils.run (['losetup', '--detach', partloop]) except: raise Exception ('Liberando la particion virtual despues del formateo: ERROR') @@ -278,7 +230,7 @@ def btogSetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisks #btogSetFsAcces: habilita el acceso al sistema root del cliente con schroot def btogSetFsAccess (btrootfsimg): - if (_grep (btrootfsimg, '/etc/schroot/schroot.conf')): + if (utils.grep (btrootfsimg, '/etc/schroot/schroot.conf')): return logger.info ('Iniciando la configuración del schroot') @@ -305,7 +257,7 @@ def btogSetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp): logger.info ('debootstrapping system') logger.debug (f'debootstrap --arch="{osarch}" --components=main,universe "{oscodename}" "{btrootfsmnt}" "{oshttp}"') - try: _run (['debootstrap', f'--arch={osarch}', '--components=main,universe', oscodename, btrootfsmnt, oshttp]) + try: utils.run (['debootstrap', f'--arch={osarch}', '--components=main,universe', oscodename, btrootfsmnt, oshttp]) except: raise Exception (f'debootstrap --arch="{osarch}" --components=main,universe "{oscodename}" "{btrootfsmnt}" "{oshttp}" : ha fallado!') logger.info (f'debootstrap --arch="{osarch}" --components=main,universe "{oscodename}" "{btrootfsmnt}" "{oshttp}" : ok') @@ -351,7 +303,7 @@ def boottoolsFsOpengnsys (builder, og_shared, og_engine, ogclientmount, osdistri def boottoolsSshServer (btrootfsmnt): if not os.path.exists ('/root/.ssh/id_rsa'): ## crea un par de claves en la VM, no en el chroot - _run (['ssh-keygen', '-q', '-f', '/root/.ssh/id_rsa', '-N', '']) + utils.run (['ssh-keygen', '-q', '-f', '/root/.ssh/id_rsa', '-N', '']) logger.debug ('comprobando directorio .ssh del root') if not os.path.exists (f'{btrootfsmnt}/root/.ssh'): ## crea directorio dentro del chroot @@ -381,7 +333,7 @@ def boottoolsSshServer (btrootfsmnt): def boottoolsSshClient (btrootfsmnt): if not os.path.exists (f'{btrootfsmnt}/root/.ssh/id_rsa'): - _run (['ssh-keygen', '-q', '-f', f'{btrootfsmnt}/root/.ssh/id_rsa', '-N', '']) ## crea un par de claves en el chroot + utils.run (['ssh-keygen', '-q', '-f', f'{btrootfsmnt}/root/.ssh/id_rsa', '-N', '']) ## crea un par de claves en el chroot #cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys ## coge la publica y se la autoriza a sí mismo fdin = open (f'{btrootfsmnt}//root/.ssh/id_rsa.pub', 'r') @@ -413,8 +365,8 @@ def btogFsInitrd (bttargetdir, osrelease): shutil.copy (f'/tmp/initrd.img-{osrelease}', f'{bttargetdir}/oginitrd.img') shutil.copy (f'/tmp/vmlinuz-{osrelease}', f'{bttargetdir}/ogvmlinuz') - _write_md5 (f'{bttargetdir}/oginitrd.img') - _write_md5 (f'{bttargetdir}/ogvmlinuz') + utils.write_md5 (f'{bttargetdir}/oginitrd.img') + utils.write_md5 (f'{bttargetdir}/ogvmlinuz') #chmod -R 755 $BTTARGETDIR for f in glob.glob (f'{bttargetdir}/oginitrd*') + glob.glob (f'{bttargetdir}/vmlinuz*'): @@ -429,10 +381,10 @@ def btogFsSqfs (bttargetdir, btrootfsmnt): now = datetime.datetime.now(datetime.timezone.utc).strftime ('%Y%m%d-%H%M%S%z') os.rename (f'{bttargetdir}/ogclient.sqfs', f'{bttargetdir}/ogclient.sqfs.{now}') - _run (['mksquashfs', btrootfsmnt, f'{bttargetdir}/ogclient.sqfs']) + utils.run (['mksquashfs', btrootfsmnt, f'{bttargetdir}/ogclient.sqfs']) os.chmod (f'{bttargetdir}/ogclient.sqfs', 0o744) - _write_md5 (f'{bttargetdir}/ogclient.sqfs') + utils.write_md5 (f'{bttargetdir}/ogclient.sqfs') # btogIsoGenerator genera la iso del cliente def btogIsoGenerator (pxepkg, isolinux_tpl, bttargetdir, nameisoclient): @@ -452,9 +404,9 @@ def btogIsoGenerator (pxepkg, isolinux_tpl, bttargetdir, nameisoclient): # Si existe el fichero ISO, montarlo para extraer isolinux.bin. if os.path.exists (f'/tmp/iso/isolinux/{pxepkg}.iso'): os.mkdir ('/tmp/iso/isolinux/mount') - _run (['mount', '-o', 'loop', f'/tmp/iso/isolinux/{pxepkg}.iso', '/tmp/iso/isolinux/mount']) + utils.run (['mount', '-o', 'loop', f'/tmp/iso/isolinux/{pxepkg}.iso', '/tmp/iso/isolinux/mount']) subprocess.run (['cp -a /tmp/iso/isolinux/mount/* /tmp/iso/isolinux'], shell=True) - _umount ('/tmp/iso/isolinux/mount') + utils.umount ('/tmp/iso/isolinux/mount') os.rmdir ('/tmp/iso/isolinux/mount') os.unlink (f'/tmp/iso/isolinux/{pxepkg}.iso') @@ -478,9 +430,9 @@ def btogIsoGenerator (pxepkg, isolinux_tpl, bttargetdir, nameisoclient): oldpwd = os.getcwd() os.chdir ('/tmp') logger.debug (f'mkisofs -V ogClient -o {nameisoclient}.iso -b isolinux/isolinux.bin -c isolinux/boot.cat -J -no-emul-boot -boot-load-size 4 -boot-info-table /tmp/iso') - _run (['mkisofs', '-V', 'ogClient', '-o', f'{nameisoclient}.iso', '-b', 'isolinux/isolinux.bin', '-c', 'isolinux/boot.cat', '-J', '-no-emul-boot', '-boot-load-size', '4', '-boot-info-table', '/tmp/iso']) + utils.run (['mkisofs', '-V', 'ogClient', '-o', f'{nameisoclient}.iso', '-b', 'isolinux/isolinux.bin', '-c', 'isolinux/boot.cat', '-J', '-no-emul-boot', '-boot-load-size', '4', '-boot-info-table', '/tmp/iso']) - _write_md5 (f'{nameisoclient}.iso') + utils.write_md5 (f'{nameisoclient}.iso') os.chdir (oldpwd) os.rename (f'/tmp/{nameisoclient}.iso' , f'/var/lib/tftpboot/ogclient/{nameisoclient}.iso') diff --git a/boottoolsgenerator.py b/boottoolsgenerator.py index 2dc47ca..dbd6e60 100755 --- a/boottoolsgenerator.py +++ b/boottoolsgenerator.py @@ -10,7 +10,7 @@ import shutil curdir = os.path.dirname (__file__) sys.path.insert (0, curdir) -from boottoolsfunctions import _run, _mount, _umount, _read_config, btogGetOsInfo1, btogGetOsInfo2, btogGetVar, btogSetFsVirtual, btogSetFsAccess, btogSetFsBase, boottoolsFsOpengnsys, boottoolsSshServer, boottoolsSshClient, btogFsInitrd, btogFsSqfs, btogIsoGenerator +from boottoolsfunctions import utils, btogGetOsInfo1, btogGetOsInfo2, btogGetVar, btogSetFsVirtual, btogSetFsAccess, btogSetFsBase, boottoolsFsOpengnsys, boottoolsSshServer, boottoolsSshClient, btogFsInitrd, btogFsSqfs, btogIsoGenerator def _logging(): #logging.root.handlers = [] @@ -31,21 +31,21 @@ def _logging(): def clone_client_dirs (ogrepo_url, ogrepo_branch, ogrepo_dir): if not os.path.exists ('/tmp/opengnsys/engine'): - _run (['git', 'clone', '-c', 'http.sslVerify=false', '--branch', ogrepo_branch, ogrepo_url, ogrepo_dir]) - _run (['rsync', '-aH', f'{ogrepo_dir}/client/engine', f'{ogrepo_dir}/client/shared', '/tmp/opengnsys/']) + utils.run (['git', 'clone', '-c', 'http.sslVerify=false', '--branch', ogrepo_branch, ogrepo_url, ogrepo_dir]) + utils.run (['rsync', '-aH', f'{ogrepo_dir}/client/engine', f'{ogrepo_dir}/client/shared', '/tmp/opengnsys/']) def _mount_rootfs(): global btrootfsimg, btrootfsmnt - try: _mount (btrootfsimg, btrootfsmnt, opts=['-o', 'loop,offset=32256']) + try: utils.mount (btrootfsimg, btrootfsmnt, opts=['-o', 'loop,offset=32256']) except: logger.error ('mount failed') sys.exit (1) def _get_pxepkg(): #grep "http://free.nchc.org.tw/drbl-core" /etc/apt/sources.list || echo "deb http://free.nchc.org.tw/drbl-core drbl stable" >> /etc/apt/sources.list - _run (['apt-get', 'update']) - acse_gpxe, _ = _run (['apt-cache', 'search', 'gpxe']) - acse_ipxe, _ = _run (['apt-cache', 'search', 'ipxe']) + utils.run (['apt-get', 'update']) + acse_gpxe, _ = utils.run (['apt-cache', 'search', 'gpxe']) + acse_ipxe, _ = utils.run (['apt-cache', 'search', 'ipxe']) if acse_ipxe: pxepkg = 'ipxe' elif acse_gpxe: @@ -90,7 +90,7 @@ def _debootstrap(): def _initramfs_version (gitrelease, osrelease, btdir): ## FIXME esto la incluye incondicionalmente, y luego terminamos con "OpenGnsys Client 1.2.0-rc1 gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) ..." #sed -i "1 s/$/ $GITRELEASE ($OSRELEASE)/" ${BTDIR}/includes/etc/initramfs-tools/scripts/VERSION.txt - _run (['sed', '-i', f'1 s/$/ {gitrelease} ({osrelease})/', f'{btdir}/includes/etc/initramfs-tools/scripts/VERSION.txt']) + utils.run (['sed', '-i', f'1 s/$/ {gitrelease} ({osrelease})/', f'{btdir}/includes/etc/initramfs-tools/scripts/VERSION.txt']) def _copy_files (btrootfsmnt, osdistrib, oscodename): _mount_rootfs() @@ -98,31 +98,31 @@ def _copy_files (btrootfsmnt, osdistrib, oscodename): og_shared = '/tmp/opengnsys/shared' og_engine = '/tmp/opengnsys/engine' boottoolsFsOpengnsys (builder, og_shared, og_engine, btrootfsmnt, osdistrib, oscodename) - _umount (btrootfsmnt) + utils.umount (btrootfsmnt) def _install_compile_software (curdir, osrelease, osarch): logger.info ('Fase 5.1 instalar paquetes deb con apt-get') logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/03-boottoolsSoftwareInstall.py --osrelease "{osrelease}" --osarch "{osarch}"\'') - stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/03-boottoolsSoftwareInstall.py', '--osrelease', osrelease, '--osarch', osarch]) + stdout, _ = utils.run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/03-boottoolsSoftwareInstall.py', '--osrelease', osrelease, '--osarch', osarch]) logger.debug (f'03-boottoolsSoftwareInstall stdout follows:') for i in stdout.strip().split('\n'): logger.debug (' ' + i) logger.info ('Fase 5.2 compilar software.') logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/04-boottoolsSoftwareCompile.py\'') - stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/04-boottoolsSoftwareCompile.py']) + stdout, _ = utils.run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/04-boottoolsSoftwareCompile.py']) logger.debug (f'04-boottoolsSoftwareCompile stdout follows:') for i in stdout.strip().split('\n'): logger.debug (' ' + i) def _debconf2 (curdir): logger.info ('Fase 6.1 Configurar las locales') logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/05-boottoolsFsLocales.py\'') - stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/05-boottoolsFsLocales.py']) + stdout, _ = utils.run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/05-boottoolsFsLocales.py']) logger.debug (f'05-boottoolsFsLocales stdout follows:') for i in stdout.strip().split('\n'): logger.debug (' ' + i) logger.info ('Fase 6.3 Crear initrd') logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/06-boottoolsInitrdGenerate.py --osrelease "{osrelease}"\'') - stdout, _ = _run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/06-boottoolsInitrdGenerate.py', '--osrelease', osrelease]) + stdout, _ = utils.run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/06-boottoolsInitrdGenerate.py', '--osrelease', osrelease]) logger.debug (f'06-boottoolsInitrdGenerate stdout follows:') for i in stdout.strip().split('\n'): logger.debug (' ' + i) ## esto deja initrd.img-6.8.0-31-generic y vmlinuz-6.8.0-31-generic en /tmp @@ -132,27 +132,27 @@ def _ssh_stuff(): _mount_rootfs() boottoolsSshServer (btrootfsmnt) boottoolsSshClient (btrootfsmnt) - _umount (btrootfsmnt) + utils.umount (btrootfsmnt) ## el resultado es: ## - hay un nuevo par de claves en la VM /root/.ssh ## - hay otro nuevo par de claves en el rootfs /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh ## - las dos claves públicas (una de cada par) están autorizadan en el rootfs /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh/authorized_keys -def _mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsmnt, pxepkg, isolinux_tpl nameisoclient): +def _mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsmnt, pxepkg, isolinux_tpl, nameisoclient): logger.info ('Fase 7.1 Copiar el initrd a su sitio') _mount_rootfs() btogFsInitrd (bttargetdir, osrelease) logger.info ('Fase 7.2 Generar fichero sqfs a partir del fichero img') btogFsSqfs (bttargetdir, btrootfsmnt) - _umount (btrootfsmnt) + utils.umount (btrootfsmnt) logger.info ('Fase 7.3 Generar la ISO') btogIsoGenerator (pxepkg, isolinux_tpl, bttargetdir, nameisoclient) -config = _read_config ('boottoolsgenerator.cfg') +config = utils.read_config ('boottoolsgenerator.cfg') if config is None: sys.exit (1) isolinux_tpl = config['General'].get ('isolinux_template') @@ -181,12 +181,12 @@ gitrelease, nameisoclient, namehostclient = btogGetOsInfo2(ogrepo_dir, versionbo logger.info (':'.join ([osdistrib, oscodename, osrelease, osarch, oshttp])) ## this is convenient in case the previous run failed and we want to run this program again -try: _umount (btrootfsmnt) +try: utils.umount (btrootfsmnt) except: pass logger.info ('FASE 2 - Instalación de software adicional.') pxepkg = _get_pxepkg() -_run (['apt-get', '-y', 'install', 'jq', 'syslinux', 'syslinux-efi', 'syslinux-utils', 'debootstrap', 'subversion', 'schroot', 'squashfs-tools', 'syslinux', 'genisoimage', 'qemu-utils', 'lsof', pxepkg]) ## TODO qemu no existe, hace falta? +utils.run (['apt-get', '-y', 'install', 'jq', 'syslinux', 'syslinux-efi', 'syslinux-utils', 'debootstrap', 'subversion', 'schroot', 'squashfs-tools', 'syslinux', 'genisoimage', 'qemu-utils', 'lsof', pxepkg]) ## TODO qemu no existe, hace falta? logger.info ('FASE 3 - Creación del Sistema raiz RootFS (Segundo Sistema archivos (img))') _mkrootfs() @@ -195,7 +195,7 @@ _debootstrap() logger.info ('FASE 4 - Incorporando ficheros OpenGnsys al sistema raíz rootfs') _initramfs_version (gitrelease, osrelease, btdir) -_copy_files (btrootfsmnt, osdistrib, oscodename, osrelease, osarch, oshttp) +_copy_files (btrootfsmnt, osdistrib, oscodename) logger.info ('FASE 5 - Instalar software') _install_compile_software (curdir, osrelease, osarch) @@ -205,6 +205,6 @@ _debconf2 (curdir) _ssh_stuff() logger.info ('FASE 7 - Generar distribucion') -_mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsmnt, pxepkg, isolinux_tpl nameisoclient) +_mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsmnt, pxepkg, isolinux_tpl, nameisoclient) logger.info ('OpenGnsys installation finished') From 1bd24c9733184af15737cb46d1c009c398aaa7ab Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Thu, 15 Aug 2024 17:38:00 +0200 Subject: [PATCH 27/45] refs #596 separate more functions to their own module --- boottoolsfunctions/{__init__.py => btog.py} | 37 +++++++------- boottoolsfunctions/utils.py | 54 +++++++++++++++++++++ boottoolsgenerator.py | 28 +++++------ 3 files changed, 87 insertions(+), 32 deletions(-) rename boottoolsfunctions/{__init__.py => btog.py} (95%) create mode 100644 boottoolsfunctions/utils.py diff --git a/boottoolsfunctions/__init__.py b/boottoolsfunctions/btog.py similarity index 95% rename from boottoolsfunctions/__init__.py rename to boottoolsfunctions/btog.py index d605174..3f14d92 100644 --- a/boottoolsfunctions/__init__.py +++ b/boottoolsfunctions/btog.py @@ -7,9 +7,10 @@ import shutil import glob import datetime -logger = logging.getLogger ('boottools') from . import utils +logger = logging.getLogger ('boottools') + schroot_conf_tpl = """ [IMGogclient] type=loopback @@ -38,7 +39,7 @@ proc /proc proc defaults 0 0 /tmp /tmp none rw,bind 0 0 """.strip() -def btogGetVar (osarch): +def GetVar (osarch): btdir = '/tmp/opengnsys/oglive_builder' bttargetdir = '/var/lib/tftpboot/ogclient/' btrootfsimg = os.path.join (bttargetdir, 'ogclient.img') @@ -52,7 +53,7 @@ def btogGetVar (osarch): btvirtualdisksize = '3G' return btdir, bttargetdir, btrootfsimg, btrootfsmnt, btrootfsimglabel, log_file, versionboottools, btvirtualdisksize -def btogGetOsInfo1 (type_client): +def GetOsInfo1 (type_client): if 'precise' == type_client: # ogLive 1.0.4-rc2 basado en Ubuntu 12.04 LTS. osdistrib='ubuntu' oscodename='precise' @@ -154,13 +155,13 @@ def btogGetOsInfo1 (type_client): os.exit (1) return osdistrib, oscodename, osrelease, osarch, oshttp -def btogGetOsInfo2 (ogrepo_dir, versionboottools, oscodename, osrelease, osarch): +def GetOsInfo2 (ogrepo_dir, versionboottools, oscodename, osrelease, osarch): gitrelease = subprocess.run (['git', '--git-dir', f'{ogrepo_dir}/.git', 'log', '--date', 'format:r%Y%m%d', '--format=%ad', '-1'], capture_output=True, text=True).stdout.strip() nameisoclient ='-'.join ([versionboottools, oscodename, osrelease, osarch, gitrelease]) namehostclient = '-'.join ([versionboottools, oscodename, gitrelease]) return gitrelease, nameisoclient, namehostclient -def btogSetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, bttargetdir, osarch): +def SetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, bttargetdir, osarch): logger.info (f'Creación y formateo del disco virtual "{btrootfsimg}" "{btvirtualdisksize}" MB') try: utils.umount (btrootfsmnt) @@ -228,8 +229,8 @@ def btogSetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisks logger.info (f'"{btrootfsimg}" "{btvirtualdisksize}" MB : OK') -#btogSetFsAcces: habilita el acceso al sistema root del cliente con schroot -def btogSetFsAccess (btrootfsimg): +#SetFsAcces: habilita el acceso al sistema root del cliente con schroot +def SetFsAccess (btrootfsimg): if (utils.grep (btrootfsimg, '/etc/schroot/schroot.conf')): return logger.info ('Iniciando la configuración del schroot') @@ -250,9 +251,9 @@ def btogSetFsAccess (btrootfsimg): logger.info ('Finalizado: OK') return 0 -# btogSetfsBase: Genera el sistema root base con debootstrap +# SetfsBase: Genera el sistema root base con debootstrap # trabaja sobre un rootfs ya montado -def btogSetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp): +def SetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp): logger.info ('Iniciando la generación del sistema de archivos') logger.info ('debootstrapping system') @@ -263,7 +264,7 @@ def btogSetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp): logger.info (f'debootstrap --arch="{osarch}" --components=main,universe "{oscodename}" "{btrootfsmnt}" "{oshttp}" : ok') return 0 -def boottoolsFsOpengnsys (builder, og_shared, og_engine, ogclientmount, osdistrib, oscodename): +def FsOpengnsys (builder, og_shared, og_engine, ogclientmount, osdistrib, oscodename): print ('Iniciando la personalización con datos del repositorio') sources_list_in = f'{builder}/includes/etc/apt/sources.list.{osdistrib.lower()}' @@ -301,7 +302,7 @@ def boottoolsFsOpengnsys (builder, og_shared, og_engine, ogclientmount, osdistri if os.path.exists (f'{og_shared}/bin/browser'): shutil.copy (f'{og_shared}/bin/browser', f'{ogclientmount}/bin/') if os.path.exists (f'{og_shared}/bin/ogAdmClient'): shutil.copy (f'{og_shared}/bin/ogAdmClient', f'{ogclientmount}/bin/') -def boottoolsSshServer (btrootfsmnt): +def SshServer (btrootfsmnt): if not os.path.exists ('/root/.ssh/id_rsa'): ## crea un par de claves en la VM, no en el chroot utils.run (['ssh-keygen', '-q', '-f', '/root/.ssh/id_rsa', '-N', '']) @@ -331,7 +332,7 @@ def boottoolsSshServer (btrootfsmnt): else: logger.error ('no key publica og') -def boottoolsSshClient (btrootfsmnt): +def SshClient (btrootfsmnt): if not os.path.exists (f'{btrootfsmnt}/root/.ssh/id_rsa'): utils.run (['ssh-keygen', '-q', '-f', f'{btrootfsmnt}/root/.ssh/id_rsa', '-N', '']) ## crea un par de claves en el chroot @@ -348,8 +349,8 @@ def boottoolsSshClient (btrootfsmnt): ## TODO: exportamos la publica a los repos #cp /root/.ssh/id_rsa.pub /tmp/rsa.ogclient.pub -#btogFsInitrd genera un initrd. -def btogFsInitrd (bttargetdir, osrelease): +#FsInitrd genera un initrd. +def FsInitrd (bttargetdir, osrelease): #echo "cp /tmp/*-${OSRELEASE} ${BTTARGETDIR}" #cp /tmp/*-${OSRELEASE} ${BTTARGETDIR} ## esto copia algo?? @@ -372,8 +373,8 @@ def btogFsInitrd (bttargetdir, osrelease): for f in glob.glob (f'{bttargetdir}/oginitrd*') + glob.glob (f'{bttargetdir}/vmlinuz*'): os.chmod (f, 0o755) -#btogFsSqfs convierte el sistema root en sqfs -def btogFsSqfs (bttargetdir, btrootfsmnt): +#FsSqfs convierte el sistema root en sqfs +def FsSqfs (bttargetdir, btrootfsmnt): logger.info ('Iniciando la creación del sistema de archivos en sqfs') # si ya existe un sqfs lo renombramos @@ -386,8 +387,8 @@ def btogFsSqfs (bttargetdir, btrootfsmnt): utils.write_md5 (f'{bttargetdir}/ogclient.sqfs') -# btogIsoGenerator genera la iso del cliente -def btogIsoGenerator (pxepkg, isolinux_tpl, bttargetdir, nameisoclient): +# IsoGenerator genera la iso del cliente +def IsoGenerator (pxepkg, isolinux_tpl, bttargetdir, nameisoclient): #Preparamos los gestores de arranque try: os.makedirs ('/tmp/iso/isolinux', exist_ok=True) except: raise diff --git a/boottoolsfunctions/utils.py b/boottoolsfunctions/utils.py new file mode 100644 index 0000000..b1385f2 --- /dev/null +++ b/boottoolsfunctions/utils.py @@ -0,0 +1,54 @@ +import logging +import subprocess +import re +import os +import configparser + +logger = logging.getLogger ('boottools') + +def run (args): + cp = subprocess.run (args, text=True, capture_output=True) + if cp.returncode: + logger.error ('command "{}" failed with rc "{}"'.format (' '.join(args), cp.returncode)) + + logger.error ('stdout follows:') + for i in cp.stdout.strip().split('\n'): logger.error (' ' + i) + + logger.error ('stderr follows:') + for i in cp.stderr.strip().split('\n'): logger.error (' ' + i) + + raise Exception ('command "{}" failed with rc "{}"'.format (' '.join(args), cp.returncode)) + stdout = cp.stdout.strip() + stderr = cp.stderr.strip() + return stdout, stderr + +def grep (regex, file): + with open (file, 'r') as f: + for l in f: + if (re.findall (regex, l)): return 1 + return 0 + +def is_mounted (mntpt): + return grep (mntpt, '/proc/mounts') + +def mount (dev, mntpt, opts=[]): + if not is_mounted (mntpt): + run (['mount', dev, mntpt] + opts) + +def umount (mntpt): + if (is_mounted (mntpt)): + run (['umount', mntpt]) + +def read_config (fn): + if not os.path.exists (fn): + print (f'configuration file "{fn}" not found') + return + config = configparser.ConfigParser (comment_prefixes='#', inline_comment_prefixes='#') + config.read (fn) + return config + +def write_md5 (fn): + md5, _ = run (['md5sum', fn]) + md5, rest = md5.split (' ', 1) + with open (f'{fn}.sum', 'w') as fd: + fd.write (md5 + '\n') diff --git a/boottoolsgenerator.py b/boottoolsgenerator.py index dbd6e60..03ddb78 100755 --- a/boottoolsgenerator.py +++ b/boottoolsgenerator.py @@ -10,7 +10,7 @@ import shutil curdir = os.path.dirname (__file__) sys.path.insert (0, curdir) -from boottoolsfunctions import utils, btogGetOsInfo1, btogGetOsInfo2, btogGetVar, btogSetFsVirtual, btogSetFsAccess, btogSetFsBase, boottoolsFsOpengnsys, boottoolsSshServer, boottoolsSshClient, btogFsInitrd, btogFsSqfs, btogIsoGenerator +from boottoolsfunctions import utils, btog def _logging(): #logging.root.handlers = [] @@ -61,14 +61,14 @@ def _mkrootfs(): rc = subprocess.run (f'file "{btrootfsimg}" |grep -q "partition 1 *: ID=0x83"', shell=True).returncode print (rc) if (rc): ## 'file|grep' failed - try: btogSetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, bttargetdir, osarch) + try: btog.SetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, bttargetdir, osarch) except Exception as e: logger.error (str (e)) sys.exit (1) def _schroot (btrootfsimg): logger.info ('Fase 3.2 - Configurar acceso schroot al Segundo Sistema de archivos (img)') - btogSetFsAccess (btrootfsimg) + btog.SetFsAccess (btrootfsimg) ## para hacer schroot --cosas, el mntpt tiene que estar desmontado ## si está montado da un pete tal que 'E: 10mount: mount: /run/schroot/mount/IMGogclient-7fbf51a2-e37e-48e5-8e5d-83f8901fc7ed: wrong fs type, bad option, bad superblock on /dev/loop1, missing codepage or helper program, or other error.' @@ -79,9 +79,9 @@ def _debootstrap(): logger.debug ('Running \'schroot --chroot IMGogclient -- stat /etc\'') cp = subprocess.run (['schroot', '--chroot', 'IMGogclient', '--', 'stat', '/etc']) if (cp.returncode): - logger.debug (f'schroot returned code "{cp.returncode}", calling btogSetFsBase()') + logger.debug (f'schroot returned code "{cp.returncode}", calling btog.SetFsBase()') _mount_rootfs() - try: btogSetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp) + try: btog.SetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp) except Exception as e: logger.error (str (e)) sys.exit (1) @@ -97,7 +97,7 @@ def _copy_files (btrootfsmnt, osdistrib, oscodename): builder = '/tmp/opengnsys/oglive_builder' og_shared = '/tmp/opengnsys/shared' og_engine = '/tmp/opengnsys/engine' - boottoolsFsOpengnsys (builder, og_shared, og_engine, btrootfsmnt, osdistrib, oscodename) + btog.FsOpengnsys (builder, og_shared, og_engine, btrootfsmnt, osdistrib, oscodename) utils.umount (btrootfsmnt) def _install_compile_software (curdir, osrelease, osarch): @@ -130,8 +130,8 @@ def _debconf2 (curdir): def _ssh_stuff(): logger.info ('Fase 6.2 Configurar ssh') _mount_rootfs() - boottoolsSshServer (btrootfsmnt) - boottoolsSshClient (btrootfsmnt) + btog.SshServer (btrootfsmnt) + btog.SshClient (btrootfsmnt) utils.umount (btrootfsmnt) ## el resultado es: ## - hay un nuevo par de claves en la VM /root/.ssh @@ -141,14 +141,14 @@ def _ssh_stuff(): def _mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsmnt, pxepkg, isolinux_tpl, nameisoclient): logger.info ('Fase 7.1 Copiar el initrd a su sitio') _mount_rootfs() - btogFsInitrd (bttargetdir, osrelease) + btog.FsInitrd (bttargetdir, osrelease) logger.info ('Fase 7.2 Generar fichero sqfs a partir del fichero img') - btogFsSqfs (bttargetdir, btrootfsmnt) + btog.FsSqfs (bttargetdir, btrootfsmnt) utils.umount (btrootfsmnt) logger.info ('Fase 7.3 Generar la ISO') - btogIsoGenerator (pxepkg, isolinux_tpl, bttargetdir, nameisoclient) + btog.IsoGenerator (pxepkg, isolinux_tpl, bttargetdir, nameisoclient) @@ -174,9 +174,9 @@ ogrepo_dir = '/tmp/ogrepo' clone_client_dirs (ogrepo_url, ogrepo_branch, ogrepo_dir) logger.info ('FASE 1 - Asignación de variables') -osdistrib, oscodename, osrelease, osarch, oshttp = btogGetOsInfo1(type_client) -btdir, bttargetdir, btrootfsimg, btrootfsmnt, btrootfsimglabel, log_file, versionboottools, btvirtualdisksize = btogGetVar(osarch) -gitrelease, nameisoclient, namehostclient = btogGetOsInfo2(ogrepo_dir, versionboottools, oscodename, osrelease, osarch) +osdistrib, oscodename, osrelease, osarch, oshttp = btog.GetOsInfo1(type_client) +btdir, bttargetdir, btrootfsimg, btrootfsmnt, btrootfsimglabel, log_file, versionboottools, btvirtualdisksize = btog.GetVar(osarch) +gitrelease, nameisoclient, namehostclient = btog.GetOsInfo2(ogrepo_dir, versionboottools, oscodename, osrelease, osarch) logger.info (':'.join ([osdistrib, oscodename, osrelease, osarch, oshttp])) From 9a6fecf90d25f1b16b551405e25021b6b5344ea1 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Fri, 16 Aug 2024 12:15:10 +0200 Subject: [PATCH 28/45] refs #596 use apt modules rather than forking apt-get --- 03-boottoolsSoftwareInstall.py | 36 ++++++---------------------------- 04-boottoolsSoftwareCompile.py | 6 +++--- boottoolsgenerator.py | 22 +++++++++++---------- 3 files changed, 21 insertions(+), 43 deletions(-) diff --git a/03-boottoolsSoftwareInstall.py b/03-boottoolsSoftwareInstall.py index 6d0cefe..e93cabc 100755 --- a/03-boottoolsSoftwareInstall.py +++ b/03-boottoolsSoftwareInstall.py @@ -6,31 +6,13 @@ import sys import re import subprocess -from boottoolsfunctions import utils +from boottoolsfunctions import utils, apt config = utils.read_config ('boottoolsgenerator.cfg') if config is None: sys.exit (1) debconf_settings = config['General'].get ('debconf_settings') -def _aptget_clean(): - utils.run (['apt-get', '--yes', 'clean']) - -def _aptget_autoclean(): - utils.run (['apt-get', '--yes', 'autoclean']) - -def _aptget_autoremove(): - utils.run (['apt-get', '--yes', 'autoremove']) - -def _aptget_update(): - utils.run (['apt-get', '--yes', 'update']) - -def _aptget_upgrade(): - utils.run (['apt-get', '--yes', 'upgrade']) - -def _aptget_install (pkgs, opts=[]): - utils.run (['apt-get', '--yes'] + opts + ['install'] + pkgs) - def _oghook_deactivate(): #Desactivamos el hook del oginitrd.img para evitar problemas, al final de este escripts se activará os.rename ('/etc/initramfs-tools/hooks/oghooks', '/etc/initramfs-tools/oghooks') @@ -60,29 +42,24 @@ def boottoolsSoftwareInstall (osarch, osrelease): utils.run (['dpkg-divert', '--local', '--rename', '--add', '/sbin/initctl']) os.symlink ('/bin/true', '/sbin/initctl') - _aptget_clean() pkgs32 = [] if 'i386' != osarch: utils.run (['dpkg', '--add-architecture', 'i386']) pkgs32 = 'lib32gcc-s1 lib32stdc++6 lib32z1 libc6-i386'.split (' ') ## he cambiado lib32gcc1 por lib32gcc-s1 pero como queramos crear un oglive viejo, esto va a petar - _aptget_update() ## esto ya esta hecho... _oghook_deactivate() - _aptget_upgrade() #_mock_mtab() - _aptget_install ([f'linux-image-{osrelease}', f'linux-headers-{osrelease}', f'linux-modules-{osrelease}', f'linux-modules-extra-{osrelease}', 'dkms', 'shim-signed', 'openssl']) + apt.install ([f'linux-image-{osrelease}', f'linux-headers-{osrelease}', f'linux-modules-{osrelease}', f'linux-modules-extra-{osrelease}', 'dkms', 'shim-signed', 'openssl']) subprocess.run (['debconf-set-selections'], input=debconf_settings, text=True) - _aptget_install (['sshfs', 'kexec-tools'] + pkgs32, opts=['-o', 'DPkg::Options::=--force-confdef']) ## hace falta --force-confdef para evitar un tema interactivo del /etc/ssh/ssh_config + apt.install (['sshfs', 'kexec-tools'] + pkgs32, opts={'DPkg::Options::': '--force-confdef'}) ## hace falta --force-confdef para evitar un tema interactivo del /etc/ssh/ssh_config pkgs = [] for section in config.options('Packages'): pkgs += re.split (r'[ \n]', config['Packages'].get(section).strip()) - print ('about to install these packages: "{}"'.format (' '.join (pkgs))) - ## TODO don't fail when there are unknown/uninstallable packages - _aptget_install (pkgs) + apt.install (pkgs) # Instalar módulos que algunos paquetes puedan tener pendientes de compilar. stdout, _ = utils.run (['dkms', 'status']) @@ -96,9 +73,8 @@ def boottoolsSoftwareInstall (osarch, osrelease): _oghook_activate() #_restore_mtab() - _aptget_clean() - _aptget_autoclean() - _aptget_autoremove() + apt.clean() + apt.autoremove() if __name__ == '__main__': parser = argparse.ArgumentParser() diff --git a/04-boottoolsSoftwareCompile.py b/04-boottoolsSoftwareCompile.py index 484cde5..4456066 100755 --- a/04-boottoolsSoftwareCompile.py +++ b/04-boottoolsSoftwareCompile.py @@ -2,7 +2,7 @@ import os import subprocess -from boottoolsfunctions import utils +from boottoolsfunctions import utils, apt os.environ['LANGUAGE'] = os.environ['LC_ALL'] = os.environ['LANG'] = 'C' os.chdir ('/tmp') @@ -28,9 +28,9 @@ except: if not os.path.exists ('python-libfdisk'): print ('python-libfdisk') - utils.run (['apt-get', '-y', 'install', 'python3-psutil', 'python3-dev', 'libfdisk-dev', 'python3-setuptools']) + apt.install (['python3-psutil', 'python3-dev', 'libfdisk-dev', 'python3-setuptools']) utils.run (['git', 'clone', 'git://git.48k.eu/python-libfdisk']) os.chdir ('python-libfdisk') utils.run (['python3', 'setup.py', 'install']) os.chdir ('..') - utils.run (['apt-get', '-y', 'remove', 'python3-dev', 'python3-setuptools']) + apt.remove (['python3-dev', 'python3-setuptools']) diff --git a/boottoolsgenerator.py b/boottoolsgenerator.py index 03ddb78..dc443f9 100755 --- a/boottoolsgenerator.py +++ b/boottoolsgenerator.py @@ -10,7 +10,7 @@ import shutil curdir = os.path.dirname (__file__) sys.path.insert (0, curdir) -from boottoolsfunctions import utils, btog +from boottoolsfunctions import utils, apt, btog def _logging(): #logging.root.handlers = [] @@ -41,17 +41,18 @@ def _mount_rootfs(): logger.error ('mount failed') sys.exit (1) +def _apt_update_upgrade(): + apt.update() + apt.upgrade() + def _get_pxepkg(): #grep "http://free.nchc.org.tw/drbl-core" /etc/apt/sources.list || echo "deb http://free.nchc.org.tw/drbl-core drbl stable" >> /etc/apt/sources.list - utils.run (['apt-get', 'update']) - acse_gpxe, _ = utils.run (['apt-cache', 'search', 'gpxe']) - acse_ipxe, _ = utils.run (['apt-cache', 'search', 'ipxe']) - if acse_ipxe: - pxepkg = 'ipxe' - elif acse_gpxe: - pxepkg = 'gpxe' - else: - logger.error ('neither gpxe nor ipxe found in apt-cache') + pxepkg = None + cache = apt.cache_search (['gpxe', 'ipxe']) + if cache['gpxe']: pxepkg = 'gpxe' + if cache['ipxe']: pxepkg = 'ipxe' + if pxepkg is None: + logger.error ('neither gpxe nor ipxe found in apt cache') sys.exit (1) logger.info (f'PXE package is "{pxepkg}"') return pxepkg @@ -185,6 +186,7 @@ try: utils.umount (btrootfsmnt) except: pass logger.info ('FASE 2 - Instalación de software adicional.') +_apt_update_upgrade() pxepkg = _get_pxepkg() utils.run (['apt-get', '-y', 'install', 'jq', 'syslinux', 'syslinux-efi', 'syslinux-utils', 'debootstrap', 'subversion', 'schroot', 'squashfs-tools', 'syslinux', 'genisoimage', 'qemu-utils', 'lsof', pxepkg]) ## TODO qemu no existe, hace falta? From feae76840596623d5be18ddbfda820422215ef43 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Fri, 16 Aug 2024 12:43:26 +0200 Subject: [PATCH 29/45] refs #596 group chroot tasks in a single script --- 03-boottoolsSoftwareInstall.py | 62 ++++++++++++++++++++++++-- 04-boottoolsSoftwareCompile.py | 36 --------------- 05-boottoolsFsLocales.py | 15 ------- 06-boottoolsInitrdGenerate.py | 22 ---------- boottoolsgenerator.py | 80 ++++++++++++---------------------- 5 files changed, 86 insertions(+), 129 deletions(-) diff --git a/03-boottoolsSoftwareInstall.py b/03-boottoolsSoftwareInstall.py index e93cabc..70ea143 100755 --- a/03-boottoolsSoftwareInstall.py +++ b/03-boottoolsSoftwareInstall.py @@ -1,7 +1,9 @@ #!/usr/bin/python3 import argparse +import shutil import os +import glob import sys import re import subprocess @@ -12,6 +14,7 @@ config = utils.read_config ('boottoolsgenerator.cfg') if config is None: sys.exit (1) debconf_settings = config['General'].get ('debconf_settings') +debconf_settings2 = config['General'].get ('debconf_settings2') def _oghook_deactivate(): #Desactivamos el hook del oginitrd.img para evitar problemas, al final de este escripts se activará @@ -76,10 +79,63 @@ def boottoolsSoftwareInstall (osarch, osrelease): apt.clean() apt.autoremove() +def boottoolsSoftwareCompile(): + os.environ['LANGUAGE'] = os.environ['LC_ALL'] = os.environ['LANG'] = 'C' + os.chdir ('/tmp') + + print ('ms-sys') + try: utils.run (['which', 'ms-sys']) + except: + utils.run (['wget', 'https://sourceforge.net/projects/ms-sys/files/latest/download', '-O', 'ms-sys.tar.gz']) + utils.run (['tar', '-xpzf', 'ms-sys.tar.gz']) + mssys_dir = subprocess.run (['tar tzf ms-sys.tar.gz |head -n 1'], shell=True, capture_output=True, text=True).stdout.strip() + print (f'mssys_dir "{mssys_dir}"') + os.chdir (mssys_dir) + utils.run (['make', 'install']) + os.chdir ('..') + + print ('spartlnx') + try: utils.run (['which', 'spartl64.run']) + except: + utils.run (['wget', 'http://damien.guibouret.free.fr/savepart.zip']) + utils.run (['unzip', '-o', 'savepart.zip', '-d', '/sbin/', 'spartl64.run']) + utils.run (['mkdir', '/usr/share/doc/spartlnx']) + utils.run (['unzip', '-j', '-o', 'savepart.zip', '-d', '/usr/share/doc/spartlnx/', 'doc/en/*']) + + if not os.path.exists ('python-libfdisk'): + print ('python-libfdisk') + apt.install (['python3-psutil', 'python3-dev', 'libfdisk-dev', 'python3-setuptools']) + utils.run (['git', 'clone', 'git://git.48k.eu/python-libfdisk']) + os.chdir ('python-libfdisk') + utils.run (['python3', 'setup.py', 'install']) + os.chdir ('..') + apt.remove (['python3-dev', 'python3-setuptools']) + + ## TODO restore os.environ and pwd + os.environ['LANGUAGE'] = os.environ['LC_ALL'] = os.environ['LANG'] = 'C' + os.chdir ('/tmp') + +def boottoolsFsLocales(): + subprocess.run (['debconf-set-selections'], input=debconf_settings2, text=True) + + ## despues de esto, debconf-get-selections devuelve los valores antiguos, no se por que... + utils.run (['dpkg-reconfigure', '--frontend', 'noninteractive', 'console-setup', 'locales']) + +def boottoolsInitrdGenerate (osrelease): + for f in glob.glob ('/usr/lib/initramfs-tools/bin/*'): + os.unlink (f) + shutil.copy ('/bin/busybox', '/usr/lib/initramfs-tools/bin') + + os.chdir ('/tmp') + utils.run (['mkinitramfs', '-o', f'/tmp/initrd.img-{osrelease}', '-v', osrelease]) + shutil.copy (f'/boot/vmlinuz-{osrelease}', '/tmp/') + if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument ('--osarch', help='OS architecture', action='store', required=True) - parser.add_argument ('--osrelease', help='OS release', action='store', required=True) + parser.add_argument ('--osarch', help='OS architecture', action='store', required=True) + parser.add_argument ('--osrelease', help='OS release', action='store', required=True) args = parser.parse_args() boottoolsSoftwareInstall (args.osarch, args.osrelease) - + boottoolsSoftwareCompile() + boottoolsFsLocales() + boottoolsInitrdGenerate (args.osrelease) diff --git a/04-boottoolsSoftwareCompile.py b/04-boottoolsSoftwareCompile.py index 4456066..e69de29 100755 --- a/04-boottoolsSoftwareCompile.py +++ b/04-boottoolsSoftwareCompile.py @@ -1,36 +0,0 @@ -#!/usr/bin/python3 - -import os -import subprocess -from boottoolsfunctions import utils, apt - -os.environ['LANGUAGE'] = os.environ['LC_ALL'] = os.environ['LANG'] = 'C' -os.chdir ('/tmp') - -print ('ms-sys') -try: utils.run (['which', 'ms-sys']) -except: - utils.run (['wget', 'https://sourceforge.net/projects/ms-sys/files/latest/download', '-O', 'ms-sys.tar.gz']) - utils.run (['tar', '-xpzf', 'ms-sys.tar.gz']) - mssys_dir = subprocess.run (['tar tzf ms-sys.tar.gz |head -n 1'], shell=True, capture_output=True, text=True).stdout.strip() - print (f'mssys_dir "{mssys_dir}"') - os.chdir (mssys_dir) - utils.run (['make', 'install']) - os.chdir ('..') - -print ('spartlnx') -try: utils.run (['which', 'spartl64.run']) -except: - utils.run (['wget', 'http://damien.guibouret.free.fr/savepart.zip']) - utils.run (['unzip', '-o', 'savepart.zip', '-d', '/sbin/', 'spartl64.run']) - utils.run (['mkdir', '/usr/share/doc/spartlnx']) - utils.run (['unzip', '-j', '-o', 'savepart.zip', '-d', '/usr/share/doc/spartlnx/', 'doc/en/*']) - -if not os.path.exists ('python-libfdisk'): - print ('python-libfdisk') - apt.install (['python3-psutil', 'python3-dev', 'libfdisk-dev', 'python3-setuptools']) - utils.run (['git', 'clone', 'git://git.48k.eu/python-libfdisk']) - os.chdir ('python-libfdisk') - utils.run (['python3', 'setup.py', 'install']) - os.chdir ('..') - apt.remove (['python3-dev', 'python3-setuptools']) diff --git a/05-boottoolsFsLocales.py b/05-boottoolsFsLocales.py index 02684ca..e69de29 100755 --- a/05-boottoolsFsLocales.py +++ b/05-boottoolsFsLocales.py @@ -1,15 +0,0 @@ -#!/usr/bin/python3 - -import os -import subprocess -from boottoolsfunctions import utils - -config = utils.read_config ('boottoolsgenerator.cfg') -if config is None: - sys.exit (1) -debconf_settings2 = config['General'].get ('debconf_settings2') - -subprocess.run (['debconf-set-selections'], input=debconf_settings2, text=True) - -## despues de esto, debconf-get-selections devuelve los valores antiguos, no se por que... -utils.run (['dpkg-reconfigure', '--frontend', 'noninteractive', 'console-setup', 'locales']) diff --git a/06-boottoolsInitrdGenerate.py b/06-boottoolsInitrdGenerate.py index 02c94b1..e69de29 100755 --- a/06-boottoolsInitrdGenerate.py +++ b/06-boottoolsInitrdGenerate.py @@ -1,22 +0,0 @@ -#!/usr/bin/python3 - -import argparse -import shutil -import os -import glob -from boottoolsfunctions import utils - -def boottoolsInitrdGenerate (osrelease): - for f in glob.glob ('/usr/lib/initramfs-tools/bin/*'): - os.unlink (f) - shutil.copy ('/bin/busybox', '/usr/lib/initramfs-tools/bin') - - os.chdir ('/tmp') - utils.run (['mkinitramfs', '-o', f'/tmp/initrd.img-{osrelease}', '-v', osrelease]) - shutil.copy (f'/boot/vmlinuz-{osrelease}', '/tmp/') - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument ('--osrelease', help='OS release', action='store', required=True) - args = parser.parse_args() - boottoolsInitrdGenerate (args.osrelease) diff --git a/boottoolsgenerator.py b/boottoolsgenerator.py index dc443f9..c18046e 100755 --- a/boottoolsgenerator.py +++ b/boottoolsgenerator.py @@ -41,10 +41,6 @@ def _mount_rootfs(): logger.error ('mount failed') sys.exit (1) -def _apt_update_upgrade(): - apt.update() - apt.upgrade() - def _get_pxepkg(): #grep "http://free.nchc.org.tw/drbl-core" /etc/apt/sources.list || echo "deb http://free.nchc.org.tw/drbl-core drbl stable" >> /etc/apt/sources.list pxepkg = None @@ -58,7 +54,7 @@ def _get_pxepkg(): return pxepkg def _mkrootfs(): - logger.info ('Fase 3.1 Generar y formatear el disco virtual. Generar el dispositivo loop.') + logger.info ('Stage 3.1 - create, partition and format the rootfs') rc = subprocess.run (f'file "{btrootfsimg}" |grep -q "partition 1 *: ID=0x83"', shell=True).returncode print (rc) if (rc): ## 'file|grep' failed @@ -68,14 +64,14 @@ def _mkrootfs(): sys.exit (1) def _schroot (btrootfsimg): - logger.info ('Fase 3.2 - Configurar acceso schroot al Segundo Sistema de archivos (img)') + logger.info ('Stage 3.2 - configure schroot in the VM') btog.SetFsAccess (btrootfsimg) ## para hacer schroot --cosas, el mntpt tiene que estar desmontado ## si está montado da un pete tal que 'E: 10mount: mount: /run/schroot/mount/IMGogclient-7fbf51a2-e37e-48e5-8e5d-83f8901fc7ed: wrong fs type, bad option, bad superblock on /dev/loop1, missing codepage or helper program, or other error.' def _debootstrap(): - logger.info ('Fase 3.3 Generar sistema de archivos con debootstrap') + logger.info ('Stage 3.3 - debootstrap system') logger.debug ('Try creation of a file within chroot (this operation may fail with "... etc/resolv.conf: No such file or directory"--that is ok)') logger.debug ('Running \'schroot --chroot IMGogclient -- stat /etc\'') cp = subprocess.run (['schroot', '--chroot', 'IMGogclient', '--', 'stat', '/etc']) @@ -101,35 +97,14 @@ def _copy_files (btrootfsmnt, osdistrib, oscodename): btog.FsOpengnsys (builder, og_shared, og_engine, btrootfsmnt, osdistrib, oscodename) utils.umount (btrootfsmnt) -def _install_compile_software (curdir, osrelease, osarch): - logger.info ('Fase 5.1 instalar paquetes deb con apt-get') +def _chroot_tasks (curdir, osrelease, osarch): logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/03-boottoolsSoftwareInstall.py --osrelease "{osrelease}" --osarch "{osarch}"\'') stdout, _ = utils.run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/03-boottoolsSoftwareInstall.py', '--osrelease', osrelease, '--osarch', osarch]) logger.debug (f'03-boottoolsSoftwareInstall stdout follows:') - for i in stdout.strip().split('\n'): logger.debug (' ' + i) - - logger.info ('Fase 5.2 compilar software.') - logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/04-boottoolsSoftwareCompile.py\'') - stdout, _ = utils.run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/04-boottoolsSoftwareCompile.py']) - logger.debug (f'04-boottoolsSoftwareCompile stdout follows:') - for i in stdout.strip().split('\n'): logger.debug (' ' + i) - -def _debconf2 (curdir): - logger.info ('Fase 6.1 Configurar las locales') - logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/05-boottoolsFsLocales.py\'') - stdout, _ = utils.run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/05-boottoolsFsLocales.py']) - logger.debug (f'05-boottoolsFsLocales stdout follows:') - for i in stdout.strip().split('\n'): logger.debug (' ' + i) - - logger.info ('Fase 6.3 Crear initrd') - logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/06-boottoolsInitrdGenerate.py --osrelease "{osrelease}"\'') - stdout, _ = utils.run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/06-boottoolsInitrdGenerate.py', '--osrelease', osrelease]) - logger.debug (f'06-boottoolsInitrdGenerate stdout follows:') - for i in stdout.strip().split('\n'): logger.debug (' ' + i) ## esto deja initrd.img-6.8.0-31-generic y vmlinuz-6.8.0-31-generic en /tmp + for i in stdout.strip().split ('\n'): logger.debug (' ' + i) def _ssh_stuff(): - logger.info ('Fase 6.2 Configurar ssh') _mount_rootfs() btog.SshServer (btrootfsmnt) btog.SshClient (btrootfsmnt) @@ -140,15 +115,15 @@ def _ssh_stuff(): ## - las dos claves públicas (una de cada par) están autorizadan en el rootfs /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh/authorized_keys def _mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsmnt, pxepkg, isolinux_tpl, nameisoclient): - logger.info ('Fase 7.1 Copiar el initrd a su sitio') + logger.info ('Stage 6.1 - Put initrd in place') _mount_rootfs() btog.FsInitrd (bttargetdir, osrelease) - logger.info ('Fase 7.2 Generar fichero sqfs a partir del fichero img') + logger.info ('Stage 6.2 - make squash filesystem') btog.FsSqfs (bttargetdir, btrootfsmnt) utils.umount (btrootfsmnt) - logger.info ('Fase 7.3 Generar la ISO') + logger.info ('Stage 6.3 - make iso filesystem') btog.IsoGenerator (pxepkg, isolinux_tpl, bttargetdir, nameisoclient) @@ -160,7 +135,7 @@ isolinux_tpl = config['General'].get ('isolinux_template') lvl = config['General'].get ('logging_level') logger = _logging() -type_client = sys.argv[1] if len(sys.argv)>1 else 'host' +type_client = sys.argv[1] if len (sys.argv)>1 else 'host' if os.getuid(): logger.error ('ERROR: this program must run under root privileges!!') @@ -168,45 +143,44 @@ if os.getuid(): #os.chdir ('/tmp') +## this is convenient in case the previous run failed and we want to run this program again +try: utils.umount (btrootfsmnt) +except: pass + logger.info ('OpenGnsys CLIENT installation begins') + +logger.info ('STAGE 1 - clone opengnsys repo') ogrepo_url = 'https://ognproject.evlt.uma.es/gitea/opengnsys/opengnsys.git' ogrepo_branch = 'main' ogrepo_dir = '/tmp/ogrepo' clone_client_dirs (ogrepo_url, ogrepo_branch, ogrepo_dir) -logger.info ('FASE 1 - Asignación de variables') -osdistrib, oscodename, osrelease, osarch, oshttp = btog.GetOsInfo1(type_client) -btdir, bttargetdir, btrootfsimg, btrootfsmnt, btrootfsimglabel, log_file, versionboottools, btvirtualdisksize = btog.GetVar(osarch) -gitrelease, nameisoclient, namehostclient = btog.GetOsInfo2(ogrepo_dir, versionboottools, oscodename, osrelease, osarch) - +osdistrib, oscodename, osrelease, osarch, oshttp = btog.GetOsInfo1 (type_client) +btdir, bttargetdir, btrootfsimg, btrootfsmnt, btrootfsimglabel, log_file, versionboottools, btvirtualdisksize = btog.GetVar (osarch) +gitrelease, nameisoclient, namehostclient = btog.GetOsInfo2 (ogrepo_dir, versionboottools, oscodename, osrelease, osarch) logger.info (':'.join ([osdistrib, oscodename, osrelease, osarch, oshttp])) -## this is convenient in case the previous run failed and we want to run this program again -try: utils.umount (btrootfsmnt) -except: pass - -logger.info ('FASE 2 - Instalación de software adicional.') -_apt_update_upgrade() +logger.info ('STAGE 2 - install software in the VM') +apt.update() +apt.upgrade() pxepkg = _get_pxepkg() -utils.run (['apt-get', '-y', 'install', 'jq', 'syslinux', 'syslinux-efi', 'syslinux-utils', 'debootstrap', 'subversion', 'schroot', 'squashfs-tools', 'syslinux', 'genisoimage', 'qemu-utils', 'lsof', pxepkg]) ## TODO qemu no existe, hace falta? +apt.install (['jq', 'syslinux', 'syslinux-efi', 'syslinux-utils', 'debootstrap', 'subversion', 'schroot', 'squashfs-tools', 'syslinux', 'genisoimage', 'qemu-utils', 'lsof', pxepkg]) ## TODO qemu no existe, hace falta? -logger.info ('FASE 3 - Creación del Sistema raiz RootFS (Segundo Sistema archivos (img))') +logger.info ('STAGE 3 - create and bootstrap rootfs') _mkrootfs() _schroot (btrootfsimg) _debootstrap() -logger.info ('FASE 4 - Incorporando ficheros OpenGnsys al sistema raíz rootfs') +logger.info ('STAGE 4 - copy files to the rootfs') _initramfs_version (gitrelease, osrelease, btdir) _copy_files (btrootfsmnt, osdistrib, oscodename) -logger.info ('FASE 5 - Instalar software') -_install_compile_software (curdir, osrelease, osarch) +logger.info ('STAGE 5 - perform tasks within the chroot') +_chroot_tasks (curdir, osrelease, osarch) -logger.info ('FASE 6 - Personalizar el sistema creado') -_debconf2 (curdir) _ssh_stuff() -logger.info ('FASE 7 - Generar distribucion') +logger.info ('STAGE 6 - generate distribution files') _mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsmnt, pxepkg, isolinux_tpl, nameisoclient) logger.info ('OpenGnsys installation finished') From 0aa118a88b7cce2b7b7330e20bdf95b4f46939d8 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Fri, 16 Aug 2024 12:45:06 +0200 Subject: [PATCH 30/45] refs #596 add a forgotten file --- boottoolsfunctions/apt.py | 87 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 boottoolsfunctions/apt.py diff --git a/boottoolsfunctions/apt.py b/boottoolsfunctions/apt.py new file mode 100644 index 0000000..ddb6662 --- /dev/null +++ b/boottoolsfunctions/apt.py @@ -0,0 +1,87 @@ +#!/usr/bin/python3 + +import apt +import apt.progress.base +import apt_pkg +import shutil +import os + +def clean(): + apt_cache = apt.Cache() + + apt_pkg.init() + cache_dir = apt_pkg.config.find_dir ('Dir::Cache::archives') + + for fn in os.listdir (cache_dir): + file_path = os.path.join (cache_dir, fn) + try: + if os.path.isfile (file_path): + os.unlink (file_path) + elif os.path.isdir (file_path): + shutil.rmtree (file_path) + except Exception as e: + print (f'Failed to delete {file_path}. Reason: {e}') + +def autoremove(): + apt_cache = apt.Cache() + for pkg in apt_cache: + if pkg.is_installed and pkg.is_auto_removable: + pkg.mark_delete() + try: + apt_cache.commit() + except Exception as e: + print ('installation failed: {}'.format (str (e))) + +def remove (pkgs): + apt_cache = apt.Cache() + for p in pkgs: + if p in apt_cache: + p.mark_delete() + try: + apt_cache.commit() + except Exception as e: + print ('removal failed: {}'.format (str (e))) + +def update(): + apt_cache = apt.Cache() + apt_cache.update() + +def upgrade(): + apt_cache = apt.Cache() + apt_cache.upgrade() + +def cache_search (pkgs): + apt_cache = apt.Cache() + res = {} + for p in pkgs: + res[p] = True if p in apt_cache else False + return res + +def install (pkgs, opts={}): + apt_pkg.init() + + if opts: + apt_pkg.init_config() + for k in opts: + apt_pkg.config.set (k, opts[k]) + + cache = apt_pkg.Cache() + sl = apt_pkg.SourceList() + sl.read_main_list() + cache.update (apt.progress.base.AcquireProgress(), sl) + + _to_install = [] + dep_cache = apt_pkg.DepCache(cache) + for p in pkgs: + package = cache[p] + if not package: + print (f'package "{p}" not found') + continue + _to_install.append (p) + dep_cache.mark_install(package) + + if _to_install: + print ('about to install these packages: "{}"'.format (' '.join (_to_install))) + fetcher = apt_pkg.Acquire() + install_progress = apt.progress.base.InstallProgress() + dep_cache.commit(fetcher, install_progress) From d1822ec5fb38cf6ec5869037fdbdb463302c8209 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Fri, 16 Aug 2024 13:00:10 +0200 Subject: [PATCH 31/45] refs #596 rename files and functions --- 04-boottoolsSoftwareCompile.py | 0 05-boottoolsFsLocales.py | 0 06-boottoolsInitrdGenerate.py | 0 {boottoolsfunctions => boottools}/apt.py | 0 {boottoolsfunctions => boottools}/btog.py | 24 ++++++------- {boottoolsfunctions => boottools}/utils.py | 0 ...toolsSoftwareInstall.py => chroot-tasks.py | 4 +-- boottoolsgenerator.cfg => mkoglive.cfg | 0 boottoolsgenerator.py => mkoglive.py | 34 +++++++++---------- 9 files changed, 31 insertions(+), 31 deletions(-) delete mode 100755 04-boottoolsSoftwareCompile.py delete mode 100755 05-boottoolsFsLocales.py delete mode 100755 06-boottoolsInitrdGenerate.py rename {boottoolsfunctions => boottools}/apt.py (100%) rename {boottoolsfunctions => boottools}/btog.py (97%) rename {boottoolsfunctions => boottools}/utils.py (100%) rename 03-boottoolsSoftwareInstall.py => chroot-tasks.py (98%) rename boottoolsgenerator.cfg => mkoglive.cfg (100%) rename boottoolsgenerator.py => mkoglive.py (86%) diff --git a/04-boottoolsSoftwareCompile.py b/04-boottoolsSoftwareCompile.py deleted file mode 100755 index e69de29..0000000 diff --git a/05-boottoolsFsLocales.py b/05-boottoolsFsLocales.py deleted file mode 100755 index e69de29..0000000 diff --git a/06-boottoolsInitrdGenerate.py b/06-boottoolsInitrdGenerate.py deleted file mode 100755 index e69de29..0000000 diff --git a/boottoolsfunctions/apt.py b/boottools/apt.py similarity index 100% rename from boottoolsfunctions/apt.py rename to boottools/apt.py diff --git a/boottoolsfunctions/btog.py b/boottools/btog.py similarity index 97% rename from boottoolsfunctions/btog.py rename to boottools/btog.py index 3f14d92..bb2049b 100644 --- a/boottoolsfunctions/btog.py +++ b/boottools/btog.py @@ -161,7 +161,7 @@ def GetOsInfo2 (ogrepo_dir, versionboottools, oscodename, osrelease, osarch): namehostclient = '-'.join ([versionboottools, oscodename, gitrelease]) return gitrelease, nameisoclient, namehostclient -def SetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, bttargetdir, osarch): +def mkrootfs (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, bttargetdir, osarch): logger.info (f'Creación y formateo del disco virtual "{btrootfsimg}" "{btvirtualdisksize}" MB') try: utils.umount (btrootfsmnt) @@ -230,7 +230,7 @@ def SetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, logger.info (f'"{btrootfsimg}" "{btvirtualdisksize}" MB : OK') #SetFsAcces: habilita el acceso al sistema root del cliente con schroot -def SetFsAccess (btrootfsimg): +def configure_schroot (btrootfsimg): if (utils.grep (btrootfsimg, '/etc/schroot/schroot.conf')): return logger.info ('Iniciando la configuración del schroot') @@ -264,8 +264,8 @@ def SetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp): logger.info (f'debootstrap --arch="{osarch}" --components=main,universe "{oscodename}" "{btrootfsmnt}" "{oshttp}" : ok') return 0 -def FsOpengnsys (builder, og_shared, og_engine, ogclientmount, osdistrib, oscodename): - print ('Iniciando la personalización con datos del repositorio') +def copy_og_files (builder, og_shared, og_engine, ogclientmount, osdistrib, oscodename): + logger.info ('Iniciando la personalización con datos del repositorio') sources_list_in = f'{builder}/includes/etc/apt/sources.list.{osdistrib.lower()}' sources_list_out = f'{builder}/includes/etc/apt/sources.list' @@ -302,7 +302,7 @@ def FsOpengnsys (builder, og_shared, og_engine, ogclientmount, osdistrib, oscode if os.path.exists (f'{og_shared}/bin/browser'): shutil.copy (f'{og_shared}/bin/browser', f'{ogclientmount}/bin/') if os.path.exists (f'{og_shared}/bin/ogAdmClient'): shutil.copy (f'{og_shared}/bin/ogAdmClient', f'{ogclientmount}/bin/') -def SshServer (btrootfsmnt): +def ssh_server (btrootfsmnt): if not os.path.exists ('/root/.ssh/id_rsa'): ## crea un par de claves en la VM, no en el chroot utils.run (['ssh-keygen', '-q', '-f', '/root/.ssh/id_rsa', '-N', '']) @@ -332,7 +332,7 @@ def SshServer (btrootfsmnt): else: logger.error ('no key publica og') -def SshClient (btrootfsmnt): +def ssh_client (btrootfsmnt): if not os.path.exists (f'{btrootfsmnt}/root/.ssh/id_rsa'): utils.run (['ssh-keygen', '-q', '-f', f'{btrootfsmnt}/root/.ssh/id_rsa', '-N', '']) ## crea un par de claves en el chroot @@ -349,8 +349,8 @@ def SshClient (btrootfsmnt): ## TODO: exportamos la publica a los repos #cp /root/.ssh/id_rsa.pub /tmp/rsa.ogclient.pub -#FsInitrd genera un initrd. -def FsInitrd (bttargetdir, osrelease): +#move_initrd genera un initrd. +def move_initrd (bttargetdir, osrelease): #echo "cp /tmp/*-${OSRELEASE} ${BTTARGETDIR}" #cp /tmp/*-${OSRELEASE} ${BTTARGETDIR} ## esto copia algo?? @@ -373,8 +373,8 @@ def FsInitrd (bttargetdir, osrelease): for f in glob.glob (f'{bttargetdir}/oginitrd*') + glob.glob (f'{bttargetdir}/vmlinuz*'): os.chmod (f, 0o755) -#FsSqfs convierte el sistema root en sqfs -def FsSqfs (bttargetdir, btrootfsmnt): +#mksquashfs convierte el sistema root en sqfs +def mksquashfs (bttargetdir, btrootfsmnt): logger.info ('Iniciando la creación del sistema de archivos en sqfs') # si ya existe un sqfs lo renombramos @@ -387,8 +387,8 @@ def FsSqfs (bttargetdir, btrootfsmnt): utils.write_md5 (f'{bttargetdir}/ogclient.sqfs') -# IsoGenerator genera la iso del cliente -def IsoGenerator (pxepkg, isolinux_tpl, bttargetdir, nameisoclient): +# mkisofs genera la iso del cliente +def mkisofs (pxepkg, isolinux_tpl, bttargetdir, nameisoclient): #Preparamos los gestores de arranque try: os.makedirs ('/tmp/iso/isolinux', exist_ok=True) except: raise diff --git a/boottoolsfunctions/utils.py b/boottools/utils.py similarity index 100% rename from boottoolsfunctions/utils.py rename to boottools/utils.py diff --git a/03-boottoolsSoftwareInstall.py b/chroot-tasks.py similarity index 98% rename from 03-boottoolsSoftwareInstall.py rename to chroot-tasks.py index 70ea143..25de7c2 100755 --- a/03-boottoolsSoftwareInstall.py +++ b/chroot-tasks.py @@ -8,9 +8,9 @@ import sys import re import subprocess -from boottoolsfunctions import utils, apt +from boottools import utils, apt -config = utils.read_config ('boottoolsgenerator.cfg') +config = utils.read_config ('mkoglive.cfg') if config is None: sys.exit (1) debconf_settings = config['General'].get ('debconf_settings') diff --git a/boottoolsgenerator.cfg b/mkoglive.cfg similarity index 100% rename from boottoolsgenerator.cfg rename to mkoglive.cfg diff --git a/boottoolsgenerator.py b/mkoglive.py similarity index 86% rename from boottoolsgenerator.py rename to mkoglive.py index c18046e..32eeef2 100755 --- a/boottoolsgenerator.py +++ b/mkoglive.py @@ -10,7 +10,7 @@ import shutil curdir = os.path.dirname (__file__) sys.path.insert (0, curdir) -from boottoolsfunctions import utils, apt, btog +from boottools import utils, apt, btog def _logging(): #logging.root.handlers = [] @@ -58,14 +58,14 @@ def _mkrootfs(): rc = subprocess.run (f'file "{btrootfsimg}" |grep -q "partition 1 *: ID=0x83"', shell=True).returncode print (rc) if (rc): ## 'file|grep' failed - try: btog.SetFsVirtual (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, bttargetdir, osarch) + try: btog.mkrootfs (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, bttargetdir, osarch) except Exception as e: logger.error (str (e)) sys.exit (1) def _schroot (btrootfsimg): logger.info ('Stage 3.2 - configure schroot in the VM') - btog.SetFsAccess (btrootfsimg) + btog.configure_schroot (btrootfsimg) ## para hacer schroot --cosas, el mntpt tiene que estar desmontado ## si está montado da un pete tal que 'E: 10mount: mount: /run/schroot/mount/IMGogclient-7fbf51a2-e37e-48e5-8e5d-83f8901fc7ed: wrong fs type, bad option, bad superblock on /dev/loop1, missing codepage or helper program, or other error.' @@ -76,9 +76,9 @@ def _debootstrap(): logger.debug ('Running \'schroot --chroot IMGogclient -- stat /etc\'') cp = subprocess.run (['schroot', '--chroot', 'IMGogclient', '--', 'stat', '/etc']) if (cp.returncode): - logger.debug (f'schroot returned code "{cp.returncode}", calling btog.SetFsBase()') + logger.debug (f'schroot returned code "{cp.returncode}", calling btog.debootstrap()') _mount_rootfs() - try: btog.SetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp) + try: btog.debootstrap (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp) except Exception as e: logger.error (str (e)) sys.exit (1) @@ -89,25 +89,25 @@ def _initramfs_version (gitrelease, osrelease, btdir): #sed -i "1 s/$/ $GITRELEASE ($OSRELEASE)/" ${BTDIR}/includes/etc/initramfs-tools/scripts/VERSION.txt utils.run (['sed', '-i', f'1 s/$/ {gitrelease} ({osrelease})/', f'{btdir}/includes/etc/initramfs-tools/scripts/VERSION.txt']) -def _copy_files (btrootfsmnt, osdistrib, oscodename): +def _copy_og_files (btrootfsmnt, osdistrib, oscodename): _mount_rootfs() builder = '/tmp/opengnsys/oglive_builder' og_shared = '/tmp/opengnsys/shared' og_engine = '/tmp/opengnsys/engine' - btog.FsOpengnsys (builder, og_shared, og_engine, btrootfsmnt, osdistrib, oscodename) + btog.copy_og_files (builder, og_shared, og_engine, btrootfsmnt, osdistrib, oscodename) utils.umount (btrootfsmnt) def _chroot_tasks (curdir, osrelease, osarch): - logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/03-boottoolsSoftwareInstall.py --osrelease "{osrelease}" --osarch "{osarch}"\'') - stdout, _ = utils.run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/03-boottoolsSoftwareInstall.py', '--osrelease', osrelease, '--osarch', osarch]) - logger.debug (f'03-boottoolsSoftwareInstall stdout follows:') + logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/chroot-tasks.py --osrelease "{osrelease}" --osarch "{osarch}"\'') + stdout, _ = utils.run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/chroot-tasks.py', '--osrelease', osrelease, '--osarch', osarch]) + logger.debug (f'chroot-tasks.py stdout follows:') ## esto deja initrd.img-6.8.0-31-generic y vmlinuz-6.8.0-31-generic en /tmp for i in stdout.strip().split ('\n'): logger.debug (' ' + i) def _ssh_stuff(): _mount_rootfs() - btog.SshServer (btrootfsmnt) - btog.SshClient (btrootfsmnt) + btog.ssh_server (btrootfsmnt) + btog.ssh_client (btrootfsmnt) utils.umount (btrootfsmnt) ## el resultado es: ## - hay un nuevo par de claves en la VM /root/.ssh @@ -117,18 +117,18 @@ def _ssh_stuff(): def _mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsmnt, pxepkg, isolinux_tpl, nameisoclient): logger.info ('Stage 6.1 - Put initrd in place') _mount_rootfs() - btog.FsInitrd (bttargetdir, osrelease) + btog.move_initrd (bttargetdir, osrelease) logger.info ('Stage 6.2 - make squash filesystem') - btog.FsSqfs (bttargetdir, btrootfsmnt) + btog.mksquashfs (bttargetdir, btrootfsmnt) utils.umount (btrootfsmnt) logger.info ('Stage 6.3 - make iso filesystem') - btog.IsoGenerator (pxepkg, isolinux_tpl, bttargetdir, nameisoclient) + btog.mkisofs (pxepkg, isolinux_tpl, bttargetdir, nameisoclient) -config = utils.read_config ('boottoolsgenerator.cfg') +config = utils.read_config ('mkoglive.cfg') if config is None: sys.exit (1) isolinux_tpl = config['General'].get ('isolinux_template') @@ -173,7 +173,7 @@ _debootstrap() logger.info ('STAGE 4 - copy files to the rootfs') _initramfs_version (gitrelease, osrelease, btdir) -_copy_files (btrootfsmnt, osdistrib, oscodename) +_copy_og_files (btrootfsmnt, osdistrib, oscodename) logger.info ('STAGE 5 - perform tasks within the chroot') _chroot_tasks (curdir, osrelease, osarch) From ffc24fb48b61275496657302ac1b1ca67a3226bb Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Fri, 16 Aug 2024 13:02:58 +0200 Subject: [PATCH 32/45] refs #596 fix bug when removing packages --- boottools/apt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boottools/apt.py b/boottools/apt.py index ddb6662..d50f6dd 100644 --- a/boottools/apt.py +++ b/boottools/apt.py @@ -36,7 +36,7 @@ def remove (pkgs): apt_cache = apt.Cache() for p in pkgs: if p in apt_cache: - p.mark_delete() + apt_cache[p].mark_delete() try: apt_cache.commit() except Exception as e: From 53310a8e5bf5f8e761f925b6722cb2b0365c881a Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Fri, 16 Aug 2024 13:22:34 +0200 Subject: [PATCH 33/45] refs #596 remove debconf_settings2, do chdir first --- boottools/btog.py | 4 ++-- chroot-tasks.py | 40 ++++++++++++---------------------------- mkoglive.cfg | 2 -- mkoglive.py | 4 ++-- 4 files changed, 16 insertions(+), 34 deletions(-) diff --git a/boottools/btog.py b/boottools/btog.py index bb2049b..2a36f49 100644 --- a/boottools/btog.py +++ b/boottools/btog.py @@ -251,9 +251,9 @@ def configure_schroot (btrootfsimg): logger.info ('Finalizado: OK') return 0 -# SetfsBase: Genera el sistema root base con debootstrap +# debootstrap: Genera el sistema root base con debootstrap # trabaja sobre un rootfs ya montado -def SetFsBase (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp): +def debootstrap (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp): logger.info ('Iniciando la generación del sistema de archivos') logger.info ('debootstrapping system') diff --git a/chroot-tasks.py b/chroot-tasks.py index 25de7c2..fb63750 100755 --- a/chroot-tasks.py +++ b/chroot-tasks.py @@ -14,7 +14,6 @@ config = utils.read_config ('mkoglive.cfg') if config is None: sys.exit (1) debconf_settings = config['General'].get ('debconf_settings') -debconf_settings2 = config['General'].get ('debconf_settings2') def _oghook_deactivate(): #Desactivamos el hook del oginitrd.img para evitar problemas, al final de este escripts se activará @@ -24,16 +23,6 @@ def _oghook_activate(): #Activamos el hook del oginitrd.img os.rename ('/etc/initramfs-tools/oghooks', '/etc/initramfs-tools/hooks/oghooks') -#def _mock_mtab(): -# # Preparamos el mtab necesario para la instalacion correcta de paquetes. -# #echo "/dev/sda1 / ext4 rw,errors=remount-ro 0 0" > /etc/mtab ## nati: falla porque es un symlink a ../proc/self/mounts -# pass - -#def _restore_mtab(): -# # Dejamos el mtab como al principio -# #echo " " > /etc/mtab -# pass - def boottoolsSoftwareInstall (osarch, osrelease): os.environ['LANGUAGE'] = 'C' os.environ['LC_ALL'] = 'C' @@ -51,18 +40,18 @@ def boottoolsSoftwareInstall (osarch, osrelease): pkgs32 = 'lib32gcc-s1 lib32stdc++6 lib32z1 libc6-i386'.split (' ') ## he cambiado lib32gcc1 por lib32gcc-s1 pero como queramos crear un oglive viejo, esto va a petar _oghook_deactivate() - #_mock_mtab() - - apt.install ([f'linux-image-{osrelease}', f'linux-headers-{osrelease}', f'linux-modules-{osrelease}', f'linux-modules-extra-{osrelease}', 'dkms', 'shim-signed', 'openssl']) subprocess.run (['debconf-set-selections'], input=debconf_settings, text=True) - - apt.install (['sshfs', 'kexec-tools'] + pkgs32, opts={'DPkg::Options::': '--force-confdef'}) ## hace falta --force-confdef para evitar un tema interactivo del /etc/ssh/ssh_config + utils.run (['dpkg-reconfigure', '--frontend', 'noninteractive', 'console-setup', 'locales']) ## despues de esto, debconf-get-selections devuelve los valores antiguos, no se por que... pkgs = [] for section in config.options('Packages'): pkgs += re.split (r'[ \n]', config['Packages'].get(section).strip()) - apt.install (pkgs) + + apt.install ( + [f'linux-image-{osrelease}', f'linux-headers-{osrelease}', f'linux-modules-{osrelease}', f'linux-modules-extra-{osrelease}', 'dkms', 'shim-signed', 'openssl', 'sshfs', 'kexec-tools'] + pkgs32 + pkgs, + opts={'DPkg::Options::': '--force-confdef'} ## hace falta --force-confdef para evitar un tema interactivo del /etc/ssh/ssh_config + ) # Instalar módulos que algunos paquetes puedan tener pendientes de compilar. stdout, _ = utils.run (['dkms', 'status']) @@ -75,11 +64,13 @@ def boottoolsSoftwareInstall (osarch, osrelease): utils.run (['dkms', 'install', '-m', mod.strip(), '-v', vers.strip()]) _oghook_activate() - #_restore_mtab() apt.clean() apt.autoremove() def boottoolsSoftwareCompile(): + env_language = os.environ['LANGUAGE'] + env_lc_all = os.environ['LC_ALL'] + env_lang = os.environ['LANG'] os.environ['LANGUAGE'] = os.environ['LC_ALL'] = os.environ['LANG'] = 'C' os.chdir ('/tmp') @@ -111,15 +102,9 @@ def boottoolsSoftwareCompile(): os.chdir ('..') apt.remove (['python3-dev', 'python3-setuptools']) - ## TODO restore os.environ and pwd - os.environ['LANGUAGE'] = os.environ['LC_ALL'] = os.environ['LANG'] = 'C' - os.chdir ('/tmp') - -def boottoolsFsLocales(): - subprocess.run (['debconf-set-selections'], input=debconf_settings2, text=True) - - ## despues de esto, debconf-get-selections devuelve los valores antiguos, no se por que... - utils.run (['dpkg-reconfigure', '--frontend', 'noninteractive', 'console-setup', 'locales']) + os.environ['LANGUAGE'] = env_language + os.environ['LC_ALL'] = env_lc_all + os.environ['LANG'] = env_lang def boottoolsInitrdGenerate (osrelease): for f in glob.glob ('/usr/lib/initramfs-tools/bin/*'): @@ -137,5 +122,4 @@ if __name__ == '__main__': args = parser.parse_args() boottoolsSoftwareInstall (args.osarch, args.osrelease) boottoolsSoftwareCompile() - boottoolsFsLocales() boottoolsInitrdGenerate (args.osrelease) diff --git a/mkoglive.cfg b/mkoglive.cfg index 018996a..708a93a 100644 --- a/mkoglive.cfg +++ b/mkoglive.cfg @@ -6,8 +6,6 @@ debconf_settings = kexec-tools kexec-tools/load_kexec boolean true openssh-server openssh-server/permit-root-login boolean true refind refind/install_to_esp boolean false - -debconf_settings2 = console-setup console-setup/charmap47 select UTF-8 console-setup console-setup/codeset47 select . Combined - Latin; Slavic Cyrillic; Greek console-setup console-setup/fontface47 select VGA diff --git a/mkoglive.py b/mkoglive.py index 32eeef2..9283dce 100755 --- a/mkoglive.py +++ b/mkoglive.py @@ -12,6 +12,8 @@ curdir = os.path.dirname (__file__) sys.path.insert (0, curdir) from boottools import utils, apt, btog +os.chdir (curdir) + def _logging(): #logging.root.handlers = [] @@ -141,8 +143,6 @@ if os.getuid(): logger.error ('ERROR: this program must run under root privileges!!') sys.exit (1) -#os.chdir ('/tmp') - ## this is convenient in case the previous run failed and we want to run this program again try: utils.umount (btrootfsmnt) except: pass From c2fe8f7695a9fe1d51b86c9641b589c29f78b370 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Fri, 16 Aug 2024 14:54:18 +0200 Subject: [PATCH 34/45] refs #596 remove deprecated sshd config items, add shell code for some unused functions --- boottools/btog.py | 49 ++- boottoolsfunctions.lib | 648 ----------------------------------- includes/etc/ssh/sshd_config | 9 - mkoglive.py | 1 - 4 files changed, 40 insertions(+), 667 deletions(-) delete mode 100755 boottoolsfunctions.lib diff --git a/boottools/btog.py b/boottools/btog.py index 2a36f49..79d01e5 100644 --- a/boottools/btog.py +++ b/boottools/btog.py @@ -156,7 +156,7 @@ def GetOsInfo1 (type_client): return osdistrib, oscodename, osrelease, osarch, oshttp def GetOsInfo2 (ogrepo_dir, versionboottools, oscodename, osrelease, osarch): - gitrelease = subprocess.run (['git', '--git-dir', f'{ogrepo_dir}/.git', 'log', '--date', 'format:r%Y%m%d', '--format=%ad', '-1'], capture_output=True, text=True).stdout.strip() + gitrelease, _ = utils.run (['git', '--git-dir', f'{ogrepo_dir}/.git', 'log', '--date', 'format:r%Y%m%d', '--format=%ad', '-1']) nameisoclient ='-'.join ([versionboottools, oscodename, osrelease, osarch, gitrelease]) namehostclient = '-'.join ([versionboottools, oscodename, gitrelease]) return gitrelease, nameisoclient, namehostclient @@ -175,18 +175,18 @@ def mkrootfs (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, btt raise Exception (f'Creando directorio "{btrootfsmnt}" : ERROR') try: utils.run (['chown', '-R', 'root:opengnsys', bttargetdir]) - except: - raise Exception (f'Failed to chown root:opengnsys "{btrootfsmnt}"') + except Exception as e: + raise Exception (f'Failed to chown root:opengnsys "{btrootfsmnt}": {str(e)}') logger.info (f'Creating disk image "{btrootfsimg}"') if 'i386' == osarch: try: utils.run (['dd', 'if=/dev/zero', f'of={btrootfsimg}', 'bs=1048576', f'count={btvirtualdisksize}']) - except: - raise Exception (f'Creando el disco virtual "{btrootfsimg}" con tamaño maxima "{btvirtualdisksize}" MB : ERROR') + except Exception as e: + raise Exception (f'Creando el disco virtual "{btrootfsimg}" con tamaño maxima "{btvirtualdisksize}" MB : ERROR: {str(e)}') else: try: utils.run (['qemu-img', 'create', btrootfsimg, btvirtualdisksize]) - except: - raise Exception (f'Creando el disco virtual "{btrootfsimg}" con tamaño maxima "{btvirtualdisksize}" MB : ERROR') + except Exception as e: + raise Exception (f'Creando el disco virtual "{btrootfsimg}" con tamaño maxima "{btvirtualdisksize}" MB : ERROR: {str(e)}') logger.debug ('losetup --find') diskloop, _ = utils.run (['losetup', '--find']) @@ -286,8 +286,8 @@ def copy_og_files (builder, og_shared, og_engine, ogclientmount, osdistrib, osco os.makedirs (f'{ogclientmount}/usr/local/plugins', exist_ok=True) subprocess.run (f'rsync -aH {builder}/includes/* {ogclientmount}/' , shell=True) - subprocess.run (f'rsync -aH {og_shared}/* {ogclientmount}/opt/opengnsys/' , shell=True) - subprocess.run (f'rsync -aH {og_engine}/* {ogclientmount}/opt/opengnsys/lib/engine/bin/', shell=True) + subprocess.run (f'rsync -aH {og_shared}/* {ogclientmount}/opt/opengnsys/' , shell=True) + subprocess.run (f'rsync -aH {og_engine}/* {ogclientmount}/opt/opengnsys/lib/engine/bin/', shell=True) if not os.path.exists (f'{ogclientmount}/etc/pci.ids'): shutil.copy (f'{og_shared}/lib/pci.ids', f'{ogclientmount}/etc/') @@ -446,3 +446,34 @@ def mkisofs (pxepkg, isolinux_tpl, bttargetdir, nameisoclient): # update-alternatives --set default.plymouth /lib/plymouth/themes/opengnsys/opengnsys.plymouth # mkdir -p /etc/initramfs-tools/conf.d # echo "FRAMEBUFFER=y" > /etc/initramfs-tools/conf.d/splash + + +############### No usados en el instalador, solo en actualizaciones. + +# ogClientMount [str_program] +#@brief Acceso al 2nd FS del cliente desde el Servidor Opengnsys +#@param 1 Opciona: scripts o programa a ejecutar para automatizaciones +#@return Si no hay parametros: login de acceso. +#@return con un parametro: La salida del programa ejecutado +#def ogClientMount(): +# TODO comprobar que OGFILE y OGFILEMOUNT existe. +# mount | grep $OGCLIENTFILE > /dev/null || mount $OGCLIENTFILE $OGCLIENTMOUNT -o loop,offset=32256 +# mount | grep $OGCLIENTMOUNT/proc > /dev/null || mount --bind /proc $OGCLIENTMOUNT/proc +# mount | grep $OGCLIENTMOUNT/sys > /dev/null || mount --bind /sys $OGCLIENTMOUNT/sys +# mount | grep $OGCLIENTMOUNT/tmp > /dev/null || mount --bind /tmp $OGCLIENTMOUNT/tmp +# mount | grep $OGCLIENTMOUNT/dev > /dev/null || mount --bind /dev $OGCLIENTMOUNT/dev +# mount | grep $OGCLIENTMOUNT/dev/pts > /dev/null || mount --bind /dev/pts $OGCLIENTMOUNT/dev/pts +# [ $# = 0 ] && $(chroot $OGCLIENTMOUNT /sbin/getty 38400 `tty`) +# [ $# = 1 ] && chroot $OGCLIENTMOUNT $1 + +#@brief Desmonta el 2nd FS del cliente desde el Servidor Opengnsys +#def ogClientUnmount(): +# cd /tmp +# echo "desmontando cliente espere" +# sleep 5 +# mount | grep $OGCLIENTMOUNT/dev > /dev/null && umount $OGCLIENTMOUNT/dev || ogClientUnmount +# mount | grep $OGCLIENTMOUNT/dev/pts > /dev/null && umount $OGCLIENTMOUNT/dev/pts || ogClientUnmount +# mount | grep $OGCLIENTMOUNT/proc > /dev/null && umount $OGCLIENTMOUNT/proc || ogClientUnmount +# mount | grep $OGCLIENTMOUNT/sys > /dev/null && umount $OGCLIENTMOUNT/sys || ogClientUnmount +# mount | grep $OGCLIENTMOUNT/tmp > /dev/null && umount $OGCLIENTMOUNT/tmp || ogClientUnmount +# mount | grep $OGCLIENTMOUNT > /dev/null && umount $OGCLIENTMOUNT || ogClientUnmount diff --git a/boottoolsfunctions.lib b/boottoolsfunctions.lib deleted file mode 100755 index f1c5c58..0000000 --- a/boottoolsfunctions.lib +++ /dev/null @@ -1,648 +0,0 @@ -#!/bin/bash -#/** -#@file boottoolsfunctions.lib -#@brief Librería o clase Boot-Tools -#@class Boot-Tools -#@brief Funciones para generar un sistema operativo cliente para opengnsys -#@version 1.1.1 -#@warning License: GNU GPLv3+ -#*/ - - -# btGetVariables: define las constantes a utilizar -# Autor: Antonio J. Doblas Viso. Universidad de Málaga. -function btogGetVar() -{ -export BTDIR=/tmp/opengnsys_installer/opengnsys/client/boot-tools -export BTTARGETDIR=/var/lib/tftpboot/ogclient/ -export BTROOTFSIMG=${BTTARGETDIR}ogclient.img -export BTROOTFSMNT=${BTTARGETDIR}ogclientmount -if [ "$OSARCH" = "i386" ]; then - # tamaño maximo limitado por schroot 2GB para 32 bits - export BTVIRTUALDISKSIZE=2000 -else - export BTVIRTUALDISKSIZE="3G" -fi -export BTROOTFSIMGLABEL=ogClient - -export LOG_FILE=/tmp/boot-tools_installation.log - -export VERSIONBOOTTOOLS="ogLive" -} - - -# btGetOsInfo: detecta la version del host, para tenerla de referencia en el boot-tools a generar. -#@version 0.9 - Prototipo de sistema operativo multiarranque de opengnsys. -#@author Antonio J. Doblas Viso. Universidad de Malaga. -#@date 2010/05/24 -#@version 1.0 - Compatibilidad OpengGnsys X. -#@author Antonio J. Doblas Viso. Universidad de Malaga. -#@date 2011/08/03 -#@version 1.0.5 - Fichero general de configuración, Kernel 3.7 para Ubuntu 12.10. -#@author Ramón Gómez, ETSII Univ. Sevilla -#@date 2013/02/06 -function btogGetOsInfo () -{ -OGCLIENTCFG=/tmp/ogclient.cfg -case "${1,,}" in - precise) # ogLive 1.0.4-rc2 basado en Ubuntu 12.04 LTS. - OSDISTRIB="ubuntu" - OSCODENAME="precise" - OSRELEASE="3.2.0-23-generic" - #OSRELEASE="3.11.0-26-generic" - OSARCH="i386" - OSHTTP="http://es.archive.ubuntu.com/ubuntu/" - ;; - wheezy) # ogLive basado en Debian 7.3. - OSDISTRIB="debian" - OSCODENAME="wheezy" - OSRELEASE="3.2.0-4-i386" - OSARCH="i386" - OSHTTP="http://ftp.es.debian.org/debian/" - ;; - quantal) # ogLive 1.0.5-rc2 basado en Ubuntu 12.10 con Kernel descargado. - OSDISTRIB="ubuntu" - OSCODENAME="quantal" - OSRELEASE="3.7.6-030706-generic" - OSARCH="i386" - OSHTTP="http://es.archive.ubuntu.com/ubuntu/" - ;; - raring) # ogLive 1.0.5-rc3 basado en Ubuntu 13.04. - OSDISTRIB="ubuntu" - OSCODENAME="raring" - OSRELEASE="3.8.0-22-generic" - OSARCH="i386" - OSHTTP="http://es.archive.ubuntu.com/ubuntu/" - ;; - trusty) # ogLive 1.0.5-rc4 basado en Ubuntu 14.04. - OSDISTRIB="ubuntu" - OSCODENAME="trusty" - OSRELEASE="3.13.0-24-generic" - OSARCH="i386" - OSHTTP="http://es.archive.ubuntu.com/ubuntu/" - ;; - vivid) # ogLive 1.1.0-rc2 basado en Ubuntu 15.04. - OSDISTRIB="ubuntu" - OSCODENAME="vivid" - OSRELEASE="3.19.0-49-generic" - OSARCH="i386" - OSHTTP="http://es.archive.ubuntu.com/ubuntu/" - ;; - wily) # ogLive 1.1.0-rc1 basado en Ubuntu 15.10. - OSDISTRIB="ubuntu" - OSCODENAME="wily" - OSRELEASE="4.2.0-35-generic" - OSARCH="i386" - OSHTTP="http://es.archive.ubuntu.com/ubuntu/" - ;; - xenial|xenial-4.4) # ogLive 1.1.0-rc4 basado en Ubuntu 16.04 y Kernel 4.4. - OSDISTRIB="ubuntu" - OSCODENAME="xenial" - OSRELEASE="4.4.0-34-generic" - OSARCH="i386" - OSHTTP="http://es.archive.ubuntu.com/ubuntu/" - ;; - xenial-4.8) # ogLive 1.1.0-rc5 basado en Ubuntu 16.04 y Kernel 4.8. - OSDISTRIB="ubuntu" - OSCODENAME="xenial" - OSRELEASE="4.8.0-39-generic" - OSARCH="amd64" - OSHTTP="http://es.archive.ubuntu.com/ubuntu/" - ;; - xenial-4.13) # ogLive 1.1.0-rc6 basado en Ubuntu 16.04 y Kernel 4.13. - OSDISTRIB="ubuntu" - OSCODENAME="xenial" - OSRELEASE="4.13.0-17-generic" - OSARCH="amd64" - OSHTTP="http://es.archive.ubuntu.com/ubuntu/" - ;; - bionic|bionic-4.15) # ogLive 1.1.1-rc1 basado en Ubuntu 18.04 y Kernel 4.15. - OSDISTRIB="ubuntu" - OSCODENAME="bionic" - OSRELEASE="4.15.0-32-generic" - OSARCH="amd64" - OSHTTP="http://es.archive.ubuntu.com/ubuntu/" - ;; - bionic-4.18) # ogLive 1.1.1-rc3 basado en Ubuntu 18.04 y Kernel 4.18. - OSDISTRIB="ubuntu" - OSCODENAME="bionic" - OSRELEASE="4.18.0-20-generic" - OSARCH="amd64" - OSHTTP="http://es.archive.ubuntu.com/ubuntu/" - ;; - bionic-5.0) # ogLive 1.1.1-rc5 basado en Ubuntu 18.04 y Kernel 5.0. - OSDISTRIB="ubuntu" - OSCODENAME="bionic" - OSRELEASE="5.0.0-27-generic" - OSARCH="amd64" - OSHTTP="http://es.archive.ubuntu.com/ubuntu/" - ;; - focal) # ogLive 1.2.0-rc1 basado en Ubuntu 20.04 y Kernel 5.4. - OSDISTRIB="ubuntu" - OSCODENAME="focal" - OSRELEASE="5.4.0-42-generic" - OSARCH="amd64" - OSHTTP="http://es.archive.ubuntu.com/ubuntu/" - ;; - host) # ogLive basado en la distribución del servidor. - OSDISTRIB=$(lsb_release -is) - OSCODENAME=$(lsb_release -cs) - OSRELEASE=$(uname -r) - OSARCH=$(dpkg --print-architecture) - export OSHTTP="http://es.archive.ubuntu.com/ubuntu/" - ;; - *) # Parámetro desconocido - echo "Parámetro no válido." - exit 1 - ;; -esac -BRANCH="master" -GITURL="https://api.github.com/repos/opengnsys/OpenGnsys/commits?sha=$BRANCH&path=/client" -GITRELEASE=$(curl -s "$GITURL" | jq -r '"r" + (.[0].commit.committer.date | split("-") | join("")[:8]) + "." + (.[0].sha[:7])') -NAMEISOCLIENT="$VERSIONBOOTTOOLS-$OSCODENAME-$OSRELEASE-$OSARCH-$GITRELEASE" -NAMEHOSTCLIENT="$VERSIONBOOTTOOLS-$OSCODENAME-$GITRELEASE" -export OSDISTRIB OSCODENAME OSRELEASE OSARCH OSHTTP -export TYPECLIENT GITRELEASE NAMEISOCLIENT NAMEHOSTCLIENT -### El fichero de configuración debe sustituir a estos 3 ficheros (borrar las 3 líneas siguientes). -echo "$NAMEISOCLIENT" > /tmp/opengnsys_info_rootfs -echo "$NAMEHOSTCLIENT" > /tmp/opengnsys_chroot -# Generar fichero de configuración. -cat << EOT > $OGCLIENTCFG -TYPECLIENT="$1" -OSDISTRIB="$OSDISTRIB" -OSCODENAME="$OSCODENAME" -OSRELEASE="$OSRELEASE" -OSARCH="$OSARCH" -OSHTTP="$OSHTTP" -GITRELEASE="$GITRELEASE" -NAMEISOCLIENT="$NAMEISOCLIENT" -NAMEHOSTCLIENT="$NAMEHOSTCLIENT" -EOT -} - -# btogSetFsVirtual: Crea y formatea el fichero - disco duro virtual -#@version 0.9 - Prototipo de sistema operativo multiarranque de opengnsys. -#@author Antonio J. Doblas Viso. Universidad de Malaga. -#@date 2010/05/24 -#@version 1.0 - Compatibilidad OpengGnsys X. -#@author Antonio J. Doblas Viso. Universidad de Malaga. -#@date 2011/08/03 -# error code return 2 -function btogSetFsVirtual () -{ - -local RERROR DISKLOOP PARTLOOP #return code error - -#Dependencias. -[ -z "$BTROOTFSIMG" ] && btogGetVar - -echoAndLog "$FUNCNAME(): Creación y formateo del disco virtual $BTVIRTUALDISKSIZE MB " - -#Desmontamos el dispositivo virtual -mount | grep $BTROOTFSMNT && umount $BTROOTFSMNT -mount | grep $BTROOTFSMNT && umount $BTROOTFSMNT - -#echo "$FUNCNAME(): Creando el directorio donde se montará el disco virtual $BTROOTFSMNT" -mkdir -p $BTROOTFSMNT - -if [ $? -ne 0 ] -then - errorAndLog "$FUNCNAME(): Creando directorio $BTROOTFSMNT : ERROR" - return 1 -fi - -# Permisos para el usuario opengnsys. -chown -R root:opengnsys $BTTARGETDIR - -#echo "$FUNCNAME(): Creando el disco virtual con tamaño máximo de $BTVIRTUALDISKSIZE MB" -if [ "$OSARCH" = "i386" ]; then - dd if=/dev/zero of=$BTROOTFSIMG bs=1048576 count=$BTVIRTUALDISKSIZE - #dd if=/dev/zero of=$BTROOTFSIMG bs=1k count=$OGCLIENTSIZEKB # necesita 500MB -else - qemu-img create $BTROOTFSIMG $BTVIRTUALDISKSIZE -fi -if [ $? -ne 0 ] -then - errorAndLog "$FUNCNAME(): Creando el disco virtual con tamaño maxima $BTVIRTUALDISKSIZE MB : ERROR" - return 2 -fi - -# Particionamos el disco virtual con una sola particion primaria. -DISKLOOP=$(losetup -f) -#TODO: si no DISKLOOP error -losetup $DISKLOOP $BTROOTFSIMG -#echo "$FUNCNAME(): particiondo el disco virtual - $DISKLOOP - con una particion primaria" -echo -e "n\np\n1\n\n\nt\n83\nw" | fdisk $DISKLOOP - - -#echoAndLog "$FUNCNAME(): Liberamos Desmontando $DISKLOOP despues del particionado " -sleep 10 -losetup -d $DISKLOOP -if [ $? -ne 0 ] -then - errorAndLog "$FUNCNAME(): Liberando disco virtual despues del particionado: ERROR" - return 2 -fi - - -PARTLOOP=$(losetup -f) -#echo "$FUNCNAME(): Formateando la particion principal $PARTLOOP" -losetup -o 32256 $PARTLOOP $BTROOTFSIMG && mkfs.ext4 -b 4096 -L $BTROOTFSIMGLABEL $PARTLOOP -if [ $? -ne 0 ] -then - errorAndLog "$FUNCNAME(): Formateando la particion principal del disco virtual: ERROR" - return 2 -fi -#echoAndLog "$FUNCNAME(): Desmontando $PARTLOOP despues del formateo " -sleep 10 -losetup -d $PARTLOOP -if [ $? -ne 0 ] -then - errorAndLog "$FUNCNAME(): Liberando la particion virtual despues del formateo: ERROR" - return 2 -else - echoAndLog "$FUNCNAME(): $BTROOTFSIMG $BTVIRTUALDISKSIZE MB : OK" -fi -} - - -# btogSetfsBase: Genera el sistema root base con debootstrap -#@version 0.9 - Prototipo de sistema operativo multiarranque de opengnsys. -#@author Antonio J. Doblas Viso. Universidad de Malaga. -#@date 2010/05/24 -#@version 1.0 - Compatibilidad OpengGnsys X. -#@author Antonio J. Doblas Viso. Universidad de Malaga. -#@date 2011/08/03 -# error code return 3 -function btogSetFsBase () -{ - -# Dependencias -[ -z "$OSCODENAME" ] && btogGetOsInfo $TYPECLIENT - -echoAndLog "$FUNCNAME: Iniciando la generación del sistema de archivos " - -#Montamos el dispositivo virtual en su punto de montaje. -mount | grep $BTROOTFSMNT || mount $BTROOTFSIMG $BTROOTFSMNT -o loop,offset=32256 -mount | grep $BTROOTFSMNT && echoAndLog "$FUNCNAME: mount $BTROOTFSIMG $BTROOTFSMNT -o loop,offset=32256 OK " || errorAndLog "$FUNCNAME: mount $BTROOTFSIMG $BTROOTFSMNT -o loop,offset=32256 : FAILURE " - -debootstrap --arch=$OSARCH --components=main,universe ${OSCODENAME} ${BTROOTFSMNT} ${OSHTTP} -#debootstrap --arch=$OSARCH ${OSCODENAME} ${BTROOTFSMNT} http://archive.ubuntu.com/ubuntu/ - - -if [ $? -ne 0 ]; then - errorAndLog "$FUNCNAME: debootstrap --arch=$OSARCH --components=main,universe $OSCODENAME $BTROOTFSMNT $OSHTTP : ha fallado!" - mount | grep $BTROOTFSMNT && umount $BTROOTFSMNT - return 3 -else - echoAndLog "$FUNCNAME: debootstrap --include=linux-image-${OSRELEASE},linux-headers-${OSRELEASE} --arch=$OSARCH --components=main,universe $OSCODENAME $BTROOTFSMNT $OSHTTP : ok" - mount | grep $BTROOTFSMNT && umount $BTROOTFSMNT - return 0 -fi - sleep 5 - -##preubas revisar OSRELEASE -#debootstrap --include=linux-image-${OSRELEASE},linux-headers-${OSRELEASE} --arch=$OSARCH --components=main,universe ${OSCODENAME} ${OGCLIENTMOUNT} ${OSHTTP} -#debootstrap --include=linux-image-${OSRELEASE} --arch=i386 --variant=minbase $OSVERSION $OGCLIENTMOUNT http://es.archive.ubuntu.com/ubuntu/ -#debootstrap --variant=minbase --include=linux-image-${OGRELEASE} --arch=i386 $OGVERSION $OGCLIENTMOUNT http://es.archive.ubuntu.com/ubuntu/ -#echo debootstrap --include=linux-image-${OGRELEASE},dbus --arch=i386 --components=main,universe $OGVERSION $OGCLIENTMOUNT http://es.archive.ubuntu.com/ubuntu/ -} - -#btogSetFsAcces: habilita el acceso al sistema root del cliente con schroot -#@version 0.9 - Prototipo de sistema operativo multiarranque de opengnsys. -#@author Antonio J. Doblas Viso. Universidad de Malaga. -#@date 2010/05/24 -#@version 1.0 - Compatibilidad OpengGnsys X. -#@author Antonio J. Doblas Viso. Universidad de Malaga. -#@date 2011/08/03 -function btogSetFsAccess() -{ -echoAndLog "$FUNCNAME: Iniciando la configuración del schroot " -cp /etc/schroot/schroot.conf /etc/schroot/schroot.conf.`getDateTime` -cat << EOF > /etc/schroot/schroot.conf -[IMGogclient] -type=loopback -file=/var/lib/tftpboot/ogclient/ogclient.img -description=ogclient Ubuntu image -#priority=1 -users=root -groups=root -root-groups=root -mount-options=-o offset=32256 -root-users=root -EOF - -cp /etc/schroot/mount-defaults /etc/schroot/mount-defaults.`getDateTime` -cat << EOF > /etc/schroot/mount-defaults -# mount.defaults: static file system information for chroots. -# Note that the mount point will be prefixed by the chroot path -# (CHROOT_PATH) -# -# -proc /proc proc defaults 0 0 -#procbususb /proc/bus/usb usbfs defaults 0 0 -#/dev /dev none rw,bind 0 0 -/dev/pts /dev/pts none rw,bind 0 0 -/dev/shm /dev/shm none rw,bind 0 0 -#/home /home none rw,bind 0 0 -/tmp /tmp none rw,bind 0 0 -EOF - -rm /etc/schroot/setup.d/*chrootname - -echoAndLog "$FUNCNAME: Finalizado: OK " -return 0 -} - - - -#btogFsInitr genera un initrd. -#@version 0.9 - Prototipo de sistema operativo multiarranque de opengnsys. -#@author Antonio J. Doblas Viso. Universidad de Malaga. -#@date 2010/05/24 -#@version 1.0 - Compatibilidad OpengGnsys X. -#@author Antonio J. Doblas Viso. Universidad de Malaga. -#@date 2011/08/03 -function btogFsInitrd () -{ -#DEPENDENCIAS -[ -z "$BTROOTFSIMG" ] && btogGetVar -[ -z "$OSCODENAME" ] && btogGetOsInfo $TYPECLIENT - -cd / -schroot -c IMGogclient -- /usr/bin/boot-tools/boottoolsInitrdGenerate.sh -echo "cp /tmp/*-${OSRELEASE} ${BTTARGETDIR}" -cp /tmp/*-${OSRELEASE} ${BTTARGETDIR} -#Creando copias de las version oginitr y ogvmlinux previas -TIMEMOD=`date +%Y%m%d-%H%M%S` -[ -f $BTTARGETDIR/oginitrd.img ] && mv $BTTARGETDIR/oginitrd.img $BTTARGETDIR/oginitrd.img.$TIMEMOD; mv $BTTARGETDIR/oginitrd.img.sum $BTTARGETDIR/oginitrd.img.sum.$TIMEMOD -[ -f $BTTARGETDIR/ogvmlinuz ] && mv $BTTARGETDIR/ogvmlinuz $BTTARGETDIR/ogvmlinuz.$TIMEMOD; mv $BTTARGETDIR/ogvmlinuz.sum $BTTARGETDIR/ogvmlinuz.sum.$TIMEMOD - -#copiando -cp /tmp/initrd.img-${OSRELEASE} ${BTTARGETDIR}/oginitrd.img -cp /tmp/vmlinuz-${OSRELEASE} ${BTTARGETDIR}/ogvmlinuz -#generamos md5 del kernel y del initrd. -DATASUM=`md5sum "${BTTARGETDIR}/oginitrd.img" | cut -f1 -d" "` -echo $DATASUM > ${BTTARGETDIR}/oginitrd.img.sum -DATASUM=`md5sum "${BTTARGETDIR}/ogvmlinuz" | cut -f1 -d" "` -echo $DATASUM > ${BTTARGETDIR}/ogvmlinuz.sum -#Damos permisos -cd - -chmod -R 755 $BTTARGETDIR -} - -#btogFsSqfs convierte el sistema root en sqfs -#@version 0.9 - Prototipo de sistema operativo multiarranque de opengnsys. -#@author Antonio J. Doblas Viso. Universidad de Malaga. -#@date 2010/05/24 -#@version 1.0 - Compatibilidad OpengGnsys X. -#@author Antonio J. Doblas Viso. Universidad de Malaga. -#@date 2011/08/03 -function btogFsSqfs () -{ -#Dependencias. -[ -z "$BTROOTFSIMG" ] && btogGetVar -[ -z "$OSCODENAME" ] && btogGetOsInfo $TYPECLIENT - -echoAndLog "$FUNCNAME: Iniciando la creación del sistema de archivos en sqfs " -# si ya existe un sqfs lo renombramos -[ -f $BTTARGETDIR/ogclient.sqfs ] && mv $BTTARGETDIR/ogclient.sqfs $BTTARGETDIR/ogclient.sqfs.`date +%Y%m%d-%H%M%S` -mount | grep $BTROOTFSMNT || mount $BTROOTFSIMG $BTROOTFSMNT -o loop,offset=32256 -mksquashfs $BTROOTFSMNT $BTTARGETDIR/ogclient.sqfs -mount | grep $BTROOTFSMNT && umount $BTROOTFSMNT -chmod 744 $BTTARGETDIR/ogclient.sqfs -DATASUM=`md5sum "${BTTARGETDIR}/ogclient.sqfs" | cut -f1 -d" "` -echo $DATASUM > ${BTTARGETDIR}/ogclient.sqfs.sum -} - - - - - -# btogIsoGenerator genera la iso del cliente -#@version 0.9 - Prototipo de sistema operativo multiarranque de opengnsys. -#@author Antonio J. Doblas Viso. Universidad de Malaga. -#@date 2010/05/24 -#@version 1.0 - Compatibilidad OpengGnsys X. -#@author Antonio J. Doblas Viso. Universidad de Malaga. -#@date 2011/08/03 -function btogIsoGenerator { -#Dependencias. -#[ -z "$BTROOTFSIMG" ] && btogGetVar -btogGetVar -#Preparamos los gestores de arranque -mkdir -p /tmp/iso/isolinux -cp -av /usr/lib/syslinux/* /tmp/iso/isolinux/ -if [ -d /usr/share/gpxe ]; then - cp -av /usr/share/gpxe/* /tmp/iso/isolinux - PXE=gpxe -elif [ -d /usr/lib/ipxe ]; then - cp -av /usr/lib/ipxe/* /tmp/iso/isolinux - PXE=ipxe -fi - -# Si existe el fichero ISO, montarlo para extraer isolinux.bin. -if [ -f /tmp/iso/isolinux/$PXE.iso ]; then - mkdir -p /tmp/iso/isolinux/mount - mount -o loop /tmp/iso/isolinux/$PXE.iso /tmp/iso/isolinux/mount - cp -va /tmp/iso/isolinux/mount/* /tmp/iso/isolinux - umount /tmp/iso/isolinux/mount - rmdir /tmp/iso/isolinux/mount - rm -f /tmp/iso/isolinux/$PXE.iso -fi - -cat << FIN > /tmp/iso/isolinux/isolinux.cfg -DEFAULT menu.c32 -PROMPT 0 -ALLOWOPTIONS 1 - -MENU TITLE OpenGnsys ${NAMEISOCLIENT} - - -LABEL 0 -MENU LABEL ogLive vga irqpool acpi ogdebug ip:none -KERNEL /ogclient/linuxISO -APPEND initrd=/ogclient/oginitrd.img ro vga=788 irqpoll acpi=on boot=oginit og2nd=sqfs ogprotocol=local ogactiveadmin=true ogdebug=true ip=none - -LABEL 1 -MENU LABEL ogLive irqpoll acpi ip:none -KERNEL /ogclient/linuxISO -APPEND initrd=/ogclient/oginitrd.img ro irqpoll acpi=on boot=oginit og2nd=sqfs ogprotocol=local ogactiveadmin=true ogdebug=false ip=none - -LABEL 2 -MENU LABEL ogLive acpi debug ip=dhcp -KERNEL /ogclient/linuxISO -APPEND initrd=/ogclient/oginitrd.img ro acpi=on boot=oginit og2nd=sqfs ogprotocol=local ogactiveadmin=true ogdebug=true ip=dhcp - -LABEL 3 -MENU LABEL ogLive ip=dhcp -KERNEL /ogclient/linuxISO -APPEND initrd=/ogclient/oginitrd.img ro acpi=off boot=oginit og2nd=sqfs ogprotocol=local ogactiveadmin=true ogdebug=false ip=dhcp - -LABEL 4 -MENU LABEL local -localboot 0x80 -append - - -label 5 -MENU LABEL Network boot via gPXE lkrn -KERNEL $PXE.lkrn - -label 6 -MENU LABEL Network boot via gPXE usb -KERNEL $PXE.usb - -label 7 -MENU LABEL Network boot via gPXE pxe -KERNEL $PXE.pxe - -label 8 -MENU LABEL Network boot via gPXE iso -KERNEL $PXE.iso - -label 9 -MENU LABEL pxe -KERNEL /clonezilla/live/vmlinuz1 -APPEND initrd=/clonezilla/live/initrd1.img boot=live union=aufs noswap vga=788 ip=frommedia -FIN - - - - -# preparamos el directorio boot-tools. -mkdir -p /tmp/iso/ogclient -cp ${BTTARGETDIR}/ogclient.sqfs /tmp/iso/ogclient/ -cp ${BTTARGETDIR}/ogclient.sqfs.sum /tmp/iso/ogclient/ -cp ${BTTARGETDIR}/ogvmlinuz /tmp/iso/ogclient/ -cp ${BTTARGETDIR}/ogvmlinuz.sum /tmp/iso/ogclient/ -cp ${BTTARGETDIR}/ogvmlinuz /tmp/iso/ogclient/linuxISO -cp ${BTTARGETDIR}/ogvmlinuz.sum /tmp/iso/ogclient/linuxISO.sum -cp ${BTTARGETDIR}/oginitrd.img /tmp/iso/ogclient/ -cp ${BTTARGETDIR}/oginitrd.img.sum /tmp/iso/ogclient/ -#el ogclienteToISO debe tener una copia del ogvmlinuz como linuxISO -#cp -prv /var/lib/tftpboot/ogclientToIso/* /tmp/iso/ogclient - -cd /tmp -#TIMEMOD=`date +%Y%m%d-%H%M%S` -####Generamos la iso -echo "mkisofs -V ogClient -o ${NAMEISOCLIENT}.iso -b isolinux/isolinux.bin -c isolinux/boot.cat -J -no-emul-boot -boot-load-size 4 -boot-info-table /tmp/iso" -mkisofs -V ogClient -o ${NAMEISOCLIENT}.iso -b isolinux/isolinux.bin -c isolinux/boot.cat -J -no-emul-boot -boot-load-size 4 -boot-info-table /tmp/iso -##Generamos el md5 -DATASUM=`md5sum ${NAMEISOCLIENT}.iso | cut -f1 -d" "` -echo $DATASUM > ${NAMEISOCLIENT}.iso.sum - -mv /tmp/${NAMEISOCLIENT}.iso /var/lib/tftpboot/ogclient/ -mv /tmp/${NAMEISOCLIENT}.iso.sum /var/lib/tftpboot/ogclient/ - - -### vi /etc/grub.d/40_custom -## -#menuentry "og cache " { -#set root=(hd0,4) -#linux /ogvmlinuz ro vga=788 irqpoll acpi=on boot=oginit og2nd=sqfs ogprotocol=local ogactiveadmin=true ogdebug=true ip=none -#initrd /oginitrd.img -#} - -} - - -#@version 0.9 - Prototipo de sistema operativo multiarranque de opengnsys. -#@author Antonio J. Doblas Viso. Universidad de Malaga. -#@date 2010/05/24 -#@version 1.0 - Compatibilidad OpengGnsys X. -#@author Antonio J. Doblas Viso. Universidad de Malaga. -#@date 2011/08/03 -function btogIsoTest { -#/tmp/iso -qemu -m 256 -boot d -cdrom ogClient.iso -} - - - - -function getDateTime() -{ - echo `date +%Y%m%d-%H%M%S` -} - -# Escribe a fichero y muestra por pantalla -function echoAndLog() -{ - echo $1 - FECHAHORA=`getDateTime` - echo "$FECHAHORA;$SSH_CLIENT;$1" >> $LOG_FILE -} - -function errorAndLog() -{ - echo "ERROR: $1" - FECHAHORA=`getDateTime` - echo "$FECHAHORA;$SSH_CLIENT;ERROR: $1" >> $LOG_FILE -} - - - - - - - - - - -############### No usados en el instalador, solo en actualizaciones. -#/** -# ogClientMount [str_program] -#@brief Acceso al 2nd FS del cliente desde el Servidor Opengnsys -#@param 1 Opciona: scripts o programa a ejecutar para automatizaciones -#@return Si no hay parametros: login de acceso. -#@return con un parametro: La salida del programa ejecutado -#@exception -#@note -#@todo -#@version 0.9 - Primera versión para OpenGnSys -#@author Antonio J. Doblas Viso, Universidad de Málaga -#@date 2010/02/15 -#*/ ## -function ogClientMount () -{ - #TODO comprobar que OGFILE y OGFILEMOUNT existe. - mount | grep $OGCLIENTFILE > /dev/null || mount $OGCLIENTFILE $OGCLIENTMOUNT -o loop,offset=32256 - mount | grep $OGCLIENTMOUNT/proc > /dev/null || mount --bind /proc $OGCLIENTMOUNT/proc - mount | grep $OGCLIENTMOUNT/sys > /dev/null || mount --bind /sys $OGCLIENTMOUNT/sys - mount | grep $OGCLIENTMOUNT/tmp > /dev/null || mount --bind /tmp $OGCLIENTMOUNT/tmp - mount | grep $OGCLIENTMOUNT/dev > /dev/null || mount --bind /dev $OGCLIENTMOUNT/dev - mount | grep $OGCLIENTMOUNT/dev/pts > /dev/null || mount --bind /dev/pts $OGCLIENTMOUNT/dev/pts - - - [ $# = 0 ] && $(chroot $OGCLIENTMOUNT /sbin/getty 38400 `tty`) - [ $# = 1 ] && chroot $OGCLIENTMOUNT $1 - -} - -#/** -# ogClientUnmount -#@brief Desmonta el 2nd FS del cliente desde el Servidor Opengnsys -#@param -#@return -#@exception -#@note -#@todo -#@version 0.9 - Primera versión para OpenGnSys -#@author Antonio J. Doblas Viso, Universidad de Málaga -#@date 2010/02/15 -#*/ ## - -function ogClientUnmount () -{ - cd /tmp - echo "desmontando cliente espere" - sleep 5 - mount | grep $OGCLIENTMOUNT/dev > /dev/null && umount $OGCLIENTMOUNT/dev || ogClientUnmount - mount | grep $OGCLIENTMOUNT/dev/pts > /dev/null && umount $OGCLIENTMOUNT/dev/pts || ogClientUnmount - mount | grep $OGCLIENTMOUNT/proc > /dev/null && umount $OGCLIENTMOUNT/proc || ogClientUnmount - mount | grep $OGCLIENTMOUNT/sys > /dev/null && umount $OGCLIENTMOUNT/sys || ogClientUnmount - mount | grep $OGCLIENTMOUNT/tmp > /dev/null && umount $OGCLIENTMOUNT/tmp || ogClientUnmount - mount | grep $OGCLIENTMOUNT > /dev/null && umount $OGCLIENTMOUNT || ogClientUnmount - #-d -f -l - -} diff --git a/includes/etc/ssh/sshd_config b/includes/etc/ssh/sshd_config index 8090068..a7d6e53 100644 --- a/includes/etc/ssh/sshd_config +++ b/includes/etc/ssh/sshd_config @@ -10,12 +10,6 @@ Protocol 2 # HostKeys for protocol version 2 HostKey /etc/ssh/ssh_host_rsa_key HostKey /etc/ssh/ssh_host_dsa_key -#Privilege Separation is turned on for security -UsePrivilegeSeparation no - -# Lifetime and size of ephemeral version 1 server key -KeyRegenerationInterval 3600 -ServerKeyBits 768 # Logging SyslogFacility AUTH @@ -26,14 +20,11 @@ LoginGraceTime 120 PermitRootLogin yes StrictModes yes -RSAAuthentication yes PubkeyAuthentication yes #AuthorizedKeysFile %h/.ssh/authorized_keys # Don't read the user's ~/.rhosts and ~/.shosts files IgnoreRhosts yes -# For this to work you will also need host keys in /etc/ssh_known_hosts -RhostsRSAAuthentication no # similar for protocol version 2 HostbasedAuthentication no # Uncomment if you don't trust ~/.ssh/known_hosts for RhostsRSAAuthentication diff --git a/mkoglive.py b/mkoglive.py index 9283dce..2901e1e 100755 --- a/mkoglive.py +++ b/mkoglive.py @@ -58,7 +58,6 @@ def _get_pxepkg(): def _mkrootfs(): logger.info ('Stage 3.1 - create, partition and format the rootfs') rc = subprocess.run (f'file "{btrootfsimg}" |grep -q "partition 1 *: ID=0x83"', shell=True).returncode - print (rc) if (rc): ## 'file|grep' failed try: btog.mkrootfs (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, bttargetdir, osarch) except Exception as e: From ef4ff2a739395d79c0902eb6383d573a25f42db5 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Mon, 19 Aug 2024 12:27:04 +0200 Subject: [PATCH 35/45] refs #596 log subprocess' output as it is generated --- boottools/utils.py | 46 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/boottools/utils.py b/boottools/utils.py index b1385f2..f382bfa 100644 --- a/boottools/utils.py +++ b/boottools/utils.py @@ -3,23 +3,53 @@ import subprocess import re import os import configparser +import selectors logger = logging.getLogger ('boottools') def run (args): - cp = subprocess.run (args, text=True, capture_output=True) - if cp.returncode: - logger.error ('command "{}" failed with rc "{}"'.format (' '.join(args), cp.returncode)) + stdout = stderr = '' + logger.debug ('run 10 args "{}"'.format (' '.join(args))) + sel = selectors.DefaultSelector() + p = subprocess.Popen (args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + sel.register (p.stdout.fileno(), selectors.EVENT_READ) + sel.register (p.stderr.fileno(), selectors.EVENT_READ) + while True: + events = sel.select() + #logger.debug (f'got {len(events)} events') + for key, _ in events: + if key.fileobj == p.stdout.fileno(): + #logger.debug (f'reading from stdout') + line = p.stdout.readline() + if not line: break + stdout += line + logger.debug (line.rstrip()) + elif key.fileobj == p.stderr.fileno(): + #logger.debug (f'reading from stderr') + line = p.stderr.readline() + if not line: break + stderr += line + logger.warn (line.rstrip()) + if p.poll() != None: + #logger.debug ('process exited, breaking loop') + break + sel.close() + #p.stdout.close() + #p.stderr.close() - logger.error ('stdout follows:') - for i in cp.stdout.strip().split('\n'): logger.error (' ' + i) + logger.debug (f'p.returncode {p.returncode}') + if p.returncode: + logger.error ('command "{}" failed with rc "{}"'.format (' '.join(args), p.returncode)) + + #logger.error ('stdout follows:') + #for i in cp.stdout.strip().split('\n'): logger.error (' ' + i) logger.error ('stderr follows:') for i in cp.stderr.strip().split('\n'): logger.error (' ' + i) - raise Exception ('command "{}" failed with rc "{}"'.format (' '.join(args), cp.returncode)) - stdout = cp.stdout.strip() - stderr = cp.stderr.strip() + raise Exception ('command "{}" failed with rc "{}"'.format (' '.join(args), p.returncode)) + stdout = stdout.strip() + stderr = stderr.strip() return stdout, stderr def grep (regex, file): From bb2d57b6379e15423874bad05c748fd30ffb4bef Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Tue, 20 Aug 2024 10:13:14 +0200 Subject: [PATCH 36/45] refs #596 correctly handle invalid param, make mkisofs quieter --- boottools/btog.py | 14 ++++++++------ mkoglive.py | 3 +++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/boottools/btog.py b/boottools/btog.py index 79d01e5..844fb3d 100644 --- a/boottools/btog.py +++ b/boottools/btog.py @@ -151,8 +151,12 @@ def GetOsInfo1 (type_client): osarch, _=utils.run (['dpkg', '--print-architecture']) oshttp='http://es.archive.ubuntu.com/ubuntu/' else: # Parámetro desconocido - print ('Parámetro no válido.') - os.exit (1) + logger.error ('Parámetro no válido.') + osdistrib=None + oscodename=None + osrelease=None + osarch=None + oshttp=None return osdistrib, oscodename, osrelease, osarch, oshttp def GetOsInfo2 (ogrepo_dir, versionboottools, oscodename, osrelease, osarch): @@ -199,8 +203,6 @@ def mkrootfs (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, btt logger.info ('Partitioning disk image') stdout, _ = utils.run (['bash', '-c', f"echo $'n\np\n1\n\n\nt\n83\nw' |fdisk {diskloop}"]) - logger.debug (f'fdisk stdout follows:') - for i in stdout.strip().split('\n'): logger.debug (' ' + i) time.sleep (3) logger.debug ('losetup --detach') @@ -430,8 +432,8 @@ def mkisofs (pxepkg, isolinux_tpl, bttargetdir, nameisoclient): oldpwd = os.getcwd() os.chdir ('/tmp') - logger.debug (f'mkisofs -V ogClient -o {nameisoclient}.iso -b isolinux/isolinux.bin -c isolinux/boot.cat -J -no-emul-boot -boot-load-size 4 -boot-info-table /tmp/iso') - utils.run (['mkisofs', '-V', 'ogClient', '-o', f'{nameisoclient}.iso', '-b', 'isolinux/isolinux.bin', '-c', 'isolinux/boot.cat', '-J', '-no-emul-boot', '-boot-load-size', '4', '-boot-info-table', '/tmp/iso']) + logger.debug (f'mkisofs -quiet -V ogClient -o {nameisoclient}.iso -b isolinux/isolinux.bin -c isolinux/boot.cat -J -no-emul-boot -boot-load-size 4 -boot-info-table /tmp/iso') + utils.run (['mkisofs', '-quiet', '-V', 'ogClient', '-o', f'{nameisoclient}.iso', '-b', 'isolinux/isolinux.bin', '-c', 'isolinux/boot.cat', '-J', '-no-emul-boot', '-boot-load-size', '4', '-boot-info-table', '/tmp/iso']) utils.write_md5 (f'{nameisoclient}.iso') os.chdir (oldpwd) diff --git a/mkoglive.py b/mkoglive.py index 2901e1e..1040b00 100755 --- a/mkoglive.py +++ b/mkoglive.py @@ -155,6 +155,9 @@ ogrepo_dir = '/tmp/ogrepo' clone_client_dirs (ogrepo_url, ogrepo_branch, ogrepo_dir) osdistrib, oscodename, osrelease, osarch, oshttp = btog.GetOsInfo1 (type_client) +if osdistrib is None: + logger.error ('GetOsInfo1() failed') + sys.exit (1) btdir, bttargetdir, btrootfsimg, btrootfsmnt, btrootfsimglabel, log_file, versionboottools, btvirtualdisksize = btog.GetVar (osarch) gitrelease, nameisoclient, namehostclient = btog.GetOsInfo2 (ogrepo_dir, versionboottools, oscodename, osrelease, osarch) logger.info (':'.join ([osdistrib, oscodename, osrelease, osarch, oshttp])) From d4564f6ad996d46b124d00025857f0cf87674dd3 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Tue, 20 Aug 2024 10:31:44 +0200 Subject: [PATCH 37/45] refs #596 take cmdline parameters, make wget quieter --- chroot-tasks.py | 20 +++++++++++--------- mkoglive.py | 22 ++++++++++++---------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/chroot-tasks.py b/chroot-tasks.py index fb63750..056cb36 100755 --- a/chroot-tasks.py +++ b/chroot-tasks.py @@ -10,11 +10,6 @@ import subprocess from boottools import utils, apt -config = utils.read_config ('mkoglive.cfg') -if config is None: - sys.exit (1) -debconf_settings = config['General'].get ('debconf_settings') - def _oghook_deactivate(): #Desactivamos el hook del oginitrd.img para evitar problemas, al final de este escripts se activará os.rename ('/etc/initramfs-tools/hooks/oghooks', '/etc/initramfs-tools/oghooks') @@ -77,7 +72,7 @@ def boottoolsSoftwareCompile(): print ('ms-sys') try: utils.run (['which', 'ms-sys']) except: - utils.run (['wget', 'https://sourceforge.net/projects/ms-sys/files/latest/download', '-O', 'ms-sys.tar.gz']) + utils.run (['wget', '--quiet', 'https://sourceforge.net/projects/ms-sys/files/latest/download', '-O', 'ms-sys.tar.gz']) utils.run (['tar', '-xpzf', 'ms-sys.tar.gz']) mssys_dir = subprocess.run (['tar tzf ms-sys.tar.gz |head -n 1'], shell=True, capture_output=True, text=True).stdout.strip() print (f'mssys_dir "{mssys_dir}"') @@ -88,7 +83,7 @@ def boottoolsSoftwareCompile(): print ('spartlnx') try: utils.run (['which', 'spartl64.run']) except: - utils.run (['wget', 'http://damien.guibouret.free.fr/savepart.zip']) + utils.run (['wget', '--quiet', 'http://damien.guibouret.free.fr/savepart.zip']) utils.run (['unzip', '-o', 'savepart.zip', '-d', '/sbin/', 'spartl64.run']) utils.run (['mkdir', '/usr/share/doc/spartlnx']) utils.run (['unzip', '-j', '-o', 'savepart.zip', '-d', '/usr/share/doc/spartlnx/', 'doc/en/*']) @@ -117,9 +112,16 @@ def boottoolsInitrdGenerate (osrelease): if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument ('--osarch', help='OS architecture', action='store', required=True) - parser.add_argument ('--osrelease', help='OS release', action='store', required=True) + parser.add_argument ('--osarch', help='OS architecture', action='store', required=True) + parser.add_argument ('--osrelease', help='OS release', action='store', required=True) + parser.add_argument ('--config', help='Path to configuration file', action='store') args = parser.parse_args() + + config = utils.read_config (args.config or 'mkoglive.cfg') + if config is None: + sys.exit (1) + debconf_settings = config['General'].get ('debconf_settings') + boottoolsSoftwareInstall (args.osarch, args.osrelease) boottoolsSoftwareCompile() boottoolsInitrdGenerate (args.osrelease) diff --git a/mkoglive.py b/mkoglive.py index 1040b00..a926741 100755 --- a/mkoglive.py +++ b/mkoglive.py @@ -7,6 +7,7 @@ import subprocess import glob import stat import shutil +import argparse curdir = os.path.dirname (__file__) sys.path.insert (0, curdir) @@ -14,9 +15,7 @@ from boottools import utils, apt, btog os.chdir (curdir) -def _logging(): - #logging.root.handlers = [] - +def _logging (lvl='INFO'): numeric_level = getattr (logging, lvl.upper(), None) if numeric_level is None: numeric_level = getattr (logging, 'INFO') @@ -100,10 +99,8 @@ def _copy_og_files (btrootfsmnt, osdistrib, oscodename): def _chroot_tasks (curdir, osrelease, osarch): logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/chroot-tasks.py --osrelease "{osrelease}" --osarch "{osarch}"\'') - stdout, _ = utils.run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/chroot-tasks.py', '--osrelease', osrelease, '--osarch', osarch]) - logger.debug (f'chroot-tasks.py stdout follows:') + stdout, _ = utils.run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/chroot-tasks.py', '--osrelease', osrelease, '--osarch', osarch, '--config', args.config or 'mkoglive.cfg']) ## esto deja initrd.img-6.8.0-31-generic y vmlinuz-6.8.0-31-generic en /tmp - for i in stdout.strip().split ('\n'): logger.debug (' ' + i) def _ssh_stuff(): _mount_rootfs() @@ -128,15 +125,20 @@ def _mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsmnt, pxepkg, isoli btog.mkisofs (pxepkg, isolinux_tpl, bttargetdir, nameisoclient) +parser = argparse.ArgumentParser() +parser.add_argument ('--loglevel', help='Log level', action='store') +parser.add_argument ('--codename', help='OS codename', action='store') +parser.add_argument ('--config', help='Path to configuration file', action='store') +args = parser.parse_args() -config = utils.read_config ('mkoglive.cfg') +config = utils.read_config (args.config or 'mkoglive.cfg') if config is None: sys.exit (1) isolinux_tpl = config['General'].get ('isolinux_template') -lvl = config['General'].get ('logging_level') +cfg_loglevel = config['General'].get ('logging_level') -logger = _logging() -type_client = sys.argv[1] if len (sys.argv)>1 else 'host' +logger = _logging (args.loglevel or cfg_loglevel) +type_client = args.codename or 'host' if os.getuid(): logger.error ('ERROR: this program must run under root privileges!!') From f002c5678ac236cac1a11e333903552448355d00 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Tue, 20 Aug 2024 10:38:44 +0200 Subject: [PATCH 38/45] refs #597 use docker to create the oglive images --- .dockerignore | 11 ++++++++++ Dockerfile | 38 ++++++++++++++++++++++++++++++++ boottools/btog.py | 54 ++-------------------------------------------- boottools/utils.py | 11 ++++++---- mkoglive.py | 20 ++--------------- mount-defaults | 12 +++++++++++ schroot.conf | 10 +++++++++ 7 files changed, 82 insertions(+), 74 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 mount-defaults create mode 100644 schroot.conf diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..9f8c34d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +.git +.gitignore +.dockerignore +__pycache__ +Dockerfile +INSTALL.es.txt +Jenkinsfile +kernelParameterOG.es.txt +ogclientImg2Sqfs +ogclientSqfs2Img +*.swp diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..df8cd7a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,38 @@ +FROM ubuntu:noble + +ARG OPENGNSYS_BRANCH=main + +RUN apt-get update && apt-get -y install \ + debootstrap \ + fdisk \ + file \ + genisoimage \ + git \ + ipxe \ + jq \ + lsof \ + python3 \ + python3-apt \ + qemu-utils \ + rsync \ + schroot \ + squashfs-tools \ + subversion \ + syslinux \ + syslinux-efi \ + syslinux-utils \ + && rm -rf /var/lib/apt/lists/* + +RUN groupadd opengnsys +RUN mkdir -p /opt/opengnsys/client /tmp/opengnsys/oglive_builder /var/lib/tftpboot/ +RUN ln -fs /var/lib/tftpboot /opt/opengnsys + +## take client stuff from the OG repo +RUN git clone -c http.sslVerify=false --branch ${OPENGNSYS_BRANCH} https://ognproject.evlt.uma.es/gitea/opengnsys/opengnsys.git /tmp/ogrepo +RUN rsync -aH /tmp/ogrepo/client/engine /tmp/ogrepo/client/shared /tmp/opengnsys/ + +RUN echo foo4 +COPY . /tmp/opengnsys/oglive_builder/ +RUN cp /tmp/opengnsys/oglive_builder/schroot.conf /tmp/opengnsys/oglive_builder/mount-defaults /etc/schroot/ && echo '' >/etc/schroot/default/nssdatabases && rm -f /etc/schroot/setup.d/*chrootname + +ENTRYPOINT ["/tmp/opengnsys/oglive_builder/mkoglive.py"] diff --git a/boottools/btog.py b/boottools/btog.py index 844fb3d..64350dc 100644 --- a/boottools/btog.py +++ b/boottools/btog.py @@ -11,34 +11,6 @@ from . import utils logger = logging.getLogger ('boottools') -schroot_conf_tpl = """ -[IMGogclient] -type=loopback -file=/var/lib/tftpboot/ogclient/ogclient.img -description=ogclient Ubuntu image -#priority=1 -users=root -groups=root -root-groups=root -mount-options=-o offset=32256 -root-users=root -""".strip() - -mount_defaults_tpl = """ -# mount.defaults: static file system information for chroots. -# Note that the mount point will be prefixed by the chroot path -# (CHROOT_PATH) -# -# -proc /proc proc defaults 0 0 -#procbususb /proc/bus/usb usbfs defaults 0 0 -#/dev /dev none rw,bind 0 0 -/dev/pts /dev/pts none rw,bind 0 0 -/dev/shm /dev/shm none rw,bind 0 0 -#/home /home none rw,bind 0 0 -/tmp /tmp none rw,bind 0 0 -""".strip() - def GetVar (osarch): btdir = '/tmp/opengnsys/oglive_builder' bttargetdir = '/var/lib/tftpboot/ogclient/' @@ -231,28 +203,6 @@ def mkrootfs (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, btt logger.info (f'"{btrootfsimg}" "{btvirtualdisksize}" MB : OK') -#SetFsAcces: habilita el acceso al sistema root del cliente con schroot -def configure_schroot (btrootfsimg): - if (utils.grep (btrootfsimg, '/etc/schroot/schroot.conf')): - return - logger.info ('Iniciando la configuración del schroot') - - if (os.path.exists ('/etc/schroot/schroot.conf')): - shutil.copy ('/etc/schroot/schroot.conf', '/etc/schroot/schroot.conf.bak') - with open ('/etc/schroot/schroot.conf', 'w') as f: - f.write (schroot_conf_tpl + '\n') - - if (os.path.exists ('/etc/schroot/mount-defaults')): - shutil.copy ('/etc/schroot/mount-defaults', '/etc/schroot/mount-defaults.bak') - with open ('/etc/schroot/mount-defaults', 'w') as f: - f.write (mount_defaults_tpl + '\n') - - for i in glob.glob ('/etc/schroot/setup.d/*chrootname'): - os.unlink (i) - - logger.info ('Finalizado: OK') - return 0 - # debootstrap: Genera el sistema root base con debootstrap # trabaja sobre un rootfs ya montado def debootstrap (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp): @@ -438,8 +388,8 @@ def mkisofs (pxepkg, isolinux_tpl, bttargetdir, nameisoclient): utils.write_md5 (f'{nameisoclient}.iso') os.chdir (oldpwd) - os.rename (f'/tmp/{nameisoclient}.iso' , f'/var/lib/tftpboot/ogclient/{nameisoclient}.iso') - os.rename (f'/tmp/{nameisoclient}.iso.sum', f'/var/lib/tftpboot/ogclient/{nameisoclient}.iso.sum') + shutil.move (f'/tmp/{nameisoclient}.iso' , f'/var/lib/tftpboot/ogclient/{nameisoclient}.iso') + shutil.move (f'/tmp/{nameisoclient}.iso.sum', f'/var/lib/tftpboot/ogclient/{nameisoclient}.iso.sum') #def __unused_boottoolsBootGraphics(): # find /tmp/opengnsys_installer/ -name .svn -type d -exec rm -fr {} \; 2>/dev/null; diff --git a/boottools/utils.py b/boottools/utils.py index f382bfa..5e0a7fc 100644 --- a/boottools/utils.py +++ b/boottools/utils.py @@ -36,20 +36,23 @@ def run (args): sel.close() #p.stdout.close() #p.stderr.close() + #p.wait() + stdout = stdout.strip() + stderr = stderr.strip() logger.debug (f'p.returncode {p.returncode}') if p.returncode: logger.error ('command "{}" failed with rc "{}"'.format (' '.join(args), p.returncode)) #logger.error ('stdout follows:') - #for i in cp.stdout.strip().split('\n'): logger.error (' ' + i) + #if stdout: + # for i in stdout.split('\n'): logger.error (' ' + i) logger.error ('stderr follows:') - for i in cp.stderr.strip().split('\n'): logger.error (' ' + i) + if stderr: + for i in stderr.split('\n'): logger.error (' ' + i) raise Exception ('command "{}" failed with rc "{}"'.format (' '.join(args), p.returncode)) - stdout = stdout.strip() - stderr = stderr.strip() return stdout, stderr def grep (regex, file): diff --git a/mkoglive.py b/mkoglive.py index a926741..12b5704 100755 --- a/mkoglive.py +++ b/mkoglive.py @@ -30,11 +30,6 @@ def _logging (lvl='INFO'): ) return logging.getLogger ('boottools') -def clone_client_dirs (ogrepo_url, ogrepo_branch, ogrepo_dir): - if not os.path.exists ('/tmp/opengnsys/engine'): - utils.run (['git', 'clone', '-c', 'http.sslVerify=false', '--branch', ogrepo_branch, ogrepo_url, ogrepo_dir]) - utils.run (['rsync', '-aH', f'{ogrepo_dir}/client/engine', f'{ogrepo_dir}/client/shared', '/tmp/opengnsys/']) - def _mount_rootfs(): global btrootfsimg, btrootfsmnt try: utils.mount (btrootfsimg, btrootfsmnt, opts=['-o', 'loop,offset=32256']) @@ -63,15 +58,11 @@ def _mkrootfs(): logger.error (str (e)) sys.exit (1) -def _schroot (btrootfsimg): - logger.info ('Stage 3.2 - configure schroot in the VM') - btog.configure_schroot (btrootfsimg) - ## para hacer schroot --cosas, el mntpt tiene que estar desmontado ## si está montado da un pete tal que 'E: 10mount: mount: /run/schroot/mount/IMGogclient-7fbf51a2-e37e-48e5-8e5d-83f8901fc7ed: wrong fs type, bad option, bad superblock on /dev/loop1, missing codepage or helper program, or other error.' def _debootstrap(): - logger.info ('Stage 3.3 - debootstrap system') + logger.info ('Stage 3.2 - debootstrap system') logger.debug ('Try creation of a file within chroot (this operation may fail with "... etc/resolv.conf: No such file or directory"--that is ok)') logger.debug ('Running \'schroot --chroot IMGogclient -- stat /etc\'') cp = subprocess.run (['schroot', '--chroot', 'IMGogclient', '--', 'stat', '/etc']) @@ -151,10 +142,7 @@ except: pass logger.info ('OpenGnsys CLIENT installation begins') logger.info ('STAGE 1 - clone opengnsys repo') -ogrepo_url = 'https://ognproject.evlt.uma.es/gitea/opengnsys/opengnsys.git' -ogrepo_branch = 'main' -ogrepo_dir = '/tmp/ogrepo' -clone_client_dirs (ogrepo_url, ogrepo_branch, ogrepo_dir) +ogrepo_dir = '/tmp/ogrepo' ## per the Dockerfile osdistrib, oscodename, osrelease, osarch, oshttp = btog.GetOsInfo1 (type_client) if osdistrib is None: @@ -165,14 +153,10 @@ gitrelease, nameisoclient, namehostclient = btog.GetOsInfo2 (ogrepo_dir, version logger.info (':'.join ([osdistrib, oscodename, osrelease, osarch, oshttp])) logger.info ('STAGE 2 - install software in the VM') -apt.update() -apt.upgrade() pxepkg = _get_pxepkg() -apt.install (['jq', 'syslinux', 'syslinux-efi', 'syslinux-utils', 'debootstrap', 'subversion', 'schroot', 'squashfs-tools', 'syslinux', 'genisoimage', 'qemu-utils', 'lsof', pxepkg]) ## TODO qemu no existe, hace falta? logger.info ('STAGE 3 - create and bootstrap rootfs') _mkrootfs() -_schroot (btrootfsimg) _debootstrap() logger.info ('STAGE 4 - copy files to the rootfs') diff --git a/mount-defaults b/mount-defaults new file mode 100644 index 0000000..684b43a --- /dev/null +++ b/mount-defaults @@ -0,0 +1,12 @@ +# mount.defaults: static file system information for chroots. +# Note that the mount point will be prefixed by the chroot path +# (CHROOT_PATH) +# +# +proc /proc proc defaults 0 0 +#procbususb /proc/bus/usb usbfs defaults 0 0 +#/dev /dev none rw,bind 0 0 +/dev/pts /dev/pts none rw,bind 0 0 +/dev/shm /dev/shm none rw,bind 0 0 +#/home /home none rw,bind 0 0 +/tmp /tmp none rw,bind 0 0 diff --git a/schroot.conf b/schroot.conf new file mode 100644 index 0000000..ec18111 --- /dev/null +++ b/schroot.conf @@ -0,0 +1,10 @@ +[IMGogclient] +type=loopback +file=/var/lib/tftpboot/ogclient/ogclient.img +description=ogclient Ubuntu image +#priority=1 +users=root +groups=root +root-groups=root +mount-options=-o offset=32256 +root-users=root From 866cdb4d861cb5c84c8eed713d1945b63e0309f3 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Tue, 20 Aug 2024 12:16:04 +0200 Subject: [PATCH 39/45] refs #597 make image smaller by not keeping a copy of the opengnsys repo --- Dockerfile | 10 ++++++---- boottools/btog.py | 5 ++--- mkoglive.py | 7 +++++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index df8cd7a..ccab095 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,11 +27,13 @@ RUN groupadd opengnsys RUN mkdir -p /opt/opengnsys/client /tmp/opengnsys/oglive_builder /var/lib/tftpboot/ RUN ln -fs /var/lib/tftpboot /opt/opengnsys -## take client stuff from the OG repo -RUN git clone -c http.sslVerify=false --branch ${OPENGNSYS_BRANCH} https://ognproject.evlt.uma.es/gitea/opengnsys/opengnsys.git /tmp/ogrepo -RUN rsync -aH /tmp/ogrepo/client/engine /tmp/ogrepo/client/shared /tmp/opengnsys/ +## take stuff from the OG repo: a) cloning engine, b) shared stuff and c) gitrelease +RUN git clone -c http.sslVerify=false --branch ${OPENGNSYS_BRANCH} https://ognproject.evlt.uma.es/gitea/opengnsys/opengnsys.git /tmp/ogrepo && \ + mv /tmp/ogrepo/client/engine /tmp/ogrepo/client/shared /tmp/opengnsys/ && \ + git --git-dir /tmp/ogrepo/.git log --date format:r%Y%m%d --format=%ad -1 >/tmp/opengnsys/oglive_builder/gitrelease && \ + rm -rf /tmp/ogrepo/ -RUN echo foo4 +RUN echo foo2 ## development: invalidate docker cache at this point, so that the 'COPY' below actually copies files COPY . /tmp/opengnsys/oglive_builder/ RUN cp /tmp/opengnsys/oglive_builder/schroot.conf /tmp/opengnsys/oglive_builder/mount-defaults /etc/schroot/ && echo '' >/etc/schroot/default/nssdatabases && rm -f /etc/schroot/setup.d/*chrootname diff --git a/boottools/btog.py b/boottools/btog.py index 64350dc..5e2218b 100644 --- a/boottools/btog.py +++ b/boottools/btog.py @@ -131,11 +131,10 @@ def GetOsInfo1 (type_client): oshttp=None return osdistrib, oscodename, osrelease, osarch, oshttp -def GetOsInfo2 (ogrepo_dir, versionboottools, oscodename, osrelease, osarch): - gitrelease, _ = utils.run (['git', '--git-dir', f'{ogrepo_dir}/.git', 'log', '--date', 'format:r%Y%m%d', '--format=%ad', '-1']) +def GetOsInfo2 (versionboottools, oscodename, osrelease, osarch, gitrelease): nameisoclient ='-'.join ([versionboottools, oscodename, osrelease, osarch, gitrelease]) namehostclient = '-'.join ([versionboottools, oscodename, gitrelease]) - return gitrelease, nameisoclient, namehostclient + return nameisoclient, namehostclient def mkrootfs (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, bttargetdir, osarch): logger.info (f'Creación y formateo del disco virtual "{btrootfsimg}" "{btvirtualdisksize}" MB') diff --git a/mkoglive.py b/mkoglive.py index 12b5704..58e24ad 100755 --- a/mkoglive.py +++ b/mkoglive.py @@ -142,14 +142,17 @@ except: pass logger.info ('OpenGnsys CLIENT installation begins') logger.info ('STAGE 1 - clone opengnsys repo') -ogrepo_dir = '/tmp/ogrepo' ## per the Dockerfile + +fd = open (f'{curdir}/gitrelease', 'r') ## per the Dockerfile +gitrelease = fd.readline().strip() +fd.close() osdistrib, oscodename, osrelease, osarch, oshttp = btog.GetOsInfo1 (type_client) if osdistrib is None: logger.error ('GetOsInfo1() failed') sys.exit (1) btdir, bttargetdir, btrootfsimg, btrootfsmnt, btrootfsimglabel, log_file, versionboottools, btvirtualdisksize = btog.GetVar (osarch) -gitrelease, nameisoclient, namehostclient = btog.GetOsInfo2 (ogrepo_dir, versionboottools, oscodename, osrelease, osarch) +nameisoclient, namehostclient = btog.GetOsInfo2 (versionboottools, oscodename, osrelease, osarch, gitrelease) logger.info (':'.join ([osdistrib, oscodename, osrelease, osarch, oshttp])) logger.info ('STAGE 2 - install software in the VM') From 5ecf9f3b2af51b5dd569a94749dba1f809819c53 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Tue, 20 Aug 2024 12:54:50 +0200 Subject: [PATCH 40/45] refs #596 simplify some variables, pass parameters to functions --- boottools/btog.py | 13 ++--------- mkoglive.cfg | 1 + mkoglive.py | 55 ++++++++++++++++++++++------------------------- 3 files changed, 29 insertions(+), 40 deletions(-) diff --git a/boottools/btog.py b/boottools/btog.py index 5e2218b..f42c001 100644 --- a/boottools/btog.py +++ b/boottools/btog.py @@ -12,20 +12,16 @@ from . import utils logger = logging.getLogger ('boottools') def GetVar (osarch): - btdir = '/tmp/opengnsys/oglive_builder' bttargetdir = '/var/lib/tftpboot/ogclient/' btrootfsimg = os.path.join (bttargetdir, 'ogclient.img') btrootfsmnt = os.path.join (bttargetdir, 'ogclientmount') - btrootfsimglabel = 'ogClient' - log_file = '/tmp/boot-tools_installation.log' - versionboottools = 'ogLive' if 'i386' == osarch: btvirtualdisksize = '2000' # tamaño maximo limitado por schroot 2GB para 32 bits else: btvirtualdisksize = '3G' - return btdir, bttargetdir, btrootfsimg, btrootfsmnt, btrootfsimglabel, log_file, versionboottools, btvirtualdisksize + return bttargetdir, btrootfsimg, btrootfsmnt, btvirtualdisksize -def GetOsInfo1 (type_client): +def GetOsInfo (type_client): if 'precise' == type_client: # ogLive 1.0.4-rc2 basado en Ubuntu 12.04 LTS. osdistrib='ubuntu' oscodename='precise' @@ -131,11 +127,6 @@ def GetOsInfo1 (type_client): oshttp=None return osdistrib, oscodename, osrelease, osarch, oshttp -def GetOsInfo2 (versionboottools, oscodename, osrelease, osarch, gitrelease): - nameisoclient ='-'.join ([versionboottools, oscodename, osrelease, osarch, gitrelease]) - namehostclient = '-'.join ([versionboottools, oscodename, gitrelease]) - return nameisoclient, namehostclient - def mkrootfs (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, bttargetdir, osarch): logger.info (f'Creación y formateo del disco virtual "{btrootfsimg}" "{btvirtualdisksize}" MB') diff --git a/mkoglive.cfg b/mkoglive.cfg index 708a93a..16507d7 100644 --- a/mkoglive.cfg +++ b/mkoglive.cfg @@ -1,6 +1,7 @@ [General] logging_level = INFO +rootfs_image_label = ogClient debconf_settings = kexec-tools kexec-tools/load_kexec boolean true diff --git a/mkoglive.py b/mkoglive.py index 58e24ad..71a7f37 100755 --- a/mkoglive.py +++ b/mkoglive.py @@ -15,7 +15,7 @@ from boottools import utils, apt, btog os.chdir (curdir) -def _logging (lvl='INFO'): +def _logging (lvl='INFO', logfile='/tmp/boot-tools_installation.log'): numeric_level = getattr (logging, lvl.upper(), None) if numeric_level is None: numeric_level = getattr (logging, 'INFO') @@ -24,7 +24,7 @@ def _logging (lvl='INFO'): format='%(levelname)s %(asctime)s (%(funcName)s) %(message)s', level=numeric_level, handlers = [ - logging.FileHandler ('/tmp/boot-tools_installation.log'), + logging.FileHandler (logfile), logging.StreamHandler (sys.stdout), ], ) @@ -49,8 +49,8 @@ def _get_pxepkg(): logger.info (f'PXE package is "{pxepkg}"') return pxepkg -def _mkrootfs(): - logger.info ('Stage 3.1 - create, partition and format the rootfs') +def _mkrootfs (btrootfsimg, btrootfsmnt, btrootfsimglabel, btvirtualdisksize, bttargetdir, osarch): + logger.info ('Stage 1.1 - create, partition and format the rootfs') rc = subprocess.run (f'file "{btrootfsimg}" |grep -q "partition 1 *: ID=0x83"', shell=True).returncode if (rc): ## 'file|grep' failed try: btog.mkrootfs (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, bttargetdir, osarch) @@ -61,8 +61,8 @@ def _mkrootfs(): ## para hacer schroot --cosas, el mntpt tiene que estar desmontado ## si está montado da un pete tal que 'E: 10mount: mount: /run/schroot/mount/IMGogclient-7fbf51a2-e37e-48e5-8e5d-83f8901fc7ed: wrong fs type, bad option, bad superblock on /dev/loop1, missing codepage or helper program, or other error.' -def _debootstrap(): - logger.info ('Stage 3.2 - debootstrap system') +def _debootstrap (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp): + logger.info ('Stage 1.2 - debootstrap system') logger.debug ('Try creation of a file within chroot (this operation may fail with "... etc/resolv.conf: No such file or directory"--that is ok)') logger.debug ('Running \'schroot --chroot IMGogclient -- stat /etc\'') cp = subprocess.run (['schroot', '--chroot', 'IMGogclient', '--', 'stat', '/etc']) @@ -75,10 +75,10 @@ def _debootstrap(): sys.exit (1) # Incluir revisión. -def _initramfs_version (gitrelease, osrelease, btdir): +def _initramfs_version (gitrelease, osrelease, curdir): ## FIXME esto la incluye incondicionalmente, y luego terminamos con "OpenGnsys Client 1.2.0-rc1 gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) ..." #sed -i "1 s/$/ $GITRELEASE ($OSRELEASE)/" ${BTDIR}/includes/etc/initramfs-tools/scripts/VERSION.txt - utils.run (['sed', '-i', f'1 s/$/ {gitrelease} ({osrelease})/', f'{btdir}/includes/etc/initramfs-tools/scripts/VERSION.txt']) + utils.run (['sed', '-i', f'1 s/$/ {gitrelease} ({osrelease})/', f'{curdir}/includes/etc/initramfs-tools/scripts/VERSION.txt']) def _copy_og_files (btrootfsmnt, osdistrib, oscodename): _mount_rootfs() @@ -93,7 +93,7 @@ def _chroot_tasks (curdir, osrelease, osarch): stdout, _ = utils.run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/chroot-tasks.py', '--osrelease', osrelease, '--osarch', osarch, '--config', args.config or 'mkoglive.cfg']) ## esto deja initrd.img-6.8.0-31-generic y vmlinuz-6.8.0-31-generic en /tmp -def _ssh_stuff(): +def _ssh_stuff (btrootfsmnt): _mount_rootfs() btog.ssh_server (btrootfsmnt) btog.ssh_client (btrootfsmnt) @@ -104,15 +104,15 @@ def _ssh_stuff(): ## - las dos claves públicas (una de cada par) están autorizadan en el rootfs /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh/authorized_keys def _mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsmnt, pxepkg, isolinux_tpl, nameisoclient): - logger.info ('Stage 6.1 - Put initrd in place') + logger.info ('Stage 4.1 - Put initrd in place') _mount_rootfs() btog.move_initrd (bttargetdir, osrelease) - logger.info ('Stage 6.2 - make squash filesystem') + logger.info ('Stage 4.2 - make squash filesystem') btog.mksquashfs (bttargetdir, btrootfsmnt) utils.umount (btrootfsmnt) - logger.info ('Stage 6.3 - make iso filesystem') + logger.info ('Stage 4.3 - make iso filesystem') btog.mkisofs (pxepkg, isolinux_tpl, bttargetdir, nameisoclient) @@ -127,6 +127,7 @@ if config is None: sys.exit (1) isolinux_tpl = config['General'].get ('isolinux_template') cfg_loglevel = config['General'].get ('logging_level') +btrootfsimglabel = config['General'].get ('rootfs_image_label') logger = _logging (args.loglevel or cfg_loglevel) type_client = args.codename or 'host' @@ -141,37 +142,33 @@ except: pass logger.info ('OpenGnsys CLIENT installation begins') -logger.info ('STAGE 1 - clone opengnsys repo') - fd = open (f'{curdir}/gitrelease', 'r') ## per the Dockerfile gitrelease = fd.readline().strip() fd.close() -osdistrib, oscodename, osrelease, osarch, oshttp = btog.GetOsInfo1 (type_client) +osdistrib, oscodename, osrelease, osarch, oshttp = btog.GetOsInfo (type_client) if osdistrib is None: - logger.error ('GetOsInfo1() failed') + logger.error ('GetOsInfo() failed') sys.exit (1) -btdir, bttargetdir, btrootfsimg, btrootfsmnt, btrootfsimglabel, log_file, versionboottools, btvirtualdisksize = btog.GetVar (osarch) -nameisoclient, namehostclient = btog.GetOsInfo2 (versionboottools, oscodename, osrelease, osarch, gitrelease) +bttargetdir, btrootfsimg, btrootfsmnt, btvirtualdisksize = btog.GetVar (osarch) logger.info (':'.join ([osdistrib, oscodename, osrelease, osarch, oshttp])) -logger.info ('STAGE 2 - install software in the VM') -pxepkg = _get_pxepkg() +logger.info ('STAGE 1 - create and bootstrap rootfs') +_mkrootfs (btrootfsimg, btrootfsmnt, btrootfsimglabel, btvirtualdisksize, bttargetdir, osarch) +_debootstrap (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp) -logger.info ('STAGE 3 - create and bootstrap rootfs') -_mkrootfs() -_debootstrap() - -logger.info ('STAGE 4 - copy files to the rootfs') -_initramfs_version (gitrelease, osrelease, btdir) +logger.info ('STAGE 2 - copy files to the rootfs') +_initramfs_version (gitrelease, osrelease, curdir) _copy_og_files (btrootfsmnt, osdistrib, oscodename) -logger.info ('STAGE 5 - perform tasks within the chroot') +logger.info ('STAGE 3 - perform tasks within the chroot') _chroot_tasks (curdir, osrelease, osarch) -_ssh_stuff() +_ssh_stuff (btrootfsmnt) -logger.info ('STAGE 6 - generate distribution files') +logger.info ('STAGE 4 - generate distribution files') +pxepkg = _get_pxepkg() +nameisoclient = '-'.join (['ogLive', oscodename, osrelease, osarch, gitrelease]) _mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsmnt, pxepkg, isolinux_tpl, nameisoclient) logger.info ('OpenGnsys installation finished') From c79d715ccc8ba9ab4a63b775753dcb7d4ae74f99 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Tue, 20 Aug 2024 14:09:54 +0200 Subject: [PATCH 41/45] refs #596 use a main() function --- .dockerignore | 1 + boottools/btog.py | 5 +- chroot-tasks.py | 20 ++++---- mkoglive.py | 113 ++++++++++++++++++++++++---------------------- 4 files changed, 74 insertions(+), 65 deletions(-) diff --git a/.dockerignore b/.dockerignore index 9f8c34d..472c70e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,6 +6,7 @@ Dockerfile INSTALL.es.txt Jenkinsfile kernelParameterOG.es.txt +ogclient/ ogclientImg2Sqfs ogclientSqfs2Img *.swp diff --git a/boottools/btog.py b/boottools/btog.py index f42c001..3b85b76 100644 --- a/boottools/btog.py +++ b/boottools/btog.py @@ -21,7 +21,7 @@ def GetVar (osarch): btvirtualdisksize = '3G' return bttargetdir, btrootfsimg, btrootfsmnt, btvirtualdisksize -def GetOsInfo (type_client): +def GetOsInfo (type_client='host'): if 'precise' == type_client: # ogLive 1.0.4-rc2 basado en Ubuntu 12.04 LTS. osdistrib='ubuntu' oscodename='precise' @@ -324,7 +324,8 @@ def mksquashfs (bttargetdir, btrootfsmnt): now = datetime.datetime.now(datetime.timezone.utc).strftime ('%Y%m%d-%H%M%S%z') os.rename (f'{bttargetdir}/ogclient.sqfs', f'{bttargetdir}/ogclient.sqfs.{now}') - utils.run (['mksquashfs', btrootfsmnt, f'{bttargetdir}/ogclient.sqfs']) + ## uses all CPU cores available, even within docker + utils.run (['mksquashfs', btrootfsmnt, f'{bttargetdir}/ogclient.sqfs', '-e', 'var/lib/apt/lists', '-e', 'usr/share/doc']) os.chmod (f'{bttargetdir}/ogclient.sqfs', 0o744) utils.write_md5 (f'{bttargetdir}/ogclient.sqfs') diff --git a/chroot-tasks.py b/chroot-tasks.py index 056cb36..bcdd9d9 100755 --- a/chroot-tasks.py +++ b/chroot-tasks.py @@ -36,6 +36,7 @@ def boottoolsSoftwareInstall (osarch, osrelease): _oghook_deactivate() + print ('boottoolsSoftwareInstall: debconf-set-selections', file=sys.stderr) subprocess.run (['debconf-set-selections'], input=debconf_settings, text=True) utils.run (['dpkg-reconfigure', '--frontend', 'noninteractive', 'console-setup', 'locales']) ## despues de esto, debconf-get-selections devuelve los valores antiguos, no se por que... @@ -43,16 +44,16 @@ def boottoolsSoftwareInstall (osarch, osrelease): for section in config.options('Packages'): pkgs += re.split (r'[ \n]', config['Packages'].get(section).strip()) - apt.install ( - [f'linux-image-{osrelease}', f'linux-headers-{osrelease}', f'linux-modules-{osrelease}', f'linux-modules-extra-{osrelease}', 'dkms', 'shim-signed', 'openssl', 'sshfs', 'kexec-tools'] + pkgs32 + pkgs, - opts={'DPkg::Options::': '--force-confdef'} ## hace falta --force-confdef para evitar un tema interactivo del /etc/ssh/ssh_config - ) + pkgs = [f'linux-image-{osrelease}', f'linux-headers-{osrelease}', f'linux-modules-{osrelease}', f'linux-modules-extra-{osrelease}', 'dkms', 'shim-signed', 'openssl', 'sshfs', 'kexec-tools'] + pkgs32 + pkgs + print (f'boottoolsSoftwareInstall: installing packages: {str(pkgs)}', file=sys.stderr) + apt.install (pkgs, opts={'DPkg::Options::': '--force-confdef'}) ## hace falta --force-confdef para evitar un tema interactivo del /etc/ssh/ssh_config # Instalar módulos que algunos paquetes puedan tener pendientes de compilar. + print ('boottoolsSoftwareInstall: dkms', file=sys.stderr) stdout, _ = utils.run (['dkms', 'status']) for l in stdout.strip().split ('\n'): if not l: continue - print (f'l "{l}"') + #print (f'l "{l}"') mod, vers, status = l.split (',') if 'added' in status: print (f'dkms installing {mod} {vers}') @@ -69,7 +70,7 @@ def boottoolsSoftwareCompile(): os.environ['LANGUAGE'] = os.environ['LC_ALL'] = os.environ['LANG'] = 'C' os.chdir ('/tmp') - print ('ms-sys') + print ('boottoolsSoftwareCompile: ms-sys', file=sys.stderr) try: utils.run (['which', 'ms-sys']) except: utils.run (['wget', '--quiet', 'https://sourceforge.net/projects/ms-sys/files/latest/download', '-O', 'ms-sys.tar.gz']) @@ -80,7 +81,7 @@ def boottoolsSoftwareCompile(): utils.run (['make', 'install']) os.chdir ('..') - print ('spartlnx') + print ('boottoolsSoftwareCompile: spartlnx', file=sys.stderr) try: utils.run (['which', 'spartl64.run']) except: utils.run (['wget', '--quiet', 'http://damien.guibouret.free.fr/savepart.zip']) @@ -89,7 +90,7 @@ def boottoolsSoftwareCompile(): utils.run (['unzip', '-j', '-o', 'savepart.zip', '-d', '/usr/share/doc/spartlnx/', 'doc/en/*']) if not os.path.exists ('python-libfdisk'): - print ('python-libfdisk') + print ('boottoolsSoftwareCompile: python-libfdisk', file=sys.stderr) apt.install (['python3-psutil', 'python3-dev', 'libfdisk-dev', 'python3-setuptools']) utils.run (['git', 'clone', 'git://git.48k.eu/python-libfdisk']) os.chdir ('python-libfdisk') @@ -102,12 +103,13 @@ def boottoolsSoftwareCompile(): os.environ['LANG'] = env_lang def boottoolsInitrdGenerate (osrelease): + print ('boottoolsInitrdGenerate', file=sys.stderr) for f in glob.glob ('/usr/lib/initramfs-tools/bin/*'): os.unlink (f) shutil.copy ('/bin/busybox', '/usr/lib/initramfs-tools/bin') os.chdir ('/tmp') - utils.run (['mkinitramfs', '-o', f'/tmp/initrd.img-{osrelease}', '-v', osrelease]) + utils.run (['mkinitramfs', '-o', f'/tmp/initrd.img-{osrelease}', osrelease]) shutil.copy (f'/boot/vmlinuz-{osrelease}', '/tmp/') if __name__ == '__main__': diff --git a/mkoglive.py b/mkoglive.py index 71a7f37..0151645 100755 --- a/mkoglive.py +++ b/mkoglive.py @@ -30,8 +30,7 @@ def _logging (lvl='INFO', logfile='/tmp/boot-tools_installation.log'): ) return logging.getLogger ('boottools') -def _mount_rootfs(): - global btrootfsimg, btrootfsmnt +def _mount_rootfs (btrootfsimg, btrootfsmnt): try: utils.mount (btrootfsimg, btrootfsmnt, opts=['-o', 'loop,offset=32256']) except: logger.error ('mount failed') @@ -68,7 +67,7 @@ def _debootstrap (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp): cp = subprocess.run (['schroot', '--chroot', 'IMGogclient', '--', 'stat', '/etc']) if (cp.returncode): logger.debug (f'schroot returned code "{cp.returncode}", calling btog.debootstrap()') - _mount_rootfs() + _mount_rootfs (btrootfsimg, btrootfsmnt) try: btog.debootstrap (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp) except Exception as e: logger.error (str (e)) @@ -80,21 +79,21 @@ def _initramfs_version (gitrelease, osrelease, curdir): #sed -i "1 s/$/ $GITRELEASE ($OSRELEASE)/" ${BTDIR}/includes/etc/initramfs-tools/scripts/VERSION.txt utils.run (['sed', '-i', f'1 s/$/ {gitrelease} ({osrelease})/', f'{curdir}/includes/etc/initramfs-tools/scripts/VERSION.txt']) -def _copy_og_files (btrootfsmnt, osdistrib, oscodename): - _mount_rootfs() +def _copy_og_files (btrootfsimg, btrootfsmnt, osdistrib, oscodename): + _mount_rootfs (btrootfsimg, btrootfsmnt) builder = '/tmp/opengnsys/oglive_builder' og_shared = '/tmp/opengnsys/shared' og_engine = '/tmp/opengnsys/engine' btog.copy_og_files (builder, og_shared, og_engine, btrootfsmnt, osdistrib, oscodename) utils.umount (btrootfsmnt) -def _chroot_tasks (curdir, osrelease, osarch): +def _chroot_tasks (cfgfile, curdir, osrelease, osarch): logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/chroot-tasks.py --osrelease "{osrelease}" --osarch "{osarch}"\'') - stdout, _ = utils.run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/chroot-tasks.py', '--osrelease', osrelease, '--osarch', osarch, '--config', args.config or 'mkoglive.cfg']) + stdout, _ = utils.run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/chroot-tasks.py', '--osrelease', osrelease, '--osarch', osarch, '--config', cfgfile]) ## esto deja initrd.img-6.8.0-31-generic y vmlinuz-6.8.0-31-generic en /tmp -def _ssh_stuff (btrootfsmnt): - _mount_rootfs() +def _ssh_stuff (btrootfsimg, btrootfsmnt): + _mount_rootfs (btrootfsimg, btrootfsmnt) btog.ssh_server (btrootfsmnt) btog.ssh_client (btrootfsmnt) utils.umount (btrootfsmnt) @@ -103,9 +102,9 @@ def _ssh_stuff (btrootfsmnt): ## - hay otro nuevo par de claves en el rootfs /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh ## - las dos claves públicas (una de cada par) están autorizadan en el rootfs /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh/authorized_keys -def _mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsmnt, pxepkg, isolinux_tpl, nameisoclient): +def _mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsimg, btrootfsmnt, pxepkg, isolinux_tpl, nameisoclient): logger.info ('Stage 4.1 - Put initrd in place') - _mount_rootfs() + _mount_rootfs (btrootfsimg, btrootfsmnt) btog.move_initrd (bttargetdir, osrelease) logger.info ('Stage 4.2 - make squash filesystem') @@ -115,60 +114,66 @@ def _mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsmnt, pxepkg, isoli logger.info ('Stage 4.3 - make iso filesystem') btog.mkisofs (pxepkg, isolinux_tpl, bttargetdir, nameisoclient) +def _main (cfgfile, config, type_client): + isolinux_tpl = config['General'].get ('isolinux_template') + btrootfsimglabel = config['General'].get ('rootfs_image_label') -parser = argparse.ArgumentParser() -parser.add_argument ('--loglevel', help='Log level', action='store') -parser.add_argument ('--codename', help='OS codename', action='store') -parser.add_argument ('--config', help='Path to configuration file', action='store') -args = parser.parse_args() + logger.info ('OpenGnsys CLIENT installation begins') -config = utils.read_config (args.config or 'mkoglive.cfg') -if config is None: - sys.exit (1) -isolinux_tpl = config['General'].get ('isolinux_template') -cfg_loglevel = config['General'].get ('logging_level') -btrootfsimglabel = config['General'].get ('rootfs_image_label') + fd = open (f'{curdir}/gitrelease', 'r') ## per the Dockerfile + gitrelease = fd.readline().strip() + fd.close() -logger = _logging (args.loglevel or cfg_loglevel) -type_client = args.codename or 'host' + osdistrib, oscodename, osrelease, osarch, oshttp = btog.GetOsInfo (type_client) + if osdistrib is None: + logger.error ('GetOsInfo() failed') + sys.exit (1) + bttargetdir, btrootfsimg, btrootfsmnt, btvirtualdisksize = btog.GetVar (osarch) + logger.info (':'.join ([osdistrib, oscodename, osrelease, osarch, oshttp])) -if os.getuid(): - logger.error ('ERROR: this program must run under root privileges!!') - sys.exit (1) + ## this is convenient in case the previous run failed and we want to run this program again + try: utils.umount (btrootfsmnt) + except: pass -## this is convenient in case the previous run failed and we want to run this program again -try: utils.umount (btrootfsmnt) -except: pass + logger.info ('STAGE 1 - create and bootstrap rootfs') + _mkrootfs (btrootfsimg, btrootfsmnt, btrootfsimglabel, btvirtualdisksize, bttargetdir, osarch) + _debootstrap (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp) -logger.info ('OpenGnsys CLIENT installation begins') + logger.info ('STAGE 2 - copy files to the rootfs') + _initramfs_version (gitrelease, osrelease, curdir) + _copy_og_files (btrootfsimg, btrootfsmnt, osdistrib, oscodename) -fd = open (f'{curdir}/gitrelease', 'r') ## per the Dockerfile -gitrelease = fd.readline().strip() -fd.close() + logger.info ('STAGE 3 - perform tasks within the chroot') + _chroot_tasks (cfgfile, curdir, osrelease, osarch) -osdistrib, oscodename, osrelease, osarch, oshttp = btog.GetOsInfo (type_client) -if osdistrib is None: - logger.error ('GetOsInfo() failed') - sys.exit (1) -bttargetdir, btrootfsimg, btrootfsmnt, btvirtualdisksize = btog.GetVar (osarch) -logger.info (':'.join ([osdistrib, oscodename, osrelease, osarch, oshttp])) + _ssh_stuff (btrootfsimg, btrootfsmnt) -logger.info ('STAGE 1 - create and bootstrap rootfs') -_mkrootfs (btrootfsimg, btrootfsmnt, btrootfsimglabel, btvirtualdisksize, bttargetdir, osarch) -_debootstrap (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp) + logger.info ('STAGE 4 - generate distribution files') + pxepkg = _get_pxepkg() + nameisoclient = '-'.join (['ogLive', oscodename, osrelease, osarch, gitrelease]) + _mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsimg, btrootfsmnt, pxepkg, isolinux_tpl, nameisoclient) -logger.info ('STAGE 2 - copy files to the rootfs') -_initramfs_version (gitrelease, osrelease, curdir) -_copy_og_files (btrootfsmnt, osdistrib, oscodename) + logger.info ('OpenGnsys installation finished') -logger.info ('STAGE 3 - perform tasks within the chroot') -_chroot_tasks (curdir, osrelease, osarch) -_ssh_stuff (btrootfsmnt) +if __name__ == '__main__': + if os.getuid(): + print ('ERROR: this program must run under root privileges!!', file=sys.stderr) + sys.exit (1) -logger.info ('STAGE 4 - generate distribution files') -pxepkg = _get_pxepkg() -nameisoclient = '-'.join (['ogLive', oscodename, osrelease, osarch, gitrelease]) -_mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsmnt, pxepkg, isolinux_tpl, nameisoclient) + parser = argparse.ArgumentParser() + parser.add_argument ('--loglevel', help='Log level', action='store') + parser.add_argument ('--codename', help='OS codename', action='store') + parser.add_argument ('--config', help='Path to configuration file', action='store') + args = parser.parse_args() -logger.info ('OpenGnsys installation finished') + cfgfile = args.config or 'mkoglive.cfg' + config = utils.read_config (cfgfile) + if config is None: + print ('ERROR: no configuration found', file=sys.stderr) + sys.exit (1) + cfg_loglevel = config['General'].get ('logging_level') + + logger = _logging (args.loglevel or cfg_loglevel) + + _main (cfgfile, config, args.codename) From 0ab0e61027cbd163afb4d2b1b7e6cd0067e386a6 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Tue, 20 Aug 2024 14:57:54 +0200 Subject: [PATCH 42/45] refs #597 translate and clean up comments --- Dockerfile | 2 +- INSTALL.es.txt | 65 +++++++++++++++++++++------------------------- boottools/btog.py | 23 +++++----------- boottools/utils.py | 7 ----- chroot-tasks.py | 6 ++--- mkoglive.py | 17 ++++-------- 6 files changed, 46 insertions(+), 74 deletions(-) diff --git a/Dockerfile b/Dockerfile index ccab095..7e0169a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,7 +33,7 @@ RUN git clone -c http.sslVerify=false --branch ${OPENGNSYS_BRANCH} https://ognpr git --git-dir /tmp/ogrepo/.git log --date format:r%Y%m%d --format=%ad -1 >/tmp/opengnsys/oglive_builder/gitrelease && \ rm -rf /tmp/ogrepo/ -RUN echo foo2 ## development: invalidate docker cache at this point, so that the 'COPY' below actually copies files +#RUN echo 0 ## development: invalidate docker cache at this point, so that the 'COPY' below actually copies files COPY . /tmp/opengnsys/oglive_builder/ RUN cp /tmp/opengnsys/oglive_builder/schroot.conf /tmp/opengnsys/oglive_builder/mount-defaults /etc/schroot/ && echo '' >/etc/schroot/default/nssdatabases && rm -f /etc/schroot/setup.d/*chrootname diff --git a/INSTALL.es.txt b/INSTALL.es.txt index 6645796..86e7396 100644 --- a/INSTALL.es.txt +++ b/INSTALL.es.txt @@ -4,43 +4,37 @@ OpenGnsys Client boot-tools INSTALL.es.txt Advertencia previa: ----------------------------- -- No usar este instalador en un sistema OpenGnsys en producción. - Se puede ejecutar este script desde un sistema Ubuntu 10.04 o superior. - Mas información https://opengnsys.es/trac/wiki/ClienteInitrdDSGenerarloV1.0.2 Requisitos iniciales -------------------- -- Utilizar el usuario "root" para ejecutar el proceso de generación del cliente. - Comprobar la conexión a Internet, sobre todo si usas proxy (el hecho de que funcione apt-get no significa que tengas el proxy configurado). -Proceso de instalación ----------------------- +Proceso de creación de imagen docker +------------------------------------ -#0. Si ya tenemos configurado TFTP, renombrar el directorio /var/lib/tftpboot/ogclient. -mv /opt/opengnsys/tftpboot/ogclient /opt/opengnsys/tftpboot/ogclient-old; +Simplemente llamar a docker build sin sorpresas: -#1. Descargar del Subversion las herramientas del cliente -apt-get install subversion wget -svn checkout https://opengnsys.es/svn/branches/version1.1/client /tmp/opengnsys_installer/opengnsys/client/; -find /tmp/opengnsys_installer/ -name .svn -type d -exec rm -fr {} \; 2>/dev/null; + docker build -t mkoglive . -#2. Ejecutar el instalador: -/tmp/opengnsys_installer/opengnsys/client/boot-tools/boottoolsgenerator.sh; +Si queremos usar una rama diferente del repo de opengnsys: -Notas para ejecución del instalador en Ubuntu 12.10: -- Configuración openssh-client: - - *** ssh_config (Y/I/N/O/D/Z) [default=N] ? [intro] -- Mapa de caracteres: - - Pantalla "Configuring console-data": elegir "Select keymap from full list" - - Keymap: pc / qwerty / Spanish / Standard / Standard -- Clave publica del servidor - - verwrite (y/n)? [intro] -- Mapa de caracteres (repetición): - - Pantalla "Configuring console-data": - - Select keymap from full list - - pc / qwerty / Spanish / Standard / Standard, UTF-8, Combined - Latin; Slavic Cyrillic; Greek, Fixed, 16 + docker build --build-arg OPENGNSYS_BRANCH=mybranch -t mkoglive . + + +Proceso de creación de imagen oglive +------------------------------------ + + docker run --rm --name mkoglive --privileged=true --volume $PWD/ogclient:/var/lib/tftpboot/ogclient mkoglive --codename noble + +Hace falta --privileged=true para que mkoglive.py pueda ejecutar, entre otras cosas, losetup. + +El parámetro --codename permite especificar la versión de ubuntu. + +Las imágenes se crean en ./ogclient. @@ -48,14 +42,15 @@ Descripción de la estrucutra de boot-tools ------------------------------------------ - INSTALL.es.txt este mismo archivo. -- boottollsgenerator.sh proceso de generación del sistema operativo opengnsys. -- boottoolsfunctions.lib libreria complementaria del boottolsgenerator.sh -- includes/ directorio con ficheros específicos para incluir en la distribución. - - - -NOTA: - boottollsfuncions.lib -> btogSetfsBase - bootsoftwareinstall -> - packages ->sw.testing - +- mkoglive: + - mkoglive.py script principal para crear la imagen oglive + - mkoglive.cfg configuración de mkoglive.py + - boottools/ libreria complementaria del mkoglive.py + - includes/ directorio con ficheros específicos para incluir en la distribución +- schroot: + - schroot.conf configuración de schroot + - mount-defaults configuración de puntos de montaje para schroot + - chroot-tasks.py helper para configurar el oglive estando en schroot +- docker: + - Dockerfile para crear una imagen docker + - .dockerignore para limitar el contexto de docker-build diff --git a/boottools/btog.py b/boottools/btog.py index 3b85b76..e1e7bf9 100644 --- a/boottools/btog.py +++ b/boottools/btog.py @@ -193,8 +193,7 @@ def mkrootfs (btrootfsimg, btrootfsimglabel, btrootfsmnt, btvirtualdisksize, btt logger.info (f'"{btrootfsimg}" "{btvirtualdisksize}" MB : OK') -# debootstrap: Genera el sistema root base con debootstrap -# trabaja sobre un rootfs ya montado +# works on an already mounted rootfs def debootstrap (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp): logger.info ('Iniciando la generación del sistema de archivos') @@ -245,23 +244,22 @@ def copy_og_files (builder, og_shared, og_engine, ogclientmount, osdistrib, osco if os.path.exists (f'{og_shared}/bin/ogAdmClient'): shutil.copy (f'{og_shared}/bin/ogAdmClient', f'{ogclientmount}/bin/') def ssh_server (btrootfsmnt): - if not os.path.exists ('/root/.ssh/id_rsa'): ## crea un par de claves en la VM, no en el chroot + if not os.path.exists ('/root/.ssh/id_rsa'): ## creates a key pair in the VM (or docker container), not in the chroot utils.run (['ssh-keygen', '-q', '-f', '/root/.ssh/id_rsa', '-N', '']) logger.debug ('comprobando directorio .ssh del root') - if not os.path.exists (f'{btrootfsmnt}/root/.ssh'): ## crea directorio dentro del chroot + if not os.path.exists (f'{btrootfsmnt}/root/.ssh'): ## creates directory within the chroot logger.debug ('creando directorio .ssh 600') os.mkdir (f'{btrootfsmnt}/root/.ssh') os.chmod (f'{btrootfsmnt}/root/.ssh', 0o700) - logger.debug ('creando el fichero authorized_keys') ## crea archivo en el chroot + logger.debug ('creando el fichero authorized_keys') ## creates file within the chroot if not os.path.exists (f'{btrootfsmnt}/root/.ssh/authorized_keys'): open (f'{btrootfsmnt}/root/.ssh/authorized_keys', 'w').close() os.chmod (f'{btrootfsmnt}/root/.ssh/authorized_keys', 0o600) logger.debug ('importando la clave publica del servidor OG') - #cat /tmp/id_rsa.pub - if os.path.exists ('/root/.ssh/id_rsa.pub'): ## coge la publica de la VM y la pone en el authorized_keys del chroot + if os.path.exists ('/root/.ssh/id_rsa.pub'): ## takes the pubkey from the VM (or docker container) and puts it in the authorized_keys within the chroot fdin = open ('/root/.ssh/id_rsa.pub', 'r') fdout = open (f'{btrootfsmnt}/root/.ssh/authorized_keys', 'a') while True: @@ -276,9 +274,9 @@ def ssh_server (btrootfsmnt): def ssh_client (btrootfsmnt): if not os.path.exists (f'{btrootfsmnt}/root/.ssh/id_rsa'): - utils.run (['ssh-keygen', '-q', '-f', f'{btrootfsmnt}/root/.ssh/id_rsa', '-N', '']) ## crea un par de claves en el chroot + utils.run (['ssh-keygen', '-q', '-f', f'{btrootfsmnt}/root/.ssh/id_rsa', '-N', '']) ## creates a key pair in the chroot - #cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys ## coge la publica y se la autoriza a sí mismo + #cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys ## takes the pubkey and authorises it to itself fdin = open (f'{btrootfsmnt}//root/.ssh/id_rsa.pub', 'r') fdout = open (f'{btrootfsmnt}/root/.ssh/authorized_keys', 'a') while True: @@ -291,11 +289,7 @@ def ssh_client (btrootfsmnt): ## TODO: exportamos la publica a los repos #cp /root/.ssh/id_rsa.pub /tmp/rsa.ogclient.pub -#move_initrd genera un initrd. def move_initrd (bttargetdir, osrelease): - #echo "cp /tmp/*-${OSRELEASE} ${BTTARGETDIR}" - #cp /tmp/*-${OSRELEASE} ${BTTARGETDIR} ## esto copia algo?? - ## backup de oginitrd.img, oginitrd.img.sum, ogvmlinuz y ogvmlinuz.sum now = datetime.datetime.now(datetime.timezone.utc).strftime ('%Y%m%d-%H%M%S%z') if os.path.exists (f'{bttargetdir}/oginitrd.img'): @@ -315,11 +309,9 @@ def move_initrd (bttargetdir, osrelease): for f in glob.glob (f'{bttargetdir}/oginitrd*') + glob.glob (f'{bttargetdir}/vmlinuz*'): os.chmod (f, 0o755) -#mksquashfs convierte el sistema root en sqfs def mksquashfs (bttargetdir, btrootfsmnt): logger.info ('Iniciando la creación del sistema de archivos en sqfs') - # si ya existe un sqfs lo renombramos if os.path.exists (f'{bttargetdir}/ogclient.sqfs'): now = datetime.datetime.now(datetime.timezone.utc).strftime ('%Y%m%d-%H%M%S%z') os.rename (f'{bttargetdir}/ogclient.sqfs', f'{bttargetdir}/ogclient.sqfs.{now}') @@ -330,7 +322,6 @@ def mksquashfs (bttargetdir, btrootfsmnt): utils.write_md5 (f'{bttargetdir}/ogclient.sqfs') -# mkisofs genera la iso del cliente def mkisofs (pxepkg, isolinux_tpl, bttargetdir, nameisoclient): #Preparamos los gestores de arranque try: os.makedirs ('/tmp/iso/isolinux', exist_ok=True) diff --git a/boottools/utils.py b/boottools/utils.py index 5e0a7fc..37dc0ce 100644 --- a/boottools/utils.py +++ b/boottools/utils.py @@ -16,27 +16,20 @@ def run (args): sel.register (p.stderr.fileno(), selectors.EVENT_READ) while True: events = sel.select() - #logger.debug (f'got {len(events)} events') for key, _ in events: if key.fileobj == p.stdout.fileno(): - #logger.debug (f'reading from stdout') line = p.stdout.readline() if not line: break stdout += line logger.debug (line.rstrip()) elif key.fileobj == p.stderr.fileno(): - #logger.debug (f'reading from stderr') line = p.stderr.readline() if not line: break stderr += line logger.warn (line.rstrip()) if p.poll() != None: - #logger.debug ('process exited, breaking loop') break sel.close() - #p.stdout.close() - #p.stderr.close() - #p.wait() stdout = stdout.strip() stderr = stderr.strip() diff --git a/chroot-tasks.py b/chroot-tasks.py index bcdd9d9..e2ce26a 100755 --- a/chroot-tasks.py +++ b/chroot-tasks.py @@ -32,13 +32,13 @@ def boottoolsSoftwareInstall (osarch, osrelease): pkgs32 = [] if 'i386' != osarch: utils.run (['dpkg', '--add-architecture', 'i386']) - pkgs32 = 'lib32gcc-s1 lib32stdc++6 lib32z1 libc6-i386'.split (' ') ## he cambiado lib32gcc1 por lib32gcc-s1 pero como queramos crear un oglive viejo, esto va a petar + pkgs32 = 'lib32gcc-s1 lib32stdc++6 lib32z1 libc6-i386'.split (' ') ## nserrano: he cambiado lib32gcc1 por lib32gcc-s1 pero como queramos crear un oglive viejo, esto va a petar _oghook_deactivate() print ('boottoolsSoftwareInstall: debconf-set-selections', file=sys.stderr) subprocess.run (['debconf-set-selections'], input=debconf_settings, text=True) - utils.run (['dpkg-reconfigure', '--frontend', 'noninteractive', 'console-setup', 'locales']) ## despues de esto, debconf-get-selections devuelve los valores antiguos, no se por que... + utils.run (['dpkg-reconfigure', '--frontend', 'noninteractive', 'console-setup', 'locales']) ## XXX: despues de esto, debconf-get-selections devuelve los valores antiguos, no se por que... pkgs = [] for section in config.options('Packages'): @@ -46,7 +46,7 @@ def boottoolsSoftwareInstall (osarch, osrelease): pkgs = [f'linux-image-{osrelease}', f'linux-headers-{osrelease}', f'linux-modules-{osrelease}', f'linux-modules-extra-{osrelease}', 'dkms', 'shim-signed', 'openssl', 'sshfs', 'kexec-tools'] + pkgs32 + pkgs print (f'boottoolsSoftwareInstall: installing packages: {str(pkgs)}', file=sys.stderr) - apt.install (pkgs, opts={'DPkg::Options::': '--force-confdef'}) ## hace falta --force-confdef para evitar un tema interactivo del /etc/ssh/ssh_config + apt.install (pkgs, opts={'DPkg::Options::': '--force-confdef'}) ## --force-confdef is required to avoid an interactive question regarding /etc/ssh/ssh_config # Instalar módulos que algunos paquetes puedan tener pendientes de compilar. print ('boottoolsSoftwareInstall: dkms', file=sys.stderr) diff --git a/mkoglive.py b/mkoglive.py index 0151645..095e435 100755 --- a/mkoglive.py +++ b/mkoglive.py @@ -37,7 +37,6 @@ def _mount_rootfs (btrootfsimg, btrootfsmnt): sys.exit (1) def _get_pxepkg(): - #grep "http://free.nchc.org.tw/drbl-core" /etc/apt/sources.list || echo "deb http://free.nchc.org.tw/drbl-core drbl stable" >> /etc/apt/sources.list pxepkg = None cache = apt.cache_search (['gpxe', 'ipxe']) if cache['gpxe']: pxepkg = 'gpxe' @@ -57,9 +56,6 @@ def _mkrootfs (btrootfsimg, btrootfsmnt, btrootfsimglabel, btvirtualdisksize, bt logger.error (str (e)) sys.exit (1) -## para hacer schroot --cosas, el mntpt tiene que estar desmontado -## si está montado da un pete tal que 'E: 10mount: mount: /run/schroot/mount/IMGogclient-7fbf51a2-e37e-48e5-8e5d-83f8901fc7ed: wrong fs type, bad option, bad superblock on /dev/loop1, missing codepage or helper program, or other error.' - def _debootstrap (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp): logger.info ('Stage 1.2 - debootstrap system') logger.debug ('Try creation of a file within chroot (this operation may fail with "... etc/resolv.conf: No such file or directory"--that is ok)') @@ -73,10 +69,7 @@ def _debootstrap (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp): logger.error (str (e)) sys.exit (1) -# Incluir revisión. def _initramfs_version (gitrelease, osrelease, curdir): - ## FIXME esto la incluye incondicionalmente, y luego terminamos con "OpenGnsys Client 1.2.0-rc1 gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) gitrelease (osrelease) ..." - #sed -i "1 s/$/ $GITRELEASE ($OSRELEASE)/" ${BTDIR}/includes/etc/initramfs-tools/scripts/VERSION.txt utils.run (['sed', '-i', f'1 s/$/ {gitrelease} ({osrelease})/', f'{curdir}/includes/etc/initramfs-tools/scripts/VERSION.txt']) def _copy_og_files (btrootfsimg, btrootfsmnt, osdistrib, oscodename): @@ -90,17 +83,17 @@ def _copy_og_files (btrootfsimg, btrootfsmnt, osdistrib, oscodename): def _chroot_tasks (cfgfile, curdir, osrelease, osarch): logger.debug (f'running \'schroot --chroot IMGogclient -- {curdir}/chroot-tasks.py --osrelease "{osrelease}" --osarch "{osarch}"\'') stdout, _ = utils.run (['schroot', '--chroot', 'IMGogclient', '--', f'{curdir}/chroot-tasks.py', '--osrelease', osrelease, '--osarch', osarch, '--config', cfgfile]) - ## esto deja initrd.img-6.8.0-31-generic y vmlinuz-6.8.0-31-generic en /tmp + ## this leaves initrd.img-6.8.0-31-generic and vmlinuz-6.8.0-31-generic in /tmp def _ssh_stuff (btrootfsimg, btrootfsmnt): _mount_rootfs (btrootfsimg, btrootfsmnt) btog.ssh_server (btrootfsmnt) btog.ssh_client (btrootfsmnt) utils.umount (btrootfsmnt) - ## el resultado es: - ## - hay un nuevo par de claves en la VM /root/.ssh - ## - hay otro nuevo par de claves en el rootfs /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh - ## - las dos claves públicas (una de cada par) están autorizadan en el rootfs /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh/authorized_keys + ## the end result is: + ## - there's a new key pair in the VM (or docker container), in /root/.ssh + ## - there's another new key pair in the rootfs, in /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh + ## - the two pubkeys (one of each pair) end up being authorised in the rootfs, in /var/lib/tftpboot/ogclient/ogclientmount/root/.ssh/authorized_keys def _mkinitrd_squashfs_isofs (bttargetdir, osrelease, btrootfsimg, btrootfsmnt, pxepkg, isolinux_tpl, nameisoclient): logger.info ('Stage 4.1 - Put initrd in place') From 2ee36ca0866cfff6a5ad387f6aa5b84b6b8894b1 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Tue, 20 Aug 2024 16:16:18 +0200 Subject: [PATCH 43/45] refs #597 improve doc and logging --- INSTALL.es.txt | 12 ++++++++++++ mkoglive.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/INSTALL.es.txt b/INSTALL.es.txt index 86e7396..c33e334 100644 --- a/INSTALL.es.txt +++ b/INSTALL.es.txt @@ -34,8 +34,20 @@ Hace falta --privileged=true para que mkoglive.py pueda ejecutar, entre otras co El parámetro --codename permite especificar la versión de ubuntu. +También se puede especificar, por ejemplo, --loglevel DEBUG. + +Y también podemos indicar otro archivo de configuración usando --config. + Las imágenes se crean en ./ogclient. +Puede que durante la ejecución del contenedor salga un error como este: + + Exception ignored in: + Traceback (most recent call last): + File "/usr/lib/python3.12/logging/__init__.py", line 241, in _acquireLock + _lock.acquire() + +Es un bug conocido en la biblioteca "logging" de python, para el cual ya existe solución y solo es cuestión de tiempo que la versión de python incluida en ubuntu incluya el fix. Descripción de la estrucutra de boot-tools diff --git a/mkoglive.py b/mkoglive.py index 095e435..468884d 100755 --- a/mkoglive.py +++ b/mkoglive.py @@ -58,7 +58,7 @@ def _mkrootfs (btrootfsimg, btrootfsmnt, btrootfsimglabel, btvirtualdisksize, bt def _debootstrap (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp): logger.info ('Stage 1.2 - debootstrap system') - logger.debug ('Try creation of a file within chroot (this operation may fail with "... etc/resolv.conf: No such file or directory"--that is ok)') + logger.info ('Try creation of a file within chroot (this operation may fail with "... etc/resolv.conf: No such file or directory"--that is ok)') logger.debug ('Running \'schroot --chroot IMGogclient -- stat /etc\'') cp = subprocess.run (['schroot', '--chroot', 'IMGogclient', '--', 'stat', '/etc']) if (cp.returncode): From 770a1061b2373b717c06c2e73c922a738af6eca4 Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Wed, 21 Aug 2024 14:53:08 +0200 Subject: [PATCH 44/45] refs #597 remove a call to schroot, make reference to the final docker image name --- .dockerignore | 2 +- INSTALL.es.txt | 10 +++++----- mkoglive.py | 12 ++++++------ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.dockerignore b/.dockerignore index 472c70e..a66a176 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,4 +9,4 @@ kernelParameterOG.es.txt ogclient/ ogclientImg2Sqfs ogclientSqfs2Img -*.swp +**/*.swp diff --git a/INSTALL.es.txt b/INSTALL.es.txt index c33e334..6567527 100644 --- a/INSTALL.es.txt +++ b/INSTALL.es.txt @@ -18,23 +18,23 @@ Proceso de creación de imagen docker Simplemente llamar a docker build sin sorpresas: - docker build -t mkoglive . + docker build -t opengnsys/mkoglive . Si queremos usar una rama diferente del repo de opengnsys: - docker build --build-arg OPENGNSYS_BRANCH=mybranch -t mkoglive . + docker build --build-arg OPENGNSYS_BRANCH=mybranch -t opengnsys/mkoglive . Proceso de creación de imagen oglive ------------------------------------ - docker run --rm --name mkoglive --privileged=true --volume $PWD/ogclient:/var/lib/tftpboot/ogclient mkoglive --codename noble + docker run --rm --name mkoglive --privileged=true --volume $PWD/ogclient:/var/lib/tftpboot/ogclient opengnsys/mkoglive --codename noble -Hace falta --privileged=true para que mkoglive.py pueda ejecutar, entre otras cosas, losetup. +Hace falta --privileged=true para que mkoglive.py pueda ejecutar, entre otras cosas, losetup y mount. El parámetro --codename permite especificar la versión de ubuntu. -También se puede especificar, por ejemplo, --loglevel DEBUG. +También se puede especificar, por ejemplo, --loglevel debug. Y también podemos indicar otro archivo de configuración usando --config. diff --git a/mkoglive.py b/mkoglive.py index 468884d..37439f3 100755 --- a/mkoglive.py +++ b/mkoglive.py @@ -58,16 +58,16 @@ def _mkrootfs (btrootfsimg, btrootfsmnt, btrootfsimglabel, btvirtualdisksize, bt def _debootstrap (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp): logger.info ('Stage 1.2 - debootstrap system') - logger.info ('Try creation of a file within chroot (this operation may fail with "... etc/resolv.conf: No such file or directory"--that is ok)') - logger.debug ('Running \'schroot --chroot IMGogclient -- stat /etc\'') - cp = subprocess.run (['schroot', '--chroot', 'IMGogclient', '--', 'stat', '/etc']) - if (cp.returncode): - logger.debug (f'schroot returned code "{cp.returncode}", calling btog.debootstrap()') - _mount_rootfs (btrootfsimg, btrootfsmnt) + _mount_rootfs (btrootfsimg, btrootfsmnt) + try: os.stat (os.path.join (btrootfsmnt, 'etc')) + except: + logger.debug (f'stat failed, calling btog.debootstrap()') try: btog.debootstrap (btrootfsimg, btrootfsmnt, osarch, oscodename, oshttp) except Exception as e: + utils.umount (btrootfsmnt) logger.error (str (e)) sys.exit (1) + utils.umount (btrootfsmnt) def _initramfs_version (gitrelease, osrelease, curdir): utils.run (['sed', '-i', f'1 s/$/ {gitrelease} ({osrelease})/', f'{curdir}/includes/etc/initramfs-tools/scripts/VERSION.txt']) From 2ed2c0ad68d9362382423c827790bbaa3e9d59fd Mon Sep 17 00:00:00 2001 From: Natalia Serrano Date: Tue, 3 Sep 2024 13:44:19 +0200 Subject: [PATCH 45/45] refs #597 improve doc about permissions in docker volume --- INSTALL.es.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/INSTALL.es.txt b/INSTALL.es.txt index 6567527..41bebbb 100644 --- a/INSTALL.es.txt +++ b/INSTALL.es.txt @@ -28,6 +28,7 @@ Si queremos usar una rama diferente del repo de opengnsys: Proceso de creación de imagen oglive ------------------------------------ + mkdir ogclient docker run --rm --name mkoglive --privileged=true --volume $PWD/ogclient:/var/lib/tftpboot/ogclient opengnsys/mkoglive --codename noble Hace falta --privileged=true para que mkoglive.py pueda ejecutar, entre otras cosas, losetup y mount. @@ -38,9 +39,9 @@ También se puede especificar, por ejemplo, --loglevel debug. Y también podemos indicar otro archivo de configuración usando --config. -Las imágenes se crean en ./ogclient. +Las imágenes se crean en ./ogclient. Es conveniente crear este directorio con antelación, antes de ejecutar docker, porque de este modo el directorio queda con permisos de usuario. De lo contrario, docker lo crearía como root y necesitaríamos sudo para hacer cualquier cosa dentro de él. -Puede que durante la ejecución del contenedor salga un error como este: +Puede que durante la ejecución del contenedor salga un error como este una o más veces: Exception ignored in: Traceback (most recent call last):