source: ogClient-Git/src/live/ogOperations.py @ 7801d48

Last change on this file since 7801d48 was c010c42, checked in by Jose M. Guisado <jguisado@…>, 2 years ago

live: clear ogbrowser log before image_create

Remove any previous unrelated log message shown in the "real time log"
html page before executing this command.
  • Property mode set to 100644
File size: 18.0 KB
Line 
1#
2# Copyright (C) 2020-2021 Soleta Networks <info@soleta.eu>
3#
4# This program is free software: you can redistribute it and/or modify it under
5# the terms of the GNU Affero General Public License as published by the
6# Free Software Foundation; either version 3 of the License, or
7# (at your option) any later version.
8
9import hashlib
10import logging
11import os
12import subprocess
13import shlex
14import shutil
15
16from subprocess import Popen, PIPE
17
18import fdisk
19
20from src.ogClient import ogClient
21from src.ogRest import ThreadState
22from src.live.partcodes import GUID_MAP
23
24from src.utils.legacy import *
25from src.utils.net import ethtool
26from src.utils.menu import generate_menu
27from src.utils.fs import *
28from src.utils.probe import os_probe, cache_probe
29from src.utils.disk import *
30from src.utils.cache import generate_cache_txt
31from src.utils.tiptorrent import *
32
33
34OG_SHELL = '/bin/bash'
35
36class OgLiveOperations:
37    def __init__(self, config):
38        self._url = config['opengnsys']['url']
39        self._url_log = config['opengnsys']['url_log']
40        self._smb_user = config['samba']['user']
41        self._smb_pass = config['samba']['pass']
42
43    def _restartBrowser(self, url):
44        try:
45            proc = subprocess.call(["pkill", "-9", "browser"])
46            proc = subprocess.Popen(["browser", "-qws", url])
47        except:
48            logging.error('Cannot restart browser')
49            raise ValueError('Error: cannot restart browser')
50
51    def _refresh_payload_disk(self, cxt, part_setup, num_disk):
52        part_setup['disk'] = str(num_disk)
53        part_setup['disk_type'] = 'DISK'
54        part_setup['partition'] = '0'
55        part_setup['filesystem'] = ''
56        part_setup['os'] = ''
57        part_setup['size'] = str(cxt.nsectors * cxt.sector_size // 1024)
58        part_setup['used_size'] = '0'
59        if not cxt.label:
60            part_setup['code'] = '0'
61        else:
62            part_setup['code'] = '2' if cxt.label.name == 'gpt' else '1'
63
64    def _refresh_payload_partition(self, cxt, pa, part_setup, disk):
65        parttype = cxt.partition_to_string(pa, fdisk.FDISK_FIELD_TYPEID)
66        fstype = cxt.partition_to_string(pa, fdisk.FDISK_FIELD_FSTYPE)
67        padev = cxt.partition_to_string(pa, fdisk.FDISK_FIELD_DEVICE)
68        size = cxt.partition_to_string(pa, fdisk.FDISK_FIELD_SIZE)
69        partnum = pa.partno + 1
70        source = padev
71        target = padev.replace('dev', 'mnt')
72
73        if cxt.label.name == 'gpt':
74            code = GUID_MAP.get(parttype, 0x0)
75        else:
76            code = int(parttype, base=16)
77
78        if mount_mkdir(source, target):
79            probe_result = os_probe(target)
80            part_setup['os'] = probe_result
81            part_setup['used_size'] = get_usedperc(target)
82            umount(target)
83        else:
84            part_setup['os'] = ''
85            part_setup['used_size'] = '0'
86
87
88        part_setup['disk_type'] = ''
89        part_setup['partition'] = str(partnum)
90        part_setup['filesystem'] = fstype.upper() if fstype else 'EMPTY'
91        # part_setup['code'] = hex(code).removeprefix('0x')
92        part_setup['code'] = hex(code)[2:]
93        part_setup['size'] = str(int(size) // 1024)
94
95        if (part_setup['filesystem'] == 'VFAT'):
96            part_setup['filesystem'] = 'FAT32'
97
98    def _refresh_part_setup_cache(self, cxt, pa, part_setup, cache):
99        padev = cxt.partition_to_string(pa, fdisk.FDISK_FIELD_DEVICE)
100        if padev == cache:
101            part_setup['filesystem'] = 'CACHE'
102            part_setup['code'] = 'ca'
103
104    def _compute_md5(self, path, bs=2**20):
105        m = hashlib.md5()
106        with open(path, 'rb') as f:
107            while True:
108                buf = f.read(bs)
109                if not buf:
110                    break
111                m.update(buf)
112        return m.hexdigest()
113
114    def _write_md5_file(self, path):
115        if not os.path.exists(path):
116            logging.error('Invalid path in _write_md5_file')
117            raise ValueError('Invalid image path when computing md5 checksum')
118        filename = path + ".full.sum"
119        dig = self._compute_md5(path)
120        with open(filename, 'w') as f:
121            f.write(dig)
122
123    def _copy_image_to_cache(self, image_name):
124        """
125        Copies /opt/opengnsys/image/{image_name} into
126        /opt/opengnsys/cache/opt/opengnsys/images/
127
128        Implies a unicast transfer. Does not use tiptorrent.
129        """
130        src = f'/opt/opengnsys/images/{image_name}.img'
131        dst = f'/opt/opengnsys/cache/opt/opengnsys/images/{image_name}.img'
132        try:
133            r = shutil.copy(src, dst)
134            tip_write_csum(image_name)
135        except:
136            logging.error('Error copying image to cache', repo)
137            raise ValueError(f'Error: Cannot copy image {image_name} to cache')
138
139    def _restore_image_unicast(self, repo, name, devpath, cache=False):
140        if ogChangeRepo(repo).returncode != 0:
141            self._restartBrowser(self._url)
142            logging.error('ogChangeRepo could not change repository to %s', repo)
143            raise ValueError(f'Error: Cannot change repository to {repo}')
144        logging.debug(f'restore_image_unicast: name => {name}')
145        if cache:
146            image_path = f'/opt/opengnsys/cache/opt/opengnsys/images/{name}.img'
147            if (not os.path.exists(image_path) or
148                not tip_check_csum(repo, name)):
149                self._copy_image_to_cache(name)
150        else:
151            image_path = f'/opt/opengnsys/images/{name}.img'
152        self._restore_image(image_path, devpath)
153
154    def _restore_image_tiptorrent(self, repo, name, devpath):
155        image_path = f'/opt/opengnsys/cache/opt/opengnsys/images/{name}.img'
156        try:
157            if (not os.path.exists(image_path) or
158                not tip_check_csum(repo, name)):
159                tip_client_get(repo, name)
160        except:
161            self._restartBrowser(self._url)
162            raise ValueError('Error before restoring image')
163        self._restore_image(image_path, devpath)
164
165    def _restore_image(self, image_path, devpath):
166        logging.debug(f'Restoring image at {image_path} into {devpath}')
167        cmd_lzop = shlex.split(f'lzop -dc {image_path}')
168        cmd_pc = shlex.split(f'partclone.restore -d0 -C -I -o {devpath}')
169        cmd_mbuffer = shlex.split('mbuffer -q -m 40M') if shutil.which('mbuffer') else None
170
171        if not os.path.exists(image_path):
172            logging.error('f{image_path} does not exist, exiting.')
173            raise ValueError(f'Error: Image not found at {image_path}')
174
175        with open('/tmp/command.log', 'wb', 0) as logfile:
176            proc_lzop = subprocess.Popen(cmd_lzop,
177                                         stdout=subprocess.PIPE)
178            proc_pc = subprocess.Popen(cmd_pc,
179                                       stdin=proc_lzop.stdout,
180                                       stderr=logfile)
181            proc_lzop.stdout.close()
182            proc_pc.communicate()
183
184    def _ogbrowser_clear_logs(self):
185        logfiles = ['/tmp/command.log', '/tmp/session.log']
186        for logfile in logfiles:
187            with open(logfile, 'wb', 0) as f:
188                f.truncate(0)
189
190    def poweroff(self):
191        logging.info('Powering off client')
192        if os.path.exists('/scripts/oginit'):
193            cmd = f'source {ogClient.OG_PATH}etc/preinit/loadenviron.sh; ' \
194                  f'{ogClient.OG_PATH}scripts/poweroff'
195            subprocess.call([cmd], shell=True, executable=OG_SHELL)
196        else:
197            subprocess.call(['/sbin/poweroff'])
198
199    def reboot(self):
200        logging.info('Rebooting client')
201        if os.path.exists('/scripts/oginit'):
202            cmd = f'source {ogClient.OG_PATH}etc/preinit/loadenviron.sh; ' \
203                  f'{ogClient.OG_PATH}scripts/reboot'
204            subprocess.call([cmd], shell=True, executable=OG_SHELL)
205        else:
206            subprocess.call(['/sbin/reboot'])
207
208    def shellrun(self, request, ogRest):
209        cmd = request.getrun()
210        cmds = cmd.split(";|\n\r")
211
212        self._restartBrowser(self._url_log)
213
214        try:
215            ogRest.proc = subprocess.Popen(cmds,
216                               stdout=subprocess.PIPE,
217                               shell=True,
218                               executable=OG_SHELL)
219            (output, error) = ogRest.proc.communicate()
220        except:
221            logging.error('Exception when running "shell run" subprocess')
222            raise ValueError('Error: Incorrect command value')
223
224        if ogRest.proc.returncode != 0:
225            logging.warn('Non zero exit code when running: %s', ' '.join(cmds))
226        else:
227            logging.info('Shell run command OK')
228
229        self.refresh(ogRest)
230
231        return output.decode('utf-8')
232
233    def session(self, request, ogRest):
234        disk = request.getDisk()
235        partition = request.getPartition()
236        cmd = f'{ogClient.OG_PATH}interfaceAdm/IniciarSesion {disk} {partition}'
237
238        try:
239            ogRest.proc = subprocess.Popen([cmd],
240                               stdout=subprocess.PIPE,
241                               shell=True,
242                               executable=OG_SHELL)
243            (output, error) = ogRest.proc.communicate()
244        except:
245            logging.error('Exception when running session subprocess')
246            raise ValueError('Error: Incorrect command value')
247
248        logging.info('Starting OS at disk %s partition %s', disk, partition)
249        return output.decode('utf-8')
250
251    def software(self, request, path, ogRest):
252        disk = request.getDisk()
253        partition = request.getPartition()
254
255        self._restartBrowser(self._url_log)
256
257        try:
258            cmd = f'{ogClient.OG_PATH}interfaceAdm/InventarioSoftware {disk} ' \
259                  f'{partition} {path}'
260
261            ogRest.proc = subprocess.Popen([cmd],
262                               stdout=subprocess.PIPE,
263                               shell=True,
264                               executable=OG_SHELL)
265            (output, error) = ogRest.proc.communicate()
266        except:
267            logging.error('Exception when running software inventory subprocess')
268            raise ValueError('Error: Incorrect command value')
269
270        self._restartBrowser(self._url)
271
272        software = ''
273        with open(path, 'r') as f:
274            software = f.read()
275
276        logging.info('Software inventory command OK')
277        return software
278
279    def hardware(self, path, ogRest):
280        self._restartBrowser(self._url_log)
281
282        try:
283            cmd = f'{ogClient.OG_PATH}interfaceAdm/InventarioHardware {path}'
284            ogRest.proc = subprocess.Popen([cmd],
285                               stdout=subprocess.PIPE,
286                               shell=True,
287                               executable=OG_SHELL)
288            (output, error) = ogRest.proc.communicate()
289        except:
290            logging.error('Exception when running hardware inventory subprocess')
291            raise ValueError('Error: Incorrect command value')
292
293        self._restartBrowser(self._url)
294
295        logging.info('Hardware inventory command OK')
296        return output.decode('utf-8')
297
298    def setup(self, request, ogRest):
299        table_type = request.getType()
300        disk = request.getDisk()
301        cache = request.getCache()
302        cache_size = request.getCacheSize()
303        partlist = request.getPartitionSetup()
304        cfg = f'dis={disk}*che={cache}*tch={cache_size}!'
305
306        for part in partlist:
307            cfg += f'par={part["partition"]}*cpt={part["code"]}*' \
308                   f'sfi={part["filesystem"]}*tam={part["size"]}*' \
309                   f'ope={part["format"]}%'
310
311            if ogRest.terminated:
312                break
313
314        cmd = f'{ogClient.OG_PATH}interfaceAdm/Configurar {table_type} {cfg}'
315        try:
316            ogRest.proc = subprocess.Popen([cmd],
317                               stdout=subprocess.PIPE,
318                               shell=True,
319                               executable=OG_SHELL)
320            (output, error) = ogRest.proc.communicate()
321        except:
322            logging.error('Exception when running setup subprocess')
323            raise ValueError('Error: Incorrect command value')
324
325        logging.info('Setup command OK')
326        result = self.refresh(ogRest)
327
328        return result
329
330    def image_restore(self, request, ogRest):
331        disk = request.getDisk()
332        partition = request.getPartition()
333        name = request.getName()
334        repo = request.getRepo()
335        ctype = request.getType()
336        profile = request.getProfile()
337        cid = request.getId()
338        partdev = get_partition_device(int(disk), int(partition))
339
340        self._ogbrowser_clear_logs()
341        self._restartBrowser(self._url_log)
342
343        logging.debug('Image restore params:')
344        logging.debug(f'\tname: {name}')
345        logging.debug(f'\trepo: {repo}')
346        logging.debug(f'\tprofile: {profile}')
347        logging.debug(f'\tctype: {ctype}')
348
349        if shutil.which('restoreImageCustom'):
350            restoreImageCustom(repo, name, disk, partition, ctype)
351        elif 'UNICAST' in ctype:
352            cache = 'DIRECT' not in ctype
353            self._restore_image_unicast(repo, name, partdev, cache)
354        elif ctype == 'TIPTORRENT':
355            self._restore_image_tiptorrent(repo, name, partdev)
356
357        output = configureOs(disk, partition)
358
359        self.refresh(ogRest)
360
361        logging.info('Image restore command OK')
362        return output
363
364    def image_create(self, path, request, ogRest):
365        disk = int(request.getDisk())
366        partition = int(request.getPartition())
367        name = request.getName()
368        repo = request.getRepo()
369        cmd_software = f'{ogClient.OG_PATH}interfaceAdm/InventarioSoftware {disk} ' \
370                   f'{partition} {path}'
371        image_path = f'/opt/opengnsys/images/{name}.img'
372
373        self._ogbrowser_clear_logs()
374        self._restartBrowser(self._url_log)
375
376        if ogChangeRepo(repo).returncode != 0:
377            self._restartBrowser(self._url)
378            logging.error('ogChangeRepo could not change repository to %s', repo)
379            raise ValueError(f'Error: Cannot change repository to {repo}')
380
381        try:
382            ogRest.proc = subprocess.Popen([cmd_software],
383                               stdout=subprocess.PIPE,
384                               shell=True,
385                               executable=OG_SHELL)
386            (output, error) = ogRest.proc.communicate()
387        except:
388            self._restartBrowser(self._url)
389            logging.error('Exception when running software inventory subprocess')
390            raise ValueError('Error: Incorrect command value')
391
392        if ogRest.terminated:
393            return
394
395        try:
396            diskname = get_disks()[disk-1]
397            cxt = fdisk.Context(f'/dev/{diskname}', details=True)
398            pa = None
399
400            for i, p in enumerate(cxt.partitions):
401                if (p.partno + 1) == partition:
402                    pa = cxt.partitions[i]
403
404            if pa is None:
405                self._restartBrowser(self._url)
406                logging.error('Target partition not found')
407                raise ValueError('Target partition number not found')
408
409            padev = cxt.partition_to_string(pa, fdisk.FDISK_FIELD_DEVICE)
410            fstype = cxt.partition_to_string(pa, fdisk.FDISK_FIELD_FSTYPE)
411            if not fstype:
412                    logging.error('No filesystem detected. Aborting image creation.')
413                    raise ValueError('Target partition has no filesystem present')
414
415            cambiar_acceso(user=self._smb_user, pwd=self._smb_pass)
416            ogCopyEfiBootLoader(disk, partition)
417            ogReduceFs(disk, partition)
418
419            cmd1 = shlex.split(f'partclone.{fstype} -I -C --clone -s {padev} -O -')
420            cmd2 = shlex.split(f'lzop -1 -fo {image_path}')
421
422            logfile = open('/tmp/command.log', 'wb', 0)
423
424            p1 = Popen(cmd1, stdout=PIPE, stderr=logfile)
425            p2 = Popen(cmd2, stdin=p1.stdout)
426            p1.stdout.close()
427
428            try:
429                    retdata = p2.communicate()
430            except OSError as e:
431                    logging.error('Unexpected error when running partclone and lzop commands')
432            finally:
433                    logfile.close()
434                    p2.terminate()
435                    p1.poll()
436
437            logging.info(f'partclone process exited with code {p1.returncode}')
438            logging.info(f'lzop process exited with code {p2.returncode}')
439
440            ogExtendFs(disk, partition)
441
442            image_info = ogGetImageInfo(image_path)
443        except:
444            self._restartBrowser(self._url)
445            logging.error('Exception when running "image create" subprocess')
446            raise ValueError('Error: Incorrect command value')
447
448        self._write_md5_file(f'/opt/opengnsys/images/{name}.img')
449
450        self._restartBrowser(self._url)
451
452        logging.info('Image creation command OK')
453        return image_info
454
455    def refresh(self, ogRest):
456        self._restartBrowser(self._url_log)
457
458        cache = cache_probe()
459        disks = get_disks()
460        interface = os.getenv('DEVICE')
461        link = ethtool(interface)
462        parsed = { 'serial_number': '',
463                'disk_setup': [],
464                'partition_setup': [],
465                'link': link
466        }
467
468        for num_disk, disk in enumerate(get_disks(), start=1):
469            logging.debug('refresh: processing %s', disk)
470            part_setup = {}
471            try:
472                cxt = fdisk.Context(device=f'/dev/{disk}', details=True)
473            except:
474                continue
475
476            self._refresh_payload_disk(cxt, part_setup, num_disk)
477            parsed['disk_setup'].append(part_setup)
478
479            for pa in cxt.partitions:
480                part_setup = part_setup.copy()
481                self._refresh_payload_partition(cxt, pa, part_setup, disk)
482                self._refresh_part_setup_cache(cxt, pa, part_setup, cache)
483                parsed['partition_setup'].append(part_setup)
484
485        generate_menu(parsed['partition_setup'])
486        generate_cache_txt()
487        self._restartBrowser(self._url)
488
489        logging.info('Sending response to refresh request')
490        return parsed
491
492    def probe(self, ogRest):
493
494        interface = os.getenv('DEVICE')
495        speed = ethtool(interface)
496
497        return {'status': 'OPG' if ogRest.state != ThreadState.BUSY else 'BSY',
498                'speed': speed}
Note: See TracBrowser for help on using the repository browser.