source: ogClient-Git/src/live/ogOperations.py @ 7f6a7b6

Last change on this file since 7f6a7b6 was 70f1d0d, checked in by Jose M. Guisado <jguisado@…>, 2 years ago

live: clear ogbrowser logs before image_restore

Clears content of blue text areas in the real time log view before
executing a restore image operation.

Adds private function _ogbrowser_clear_logs, this function writes to a
couple of text files present in the ogLive environment.
The contents of this file are printed out to the blue text areas
in the "real time log" view.
  • Property mode set to 100644
File size: 18.0 KB
Line 
1#
2# Copyright (C) 2020-2021 Soleta Networks <info@soleta.eu>
3#
4# This program is free software: you can redistribute it and/or modify it under
5# the terms of the GNU Affero General Public License as published by the
6# Free Software Foundation; either version 3 of the License, or
7# (at your option) any later version.
8
9import hashlib
10import logging
11import os
12import subprocess
13import shlex
14import shutil
15
16from subprocess import Popen, PIPE
17
18import fdisk
19
20from src.ogClient import ogClient
21from src.ogRest import ThreadState
22from src.live.partcodes import GUID_MAP
23
24from src.utils.legacy import *
25from src.utils.net import ethtool
26from src.utils.menu import generate_menu
27from src.utils.fs import *
28from src.utils.probe import os_probe, cache_probe
29from src.utils.disk import *
30from src.utils.cache import generate_cache_txt
31from src.utils.tiptorrent import *
32
33
34OG_SHELL = '/bin/bash'
35
36class OgLiveOperations:
37    def __init__(self, config):
38        self._url = config['opengnsys']['url']
39        self._url_log = config['opengnsys']['url_log']
40        self._smb_user = config['samba']['user']
41        self._smb_pass = config['samba']['pass']
42
43    def _restartBrowser(self, url):
44        try:
45            proc = subprocess.call(["pkill", "-9", "browser"])
46            proc = subprocess.Popen(["browser", "-qws", url])
47        except:
48            logging.error('Cannot restart browser')
49            raise ValueError('Error: cannot restart browser')
50
51    def _refresh_payload_disk(self, cxt, part_setup, num_disk):
52        part_setup['disk'] = str(num_disk)
53        part_setup['disk_type'] = 'DISK'
54        part_setup['partition'] = '0'
55        part_setup['filesystem'] = ''
56        part_setup['os'] = ''
57        part_setup['size'] = str(cxt.nsectors * cxt.sector_size // 1024)
58        part_setup['used_size'] = '0'
59        if not cxt.label:
60            part_setup['code'] = '0'
61        else:
62            part_setup['code'] = '2' if cxt.label.name == 'gpt' else '1'
63
64    def _refresh_payload_partition(self, cxt, pa, part_setup, disk):
65        parttype = cxt.partition_to_string(pa, fdisk.FDISK_FIELD_TYPEID)
66        fstype = cxt.partition_to_string(pa, fdisk.FDISK_FIELD_FSTYPE)
67        padev = cxt.partition_to_string(pa, fdisk.FDISK_FIELD_DEVICE)
68        size = cxt.partition_to_string(pa, fdisk.FDISK_FIELD_SIZE)
69        partnum = pa.partno + 1
70        source = padev
71        target = padev.replace('dev', 'mnt')
72
73        if cxt.label.name == 'gpt':
74            code = GUID_MAP.get(parttype, 0x0)
75        else:
76            code = int(parttype, base=16)
77
78        if mount_mkdir(source, target):
79            probe_result = os_probe(target)
80            part_setup['os'] = probe_result
81            part_setup['used_size'] = get_usedperc(target)
82            umount(target)
83        else:
84            part_setup['os'] = ''
85            part_setup['used_size'] = '0'
86
87
88        part_setup['disk_type'] = ''
89        part_setup['partition'] = str(partnum)
90        part_setup['filesystem'] = fstype.upper() if fstype else 'EMPTY'
91        # part_setup['code'] = hex(code).removeprefix('0x')
92        part_setup['code'] = hex(code)[2:]
93        part_setup['size'] = str(int(size) // 1024)
94
95        if (part_setup['filesystem'] == 'VFAT'):
96            part_setup['filesystem'] = 'FAT32'
97
98    def _refresh_part_setup_cache(self, cxt, pa, part_setup, cache):
99        padev = cxt.partition_to_string(pa, fdisk.FDISK_FIELD_DEVICE)
100        if padev == cache:
101            part_setup['filesystem'] = 'CACHE'
102            part_setup['code'] = 'ca'
103
104    def _compute_md5(self, path, bs=2**20):
105        m = hashlib.md5()
106        with open(path, 'rb') as f:
107            while True:
108                buf = f.read(bs)
109                if not buf:
110                    break
111                m.update(buf)
112        return m.hexdigest()
113
114    def _write_md5_file(self, path):
115        if not os.path.exists(path):
116            logging.error('Invalid path in _write_md5_file')
117            raise ValueError('Invalid image path when computing md5 checksum')
118        filename = path + ".full.sum"
119        dig = self._compute_md5(path)
120        with open(filename, 'w') as f:
121            f.write(dig)
122
123    def _copy_image_to_cache(self, image_name):
124        """
125        Copies /opt/opengnsys/image/{image_name} into
126        /opt/opengnsys/cache/opt/opengnsys/images/
127
128        Implies a unicast transfer. Does not use tiptorrent.
129        """
130        src = f'/opt/opengnsys/images/{image_name}.img'
131        dst = f'/opt/opengnsys/cache/opt/opengnsys/images/{image_name}.img'
132        try:
133            r = shutil.copy(src, dst)
134            tip_write_csum(image_name)
135        except:
136            logging.error('Error copying image to cache', repo)
137            raise ValueError(f'Error: Cannot copy image {image_name} to cache')
138
139    def _restore_image_unicast(self, repo, name, devpath, cache=False):
140        if ogChangeRepo(repo).returncode != 0:
141            self._restartBrowser(self._url)
142            logging.error('ogChangeRepo could not change repository to %s', repo)
143            raise ValueError(f'Error: Cannot change repository to {repo}')
144        logging.debug(f'restore_image_unicast: name => {name}')
145        if cache:
146            image_path = f'/opt/opengnsys/cache/opt/opengnsys/images/{name}.img'
147            if (not os.path.exists(image_path) or
148                not tip_check_csum(repo, name)):
149                self._copy_image_to_cache(name)
150        else:
151            image_path = f'/opt/opengnsys/images/{name}.img'
152        self._restore_image(image_path, devpath)
153
154    def _restore_image_tiptorrent(self, repo, name, devpath):
155        image_path = f'/opt/opengnsys/cache/opt/opengnsys/images/{name}.img'
156        try:
157            if (not os.path.exists(image_path) or
158                not tip_check_csum(repo, name)):
159                tip_client_get(repo, name)
160        except:
161            self._restartBrowser(self._url)
162            raise ValueError('Error before restoring image')
163        self._restore_image(image_path, devpath)
164
165    def _restore_image(self, image_path, devpath):
166        logging.debug(f'Restoring image at {image_path} into {devpath}')
167        cmd_lzop = shlex.split(f'lzop -dc {image_path}')
168        cmd_pc = shlex.split(f'partclone.restore -d0 -C -I -o {devpath}')
169        cmd_mbuffer = shlex.split('mbuffer -q -m 40M') if shutil.which('mbuffer') else None
170
171        if not os.path.exists(image_path):
172            logging.error('f{image_path} does not exist, exiting.')
173            raise ValueError(f'Error: Image not found at {image_path}')
174
175        with open('/tmp/command.log', 'wb', 0) as logfile:
176            proc_lzop = subprocess.Popen(cmd_lzop,
177                                         stdout=subprocess.PIPE)
178            proc_pc = subprocess.Popen(cmd_pc,
179                                       stdin=proc_lzop.stdout,
180                                       stderr=logfile)
181            proc_lzop.stdout.close()
182            proc_pc.communicate()
183
184    def _ogbrowser_clear_logs(self):
185        logfiles = ['/tmp/command.log', '/tmp/session.log']
186        for logfile in logfiles:
187            with open(logfile, 'wb', 0) as f:
188                f.truncate(0)
189
190    def poweroff(self):
191        logging.info('Powering off client')
192        if os.path.exists('/scripts/oginit'):
193            cmd = f'source {ogClient.OG_PATH}etc/preinit/loadenviron.sh; ' \
194                  f'{ogClient.OG_PATH}scripts/poweroff'
195            subprocess.call([cmd], shell=True, executable=OG_SHELL)
196        else:
197            subprocess.call(['/sbin/poweroff'])
198
199    def reboot(self):
200        logging.info('Rebooting client')
201        if os.path.exists('/scripts/oginit'):
202            cmd = f'source {ogClient.OG_PATH}etc/preinit/loadenviron.sh; ' \
203                  f'{ogClient.OG_PATH}scripts/reboot'
204            subprocess.call([cmd], shell=True, executable=OG_SHELL)
205        else:
206            subprocess.call(['/sbin/reboot'])
207
208    def shellrun(self, request, ogRest):
209        cmd = request.getrun()
210        cmds = cmd.split(";|\n\r")
211
212        self._restartBrowser(self._url_log)
213
214        try:
215            ogRest.proc = subprocess.Popen(cmds,
216                               stdout=subprocess.PIPE,
217                               shell=True,
218                               executable=OG_SHELL)
219            (output, error) = ogRest.proc.communicate()
220        except:
221            logging.error('Exception when running "shell run" subprocess')
222            raise ValueError('Error: Incorrect command value')
223
224        if ogRest.proc.returncode != 0:
225            logging.warn('Non zero exit code when running: %s', ' '.join(cmds))
226        else:
227            logging.info('Shell run command OK')
228
229        self.refresh(ogRest)
230
231        return output.decode('utf-8')
232
233    def session(self, request, ogRest):
234        disk = request.getDisk()
235        partition = request.getPartition()
236        cmd = f'{ogClient.OG_PATH}interfaceAdm/IniciarSesion {disk} {partition}'
237
238        try:
239            ogRest.proc = subprocess.Popen([cmd],
240                               stdout=subprocess.PIPE,
241                               shell=True,
242                               executable=OG_SHELL)
243            (output, error) = ogRest.proc.communicate()
244        except:
245            logging.error('Exception when running session subprocess')
246            raise ValueError('Error: Incorrect command value')
247
248        logging.info('Starting OS at disk %s partition %s', disk, partition)
249        return output.decode('utf-8')
250
251    def software(self, request, path, ogRest):
252        disk = request.getDisk()
253        partition = request.getPartition()
254
255        self._restartBrowser(self._url_log)
256
257        try:
258            cmd = f'{ogClient.OG_PATH}interfaceAdm/InventarioSoftware {disk} ' \
259                  f'{partition} {path}'
260
261            ogRest.proc = subprocess.Popen([cmd],
262                               stdout=subprocess.PIPE,
263                               shell=True,
264                               executable=OG_SHELL)
265            (output, error) = ogRest.proc.communicate()
266        except:
267            logging.error('Exception when running software inventory subprocess')
268            raise ValueError('Error: Incorrect command value')
269
270        self._restartBrowser(self._url)
271
272        software = ''
273        with open(path, 'r') as f:
274            software = f.read()
275
276        logging.info('Software inventory command OK')
277        return software
278
279    def hardware(self, path, ogRest):
280        self._restartBrowser(self._url_log)
281
282        try:
283            cmd = f'{ogClient.OG_PATH}interfaceAdm/InventarioHardware {path}'
284            ogRest.proc = subprocess.Popen([cmd],
285                               stdout=subprocess.PIPE,
286                               shell=True,
287                               executable=OG_SHELL)
288            (output, error) = ogRest.proc.communicate()
289        except:
290            logging.error('Exception when running hardware inventory subprocess')
291            raise ValueError('Error: Incorrect command value')
292
293        self._restartBrowser(self._url)
294
295        logging.info('Hardware inventory command OK')
296        return output.decode('utf-8')
297
298    def setup(self, request, ogRest):
299        table_type = request.getType()
300        disk = request.getDisk()
301        cache = request.getCache()
302        cache_size = request.getCacheSize()
303        partlist = request.getPartitionSetup()
304        cfg = f'dis={disk}*che={cache}*tch={cache_size}!'
305
306        for part in partlist:
307            cfg += f'par={part["partition"]}*cpt={part["code"]}*' \
308                   f'sfi={part["filesystem"]}*tam={part["size"]}*' \
309                   f'ope={part["format"]}%'
310
311            if ogRest.terminated:
312                break
313
314        cmd = f'{ogClient.OG_PATH}interfaceAdm/Configurar {table_type} {cfg}'
315        try:
316            ogRest.proc = subprocess.Popen([cmd],
317                               stdout=subprocess.PIPE,
318                               shell=True,
319                               executable=OG_SHELL)
320            (output, error) = ogRest.proc.communicate()
321        except:
322            logging.error('Exception when running setup subprocess')
323            raise ValueError('Error: Incorrect command value')
324
325        logging.info('Setup command OK')
326        result = self.refresh(ogRest)
327
328        return result
329
330    def image_restore(self, request, ogRest):
331        disk = request.getDisk()
332        partition = request.getPartition()
333        name = request.getName()
334        repo = request.getRepo()
335        ctype = request.getType()
336        profile = request.getProfile()
337        cid = request.getId()
338        partdev = get_partition_device(int(disk), int(partition))
339
340        self._ogbrowser_clear_logs()
341        self._restartBrowser(self._url_log)
342
343        logging.debug('Image restore params:')
344        logging.debug(f'\tname: {name}')
345        logging.debug(f'\trepo: {repo}')
346        logging.debug(f'\tprofile: {profile}')
347        logging.debug(f'\tctype: {ctype}')
348
349        if shutil.which('restoreImageCustom'):
350            restoreImageCustom(repo, name, disk, partition, ctype)
351        elif 'UNICAST' in ctype:
352            cache = 'DIRECT' not in ctype
353            self._restore_image_unicast(repo, name, partdev, cache)
354        elif ctype == 'TIPTORRENT':
355            self._restore_image_tiptorrent(repo, name, partdev)
356
357        output = configureOs(disk, partition)
358
359        self.refresh(ogRest)
360
361        logging.info('Image restore command OK')
362        return output
363
364    def image_create(self, path, request, ogRest):
365        disk = int(request.getDisk())
366        partition = int(request.getPartition())
367        name = request.getName()
368        repo = request.getRepo()
369        cmd_software = f'{ogClient.OG_PATH}interfaceAdm/InventarioSoftware {disk} ' \
370                   f'{partition} {path}'
371        image_path = f'/opt/opengnsys/images/{name}.img'
372
373        self._restartBrowser(self._url_log)
374
375        if ogChangeRepo(repo).returncode != 0:
376            self._restartBrowser(self._url)
377            logging.error('ogChangeRepo could not change repository to %s', repo)
378            raise ValueError(f'Error: Cannot change repository to {repo}')
379
380        try:
381            ogRest.proc = subprocess.Popen([cmd_software],
382                               stdout=subprocess.PIPE,
383                               shell=True,
384                               executable=OG_SHELL)
385            (output, error) = ogRest.proc.communicate()
386        except:
387            self._restartBrowser(self._url)
388            logging.error('Exception when running software inventory subprocess')
389            raise ValueError('Error: Incorrect command value')
390
391        if ogRest.terminated:
392            return
393
394        try:
395            diskname = get_disks()[disk-1]
396            cxt = fdisk.Context(f'/dev/{diskname}', details=True)
397            pa = None
398
399            for i, p in enumerate(cxt.partitions):
400                if (p.partno + 1) == partition:
401                    pa = cxt.partitions[i]
402
403            if pa is None:
404                self._restartBrowser(self._url)
405                logging.error('Target partition not found')
406                raise ValueError('Target partition number not found')
407
408            padev = cxt.partition_to_string(pa, fdisk.FDISK_FIELD_DEVICE)
409            fstype = cxt.partition_to_string(pa, fdisk.FDISK_FIELD_FSTYPE)
410            if not fstype:
411                    logging.error('No filesystem detected. Aborting image creation.')
412                    raise ValueError('Target partition has no filesystem present')
413
414            cambiar_acceso(user=self._smb_user, pwd=self._smb_pass)
415            ogCopyEfiBootLoader(disk, partition)
416            ogReduceFs(disk, partition)
417
418            cmd1 = shlex.split(f'partclone.{fstype} -I -C --clone -s {padev} -O -')
419            cmd2 = shlex.split(f'lzop -1 -fo {image_path}')
420
421            logfile = open('/tmp/command.log', 'wb', 0)
422
423            p1 = Popen(cmd1, stdout=PIPE, stderr=logfile)
424            p2 = Popen(cmd2, stdin=p1.stdout)
425            p1.stdout.close()
426
427            try:
428                    retdata = p2.communicate()
429            except OSError as e:
430                    logging.error('Unexpected error when running partclone and lzop commands')
431            finally:
432                    logfile.close()
433                    p2.terminate()
434                    p1.poll()
435
436            logging.info(f'partclone process exited with code {p1.returncode}')
437            logging.info(f'lzop process exited with code {p2.returncode}')
438
439            ogExtendFs(disk, partition)
440
441            image_info = ogGetImageInfo(image_path)
442        except:
443            self._restartBrowser(self._url)
444            logging.error('Exception when running "image create" subprocess')
445            raise ValueError('Error: Incorrect command value')
446
447        self._write_md5_file(f'/opt/opengnsys/images/{name}.img')
448
449        self._restartBrowser(self._url)
450
451        logging.info('Image creation command OK')
452        return image_info
453
454    def refresh(self, ogRest):
455        self._restartBrowser(self._url_log)
456
457        cache = cache_probe()
458        disks = get_disks()
459        interface = os.getenv('DEVICE')
460        link = ethtool(interface)
461        parsed = { 'serial_number': '',
462                'disk_setup': [],
463                'partition_setup': [],
464                'link': link
465        }
466
467        for num_disk, disk in enumerate(get_disks(), start=1):
468            logging.debug('refresh: processing %s', disk)
469            part_setup = {}
470            try:
471                cxt = fdisk.Context(device=f'/dev/{disk}', details=True)
472            except:
473                continue
474
475            self._refresh_payload_disk(cxt, part_setup, num_disk)
476            parsed['disk_setup'].append(part_setup)
477
478            for pa in cxt.partitions:
479                part_setup = part_setup.copy()
480                self._refresh_payload_partition(cxt, pa, part_setup, disk)
481                self._refresh_part_setup_cache(cxt, pa, part_setup, cache)
482                parsed['partition_setup'].append(part_setup)
483
484        generate_menu(parsed['partition_setup'])
485        generate_cache_txt()
486        self._restartBrowser(self._url)
487
488        logging.info('Sending response to refresh request')
489        return parsed
490
491    def probe(self, ogRest):
492
493        interface = os.getenv('DEVICE')
494        speed = ethtool(interface)
495
496        return {'status': 'OPG' if ogRest.state != ThreadState.BUSY else 'BSY',
497                'speed': speed}
Note: See TracBrowser for help on using the repository browser.