source: OpenRLabs-Git/deploy/rlabs-docker/web2py-rlabs/gluon/fileutils.py

main
Last change on this file was 42bd667, checked in by David Fuertes <dfuertes@…>, 4 years ago

Historial Limpio

  • Property mode set to 100755
File size: 14.9 KB
Line 
1# -*- coding: utf-8 -*-
2
3"""
4| This file is part of the web2py Web Framework
5| Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
6| License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
7
8File operations
9---------------
10"""
11
12from gluon import storage
13import os
14import sys
15import re
16import tarfile
17import glob
18import time
19import datetime
20import logging
21import shutil
22
23from gluon.http import HTTP
24from gzip import open as gzopen
25from gluon.recfile import generate
26from gluon._compat import PY2
27from gluon.settings import global_settings
28
29__all__ = (
30    'parse_version',
31    'read_file',
32    'write_file',
33    'readlines_file',
34    'up',
35    'abspath',
36    'mktree',
37    'listdir',
38    'recursive_unlink',
39    'cleanpath',
40    'tar',
41    'untar',
42    'tar_compiled',
43    'get_session',
44    'check_credentials',
45    'w2p_pack',
46    'w2p_unpack',
47    'create_app',
48    'w2p_pack_plugin',
49    'w2p_unpack_plugin',
50    'fix_newlines',
51    'create_missing_folders',
52    'create_missing_app_folders',
53    'add_path_first',
54)
55
56
57def parse_semantic(version="Version 1.99.0-rc.1+timestamp.2011.09.19.08.23.26"):
58    """Parses a version string according to http://semver.org/ rules
59
60    Args:
61        version(str): the SemVer string
62
63    Returns:
64        tuple: Major, Minor, Patch, Release, Build Date
65
66    """
67    re_version = re.compile(r'(\d+)\.(\d+)\.(\d+)(-(?P<pre>[^\s+]*))?(\+(?P<build>\S*))')
68    m = re_version.match(version.strip().split()[-1])
69    if not m:
70        return None
71    a, b, c = int(m.group(1)), int(m.group(2)), int(m.group(3))
72    pre_release = m.group('pre') or ''
73    build = m.group('build') or ''
74    if build.startswith('timestamp'):
75        build = datetime.datetime.strptime(build.split('.', 1)[1], '%Y.%m.%d.%H.%M.%S')
76    return (a, b, c, pre_release, build)
77
78
79def parse_legacy(version="Version 1.99.0 (2011-09-19 08:23:26)"):
80    """Parses "legacy" version string
81
82    Args:
83        version(str): the version string
84
85    Returns:
86        tuple: Major, Minor, Patch, Release, Build Date
87
88    """
89    re_version = re.compile(r'[^\d]+ (\d+)\.(\d+)\.(\d+)\s*\((?P<datetime>.+?)\)\s*(?P<type>[a-z]+)?')
90    m = re_version.match(version)
91    a, b, c = int(m.group(1)), int(m.group(2)), int(m.group(3)),
92    pre_release = m.group('type') or 'dev'
93    build = datetime.datetime.strptime(m.group('datetime'), '%Y-%m-%d %H:%M:%S')
94    return (a, b, c, pre_release, build)
95
96
97def parse_version(version):
98    """Attempts to parse SemVer, fallbacks on legacy
99    """
100    version_tuple = parse_semantic(version)
101    if not version_tuple:
102        version_tuple = parse_legacy(version)
103    return version_tuple
104
105
106def open_file(filename, mode):
107    if PY2 or 'b' in mode:
108        f = open(filename, mode)
109    else:
110        f = open(filename, mode, encoding="utf8")
111    return f
112
113
114def read_file(filename, mode='r'):
115    """Returns content from filename, making sure to close the file explicitly
116    on exit.
117    """
118    with open_file(filename, mode) as f:
119        return f.read()
120
121
122def write_file(filename, value, mode='w'):
123    """Writes <value> to filename, making sure to close the file
124    explicitly on exit.
125    """
126    with open_file(filename, mode) as f:
127        return f.write(value)
128
129
130def readlines_file(filename, mode='r'):
131    """Applies .split('\n') to the output of `read_file()`
132    """
133    return read_file(filename, mode).split('\n')
134
135
136def mktree(path):
137    head, tail = os.path.split(path)
138    if head:
139        if tail:
140            mktree(head)
141        if not os.path.exists(head):
142            os.mkdir(head)
143
144
145def listdir(path,
146            expression='^.+$',
147            drop=True,
148            add_dirs=False,
149            sort=True,
150            maxnum=None,
151            exclude_content_from=None,
152            followlinks=False
153            ):
154    """
155    Like `os.listdir()` but you can specify a regex pattern to filter files.
156    If `add_dirs` is True, the returned items will have the full path.
157    """
158    if exclude_content_from is None:
159        exclude_content_from = []
160    if path[-1:] != os.path.sep:
161        path = path + os.path.sep
162    if drop:
163        n = len(path)
164    else:
165        n = 0
166    regex = re.compile(expression)
167    items = []
168    for (root, dirs, files) in os.walk(path, topdown=True, followlinks=followlinks):
169        for dir in dirs[:]:
170            if dir.startswith('.'):
171                dirs.remove(dir)
172        if add_dirs:
173            items.append(root[n:])
174        for file in sorted(files):
175            if regex.match(file) and not file.startswith('.'):
176                if root not in exclude_content_from:
177                    items.append(os.path.join(root, file)[n:])
178            if maxnum and len(items) >= maxnum:
179                break
180    if sort:
181        return sorted(items)
182    else:
183        return items
184
185
186def recursive_unlink(f):
187    """Deletes `f`. If it's a folder, also its contents will be deleted
188    """
189    if os.path.isdir(f):
190        for s in os.listdir(f):
191            recursive_unlink(os.path.join(f, s))
192        os.rmdir(f)
193    elif os.path.isfile(f):
194        os.unlink(f)
195
196
197def cleanpath(path):
198    """Turns any expression/path into a valid filename. replaces / with _ and
199    removes special characters.
200    """
201
202    items = path.split('.')
203    if len(items) > 1:
204        path = re.sub(r'[^\w.]+', '_', '_'.join(items[:-1]) + '.'
205                      + ''.join(items[-1:]))
206    else:
207        path = re.sub(r'[^\w.]+', '_', ''.join(items[-1:]))
208    return path
209
210
211def _extractall(filename, path='.', members=None):
212    tar = tarfile.TarFile(filename, 'r')
213    ret = tar.extractall(path, members)
214    tar.close()
215    return ret
216
217
218def tar(file, dir, expression='^.+$',
219        filenames=None, exclude_content_from=None):
220    """Tars dir into file, only tars file that match expression
221    """
222
223    tar = tarfile.TarFile(file, 'w')
224    try:
225        if filenames is None:
226            filenames = listdir(dir, expression, add_dirs=True,
227                exclude_content_from=exclude_content_from)
228        for file in filenames:
229            tar.add(os.path.join(dir, file), file, False)
230    finally:
231        tar.close()
232
233
234def untar(file, dir):
235    """Untar file into dir
236    """
237
238    _extractall(file, dir)
239
240
241def w2p_pack(filename, path, compiled=False, filenames=None):
242    """Packs a web2py application.
243
244    Args:
245        filename(str): path to the resulting archive
246        path(str): path to the application
247        compiled(bool): if `True` packs the compiled version
248        filenames(list): adds filenames to the archive
249    """
250    filename = abspath(filename)
251    path = abspath(path)
252    tarname = filename + '.tar'
253    if compiled:
254        tar_compiled(tarname, path, r'^[\w.-]+$',
255                     exclude_content_from=['cache', 'sessions', 'errors'])
256    else:
257        tar(tarname, path, r'^[\w.-]+$', filenames=filenames,
258            exclude_content_from=['cache', 'sessions', 'errors'])
259    with open(tarname, 'rb') as tarfp, gzopen(filename, 'wb') as gzfp:
260        shutil.copyfileobj(tarfp, gzfp, 4194304) # 4 MB buffer
261    os.unlink(tarname)
262
263
264def missing_app_folders(path):
265    for subfolder in ('models', 'views', 'controllers', 'databases',
266                      'modules', 'cron', 'errors', 'sessions',
267                      'languages', 'static', 'private', 'uploads'):
268        yield os.path.join(path, subfolder)
269
270
271def create_welcome_w2p():
272    is_newinstall = os.path.exists('NEWINSTALL')
273    if not os.path.exists('welcome.w2p') or is_newinstall:
274        logger = logging.getLogger("web2py")
275        try:
276            app_path = 'applications/welcome'
277            for amf in missing_app_folders(app_path):
278                if not os.path.exists(amf):
279                    os.mkdir(amf)
280            w2p_pack('welcome.w2p', app_path)
281            logger.info("New installation: created welcome.w2p file")
282        except:
283            logger.exception("New installation error: unable to create welcome.w2p file")
284            return
285        if is_newinstall:
286            try:
287                os.unlink('NEWINSTALL')
288                logger.info("New installation: removed NEWINSTALL file")
289            except:
290                logger.exception("New installation error: unable to remove NEWINSTALL file")
291
292
293def w2p_unpack(filename, path, delete_tar=True):
294    if filename == 'welcome.w2p':
295        create_welcome_w2p()
296    filename = abspath(filename)
297    tarname = None
298    if filename.endswith('.w2p'):
299        tarname = filename[:-4] + '.tar'
300    elif filename.endswith('.gz'):
301        tarname = filename[:-3] + '.tar'
302    if tarname is not None:
303        with gzopen(filename, 'rb') as gzfp, open(tarname, 'wb') as tarfp:
304            shutil.copyfileobj(gzfp, tarfp, 4194304) # 4 MB buffer
305    else:
306        tarname = filename
307    path = abspath(path)
308    untar(tarname, path)
309    if delete_tar:
310        os.unlink(tarname)
311
312
313def create_app(path):
314    w2p_unpack('welcome.w2p', path)
315
316
317def w2p_pack_plugin(filename, path, plugin_name):
318    """Packs the given plugin into a w2p file.
319    Will match files at::
320
321        <path>/*/plugin_[name].*
322        <path>/*/plugin_[name]/*
323
324    """
325    filename = abspath(filename)
326    path = abspath(path)
327    if not filename.endswith('web2py.plugin.%s.w2p' % plugin_name):
328        raise ValueError('Not a web2py plugin')
329    with tarfile.open(filename, 'w:gz') as plugin_tarball:
330        app_dir = path
331        while app_dir.endswith('/'):
332            app_dir = app_dir[:-1]
333        files1 = glob.glob(
334            os.path.join(app_dir, '*/plugin_%s.*' % plugin_name))
335        files2 = glob.glob(
336            os.path.join(app_dir, '*/plugin_%s/*' % plugin_name))
337        for file in files1 + files2:
338            plugin_tarball.add(file, arcname=file[len(app_dir) + 1:])
339
340
341def w2p_unpack_plugin(filename, path, delete_tar=True):
342    filename = abspath(filename)
343    path = abspath(path)
344    if not os.path.basename(filename).startswith('web2py.plugin.'):
345        raise ValueError('Not a web2py plugin')
346    w2p_unpack(filename, path, delete_tar)
347
348
349def tar_compiled(file, dir, expression='^.+$',
350                 exclude_content_from=None):
351    """Used to tar a compiled application.
352    The content of models, views, controllers is not stored in the tar file.
353    """
354
355    with tarfile.TarFile(file, 'w') as tar:
356        for file in listdir(dir, expression, add_dirs=True,
357                            exclude_content_from=exclude_content_from):
358            filename = os.path.join(dir, file)
359            if os.path.islink(filename):
360                continue
361            if os.path.isfile(filename) and not file.endswith('.pyc'):
362                if file.startswith('models'):
363                    continue
364                if file.startswith('views'):
365                    continue
366                if file.startswith('controllers'):
367                    continue
368                if file.startswith('modules'):
369                    continue
370            tar.add(filename, file, False)
371
372
373def up(path):
374    return os.path.dirname(os.path.normpath(path))
375
376
377def get_session(request, other_application='admin'):
378    """Checks that user is authorized to access other_application"""
379    if request.application == other_application:
380        raise KeyError
381    try:
382        session_id = request.cookies['session_id_' + other_application].value
383        session_filename = os.path.join(
384            up(request.folder), other_application, 'sessions', session_id)
385        if not os.path.exists(session_filename):
386            session_filename = generate(session_filename)
387        osession = storage.load_storage(session_filename)
388    except Exception:
389        osession = storage.Storage()
390    return osession
391
392
393def set_session(request, session, other_application='admin'):
394    """Checks that user is authorized to access other_application"""
395    if request.application == other_application:
396        raise KeyError
397    session_id = request.cookies['session_id_' + other_application].value
398    session_filename = os.path.join(
399        up(request.folder), other_application, 'sessions', session_id)
400    storage.save_storage(session, session_filename)
401
402
403def check_credentials(request, other_application='admin',
404                      expiration=60 * 60, gae_login=True):
405    """Checks that user is authorized to access other_application"""
406    if request.env.web2py_runtime_gae:
407        from google.appengine.api import users
408        if users.is_current_user_admin():
409            return True
410        elif gae_login:
411            login_html = '<a href="%s">Sign in with your google account</a>.' \
412                % users.create_login_url(request.env.path_info)
413            raise HTTP(200, '<html><body>%s</body></html>' % login_html)
414        else:
415            return False
416    else:
417        t0 = time.time()
418        dt = t0 - expiration
419        s = get_session(request, other_application)
420        r = (s.authorized and s.last_time and s.last_time > dt)
421        if r:
422            s.last_time = t0
423            set_session(request, s, other_application)
424        return r
425
426
427def fix_newlines(path):
428    regex = re.compile(r'''(\r
429|\r|
430)''')
431    for filename in listdir(path, r'.*\.(py|html)$', drop=False):
432        rdata = read_file(filename, 'r')
433        wdata = regex.sub('\n', rdata)
434        if wdata != rdata:
435            write_file(filename, wdata, 'w')
436
437
438# NOTE: same name as os.path.abspath (but signature is different)
439def abspath(*relpath, **kwargs):
440    """Converts relative path to absolute path based (by default) on
441    applications_parent
442    """
443    path = os.path.join(*relpath)
444    if os.path.isabs(path):
445        return path
446    if kwargs.get('gluon', False):
447        return os.path.join(global_settings.gluon_parent, path)
448    return os.path.join(global_settings.applications_parent, path)
449
450
451def try_mkdir(path):
452    if not os.path.exists(path):
453        try:
454            if os.path.islink(path):
455                # path is a broken link, try to mkdir the target of the link
456                # instead of the link itself.
457                os.mkdir(os.path.realpath(path))
458            else:
459                os.mkdir(path)
460        except OSError as e:
461            if e.errno == 17:  # "File exists" (race condition).
462                pass
463            else:
464                raise
465
466
467def create_missing_folders():
468    if not global_settings.web2py_runtime_gae:
469        for path in ('applications', 'deposit', 'site-packages', 'logs'):
470            try_mkdir(abspath(path, gluon=True))
471    """
472    OLD sys.path dance
473    paths = (global_settings.gluon_parent, abspath(
474        'site-packages', gluon=True), abspath('gluon', gluon=True), '')
475    """
476    for p in (global_settings.gluon_parent,
477              abspath('site-packages', gluon=True),
478              ''):
479        add_path_first(p)
480
481
482def create_missing_app_folders(request):
483    if not global_settings.web2py_runtime_gae:
484        if request.folder not in global_settings.app_folders:
485            for amf in missing_app_folders(request.folder):
486                try_mkdir(amf)
487            global_settings.app_folders.add(request.folder)
488
489
490def add_path_first(path):
491    sys.path = [path] + [p for p in sys.path if (
492        not p == path and not p == (path + '/'))]
493    if not global_settings.web2py_runtime_gae:
494        if not path in sys.path:
495            site.addsitedir(path)
Note: See TracBrowser for help on using the repository browser.