1 | # -*- coding: utf-8 -*- |
---|
2 | |
---|
3 | """ |
---|
4 | | This file is part of the web2py Web Framework |
---|
5 | | Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu> |
---|
6 | | License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html) |
---|
7 | |
---|
8 | File operations |
---|
9 | --------------- |
---|
10 | """ |
---|
11 | |
---|
12 | from gluon import storage |
---|
13 | import os |
---|
14 | import sys |
---|
15 | import re |
---|
16 | import tarfile |
---|
17 | import glob |
---|
18 | import time |
---|
19 | import datetime |
---|
20 | import logging |
---|
21 | import shutil |
---|
22 | |
---|
23 | from gluon.http import HTTP |
---|
24 | from gzip import open as gzopen |
---|
25 | from gluon.recfile import generate |
---|
26 | from gluon._compat import PY2 |
---|
27 | from gluon.settings import global_settings |
---|
28 | |
---|
29 | __all__ = ( |
---|
30 | 'parse_version', |
---|
31 | 'read_file', |
---|
32 | 'write_file', |
---|
33 | 'readlines_file', |
---|
34 | 'up', |
---|
35 | 'abspath', |
---|
36 | 'mktree', |
---|
37 | 'listdir', |
---|
38 | 'recursive_unlink', |
---|
39 | 'cleanpath', |
---|
40 | 'tar', |
---|
41 | 'untar', |
---|
42 | 'tar_compiled', |
---|
43 | 'get_session', |
---|
44 | 'check_credentials', |
---|
45 | 'w2p_pack', |
---|
46 | 'w2p_unpack', |
---|
47 | 'create_app', |
---|
48 | 'w2p_pack_plugin', |
---|
49 | 'w2p_unpack_plugin', |
---|
50 | 'fix_newlines', |
---|
51 | 'create_missing_folders', |
---|
52 | 'create_missing_app_folders', |
---|
53 | 'add_path_first', |
---|
54 | ) |
---|
55 | |
---|
56 | |
---|
57 | def parse_semantic(version="Version 1.99.0-rc.1+timestamp.2011.09.19.08.23.26"): |
---|
58 | """Parses a version string according to http://semver.org/ rules |
---|
59 | |
---|
60 | Args: |
---|
61 | version(str): the SemVer string |
---|
62 | |
---|
63 | Returns: |
---|
64 | tuple: Major, Minor, Patch, Release, Build Date |
---|
65 | |
---|
66 | """ |
---|
67 | re_version = re.compile(r'(\d+)\.(\d+)\.(\d+)(-(?P<pre>[^\s+]*))?(\+(?P<build>\S*))') |
---|
68 | m = re_version.match(version.strip().split()[-1]) |
---|
69 | if not m: |
---|
70 | return None |
---|
71 | a, b, c = int(m.group(1)), int(m.group(2)), int(m.group(3)) |
---|
72 | pre_release = m.group('pre') or '' |
---|
73 | build = m.group('build') or '' |
---|
74 | if build.startswith('timestamp'): |
---|
75 | build = datetime.datetime.strptime(build.split('.', 1)[1], '%Y.%m.%d.%H.%M.%S') |
---|
76 | return (a, b, c, pre_release, build) |
---|
77 | |
---|
78 | |
---|
79 | def parse_legacy(version="Version 1.99.0 (2011-09-19 08:23:26)"): |
---|
80 | """Parses "legacy" version string |
---|
81 | |
---|
82 | Args: |
---|
83 | version(str): the version string |
---|
84 | |
---|
85 | Returns: |
---|
86 | tuple: Major, Minor, Patch, Release, Build Date |
---|
87 | |
---|
88 | """ |
---|
89 | re_version = re.compile(r'[^\d]+ (\d+)\.(\d+)\.(\d+)\s*\((?P<datetime>.+?)\)\s*(?P<type>[a-z]+)?') |
---|
90 | m = re_version.match(version) |
---|
91 | a, b, c = int(m.group(1)), int(m.group(2)), int(m.group(3)), |
---|
92 | pre_release = m.group('type') or 'dev' |
---|
93 | build = datetime.datetime.strptime(m.group('datetime'), '%Y-%m-%d %H:%M:%S') |
---|
94 | return (a, b, c, pre_release, build) |
---|
95 | |
---|
96 | |
---|
97 | def parse_version(version): |
---|
98 | """Attempts to parse SemVer, fallbacks on legacy |
---|
99 | """ |
---|
100 | version_tuple = parse_semantic(version) |
---|
101 | if not version_tuple: |
---|
102 | version_tuple = parse_legacy(version) |
---|
103 | return version_tuple |
---|
104 | |
---|
105 | |
---|
106 | def open_file(filename, mode): |
---|
107 | if PY2 or 'b' in mode: |
---|
108 | f = open(filename, mode) |
---|
109 | else: |
---|
110 | f = open(filename, mode, encoding="utf8") |
---|
111 | return f |
---|
112 | |
---|
113 | |
---|
114 | def read_file(filename, mode='r'): |
---|
115 | """Returns content from filename, making sure to close the file explicitly |
---|
116 | on exit. |
---|
117 | """ |
---|
118 | with open_file(filename, mode) as f: |
---|
119 | return f.read() |
---|
120 | |
---|
121 | |
---|
122 | def write_file(filename, value, mode='w'): |
---|
123 | """Writes <value> to filename, making sure to close the file |
---|
124 | explicitly on exit. |
---|
125 | """ |
---|
126 | with open_file(filename, mode) as f: |
---|
127 | return f.write(value) |
---|
128 | |
---|
129 | |
---|
130 | def readlines_file(filename, mode='r'): |
---|
131 | """Applies .split('\n') to the output of `read_file()` |
---|
132 | """ |
---|
133 | return read_file(filename, mode).split('\n') |
---|
134 | |
---|
135 | |
---|
136 | def mktree(path): |
---|
137 | head, tail = os.path.split(path) |
---|
138 | if head: |
---|
139 | if tail: |
---|
140 | mktree(head) |
---|
141 | if not os.path.exists(head): |
---|
142 | os.mkdir(head) |
---|
143 | |
---|
144 | |
---|
145 | def listdir(path, |
---|
146 | expression='^.+$', |
---|
147 | drop=True, |
---|
148 | add_dirs=False, |
---|
149 | sort=True, |
---|
150 | maxnum=None, |
---|
151 | exclude_content_from=None, |
---|
152 | followlinks=False |
---|
153 | ): |
---|
154 | """ |
---|
155 | Like `os.listdir()` but you can specify a regex pattern to filter files. |
---|
156 | If `add_dirs` is True, the returned items will have the full path. |
---|
157 | """ |
---|
158 | if exclude_content_from is None: |
---|
159 | exclude_content_from = [] |
---|
160 | if path[-1:] != os.path.sep: |
---|
161 | path = path + os.path.sep |
---|
162 | if drop: |
---|
163 | n = len(path) |
---|
164 | else: |
---|
165 | n = 0 |
---|
166 | regex = re.compile(expression) |
---|
167 | items = [] |
---|
168 | for (root, dirs, files) in os.walk(path, topdown=True, followlinks=followlinks): |
---|
169 | for dir in dirs[:]: |
---|
170 | if dir.startswith('.'): |
---|
171 | dirs.remove(dir) |
---|
172 | if add_dirs: |
---|
173 | items.append(root[n:]) |
---|
174 | for file in sorted(files): |
---|
175 | if regex.match(file) and not file.startswith('.'): |
---|
176 | if root not in exclude_content_from: |
---|
177 | items.append(os.path.join(root, file)[n:]) |
---|
178 | if maxnum and len(items) >= maxnum: |
---|
179 | break |
---|
180 | if sort: |
---|
181 | return sorted(items) |
---|
182 | else: |
---|
183 | return items |
---|
184 | |
---|
185 | |
---|
186 | def recursive_unlink(f): |
---|
187 | """Deletes `f`. If it's a folder, also its contents will be deleted |
---|
188 | """ |
---|
189 | if os.path.isdir(f): |
---|
190 | for s in os.listdir(f): |
---|
191 | recursive_unlink(os.path.join(f, s)) |
---|
192 | os.rmdir(f) |
---|
193 | elif os.path.isfile(f): |
---|
194 | os.unlink(f) |
---|
195 | |
---|
196 | |
---|
197 | def cleanpath(path): |
---|
198 | """Turns any expression/path into a valid filename. replaces / with _ and |
---|
199 | removes special characters. |
---|
200 | """ |
---|
201 | |
---|
202 | items = path.split('.') |
---|
203 | if len(items) > 1: |
---|
204 | path = re.sub(r'[^\w.]+', '_', '_'.join(items[:-1]) + '.' |
---|
205 | + ''.join(items[-1:])) |
---|
206 | else: |
---|
207 | path = re.sub(r'[^\w.]+', '_', ''.join(items[-1:])) |
---|
208 | return path |
---|
209 | |
---|
210 | |
---|
211 | def _extractall(filename, path='.', members=None): |
---|
212 | tar = tarfile.TarFile(filename, 'r') |
---|
213 | ret = tar.extractall(path, members) |
---|
214 | tar.close() |
---|
215 | return ret |
---|
216 | |
---|
217 | |
---|
218 | def tar(file, dir, expression='^.+$', |
---|
219 | filenames=None, exclude_content_from=None): |
---|
220 | """Tars dir into file, only tars file that match expression |
---|
221 | """ |
---|
222 | |
---|
223 | tar = tarfile.TarFile(file, 'w') |
---|
224 | try: |
---|
225 | if filenames is None: |
---|
226 | filenames = listdir(dir, expression, add_dirs=True, |
---|
227 | exclude_content_from=exclude_content_from) |
---|
228 | for file in filenames: |
---|
229 | tar.add(os.path.join(dir, file), file, False) |
---|
230 | finally: |
---|
231 | tar.close() |
---|
232 | |
---|
233 | |
---|
234 | def untar(file, dir): |
---|
235 | """Untar file into dir |
---|
236 | """ |
---|
237 | |
---|
238 | _extractall(file, dir) |
---|
239 | |
---|
240 | |
---|
241 | def w2p_pack(filename, path, compiled=False, filenames=None): |
---|
242 | """Packs a web2py application. |
---|
243 | |
---|
244 | Args: |
---|
245 | filename(str): path to the resulting archive |
---|
246 | path(str): path to the application |
---|
247 | compiled(bool): if `True` packs the compiled version |
---|
248 | filenames(list): adds filenames to the archive |
---|
249 | """ |
---|
250 | filename = abspath(filename) |
---|
251 | path = abspath(path) |
---|
252 | tarname = filename + '.tar' |
---|
253 | if compiled: |
---|
254 | tar_compiled(tarname, path, r'^[\w.-]+$', |
---|
255 | exclude_content_from=['cache', 'sessions', 'errors']) |
---|
256 | else: |
---|
257 | tar(tarname, path, r'^[\w.-]+$', filenames=filenames, |
---|
258 | exclude_content_from=['cache', 'sessions', 'errors']) |
---|
259 | with open(tarname, 'rb') as tarfp, gzopen(filename, 'wb') as gzfp: |
---|
260 | shutil.copyfileobj(tarfp, gzfp, 4194304) # 4 MB buffer |
---|
261 | os.unlink(tarname) |
---|
262 | |
---|
263 | |
---|
264 | def missing_app_folders(path): |
---|
265 | for subfolder in ('models', 'views', 'controllers', 'databases', |
---|
266 | 'modules', 'cron', 'errors', 'sessions', |
---|
267 | 'languages', 'static', 'private', 'uploads'): |
---|
268 | yield os.path.join(path, subfolder) |
---|
269 | |
---|
270 | |
---|
271 | def create_welcome_w2p(): |
---|
272 | is_newinstall = os.path.exists('NEWINSTALL') |
---|
273 | if not os.path.exists('welcome.w2p') or is_newinstall: |
---|
274 | logger = logging.getLogger("web2py") |
---|
275 | try: |
---|
276 | app_path = 'applications/welcome' |
---|
277 | for amf in missing_app_folders(app_path): |
---|
278 | if not os.path.exists(amf): |
---|
279 | os.mkdir(amf) |
---|
280 | w2p_pack('welcome.w2p', app_path) |
---|
281 | logger.info("New installation: created welcome.w2p file") |
---|
282 | except: |
---|
283 | logger.exception("New installation error: unable to create welcome.w2p file") |
---|
284 | return |
---|
285 | if is_newinstall: |
---|
286 | try: |
---|
287 | os.unlink('NEWINSTALL') |
---|
288 | logger.info("New installation: removed NEWINSTALL file") |
---|
289 | except: |
---|
290 | logger.exception("New installation error: unable to remove NEWINSTALL file") |
---|
291 | |
---|
292 | |
---|
293 | def w2p_unpack(filename, path, delete_tar=True): |
---|
294 | if filename == 'welcome.w2p': |
---|
295 | create_welcome_w2p() |
---|
296 | filename = abspath(filename) |
---|
297 | tarname = None |
---|
298 | if filename.endswith('.w2p'): |
---|
299 | tarname = filename[:-4] + '.tar' |
---|
300 | elif filename.endswith('.gz'): |
---|
301 | tarname = filename[:-3] + '.tar' |
---|
302 | if tarname is not None: |
---|
303 | with gzopen(filename, 'rb') as gzfp, open(tarname, 'wb') as tarfp: |
---|
304 | shutil.copyfileobj(gzfp, tarfp, 4194304) # 4 MB buffer |
---|
305 | else: |
---|
306 | tarname = filename |
---|
307 | path = abspath(path) |
---|
308 | untar(tarname, path) |
---|
309 | if delete_tar: |
---|
310 | os.unlink(tarname) |
---|
311 | |
---|
312 | |
---|
313 | def create_app(path): |
---|
314 | w2p_unpack('welcome.w2p', path) |
---|
315 | |
---|
316 | |
---|
317 | def w2p_pack_plugin(filename, path, plugin_name): |
---|
318 | """Packs the given plugin into a w2p file. |
---|
319 | Will match files at:: |
---|
320 | |
---|
321 | <path>/*/plugin_[name].* |
---|
322 | <path>/*/plugin_[name]/* |
---|
323 | |
---|
324 | """ |
---|
325 | filename = abspath(filename) |
---|
326 | path = abspath(path) |
---|
327 | if not filename.endswith('web2py.plugin.%s.w2p' % plugin_name): |
---|
328 | raise ValueError('Not a web2py plugin') |
---|
329 | with tarfile.open(filename, 'w:gz') as plugin_tarball: |
---|
330 | app_dir = path |
---|
331 | while app_dir.endswith('/'): |
---|
332 | app_dir = app_dir[:-1] |
---|
333 | files1 = glob.glob( |
---|
334 | os.path.join(app_dir, '*/plugin_%s.*' % plugin_name)) |
---|
335 | files2 = glob.glob( |
---|
336 | os.path.join(app_dir, '*/plugin_%s/*' % plugin_name)) |
---|
337 | for file in files1 + files2: |
---|
338 | plugin_tarball.add(file, arcname=file[len(app_dir) + 1:]) |
---|
339 | |
---|
340 | |
---|
341 | def w2p_unpack_plugin(filename, path, delete_tar=True): |
---|
342 | filename = abspath(filename) |
---|
343 | path = abspath(path) |
---|
344 | if not os.path.basename(filename).startswith('web2py.plugin.'): |
---|
345 | raise ValueError('Not a web2py plugin') |
---|
346 | w2p_unpack(filename, path, delete_tar) |
---|
347 | |
---|
348 | |
---|
349 | def tar_compiled(file, dir, expression='^.+$', |
---|
350 | exclude_content_from=None): |
---|
351 | """Used to tar a compiled application. |
---|
352 | The content of models, views, controllers is not stored in the tar file. |
---|
353 | """ |
---|
354 | |
---|
355 | with tarfile.TarFile(file, 'w') as tar: |
---|
356 | for file in listdir(dir, expression, add_dirs=True, |
---|
357 | exclude_content_from=exclude_content_from): |
---|
358 | filename = os.path.join(dir, file) |
---|
359 | if os.path.islink(filename): |
---|
360 | continue |
---|
361 | if os.path.isfile(filename) and not file.endswith('.pyc'): |
---|
362 | if file.startswith('models'): |
---|
363 | continue |
---|
364 | if file.startswith('views'): |
---|
365 | continue |
---|
366 | if file.startswith('controllers'): |
---|
367 | continue |
---|
368 | if file.startswith('modules'): |
---|
369 | continue |
---|
370 | tar.add(filename, file, False) |
---|
371 | |
---|
372 | |
---|
373 | def up(path): |
---|
374 | return os.path.dirname(os.path.normpath(path)) |
---|
375 | |
---|
376 | |
---|
377 | def get_session(request, other_application='admin'): |
---|
378 | """Checks that user is authorized to access other_application""" |
---|
379 | if request.application == other_application: |
---|
380 | raise KeyError |
---|
381 | try: |
---|
382 | session_id = request.cookies['session_id_' + other_application].value |
---|
383 | session_filename = os.path.join( |
---|
384 | up(request.folder), other_application, 'sessions', session_id) |
---|
385 | if not os.path.exists(session_filename): |
---|
386 | session_filename = generate(session_filename) |
---|
387 | osession = storage.load_storage(session_filename) |
---|
388 | except Exception: |
---|
389 | osession = storage.Storage() |
---|
390 | return osession |
---|
391 | |
---|
392 | |
---|
393 | def set_session(request, session, other_application='admin'): |
---|
394 | """Checks that user is authorized to access other_application""" |
---|
395 | if request.application == other_application: |
---|
396 | raise KeyError |
---|
397 | session_id = request.cookies['session_id_' + other_application].value |
---|
398 | session_filename = os.path.join( |
---|
399 | up(request.folder), other_application, 'sessions', session_id) |
---|
400 | storage.save_storage(session, session_filename) |
---|
401 | |
---|
402 | |
---|
403 | def check_credentials(request, other_application='admin', |
---|
404 | expiration=60 * 60, gae_login=True): |
---|
405 | """Checks that user is authorized to access other_application""" |
---|
406 | if request.env.web2py_runtime_gae: |
---|
407 | from google.appengine.api import users |
---|
408 | if users.is_current_user_admin(): |
---|
409 | return True |
---|
410 | elif gae_login: |
---|
411 | login_html = '<a href="%s">Sign in with your google account</a>.' \ |
---|
412 | % users.create_login_url(request.env.path_info) |
---|
413 | raise HTTP(200, '<html><body>%s</body></html>' % login_html) |
---|
414 | else: |
---|
415 | return False |
---|
416 | else: |
---|
417 | t0 = time.time() |
---|
418 | dt = t0 - expiration |
---|
419 | s = get_session(request, other_application) |
---|
420 | r = (s.authorized and s.last_time and s.last_time > dt) |
---|
421 | if r: |
---|
422 | s.last_time = t0 |
---|
423 | set_session(request, s, other_application) |
---|
424 | return r |
---|
425 | |
---|
426 | |
---|
427 | def fix_newlines(path): |
---|
428 | regex = re.compile(r'''(\r |
---|
429 | |\r| |
---|
430 | )''') |
---|
431 | for filename in listdir(path, r'.*\.(py|html)$', drop=False): |
---|
432 | rdata = read_file(filename, 'r') |
---|
433 | wdata = regex.sub('\n', rdata) |
---|
434 | if wdata != rdata: |
---|
435 | write_file(filename, wdata, 'w') |
---|
436 | |
---|
437 | |
---|
438 | # NOTE: same name as os.path.abspath (but signature is different) |
---|
439 | def abspath(*relpath, **kwargs): |
---|
440 | """Converts relative path to absolute path based (by default) on |
---|
441 | applications_parent |
---|
442 | """ |
---|
443 | path = os.path.join(*relpath) |
---|
444 | if os.path.isabs(path): |
---|
445 | return path |
---|
446 | if kwargs.get('gluon', False): |
---|
447 | return os.path.join(global_settings.gluon_parent, path) |
---|
448 | return os.path.join(global_settings.applications_parent, path) |
---|
449 | |
---|
450 | |
---|
451 | def try_mkdir(path): |
---|
452 | if not os.path.exists(path): |
---|
453 | try: |
---|
454 | if os.path.islink(path): |
---|
455 | # path is a broken link, try to mkdir the target of the link |
---|
456 | # instead of the link itself. |
---|
457 | os.mkdir(os.path.realpath(path)) |
---|
458 | else: |
---|
459 | os.mkdir(path) |
---|
460 | except OSError as e: |
---|
461 | if e.errno == 17: # "File exists" (race condition). |
---|
462 | pass |
---|
463 | else: |
---|
464 | raise |
---|
465 | |
---|
466 | |
---|
467 | def create_missing_folders(): |
---|
468 | if not global_settings.web2py_runtime_gae: |
---|
469 | for path in ('applications', 'deposit', 'site-packages', 'logs'): |
---|
470 | try_mkdir(abspath(path, gluon=True)) |
---|
471 | """ |
---|
472 | OLD sys.path dance |
---|
473 | paths = (global_settings.gluon_parent, abspath( |
---|
474 | 'site-packages', gluon=True), abspath('gluon', gluon=True), '') |
---|
475 | """ |
---|
476 | for p in (global_settings.gluon_parent, |
---|
477 | abspath('site-packages', gluon=True), |
---|
478 | ''): |
---|
479 | add_path_first(p) |
---|
480 | |
---|
481 | |
---|
482 | def create_missing_app_folders(request): |
---|
483 | if not global_settings.web2py_runtime_gae: |
---|
484 | if request.folder not in global_settings.app_folders: |
---|
485 | for amf in missing_app_folders(request.folder): |
---|
486 | try_mkdir(amf) |
---|
487 | global_settings.app_folders.add(request.folder) |
---|
488 | |
---|
489 | |
---|
490 | def add_path_first(path): |
---|
491 | sys.path = [path] + [p for p in sys.path if ( |
---|
492 | not p == path and not p == (path + '/'))] |
---|
493 | if not global_settings.web2py_runtime_gae: |
---|
494 | if not path in sys.path: |
---|
495 | site.addsitedir(path) |
---|