source: OpenRLabs-Git/deploy/rlabs-docker/web2py-rlabs/gluon/languages.py

main
Last change on this file was 42bd667, checked in by David Fuertes <dfuertes@…>, 4 years ago

Historial Limpio
  • Property mode set to 100755
File size: 39.1 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3
4"""
5| This file is part of the web2py Web Framework
6| Copyrighted by Massimo Di Pierro <mdipierro@cs.depaul.edu>
7| License: LGPLv3 (http://www.gnu.org/licenses/lgpl.html)
8| Plural subsystem is created by Vladyslav Kozlovskyy (Ukraine) <dbdevelop@gmail.com>
9
10Translation system
11--------------------------------------------
12"""
13
14import os
15import re
16import sys
17import pkgutil
18import logging
19from threading import RLock
20
21from pydal._compat import copyreg, PY2, maketrans, iterkeys, unicodeT, to_unicode, to_bytes, iteritems, to_native, pjoin
22from pydal.contrib.portalocker import read_locked, LockedFile
23
24from yatl.sanitizer import xmlescape
25
26from gluon.fileutils import listdir
27from gluon.cfs import getcfs
28from gluon.html import XML, xmlescape
29from gluon.contrib.markmin.markmin2html import render, markmin_escape
30
31__all__ = ['translator', 'findT', 'update_all_languages']
32
33ostat = os.stat
34oslistdir = os.listdir
35pdirname = os.path.dirname
36isdir = os.path.isdir
37
38DEFAULT_LANGUAGE = 'en'
39DEFAULT_LANGUAGE_NAME = 'English'
40
41# DEFAULT PLURAL-FORMS RULES:
42# language doesn't use plural forms
43DEFAULT_NPLURALS = 1
44# only one singular/plural form is used
45DEFAULT_GET_PLURAL_ID = lambda n: 0
46# word is unchangeable
47DEFAULT_CONSTRUCT_PLURAL_FORM = lambda word, plural_id: word
48
49if PY2:
50    NUMBERS = (int, long, float)
51    from gluon.utf8 import Utf8
52else:
53    NUMBERS = (int, float)
54    Utf8 = str
55
56# pattern to find T(blah blah blah) expressions
57PY_STRING_LITERAL_RE = r'(?<=[^\w]T\()(?P<name>'\
58    + r"[uU]?[rR]?(?:'''(?:[^']|'{1,2}(?!'))*''')|"\
59    + r"(?:'(?:[^'\\]|\\.)*')|" + r'(?:"""(?:[^"]|"{1,2}(?!"))*""")|'\
60    + r'(?:"(?:[^"\\]|\\.)*"))'
61
62PY_M_STRING_LITERAL_RE = r'(?<=[^\w]T\.M\()(?P<name>'\
63    + r"[uU]?[rR]?(?:'''(?:[^']|'{1,2}(?!'))*''')|"\
64    + r"(?:'(?:[^'\\]|\\.)*')|" + r'(?:"""(?:[^"]|"{1,2}(?!"))*""")|'\
65    + r'(?:"(?:[^"\\]|\\.)*"))'
66
67regex_translate = re.compile(PY_STRING_LITERAL_RE, re.DOTALL)
68regex_translate_m = re.compile(PY_M_STRING_LITERAL_RE, re.DOTALL)
69regex_param = re.compile(r'{(?P<s>.+?)}')
70
71# pattern for a valid accept_language
72regex_language = \
73    re.compile('([a-z]{2,3}(?:\-[a-z]{2})?(?:\-[a-z]{2})?)(?:[,;]|$)')
74regex_langfile = re.compile('^[a-z]{2,3}(-[a-z]{2})?\.py$')
75regex_backslash = re.compile(r"\\([\\{}%])")
76regex_plural = re.compile('%({.+?})')
77regex_plural_dict = re.compile('^{(?P<w>[^()[\]][^()[\]]*?)\((?P<n>[^()\[\]]+)\)}$')  # %%{word(varname or number)}
78regex_plural_tuple = re.compile(
79    '^{(?P<w>[^[\]()]+)(?:\[(?P<i>\d+)\])?}$')  # %%{word[index]} or %%{word}
80regex_plural_file = re.compile('^plural-[a-zA-Z]{2}(-[a-zA-Z]{2})?\.py$')
81
82
83def is_writable():
84    """ returns True if and only if the filesystem is writable """
85    from gluon.settings import global_settings
86    return not global_settings.web2py_runtime_gae
87
88
89def safe_eval(text):
90    if text.strip():
91        try:
92            import ast
93            return ast.literal_eval(text)
94        except ImportError:
95            return eval(text, {}, {})
96    return None
97
98# used as default filter in translator.M()
99
100
101def markmin(s):
102    def markmin_aux(m):
103        return '{%s}' % markmin_escape(m.group('s'))
104    return render(regex_param.sub(markmin_aux, s),
105                  sep='br', autolinks=None, id_prefix='')
106
107# UTF8 helper functions
108
109
110def upper_fun(s):
111    return to_bytes(to_unicode(s).upper())
112
113
114def title_fun(s):
115    return to_bytes(to_unicode(s).title())
116
117
118def cap_fun(s):
119    return to_bytes(to_unicode(s).capitalize())
120
121
122ttab_in = maketrans("\\%{}", '\x1c\x1d\x1e\x1f')
123ttab_out = maketrans('\x1c\x1d\x1e\x1f', "\\%{}")
124
125# cache of translated messages:
126# global_language_cache:
127# { 'languages/xx.py':
128#     ( {"def-message": "xx-message",
129#        ...
130#        "def-message": "xx-message"}, lock_object )
131#  'languages/yy.py': ( {dict}, lock_object )
132#  ...
133# }
134
135global_language_cache = {}
136
137
138def get_from_cache(cache, val, fun):
139    lang_dict, lock = cache
140    lock.acquire()
141    try:
142        result = lang_dict.get(val)
143    finally:
144        lock.release()
145    if result:
146        return result
147    lock.acquire()
148    try:
149        result = lang_dict.setdefault(val, fun())
150    finally:
151        lock.release()
152    return result
153
154
155def clear_cache(filename):
156    cache = global_language_cache.setdefault(
157        filename, ({}, RLock()))
158    lang_dict, lock = cache
159    lock.acquire()
160    try:
161        lang_dict.clear()
162    finally:
163        lock.release()
164
165
166def read_dict_aux(filename):
167    lang_text = read_locked(filename).replace(b'\r\n', b'\n')
168    clear_cache(filename)
169    try:
170        return safe_eval(to_native(lang_text)) or {}
171    except Exception:
172        e = sys.exc_info()[1]
173        status = 'Syntax error in %s (%s)' % (filename, e)
174        logging.error(status)
175        return {'__corrupted__': status}
176
177
178def read_dict(filename):
179    """ Returns dictionary with translation messages
180    """
181    return getcfs('lang:' + filename, filename,
182                  lambda: read_dict_aux(filename))
183
184
185def read_possible_plural_rules():
186    """
187    Creates list of all possible plural rules files
188    The result is cached in PLURAL_RULES dictionary to increase speed
189    """
190    plurals = {}
191    try:
192        import gluon.contrib.plural_rules as package
193        for importer, modname, ispkg in pkgutil.iter_modules(package.__path__):
194            if len(modname) == 2:
195                module = __import__(package.__name__ + '.' + modname,
196                                    fromlist=[modname])
197                lang = modname
198                pname = modname + '.py'
199                nplurals = getattr(module, 'nplurals', DEFAULT_NPLURALS)
200                get_plural_id = getattr(
201                    module, 'get_plural_id',
202                    DEFAULT_GET_PLURAL_ID)
203                construct_plural_form = getattr(
204                    module, 'construct_plural_form',
205                    DEFAULT_CONSTRUCT_PLURAL_FORM)
206                plurals[lang] = (lang, nplurals, get_plural_id,
207                                 construct_plural_form)
208    except ImportError:
209        e = sys.exc_info()[1]
210        logging.warn('Unable to import plural rules: %s' % e)
211    return plurals
212
213PLURAL_RULES = read_possible_plural_rules()
214
215
216def read_possible_languages_aux(langdir):
217    def get_lang_struct(lang, langcode, langname, langfile_mtime):
218        if lang == 'default':
219            real_lang = langcode.lower()
220        else:
221            real_lang = lang
222        (prules_langcode,
223         nplurals,
224         get_plural_id,
225         construct_plural_form
226         ) = PLURAL_RULES.get(real_lang[:2], ('default',
227                                              DEFAULT_NPLURALS,
228                                              DEFAULT_GET_PLURAL_ID,
229                                              DEFAULT_CONSTRUCT_PLURAL_FORM))
230        if prules_langcode != 'default':
231            (pluraldict_fname,
232             pluraldict_mtime) = plurals.get(real_lang,
233                                             plurals.get(real_lang[:2],
234                                                         ('plural-%s.py' % real_lang, 0)))
235        else:
236            pluraldict_fname = None
237            pluraldict_mtime = 0
238        return (langcode,        # language code from !langcode!
239                langname,
240                # language name in national spelling from !langname!
241                langfile_mtime,  # m_time of language file
242                pluraldict_fname,  # name of plural dictionary file or None (when default.py is not exist)
243                pluraldict_mtime,  # m_time of plural dictionary file or 0 if file is not exist
244                prules_langcode,  # code of plural rules language or 'default'
245                nplurals,        # nplurals for current language
246                get_plural_id,   # get_plural_id() for current language
247                construct_plural_form)  # construct_plural_form() for current language
248
249    plurals = {}
250    flist = oslistdir(langdir) if isdir(langdir) else []
251
252    # scan languages directory for plural dict files:
253    for pname in flist:
254        if regex_plural_file.match(pname):
255            plurals[pname[7:-3]] = (pname,
256                                    ostat(pjoin(langdir, pname)).st_mtime)
257    langs = {}
258    # scan languages directory for langfiles:
259    for fname in flist:
260        if regex_langfile.match(fname) or fname == 'default.py':
261            fname_with_path = pjoin(langdir, fname)
262            d = read_dict(fname_with_path)
263            lang = fname[:-3]
264            langcode = d.get('!langcode!', lang if lang != 'default'
265                             else DEFAULT_LANGUAGE)
266            langname = d.get('!langname!', langcode)
267            langfile_mtime = ostat(fname_with_path).st_mtime
268            langs[lang] = get_lang_struct(lang, langcode,
269                                          langname, langfile_mtime)
270    if 'default' not in langs:
271        # if default.py is not found,
272        # add DEFAULT_LANGUAGE as default language:
273        langs['default'] = get_lang_struct('default', DEFAULT_LANGUAGE,
274                                           DEFAULT_LANGUAGE_NAME, 0)
275    deflang = langs['default']
276    deflangcode = deflang[0]
277    if deflangcode not in langs:
278        # create language from default.py:
279        langs[deflangcode] = deflang[:2] + (0,) + deflang[3:]
280
281    return langs
282
283
284def read_possible_languages(langpath):
285    return getcfs('langs:' + langpath, langpath,
286                  lambda: read_possible_languages_aux(langpath))
287
288
289def read_plural_dict_aux(filename):
290    lang_text = read_locked(filename).replace(b'\r\n', b'\n')
291    try:
292        return eval(lang_text) or {}
293    except Exception:
294        e = sys.exc_info()[1]
295        status = 'Syntax error in %s (%s)' % (filename, e)
296        logging.error(status)
297        return {'__corrupted__': status}
298
299
300def read_plural_dict(filename):
301    return getcfs('plurals:' + filename, filename,
302                  lambda: read_plural_dict_aux(filename))
303
304
305def write_plural_dict(filename, contents):
306    if '__corrupted__' in contents:
307        return
308    fp = None
309    try:
310        fp = LockedFile(filename, 'w')
311        fp.write('#!/usr/bin/env python\n# -*- coding: utf-8 -*-\n{\n# "singular form (0)": ["first plural form (1)", "second plural form (2)", ...],\n')
312        for key in sorted(contents, key=sort_function):
313            forms = '[' + ','.join([repr(Utf8(form))
314                                    for form in contents[key]]) + ']'
315            fp.write('%s: %s,\n' % (repr(Utf8(key)), forms))
316        fp.write('}\n')
317    except (IOError, OSError):
318        if is_writable():
319            logging.warning('Unable to write to file %s' % filename)
320        return
321    finally:
322        if fp:
323            fp.close()
324
325
326def sort_function(x):
327    return to_unicode(x, 'utf-8').lower()
328
329
330def write_dict(filename, contents):
331    if '__corrupted__' in contents:
332        return
333    fp = None
334    try:
335        fp = LockedFile(filename, 'w')
336        fp.write('# -*- coding: utf-8 -*-\n{\n')
337        for key in sorted(contents, key=lambda x: to_unicode(x, 'utf-8').lower()):
338            fp.write('%s: %s,\n' % (repr(Utf8(key)),
339                                    repr(Utf8(contents[key]))))
340        fp.write('}\n')
341    except (IOError, OSError):
342        if is_writable():
343            logging.warning('Unable to write to file %s' % filename)
344        return
345    finally:
346        if fp:
347            fp.close()
348
349
350class lazyT(object):
351    """
352    Never to be called explicitly, returned by
353    translator.__call__() or translator.M()
354    """
355    m = s = T = f = t = None
356    M = is_copy = False
357
358    def __init__(
359        self,
360        message,
361        symbols={},
362        T=None,
363        filter=None,
364        ftag=None,
365        M=False
366    ):
367        if isinstance(message, lazyT):
368            self.m = message.m
369            self.s = message.s
370            self.T = message.T
371            self.f = message.f
372            self.t = message.t
373            self.M = message.M
374            self.is_copy = True
375        else:
376            self.m = message
377            self.s = symbols
378            self.T = T
379            self.f = filter
380            self.t = ftag
381            self.M = M
382            self.is_copy = False
383
384    def __repr__(self):
385        return "<lazyT %s>" % (repr(Utf8(self.m)), )
386
387    def __str__(self):
388        return str(self.T.apply_filter(self.m, self.s, self.f, self.t) if self.M else
389                   self.T.translate(self.m, self.s))
390
391    def __eq__(self, other):
392        return str(self) == str(other)
393
394    def __lt__(self, other):
395        return str(self) < str(other)
396
397    def __gt__(self, other):
398        return str(self) > str(other)
399
400    def __ne__(self, other):
401        return str(self) != str(other)
402
403    def __add__(self, other):
404        return '%s%s' % (self, other)
405
406    def __radd__(self, other):
407        return '%s%s' % (other, self)
408
409    def __mul__(self, other):
410        return str(self) * other
411
412    def __cmp__(self, other):
413        return cmp(str(self), str(other))
414
415    def __hash__(self):
416        return hash(str(self))
417
418    def __getattr__(self, name):
419        return getattr(str(self), name)
420
421    def __getitem__(self, i):
422        return str(self)[i]
423
424    def __getslice__(self, i, j):
425        return str(self)[i:j]
426
427    def __iter__(self):
428        for c in str(self):
429            yield c
430
431    def __len__(self):
432        return len(str(self))
433
434    def xml(self):
435        return str(self) if self.M else xmlescape(str(self), quote=False)
436
437    def encode(self, *a, **b):
438        if PY2 and a[0] != 'utf8':
439            return to_unicode(str(self)).encode(*a, **b)
440        else:
441            return str(self)
442
443    def decode(self, *a, **b):
444        if PY2:
445            return str(self).decode(*a, **b)
446        else:
447            return str(self)
448
449    def read(self):
450        return str(self)
451
452    def __mod__(self, symbols):
453        if self.is_copy:
454            return lazyT(self)
455        return lazyT(self.m, symbols, self.T, self.f, self.t, self.M)
456
457
458def pickle_lazyT(c):
459    return str, (to_native(c.xml()),)
460
461copyreg.pickle(lazyT, pickle_lazyT)
462
463
464class TranslatorFactory(object):
465    """
466    This class is instantiated by gluon.compileapp.build_environment
467    as the T object
468
469    Example:
470
471        T.force(None) # turns off translation
472        T.force('fr, it') # forces web2py to translate using fr.py or it.py
473
474        T("Hello World") # translates "Hello World" using the selected file
475
476    Note:
477        - there is no need to force since, by default, T uses
478          http_accept_language to determine a translation file.
479        - en and en-en are considered different languages!
480        - if language xx-yy is not found force() probes other similar languages
481          using such algorithm: `xx-yy.py -> xx.py -> xx-yy*.py -> xx*.py`
482    """
483
484    def __init__(self, langpath, http_accept_language):
485        self.langpath = langpath
486        self.http_accept_language = http_accept_language
487        # filled in self.force():
488        # ------------------------
489        # self.cache
490        # self.accepted_language
491        # self.language_file
492        # self.plural_language
493        # self.nplurals
494        # self.get_plural_id
495        # self.construct_plural_form
496        # self.plural_file
497        # self.plural_dict
498        # self.requested_languages
499        # ----------------------------------------
500        # filled in self.set_current_languages():
501        # ----------------------------------------
502        # self.default_language_file
503        # self.default_t
504        # self.current_languages
505        self.set_current_languages()
506        self.lazy = True
507        self.otherTs = {}
508        self.filter = markmin
509        self.ftag = 'markmin'
510        self.ns = None
511        self.is_writable = True
512
513    def get_possible_languages_info(self, lang=None):
514        """
515        Returns info for selected language or dictionary with all
516        possible languages info from `APP/languages/*.py`
517        It Returns:
518
519        - a tuple containing::
520
521                langcode, langname, langfile_mtime,
522                pluraldict_fname, pluraldict_mtime,
523                prules_langcode, nplurals,
524                get_plural_id, construct_plural_form
525
526                or None
527
528        - if *lang* is NOT defined a dictionary with all possible
529          languages::
530
531            { langcode(from filename):
532                ( langcode,        # language code from !langcode!
533                  langname,
534                      # language name in national spelling from !langname!
535                  langfile_mtime,  # m_time of language file
536                  pluraldict_fname,# name of plural dictionary file or None (when default.py is not exist)
537                  pluraldict_mtime,# m_time of plural dictionary file or 0 if file is not exist
538                  prules_langcode, # code of plural rules language or 'default'
539                  nplurals,        # nplurals for current language
540                  get_plural_id,   # get_plural_id() for current language
541                  construct_plural_form) # construct_plural_form() for current language
542            }
543
544        Args:
545            lang (str): language
546
547        """
548        info = read_possible_languages(self.langpath)
549        if lang:
550            info = info.get(lang)
551        return info
552
553    def get_possible_languages(self):
554        """ Gets list of all possible languages for current application """
555        return list(set(self.current_languages +
556                        [lang for lang in read_possible_languages(self.langpath)
557                         if lang != 'default']))
558
559    def set_current_languages(self, *languages):
560        """
561        Sets current AKA "default" languages
562        Setting one of this languages makes the force() function to turn
563        translation off
564        """
565        if len(languages) == 1 and isinstance(languages[0], (tuple, list)):
566            languages = languages[0]
567        if not languages or languages[0] is None:
568            # set default language from default.py/DEFAULT_LANGUAGE
569            pl_info = self.get_possible_languages_info('default')
570            if pl_info[2] == 0:  # langfile_mtime
571                # if languages/default.py is not found
572                self.default_language_file = self.langpath
573                self.default_t = {}
574                self.current_languages = [DEFAULT_LANGUAGE]
575            else:
576                self.default_language_file = pjoin(self.langpath,
577                                                   'default.py')
578                self.default_t = read_dict(self.default_language_file)
579                self.current_languages = [pl_info[0]]  # !langcode!
580        else:
581            self.current_languages = list(languages)
582        self.force(self.http_accept_language)
583
584    def plural(self, word, n):
585        """
586        Gets plural form of word for number *n*
587        invoked from T()/T.M() in `%%{}` tag
588
589        Note:
590            "word" MUST be defined in current language (T.accepted_language)
591
592        Args:
593            word (str): word in singular
594            n (numeric): number plural form created for
595
596        Returns:
597            word (str): word in appropriate singular/plural form
598
599        """
600        if int(n) == 1:
601            return word
602        elif word:
603            id = self.get_plural_id(abs(int(n)))
604            # id = 0 singular form
605            # id = 1 first plural form
606            # id = 2 second plural form
607            # etc.
608            if id != 0:
609                forms = self.plural_dict.get(word, [])
610                if len(forms) >= id:
611                    # have this plural form:
612                    return forms[id - 1]
613                else:
614                    # guessing this plural form
615                    forms += [''] * (self.nplurals - len(forms) - 1)
616                    form = self.construct_plural_form(word, id)
617                    forms[id - 1] = form
618                    self.plural_dict[word] = forms
619                    if self.is_writable and is_writable() and self.plural_file:
620                        write_plural_dict(self.plural_file,
621                                          self.plural_dict)
622                    return form
623        return word
624
625    def force(self, *languages):
626        """
627        Selects language(s) for translation
628
629        if a list of languages is passed as a parameter,
630        the first language from this list that matches the ones
631        from the possible_languages dictionary will be
632        selected
633
634        default language will be selected if none
635        of them matches possible_languages.
636        """
637        pl_info = read_possible_languages(self.langpath)
638        def set_plural(language):
639            """
640            initialize plural forms subsystem
641            """
642            lang_info = pl_info.get(language)
643            if lang_info:
644                (pname,
645                 pmtime,
646                 self.plural_language,
647                 self.nplurals,
648                 self.get_plural_id,
649                 self.construct_plural_form
650                 ) = lang_info[3:]
651                pdict = {}
652                if pname:
653                    pname = pjoin(self.langpath, pname)
654                    if pmtime != 0:
655                        pdict = read_plural_dict(pname)
656                self.plural_file = pname
657                self.plural_dict = pdict
658            else:
659                self.plural_language = 'default'
660                self.nplurals = DEFAULT_NPLURALS
661                self.get_plural_id = DEFAULT_GET_PLURAL_ID
662                self.construct_plural_form = DEFAULT_CONSTRUCT_PLURAL_FORM
663                self.plural_file = None
664                self.plural_dict = {}
665        language = ''
666        if len(languages) == 1 and isinstance(languages[0], str):
667            languages = regex_language.findall(languages[0].lower())
668        elif not languages or languages[0] is None:
669            languages = []
670        self.requested_languages = languages = tuple(languages)
671        if languages:
672            all_languages = set(lang for lang in pl_info
673                                if lang != 'default') \
674                | set(self.current_languages)
675            for lang in languages:
676                # compare "aa-bb" | "aa" from *language* parameter
677                # with strings from langlist using such alghorythm:
678                # xx-yy.py -> xx.py -> xx*.py
679                lang5 = lang[:5]
680                if lang5 in all_languages:
681                    language = lang5
682                else:
683                    lang2 = lang[:2]
684                    if len(lang5) > 2 and lang2 in all_languages:
685                        language = lang2
686                    else:
687                        for l in all_languages:
688                            if l[:2] == lang2:
689                                language = l
690                if language:
691                    if language in self.current_languages:
692                        break
693                    self.language_file = pjoin(self.langpath, language + '.py')
694                    self.t = read_dict(self.language_file)
695                    self.cache = global_language_cache.setdefault(
696                        self.language_file,
697                        ({}, RLock()))
698                    set_plural(language)
699                    self.accepted_language = language
700                    return languages
701        self.accepted_language = language
702        if not language:
703            if self.current_languages:
704                self.accepted_language = self.current_languages[0]
705            else:
706                self.accepted_language = DEFAULT_LANGUAGE
707        self.language_file = self.default_language_file
708        self.cache = global_language_cache.setdefault(self.language_file,
709                                                      ({}, RLock()))
710        self.t = self.default_t
711        set_plural(self.accepted_language)
712        return languages
713
714    def __call__(self, message, symbols={}, language=None, lazy=None, ns=None):
715        """
716        get cached translated plain text message with inserted parameters(symbols)
717        if lazy==True lazyT object is returned
718        """
719        if lazy is None:
720            lazy = self.lazy
721        if not language and not ns:
722            if lazy:
723                return lazyT(message, symbols, self)
724            else:
725                return self.translate(message, symbols)
726        else:
727            if ns:
728                if ns != self.ns:
729                    self.langpath = os.path.join(self.langpath, ns)
730                if self.ns is None:
731                    self.ns = ns
732            otherT = self.__get_otherT__(language, ns)
733            return otherT(message, symbols, lazy=lazy)
734
735    def __get_otherT__(self, language=None, namespace=None):
736        if not language and not namespace:
737            raise Exception('Incorrect parameters')
738
739        if namespace:
740            if language:
741                index = '%s/%s' % (namespace, language)
742            else:
743                index = namespace
744        else:
745            index = language
746        try:
747            otherT = self.otherTs[index]
748        except KeyError:
749            otherT = self.otherTs[index] = TranslatorFactory(self.langpath,
750                                                             self.http_accept_language)
751            if language:
752                otherT.force(language)
753        return otherT
754
755    def apply_filter(self, message, symbols={}, filter=None, ftag=None):
756        def get_tr(message, prefix, filter):
757            s = self.get_t(message, prefix)
758            return filter(s) if filter else self.filter(s)
759        if filter:
760            prefix = '@' + (ftag or 'userdef') + '\x01'
761        else:
762            prefix = '@' + self.ftag + '\x01'
763        message = get_from_cache(
764            self.cache, prefix + message,
765            lambda: get_tr(message, prefix, filter))
766        if symbols or symbols == 0 or symbols == "":
767            if isinstance(symbols, dict):
768                symbols.update(
769                    (key, xmlescape(value).translate(ttab_in))
770                    for key, value in iteritems(symbols)
771                    if not isinstance(value, NUMBERS))
772            else:
773                if not isinstance(symbols, tuple):
774                    symbols = (symbols,)
775                symbols = tuple(
776                    value if isinstance(value, NUMBERS)
777                    else to_native(xmlescape(value)).translate(ttab_in)
778                    for value in symbols)
779            message = self.params_substitution(message, symbols)
780        return to_native(XML(message.translate(ttab_out)).xml())
781
782    def M(self, message, symbols={}, language=None,
783          lazy=None, filter=None, ftag=None, ns=None):
784        """
785        Gets cached translated markmin-message with inserted parametes
786        if lazy==True lazyT object is returned
787        """
788        if lazy is None:
789            lazy = self.lazy
790        if not language and not ns:
791            if lazy:
792                return lazyT(message, symbols, self, filter, ftag, True)
793            else:
794                return self.apply_filter(message, symbols, filter, ftag)
795        else:
796            if ns:
797                self.langpath = os.path.join(self.langpath, ns)
798            otherT = self.__get_otherT__(language, ns)
799            return otherT.M(message, symbols, lazy=lazy)
800
801    def get_t(self, message, prefix=''):
802        """
803        Use ## to add a comment into a translation string
804        the comment can be useful do discriminate different possible
805        translations for the same string (for example different locations):
806
807            T(' hello world ') -> ' hello world '
808            T(' hello world ## token') -> ' hello world '
809            T('hello ## world## token') -> 'hello ## world'
810
811        the ## notation is ignored in multiline strings and strings that
812        start with ##. This is needed to allow markmin syntax to be translated
813        """
814        message = to_native(message, 'utf8')
815        prefix = to_native(prefix, 'utf8')
816        key = prefix + message
817        mt = self.t.get(key, None)
818        if mt is not None:
819            return mt
820        # we did not find a translation
821        if message.find('##') > 0:
822            pass
823        if message.find('##') > 0 and not '\n' in message:
824            # remove comments
825            message = message.rsplit('##', 1)[0]
826        # guess translation same as original
827        self.t[key] = mt = self.default_t.get(key, message)
828        # update language file for latter translation
829        if self.is_writable and is_writable() and \
830                self.language_file != self.default_language_file:
831            write_dict(self.language_file, self.t)
832        return regex_backslash.sub(
833            lambda m: m.group(1).translate(ttab_in), to_native(mt))
834
835    def params_substitution(self, message, symbols):
836        """
837        Substitutes parameters from symbols into message using %.
838        also parse `%%{}` placeholders for plural-forms processing.
839
840        Returns:
841            string with parameters
842
843        Note:
844            *symbols* MUST BE OR tuple OR dict of parameters!
845        """
846        def sub_plural(m):
847            """String in `%{}` is transformed by this rules:
848               If string starts with  `!` or `?` such transformations
849               take place:
850
851                   "!string of words" -> "String of word" (Capitalize)
852                   "!!string of words" -> "String Of Word" (Title)
853                   "!!!string of words" -> "STRING OF WORD" (Upper)
854
855                   "?word1?number" -> "word1" or "number"
856                                 (return word1 if number == 1,
857                                  return number otherwise)
858                   "??number" or "?number" -> "" or "number"
859                                 (as above with word1 = "")
860
861                   "?word1?number?word0" -> "word1" or "number" or "word0"
862                                 (return word1 if number == 1,
863                                  return word0 if number == 0,
864                                  return number otherwise)
865                   "?word1?number?" -> "word1" or "number" or ""
866                                 (as above with word0 = "")
867                   "??number?word0" -> "number" or "word0"
868                                 (as above with word1 = "")
869                   "??number?" -> "number" or ""
870                                 (as above with word1 = word0 = "")
871
872                   "?word1?word[number]" -> "word1" or "word"
873                                 (return word1 if symbols[number] == 1,
874                                  return word otherwise)
875                   "?word1?[number]" -> "" or "word1"
876                                 (as above with word = "")
877                   "??word[number]" or "?word[number]" -> "" or "word"
878                                 (as above with word1 = "")
879
880                   "?word1?word?word0[number]" -> "word1" or "word" or "word0"
881                                 (return word1 if symbols[number] == 1,
882                                  return word0 if symbols[number] == 0,
883                                  return word otherwise)
884                   "?word1?word?[number]" -> "word1" or "word" or ""
885                                 (as above with word0 = "")
886                   "??word?word0[number]" -> "" or "word" or "word0"
887                                 (as above with word1 = "")
888                   "??word?[number]" -> "" or "word"
889                                 (as above with word1 = word0 = "")
890
891               Other strings, (those not starting with  `!` or `?`)
892               are processed by self.plural
893            """
894            def sub_tuple(m):
895                """ word
896                    !word, !!word, !!!word
897                    ?word1?number
898                         ??number, ?number
899                    ?word1?number?word0
900                    ?word1?number?
901                         ??number?word0
902                         ??number?
903
904                    word[number]
905                    !word[number], !!word[number], !!!word[number]
906                    ?word1?word[number]
907                    ?word1?[number]
908                         ??word[number], ?word[number]
909                    ?word1?word?word0[number]
910                    ?word1?word?[number]
911                         ??word?word0[number]
912                         ??word?[number]
913                """
914                w, i = m.group('w', 'i')
915                c = w[0]
916                if c not in '!?':
917                    return self.plural(w, symbols[int(i or 0)])
918                elif c == '?':
919                    (p1, sep, p2) = w[1:].partition("?")
920                    part1 = p1 if sep else ""
921                    (part2, sep, part3) = (p2 if sep else p1).partition("?")
922                    if not sep:
923                        part3 = part2
924                    if i is None:
925                        # ?[word]?number[?number] or ?number
926                        if not part2:
927                            return m.group(0)
928                        num = int(part2)
929                    else:
930                        # ?[word1]?word[?word0][number]
931                        num = int(symbols[int(i or 0)])
932                    return part1 if num == 1 else part3 if num == 0 else part2
933                elif w.startswith('!!!'):
934                    word = w[3:]
935                    fun = upper_fun
936                elif w.startswith('!!'):
937                    word = w[2:]
938                    fun = title_fun
939                else:
940                    word = w[1:]
941                    fun = cap_fun
942                if i is not None:
943                    return to_native(fun(self.plural(word, symbols[int(i)])))
944                return to_native(fun(word))
945
946            def sub_dict(m):
947                """ word(key or num)
948                    !word(key or num), !!word(key or num), !!!word(key or num)
949                    ?word1?word(key or num)
950                         ??word(key or num), ?word(key or num)
951                    ?word1?word?word0(key or num)
952                    ?word1?word?(key or num)
953                         ??word?word0(key or num)
954                    ?word1?word?(key or num)
955                         ??word?(key or num), ?word?(key or num)
956                """
957                w, n = m.group('w', 'n')
958                c = w[0]
959                n = int(n) if n.isdigit() else symbols[n]
960                if c not in '!?':
961                    return self.plural(w, n)
962                elif c == '?':
963                    # ?[word1]?word[?word0](key or num), ?[word1]?word(key or num) or ?word(key or num)
964                    (p1, sep, p2) = w[1:].partition("?")
965                    part1 = p1 if sep else ""
966                    (part2, sep, part3) = (p2 if sep else p1).partition("?")
967                    if not sep:
968                        part3 = part2
969                    num = int(n)
970                    return part1 if num == 1 else part3 if num == 0 else part2
971                elif w.startswith('!!!'):
972                    word = w[3:]
973                    fun = upper_fun
974                elif w.startswith('!!'):
975                    word = w[2:]
976                    fun = title_fun
977                else:
978                    word = w[1:]
979                    fun = cap_fun
980                s = fun(self.plural(word, n))
981                return s if PY2 else to_unicode(s)
982
983            s = m.group(1)
984            part = regex_plural_tuple.sub(sub_tuple, s)
985            if part == s:
986                part = regex_plural_dict.sub(sub_dict, s)
987                if part == s:
988                    return m.group(0)
989            return part
990        message = message % symbols
991        message = regex_plural.sub(sub_plural, message)
992        return message
993
994    def translate(self, message, symbols):
995        """
996        Gets cached translated message with inserted parameters(symbols)
997        """
998        message = get_from_cache(self.cache, message,
999                                 lambda: self.get_t(message))
1000        if symbols or symbols == 0 or symbols == "":
1001            if isinstance(symbols, dict):
1002                symbols.update(
1003                    (key, str(value).translate(ttab_in))
1004                    for key, value in iteritems(symbols)
1005                    if not isinstance(value, NUMBERS))
1006            else:
1007                if not isinstance(symbols, tuple):
1008                    symbols = (symbols,)
1009                symbols = tuple(
1010                    value if isinstance(value, NUMBERS)
1011                    else str(value).translate(ttab_in)
1012                    for value in symbols)
1013            message = self.params_substitution(message, symbols)
1014        return message.translate(ttab_out)
1015
1016
1017def findT(path, language=DEFAULT_LANGUAGE):
1018    """
1019    Note:
1020        Must be run by the admin app
1021    """
1022    from gluon.tools import Auth, Crud
1023    lang_file = pjoin(path, 'languages', language + '.py')
1024    sentences = read_dict(lang_file)
1025    mp = pjoin(path, 'models')
1026    cp = pjoin(path, 'controllers')
1027    vp = pjoin(path, 'views')
1028    mop = pjoin(path, 'modules')
1029    def add_message(message):
1030        if not message.startswith('#') and not '\n' in message:
1031            tokens = message.rsplit('##', 1)
1032        else:
1033            # this allows markmin syntax in translations
1034            tokens = [message]
1035        if len(tokens) == 2:
1036            message = tokens[0].strip() + '##' + tokens[1].strip()
1037        if message and not message in sentences:
1038            sentences[message] = message.replace("@markmin\x01", "")
1039    for filename in \
1040            listdir(mp, '^.+\.py$', 0) + listdir(cp, '^.+\.py$', 0)\
1041            + listdir(vp, '^.+\.html$', 0) + listdir(mop, '^.+\.py$', 0):
1042        data = to_native(read_locked(filename))
1043        items = regex_translate.findall(data)
1044        for x in regex_translate_m.findall(data):
1045            if x[0:3] in ["'''", '"""']: items.append("%s@markmin\x01%s" %(x[0:3], x[3:]))
1046            else: items.append("%s@markmin\x01%s" %(x[0], x[1:]))
1047        for item in items:
1048            try:
1049                message = safe_eval(item)
1050            except:
1051                continue  # silently ignore inproperly formatted strings
1052            add_message(message)
1053    gluon_msg = [Auth.default_messages, Crud.default_messages]
1054    for item in [x for m in gluon_msg for x in m.values() if x is not None]:
1055        add_message(item)
1056    if not '!langcode!' in sentences:
1057        sentences['!langcode!'] = (
1058            DEFAULT_LANGUAGE if language in ('default', DEFAULT_LANGUAGE) else language)
1059    if not '!langname!' in sentences:
1060        sentences['!langname!'] = (
1061            DEFAULT_LANGUAGE_NAME if language in ('default', DEFAULT_LANGUAGE)
1062            else sentences['!langcode!'])
1063    write_dict(lang_file, sentences)
1064
1065
1066def update_all_languages(application_path):
1067    """
1068    Note:
1069        Must be run by the admin app
1070    """
1071    path = pjoin(application_path, 'languages/')
1072    for language in oslistdir(path):
1073        if regex_langfile.match(language):
1074            findT(application_path, language[:-3])
1075
1076
1077def update_from_langfile(target, source, force_update=False):
1078    """this will update untranslated messages in target from source (where both are language files)
1079    this can be used as first step when creating language file for new but very similar language
1080        or if you want update your app from welcome app of newer web2py version
1081        or in non-standard scenarios when you work on target and from any reason you have partial translation in source
1082    Args:
1083        force_update: if False existing translations remain unchanged, if True existing translations will update from source
1084    """
1085    src = read_dict(source)
1086    sentences = read_dict(target)
1087    for key in sentences:
1088        val = sentences[key]
1089        if not val or val == key or force_update:
1090            new_val = src.get(key)
1091            if new_val and new_val != val:
1092                sentences[key] = new_val
1093    write_dict(target, sentences)
1094
1095
1096if __name__ == '__main__':
1097    import doctest
1098    doctest.testmod()
Note: See TracBrowser for help on using the repository browser.