go back to generating regex, simplified

new version uses ~2KB vs 200KB memory, is ~100x faster to load
move script to generate pattern to scripts directory
add more tests

jinja2/_identifier.py

@@ -0,0 +1,2 @@
+# generated by scripts/generate_identifier_pattern.py
+pattern = '·̀-ͯ·҃-֑҇-ׇֽֿׁׂׅׄؐ-ًؚ-ٰٟۖ-ۜ۟-۪ۤۧۨ-ܑۭܰ-݊ަ-ް߫-߳ࠖ-࠙ࠛ-ࠣࠥ-ࠧࠩ-࡙࠭-࡛ࣔ-ࣣ࣡-ःऺ-़ा-ॏ॑-ॗॢॣঁ-ঃ়া-ৄেৈো-্ৗৢৣਁ-ਃ਼ਾ-ੂੇੈੋ-੍ੑੰੱੵઁ-ઃ઼ા-ૅે-ૉો-્ૢૣଁ-ଃ଼ା-ୄେୈୋ-୍ୖୗୢୣஂா-ூெ-ைொ-்ௗఀ-ఃా-ౄె-ైొ-్ౕౖౢౣಁ-ಃ಼ಾ-ೄೆ-ೈೊ-್ೕೖೢೣഁ-ഃാ-ൄെ-ൈൊ-്ൗൢൣංඃ්ා-ුූෘ-ෟෲෳัิ-ฺ็-๎ັິ-ູົຼ່-ໍ༹༘༙༵༷༾༿ཱ-྄྆྇ྍ-ྗྙ-ྼ࿆ါ-ှၖ-ၙၞ-ၠၢ-ၤၧ-ၭၱ-ၴႂ-ႍႏႚ-ႝ፝-፟ᜒ-᜔ᜲ-᜴ᝒᝓᝲᝳ឴-៓៝᠋-᠍ᢅᢆᢩᤠ-ᤫᤰ-᤻ᨗ-ᨛᩕ-ᩞ᩠-᩿᩼᪰-᪽ᬀ-ᬄ᬴-᭄᭫-᭳ᮀ-ᮂᮡ-ᮭ᯦-᯳ᰤ-᰷᳐-᳔᳒-᳨᳭ᳲ-᳴᳸᳹᷀-᷵᷻-᷿‿⁀⁔⃐-⃥⃜⃡-⃰℘℮⳯-⵿⳱ⷠ-〪ⷿ-゙゚〯꙯ꙴ-꙽ꚞꚟ꛰꛱ꠂ꠆ꠋꠣ-ꠧꢀꢁꢴ-ꣅ꣠-꣱ꤦ-꤭ꥇ-꥓ꦀ-ꦃ꦳-꧀ꧥꨩ-ꨶꩃꩌꩍꩻ-ꩽꪰꪲ-ꪴꪷꪸꪾ꪿꫁ꫫ-ꫯꫵ꫶ꯣ-ꯪ꯬꯭ﬞ︀-️︠-︯︳︴﹍-﹏＿𐇽𐋠𐍶-𐍺𐨁-𐨃𐨅𐨆𐨌-𐨏𐨸-𐨿𐨺𐫦𐫥𑀀-𑀂𑀸-𑁆𑁿-𑂂𑂰-𑂺𑄀-𑄂𑄧-𑅳𑄴𑆀-𑆂𑆳-𑇊𑇀-𑇌𑈬-𑈷𑈾𑋟-𑋪𑌀-𑌃𑌼𑌾-𑍄𑍇𑍈𑍋-𑍍𑍗𑍢𑍣𑍦-𑍬𑍰-𑍴𑐵-𑑆𑒰-𑓃𑖯-𑖵𑖸-𑗀𑗜𑗝𑘰-𑙀𑚫-𑚷𑜝-𑜫𑰯-𑰶𑰸-𑰿𑲒-𑲧𑲩-𑲶𖫰-𖫴𖬰-𖬶𖽑-𖽾𖾏-𖾒𛲝𛲞𝅥-𝅩𝅭-𝅲𝅻-𝆂𝆅-𝆋𝆪-𝆭𝉂-𝉄𝨀-𝨶𝨻-𝩬𝩵𝪄𝪛-𝪟𝪡-𝪯𞀀-𞀆𞀈-𞀘𞀛-𞀡𞀣𞀤𞀦-𞣐𞀪-𞣖𞥄-𞥊󠄀-󠇯'

jinja2/lexer.py

@@ -32,22 +32,25 @@ string_re = re.compile(r"('([^'\\]*(?:\\.[^'\\]*)*)'"
                        r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S)
 integer_re = re.compile(r'\d+')
 
-# we use the unicode identifier rule if this python version is able
-# to handle unicode identifiers, otherwise the standard ASCII one.
 try:
+    # check if this Python supports Unicode identifiers
     compile('föö', '<unknown>', 'eval')
 except SyntaxError:
+    # no Unicode support, use ASCII identifiers
     name_re = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]*')
     check_ident = False
 else:
-    # for whatever reason these do not exist in 'some' verisons of
-    # python 3 within \w so we just add them.  These are
-    #   0x1885 MONGOLIAN LETTER ALI GALI BALUDA
-    #   0x1886 MONGOLIAN LETTER ALI GALI THREE BALUDA
-    #   0x2118 SCRIPT CAPITAL P
-    #   0x212e ESTIMATED SYMBOL
-    name_re = re.compile(r'[\w\u1885\u1886\u2118\u212e]+')
+    # Unicode support, build a pattern to match valid characters, and set flag
+    # to use str.isidentifier to validate during lexing
+    from jinja2 import _identifier
+    name_re = re.compile(r'[\w{0}]+'.format(_identifier.pattern))
     check_ident = True
+    # remove the pattern from memory after building the regex
+    import sys
+    del sys.modules['jinja2._identifier']
+    import jinja2
+    del jinja2._identifier
+    del _identifier
 
 float_re = re.compile(r'(?<!\.)\d+\.\d+')
 newline_re = re.compile(r'(\r\n|\r|\n)')

scripts/generate_identifier_pattern.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+import itertools
+import os
+import re
+import sys
+
+if sys.version_info[0] < 3:
+    raise RuntimeError('This needs to run on Python 3.')
+
+
+def get_characters():
+    """Find every Unicode character that is valid in a Python `identifier`_ but
+    is not matched by the regex ``\w`` group.
+
+    ``\w`` matches some characters that aren't valid in identifiers, but
+    :meth:`str.isidentifier` will catch that later in lexing.
+
+    All start characters are valid continue characters, so we only test for
+    continue characters.
+
+    _identifier: https://docs.python.org/3/reference/lexical_analysis.html#identifiers
+    """
+    for cp in range(sys.maxunicode + 1):
+        s = chr(cp)
+
+        if ('a' + s).isidentifier() and not re.match(r'\w', s):
+            yield s
+
+
+def collapse_ranges(data):
+    """Given a sorted list of unique characters, generate ranges representing
+    sequential code points.
+
+    Source: https://stackoverflow.com/a/4629241/400617
+    """
+    for a, b in itertools.groupby(
+        enumerate(data),
+        lambda x: ord(x[1]) - x[0]
+    ):
+        b = list(b)
+        yield b[0][1], b[-1][1]
+
+
+def build_pattern(ranges):
+    """Output the regex pattern for ranges of characters.
+
+    One and two character ranges output the individual characters.
+    """
+    out = []
+
+    for a, b in ranges:
+        if a == b:  # single char
+            out.append(a)
+        elif ord(b) - ord(a) == 1:  # two chars, range is redundant
+            out.append(a)
+            out.append(b)
+        else:
+            out.append(f'{a}-{b}')
+
+    return ''.join(out)
+
+
+def main():
+    """Build the regex pattern and write it to the file
+    :file:`jinja2/_identifier.py`."""
+    pattern = build_pattern(collapse_ranges(get_characters()))
+    filename = os.path.abspath(os.path.join(
+        os.path.dirname(__file__), '..', 'jinja2', '_identifier.py'
+    ))
+
+    with open(filename, 'w', encoding='utf8') as f:
+        f.write('# generated by scripts/generate_identifier_pattern.py\n')
+        f.write(f'pattern = \'{pattern}\'\n')
+
+
+if __name__ == '__main__':
+    main()

tests/test_lexnparse.py

@@ -131,18 +131,24 @@ class TestLexer(object):
         (u'föö', False, True),
         (u'き', False, True),
         (u'_', True, True),
-        (u'1a', False, False),
-        # special cases in addition to \w
+        (u'1a', False, False),  # invalid ascii start
+        (u'a-', False, False),  # invalid ascii continue
+        (u'🐍', False, False),  # invalid unicode start
+        (u'a🐍', False, False),  # invalid unicode continue
+        # start characters not matched by \w
         (u'\u1885', False, True),
         (u'\u1886', False, True),
         (u'\u2118', False, True),
         (u'\u212e', False, True),
+        # continue character not matched by \w
+        (u'\xb7', False, False),
+        (u'a\xb7', False, True),
     ))
     def test_name(self, env, name, valid2, valid3):
         t = u'{{ ' + name + u' }}'
 
         if (valid2 and PY2) or (valid3 and not PY2):
-            # shouldn't raise
+            # valid for version being tested, shouldn't raise
             env.from_string(t)
         else:
             pytest.raises(TemplateSyntaxError, env.from_string, t)