Switch testing to use jinja internals

.gitignore

@@ -18,6 +18,8 @@ venv-*/
 htmlcov
 .pytest_cache/
 /.vscode/
+
 tatsu_jinja.json
 tatsu_jinja.py
-parsed_jinja.py
+parsed_jinja.py
+test_tatsu.jinja

test_tatsu.py

@@ -1,54 +1,33 @@
 from datetime import datetime
-from tatsu.exceptions import FailedSemantics
-from tatsu.util import asjson
-import json
 import pprint
-import tatsu
-import sys
-from new_parser import parse_template
 from jinja2.environment import Environment
+from jinja2.parser import Parser
 
 
 with open('grammar.ebnf', 'r') as tatsu_grammar:
     with open('test_template.jinja', 'r') as test_template:
         template_string = test_template.read()
 
-        grammar_start = datetime.now()
-
-        grammar = tatsu.compile(tatsu_grammar.read())
-
-        grammar_end = datetime.now()
-
-        parse_start = datetime.now()
-
-        ast = grammar.parse(template_string, whitespace='', parseinfo=True, semantics=JinjaSemantics())
-
-        parse_end = datetime.now()
-
-        with open('tatsu_jinja.json', 'w') as tatsu_ast_file:
-            json.dump(asjson(ast), tatsu_ast_file, indent=2)
+        env = Environment(line_statement_prefix='#', line_comment_prefix='##')
+        parser = Parser(env, template_string)
 
         new_parse_start = datetime.now()
 
-        new_ast = parse_template(ast)
+        new_ast = parser.parse()
 
         new_parse_end = datetime.now()
 
         with open('tatsu_jinja.py', 'w') as new_ast_file:
             pprint.pprint(new_ast, indent=2, stream=new_ast_file)
 
-        env = Environment(line_statement_prefix='#', line_comment_prefix='##')
-
         jinja_parse_start = datetime.now()
 
-        jinja_ast = env.parse(template_string)
+        jinja_ast = parser.parse_old()
 
         jinja_parse_end = datetime.now()
 
         with open('parsed_jinja.py', 'w') as jinja_ast_file:
             pprint.pprint(jinja_ast, indent=2, stream=jinja_ast_file)
 
-        print("Grammar", grammar_end - grammar_start, file=sys.stderr)
-        print("Parser", parse_end - parse_start, file=sys.stderr)
         print("New Parser", new_parse_end - new_parse_start, file=sys.stderr)
         print("Jinja Parser", jinja_parse_end - jinja_parse_start, file=sys.stderr)

test_template.jinja

@@ -1,108 +0,0 @@
-{% from 'forms.html' import input as input_field, textarea %}
-{{ dict_var['single']["double"].dot  |test("first" ,2_000, named=3.14)|filter | lastFilter}}
-{% with a='', b=a.attribute %}...{% endwith %}
-{% for item in dict_var.values() %}
-    <li>{% block loop_item %}{{ item }}{% endblock %}</li>
-{% endfor %}
-{# comment contents
-across lines #}
-{% raw %}
-    {% block %}fake content{{ fake vars }}
-{% endraw %}
-{% if False %}
-{{ '{{' }}
-{% endif %}
-{% for href, caption in [('index.html', 'Index'), ('about.html', 'About'),
-                         ('downloads.html', 'Downloads')] %}
-    <li><a href="{{ href }}">{{ caption }}</a></li>
-{% endfor %}
-{% set navigation = [('index.html', 'Index'), ('about.html', 'About')] %}
-{% set key, value = call_something() %}
-{% for value in values %}
-    {% if loop.previtem is defined and value > loop.previtem %}
-        The value just increased!
-    {% endif %}
-    {{ value }}
-    {% if loop.nextitem is defined and loop.nextitem > value %}
-        The value will increase even more!
-    {% endif %}
-{% endfor %}
-{%- macro textarea(name, value='', rows=10, cols=40) -%}
-    <textarea name="{{ name }}" rows="{{ rows }}" cols="{{ cols
-        }}">{{ value|e }}</textarea>
-{%- endmacro -%}
-{# comment contents
-across lines #}
-<ul>
-    # for item in seq:
-    <li>{{ item }}</li>     ## this comment is ignored
-    # endfor
-</ul>
-{% set reply | wordwrap %}
-    You wrote:
-    {{ message }}
-{% endset %}
-<ul class="sitemap">
-{%- for item in sitemap recursive %}
-    <li><a href="{{ item.href|e }}">{{ item.title }}</a>
-    {%- if item.children -%}
-        <ul class="submenu">{{ loop(item.children) }}</ul>
-    {%- endif %}</li>
-{%- endfor %}
-</ul>
-{% autoescape true %}
-    Autoescaping is active within this block
-{% endautoescape %}
-
-{% autoescape false %}
-    Autoescaping is inactive within this block
-{% endautoescape %}
-{% if foo.attribute is sameas false %}
-    the foo attribute really is the `False` singleton
-{% endif %}
-<ul{{ {'class': 'my_list', 'missing': none,
-        'id': 'list-%d'|format(variable)}|xmlattr }}>
-...
-</ul>
-{{ "[{}]".format(page.title) if page.title }}
-{% extends layout_template if layout_template is defined else 'master.html' %}
-{{ "Hello " ~ name ~ "!" }}
-{{ 1 in [1, 2, 3] }}
-{{ foo is not bar }}
-{{ not (foo and bar) }}
-{{ foo not in bar }}
-{% from 'forms.html' import input with context %}
-{% include 'header.html' without context %}
-{% import 'forms.html' as forms %}
-{% include ['special_sidebar.html', 'sidebar.html'] ignore missing %}
-{% include "sidebar.html" ignore missing %}
-{% include "sidebar.html" ignore missing with context %}
-{% include "sidebar.html" ignore missing without context %}
-{% for item in seq %}
-    {{ item }}
-{% else %}
-    did not iterate
-{% endfor %}
-{% call(user) dump_users(list_of_user) %}
-    <dl>
-        <dl>Realname</dl>
-        <dd>{{ user.realname|e }}</dd>
-        <dl>Description</dl>
-        <dd>{{ user.description }}</dd>
-    </dl>
-{% endcall %}
-{% if kenny.sick %}
-    Kenny is sick.
-{% elif kenny.dead %}
-    You killed Kenny!  You bastard!!!
-{% else %}
-    Kenny looks okay --- so far
-{% endif %}
-{% for user in users if not user.hidden %}
-    <li>{{ user.username|e }}</li>
-{% endfor %}
-{% for item in seq %}
-    <li>{% block loop_item scoped %}{{ item }}{% endblock %}</li>
-{% endfor %}
-{% if loop.index is divisibleby 3 %}{% endif %}
-{% if loop.index is divisibleby(3) %}{% endif %}