Only support implicit tuples in variable expressions

They cause super inconsistent parsing behaviour if they show up
pretty much anywhere else because there is no difference between an
implicit tuple and a set of comma-separated parameters.

.azure-pipelines.yml

@@ -0,0 +1,42 @@
+trigger:
+  - master
+  - '*.x'
+
+variables:
+  vmImage: ubuntu-latest
+  python.version: '3.8'
+  TOXENV: py
+
+strategy:
+  matrix:
+    Python 3.8 Linux:
+      vmImage: ubuntu-latest
+    Python 3.8 Windows:
+      vmImage: windows-latest
+    Python 3.8 Mac:
+      vmImage: macos-latest
+    Python 3.7 Linux:
+      python.version: '3.7'
+    Python 3.6 Linux:
+      python.version: '3.6'
+    PyPy 3 Linux:
+      python.version: pypy3
+    Docs:
+      TOXENV: docs
+    Style:
+      TOXENV: style
+
+pool:
+  vmImage: $[ variables.vmImage ]
+
+steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: $(python.version)
+    displayName: Use Python $(python.version)
+
+  - script: pip --disable-pip-version-check install -U tox
+    displayName: Install tox
+
+  - script: tox
+    displayName: Run tox

.devcontainer/devcontainer.json

@@ -1,17 +0,0 @@
-{
-  "name": "pallets/jinja",
-  "image": "mcr.microsoft.com/devcontainers/python:3",
-  "customizations": {
-    "vscode": {
-      "settings": {
-        "python.defaultInterpreterPath": "${workspaceFolder}/.venv",
-        "python.terminal.activateEnvInCurrentTerminal": true,
-        "python.terminal.launchArgs": [
-          "-X",
-          "dev"
-        ]
-      }
-    }
-  },
-  "onCreateCommand": ".devcontainer/on-create-command.sh"
-}

.devcontainer/on-create-command.sh

@@ -1,17 +0,0 @@
-#!/bin/bash
-set -e
-
-# Install uv if not already installed
-if ! command -v uv &> /dev/null; then
-    echo "Installing uv..."
-    curl -LsSf https://astral.sh/uv/install.sh | sh
-    export PATH="$HOME/.cargo/bin:$PATH"
-fi
-
-# Create venv using uv and install dependencies
-echo "Creating virtual environment and installing dependencies..."
-uv sync
-
-# Install pre-commit hooks
-echo "Installing pre-commit hooks..."
-pre-commit install --install-hooks

.editorconfig

@@ -1,13 +0,0 @@
-root = true
-
-[*]
-indent_style = space
-indent_size = 4
-insert_final_newline = true
-trim_trailing_whitespace = true
-end_of_line = lf
-charset = utf-8
-max_line_length = 88
-
-[*.{css,html,js,json,jsx,scss,ts,tsx,yaml,yml}]
-indent_size = 2

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,33 @@
+The issue tracker is a tool to address bugs in Jinja itself.
+Please use the #pocoo IRC channel on freenode or Stack Overflow for general
+questions about using Jinja or issues not related to Jinja.
+
+If you'd like to report a bug in Jinja, fill out the template below and provide
+any extra information that may be useful / related to your problem.
+Ideally, you create an [MCVE](http://stackoverflow.com/help/mcve) reproducing
+the problem before opening an issue to ensure it's not caused by something in
+your code.
+
+---
+
+## Expected Behavior
+Tell us what should happen
+
+## Actual Behavior
+Tell us what happens instead
+
+## Template Code
+```jinja
+Paste the template code (ideally a minimal example) that causes the issue
+
+```
+
+## Full Traceback
+```pytb
+Paste the full traceback in case there is an exception
+
+```
+
+## Your Environment
+* Python version:
+* Jinja version:

.github/ISSUE_TEMPLATE/bug-report.md

@@ -1,27 +0,0 @@
----
-name: Bug report
-about: Report a bug in Jinja (not other projects which depend on Jinja)
----
-
-<!--
-This issue tracker is a tool to address bugs in Jinja itself. Please use
-GitHub Discussions or the Pallets Discord for questions about your own code.
-
-Replace this comment with a clear outline of what the bug is.
--->
-
-<!--
-Describe how to replicate the bug.
-
-Include a minimal reproducible example that demonstrates the bug.
-Include the full traceback if there was an exception.
--->
-
-<!--
-Describe the expected behavior that should have happened but didn't.
--->
-
-Environment:
-
-- Python version:
-- Jinja version:

.github/ISSUE_TEMPLATE/config.yml

@@ -1,8 +0,0 @@
-blank_issues_enabled: false
-contact_links:
-  - name: Questions on Discussions
-    url: https://github.com/pallets/jinja/discussions/
-    about: Ask questions about your own code on the Discussions tab.
-  - name: Questions on Chat
-    url: https://discord.gg/pallets
-    about: Ask questions about your own code on our Discord chat.

.github/ISSUE_TEMPLATE/feature-request.md

@@ -1,15 +0,0 @@
----
-name: Feature request
-about: Suggest a new feature for Jinja
----
-
-<!--
-Replace this comment with a description of what the feature should do.
-Include details such as links to relevant specs or previous discussions.
--->
-
-<!--
-Replace this comment with an example of the problem which this feature
-would resolve. Is this problem solvable without changes to Jinja, such
-as by subclassing or using an extension?
--->

.github/pull_request_template.md

@@ -1,25 +0,0 @@
-<!--
-Before opening a PR, open a ticket describing the issue or feature the
-PR will address. An issue is not required for fixing typos in
-documentation, or other simple non-code changes.
-
-Replace this comment with a description of the change. Describe how it
-addresses the linked ticket.
--->
-
-<!--
-Link to relevant issues or previous PRs, one per line. Use "fixes" to
-automatically close an issue.
-
-fixes #<issue number>
--->
-
-<!--
-Ensure each step in CONTRIBUTING.rst is complete, especially the following:
-
-- Add tests that demonstrate the correct behavior of the change. Tests
-  should fail without the change.
-- Add or update relevant docs, in the docs folder and in code.
-- Add an entry in CHANGES.rst summarizing the change and linking to the issue.
-- Add `.. versionchanged::` entries in any relevant code docs.
--->

.github/workflows/lock.yaml

@@ -1,24 +0,0 @@
-name: Lock inactive closed issues
-# Lock closed issues that have not received any further activity for two weeks.
-# This does not close open issues, only humans may do that. It is easier to
-# respond to new issues with fresh examples rather than continuing discussions
-# on old issues.
-
-on:
-  schedule:
-    - cron: '0 0 * * *'
-permissions:
-  issues: write
-  pull-requests: write
-  discussions: write
-concurrency:
-  group: lock
-jobs:
-  lock:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: dessant/lock-threads@1bf7ec25051fe7c00bdd17e6a7cf3d7bfb7dc771 # v5.0.1
-        with:
-          issue-inactive-days: 14
-          pr-inactive-days: 14
-          discussion-inactive-days: 14

.github/workflows/pre-commit.yaml

@@ -1,25 +0,0 @@
-name: pre-commit
-on:
-  pull_request:
-  push:
-    branches: [main, stable]
-jobs:
-  main:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-      - uses: astral-sh/setup-uv@f0ec1fc3b38f5e7cd731bb6ce540c5af426746bb # v6.1.0
-        with:
-          enable-cache: true
-          prune-cache: false
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        id: setup-python
-        with:
-          python-version-file: pyproject.toml
-      - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
-        with:
-          path: ~/.cache/pre-commit
-          key: pre-commit|${{ hashFiles('pyproject.toml', '.pre-commit-config.yaml') }}
-      - run: uv run --locked --group pre-commit pre-commit run --show-diff-on-failure --color=always --all-files
-      - uses: pre-commit-ci/lite-action@5d6cc0eb514c891a40562a58a8e71576c5c7fb43 # v1.1.0
-        if: ${{ !cancelled() }}

.github/workflows/publish.yaml

@@ -1,47 +0,0 @@
-name: Publish
-on:
-  push:
-    tags: ['*']
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-      - uses: astral-sh/setup-uv@f0ec1fc3b38f5e7cd731bb6ce540c5af426746bb # v6.1.0
-        with:
-          enable-cache: true
-          prune-cache: false
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version-file: pyproject.toml
-      - run: echo "SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV
-      - run: uv build
-      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
-        with:
-          path: ./dist
-  create-release:
-    needs: [build]
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
-      - name: create release
-        run: >
-          gh release create --draft --repo ${{ github.repository }}
-          ${{ github.ref_name }} artifact/*
-        env:
-          GH_TOKEN: ${{ github.token }}
-  publish-pypi:
-    needs: [build]
-    environment:
-      name: publish
-      url: https://pypi.org/project/Jinja2/${{ github.ref_name }}
-    runs-on: ubuntu-latest
-    permissions:
-      id-token: write
-    steps:
-      - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
-      - uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc # v1.12.4
-        with:
-          packages-dir: artifact/

.github/workflows/tests.yaml

@@ -1,49 +0,0 @@
-name: Tests
-on:
-  pull_request:
-    paths-ignore: ['docs/**', 'README.md']
-  push:
-    branches: [main, stable]
-    paths-ignore: ['docs/**', 'README.md']
-jobs:
-  tests:
-    name: ${{ matrix.name || matrix.python }}
-    runs-on: ${{ matrix.os || 'ubuntu-latest' }}
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - {python: '3.13'}
-          - {name: Windows, python: '3.13', os: windows-latest}
-          - {name: Mac, python: '3.13', os: macos-latest}
-          - {python: '3.12'}
-          - {python: '3.11'}
-          - {python: '3.10'}
-          - {name: PyPy, python: 'pypy-3.11', tox: pypy3.11}
-    steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-      - uses: astral-sh/setup-uv@f0ec1fc3b38f5e7cd731bb6ce540c5af426746bb # v6.1.0
-        with:
-          enable-cache: true
-          prune-cache: false
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: ${{ matrix.python }}
-      - run: uv run --locked tox run -e ${{ matrix.tox || format('py{0}', matrix.python) }}
-  typing:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-      - uses: astral-sh/setup-uv@f0ec1fc3b38f5e7cd731bb6ce540c5af426746bb # v6.1.0
-        with:
-          enable-cache: true
-          prune-cache: false
-      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version-file: pyproject.toml
-      - name: cache mypy
-        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
-        with:
-          path: ./.mypy_cache
-          key: mypy|${{ hashFiles('pyproject.toml') }}
-      - run: uv run --locked tox run -e typing

.gitignore

@@ -1,8 +1,25 @@
-.idea/
-.vscode/
-__pycache__/
-dist/
-.coverage*
-htmlcov/
-.tox/
+*.so
 docs/_build/
+*.pyc
+*.pyo
+*.egg-info/
+*.egg
+build/
+dist/
+.DS_Store
+.tox/
+.cache/
+.idea/
+env/
+venv/
+venv-*/
+.coverage
+.coverage.*
+htmlcov
+.pytest_cache/
+/.vscode/
+
+tatsu_jinja.json
+tatsu_jinja.py
+parsed_jinja.py
+test_template.jinja

.pre-commit-config.yaml

@@ -1,18 +1,26 @@
 repos:
-  - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: 76e47323a83cd9795e4ff9a1de1c0d2eef610f17  # frozen: v0.11.11
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v1.26.2
     hooks:
-      - id: ruff
-      - id: ruff-format
-  - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 648bdbfd6bb1a82f132ecc2c666e0d1b2e4b0d94  # frozen: 0.7.8
+      - id: pyupgrade
+        args: ["--py36-plus"]
+  - repo: https://github.com/asottile/reorder_python_imports
+    rev: v1.9.0
     hooks:
-      - id: uv-lock
+      - id: reorder-python-imports
+        args: ["--application-directories", "src"]
+  - repo: https://github.com/ambv/black
+    rev: 19.10b0
+    hooks:
+      - id: black
+  - repo: https://gitlab.com/pycqa/flake8
+    rev: 3.7.9
+    hooks:
+      - id: flake8
+        additional_dependencies: [flake8-bugbear]
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: cef0300fd0fc4d2a87a85fa2093c6b283ea36f4b  # frozen: v5.0.0
+    rev: v2.4.0
     hooks:
-      - id: check-merge-conflict
-      - id: debug-statements
-      - id: fix-byte-order-marker
+      - id: check-byte-order-marker
       - id: trailing-whitespace
       - id: end-of-file-fixer

.readthedocs.yaml

@@ -1,10 +0,0 @@
-version: 2
-build:
-  os: ubuntu-24.04
-  tools:
-    python: '3.13'
-  commands:
-    - asdf plugin add uv
-    - asdf install uv latest
-    - asdf global uv latest
-    - uv run --group docs sphinx-build -W -b dirhtml docs $READTHEDOCS_OUTPUT/html

CHANGES.rst

@@ -1,301 +1,16 @@
 .. currentmodule:: jinja2
 
-Version 3.2.0
--------------
-
-Unreleased
-
--   Drop support for Python 3.7, 3.8, and 3.9.
--   Update minimum MarkupSafe version to >= 3.0.
--   Update minimum Babel version to >= 2.17.
--   Deprecate the ``__version__`` attribute. Use feature detection or
-    ``importlib.metadata.version("jinja2")`` instead.
--   Use modern packaging metadata with ``pyproject.toml`` instead of ``setup.cfg``.
-    :pr:`1793`
--   Use ``flit_core`` instead of ``setuptools`` as build backend.
-
-
-Version 3.1.6
--------------
-
-Released 2025-03-05
-
--   The ``|attr`` filter does not bypass the environment's attribute lookup,
-    allowing the sandbox to apply its checks. :ghsa:`cpwx-vrp4-4pq7`
-
-
-Version 3.1.5
--------------
-
-Released 2024-12-21
-
--   The sandboxed environment handles indirect calls to ``str.format``, such as
-    by passing a stored reference to a filter that calls its argument.
-    :ghsa:`q2x7-8rv6-6q7h`
--   Escape template name before formatting it into error messages, to avoid
-    issues with names that contain f-string syntax.
-    :issue:`1792`, :ghsa:`gmj6-6f8f-6699`
--   Sandbox does not allow ``clear`` and ``pop`` on known mutable sequence
-    types. :issue:`2032`
--   Calling sync ``render`` for an async template uses ``asyncio.run``.
-    :pr:`1952`
--   Avoid unclosed ``auto_aiter`` warnings. :pr:`1960`
--   Return an ``aclose``-able ``AsyncGenerator`` from
-    ``Template.generate_async``. :pr:`1960`
--   Avoid leaving ``root_render_func()`` unclosed in
-    ``Template.generate_async``. :pr:`1960`
--   Avoid leaving async generators unclosed in blocks, includes and extends.
-    :pr:`1960`
--   The runtime uses the correct ``concat`` function for the current environment
-    when calling block references. :issue:`1701`
--   Make ``|unique`` async-aware, allowing it to be used after another
-    async-aware filter. :issue:`1781`
--   ``|int`` filter handles ``OverflowError`` from scientific notation.
-    :issue:`1921`
--   Make compiling deterministic for tuple unpacking in a ``{% set ... %}``
-    call. :issue:`2021`
--   Fix dunder protocol (`copy`/`pickle`/etc) interaction with ``Undefined``
-    objects. :issue:`2025`
--   Fix `copy`/`pickle` support for the internal ``missing`` object.
-    :issue:`2027`
--   ``Environment.overlay(enable_async)`` is applied correctly. :pr:`2061`
--   The error message from ``FileSystemLoader`` includes the paths that were
-    searched. :issue:`1661`
--   ``PackageLoader`` shows a clearer error message when the package does not
-    contain the templates directory. :issue:`1705`
--   Improve annotations for methods returning copies. :pr:`1880`
--   ``urlize`` does not add ``mailto:`` to values like `@a@b`. :pr:`1870`
--   Tests decorated with `@pass_context`` can be used with the ``|select``
-    filter. :issue:`1624`
--   Using ``set`` for multiple assignment (``a, b = 1, 2``) does not fail when the
-    target is a namespace attribute. :issue:`1413`
--   Using ``set`` in all branches of ``{% if %}{% elif %}{% else %}`` blocks
-    does not cause the variable to be considered initially undefined.
-    :issue:`1253`
-
-
-Version 3.1.4
--------------
-
-Released 2024-05-05
-
--   The ``xmlattr`` filter does not allow keys with ``/`` solidus, ``>``
-    greater-than sign, or ``=`` equals sign, in addition to disallowing spaces.
-    Regardless of any validation done by Jinja, user input should never be used
-    as keys to this filter, or must be separately validated first.
-    :ghsa:`h75v-3vvj-5mfj`
-
-
-Version 3.1.3
--------------
-
-Released 2024-01-10
-
--   Fix compiler error when checking if required blocks in parent templates are
-    empty. :pr:`1858`
--   ``xmlattr`` filter does not allow keys with spaces. :ghsa:`h5c8-rqwp-cp95`
--   Make error messages stemming from invalid nesting of ``{% trans %}`` blocks
-    more helpful. :pr:`1918`
-
-
-Version 3.1.2
--------------
-
-Released 2022-04-28
-
--   Add parameters to ``Environment.overlay`` to match ``__init__``.
-    :issue:`1645`
--   Handle race condition in ``FileSystemBytecodeCache``. :issue:`1654`
-
-
-Version 3.1.1
--------------
-
-Released 2022-03-25
-
--   The template filename on Windows uses the primary path separator.
-    :issue:`1637`
-
-
-Version 3.1.0
--------------
-
-Released 2022-03-24
-
--   Drop support for Python 3.6. :pr:`1534`
--   Remove previously deprecated code. :pr:`1544`
-
-    -   ``WithExtension`` and ``AutoEscapeExtension`` are built-in now.
-    -   ``contextfilter`` and ``contextfunction`` are replaced by
-        ``pass_context``. ``evalcontextfilter`` and
-        ``evalcontextfunction`` are replaced by ``pass_eval_context``.
-        ``environmentfilter`` and ``environmentfunction`` are replaced
-        by ``pass_environment``.
-    -   ``Markup`` and ``escape`` should be imported from MarkupSafe.
-    -   Compiled templates from very old Jinja versions may need to be
-        recompiled.
-    -   Legacy resolve mode for ``Context`` subclasses is no longer
-        supported. Override ``resolve_or_missing`` instead of
-        ``resolve``.
-    -   ``unicode_urlencode`` is renamed to ``url_quote``.
-
--   Add support for native types in macros. :issue:`1510`
--   The ``{% trans %}`` tag can use ``pgettext`` and ``npgettext`` by
-    passing a context string as the first token in the tag, like
-    ``{% trans "title" %}``. :issue:`1430`
--   Update valid identifier characters from Python 3.6 to 3.7.
-    :pr:`1571`
--   Filters and tests decorated with ``@async_variant`` are pickleable.
-    :pr:`1612`
--   Add ``items`` filter. :issue:`1561`
--   Subscriptions (``[0]``, etc.) can be used after filters, tests, and
-    calls when the environment is in async mode. :issue:`1573`
--   The ``groupby`` filter is case-insensitive by default, matching
-    other comparison filters. Added the ``case_sensitive`` parameter to
-    control this. :issue:`1463`
--   Windows drive-relative path segments in template names will not
-    result in ``FileSystemLoader`` and ``PackageLoader`` loading from
-    drive-relative paths. :pr:`1621`
-
-
-Version 3.0.3
--------------
-
-Released 2021-11-09
-
--   Fix traceback rewriting internals for Python 3.10 and 3.11.
-    :issue:`1535`
--   Fix how the native environment treats leading and trailing spaces
-    when parsing values on Python 3.10. :pr:`1537`
--   Improve async performance by avoiding checks for common types.
-    :issue:`1514`
--   Revert change to ``hash(Node)`` behavior. Nodes are hashed by id
-    again :issue:`1521`
--   ``PackageLoader`` works when the package is a single module file.
-    :issue:`1512`
-
-
-Version 3.0.2
--------------
-
-Released 2021-10-04
-
--   Fix a loop scoping bug that caused assignments in nested loops
-    to still be referenced outside of it. :issue:`1427`
--   Make ``compile_templates`` deterministic for filter and import
-    names. :issue:`1452, 1453`
--   Revert an unintended change that caused ``Undefined`` to act like
-    ``StrictUndefined`` for the ``in`` operator. :issue:`1448`
--   Imported macros have access to the current template globals in async
-    environments. :issue:`1494`
--   ``PackageLoader`` will not include a current directory (.) path
-    segment. This allows loading templates from the root of a zip
-    import. :issue:`1467`
-
-
-Version 3.0.1
--------------
-
-Released 2021-05-18
-
--   Update MarkupSafe dependency to >= 2.0. :pr:`1418`
--   Mark top-level names as exported so type checking understands
-    imports in user projects. :issue:`1426`
--   Fix some types that weren't available in Python 3.6.0. :issue:`1433`
--   The deprecation warning for unneeded ``autoescape`` and ``with_``
-    extensions shows more relevant context. :issue:`1429`
--   Fixed calling deprecated ``jinja2.Markup`` without an argument.
-    Use ``markupsafe.Markup`` instead. :issue:`1438`
--   Calling sync ``render`` for an async template uses ``asyncio.new_event_loop``
-    This fixes a deprecation that Python 3.10 introduces. :issue:`1443`
-
-
 Version 3.0.0
 -------------
 
-Released 2021-05-11
+Unreleased
 
 -   Drop support for Python 2.7 and 3.5.
 -   Bump MarkupSafe dependency to >=1.1.
 -   Bump Babel optional dependency to >=2.1.
 -   Remove code that was marked deprecated.
--   Add type hinting. :pr:`1412`
 -   Use :pep:`451` API to load templates with
     :class:`~loaders.PackageLoader`. :issue:`1168`
--   Fix a bug that caused imported macros to not have access to the
-    current template's globals. :issue:`688`
--   Add ability to ignore ``trim_blocks`` using ``+%}``. :issue:`1036`
--   Fix a bug that caused custom async-only filters to fail with
-    constant input. :issue:`1279`
--   Fix UndefinedError incorrectly being thrown on an undefined variable
-    instead of ``Undefined`` being returned on
-    ``NativeEnvironment`` on Python 3.10. :issue:`1335`
--   Blocks can be marked as ``required``. They must be overridden at
-    some point, but not necessarily by the direct child. :issue:`1147`
--   Deprecate the ``autoescape`` and ``with`` extensions, they are
-    built-in to the compiler. :issue:`1203`
--   The ``urlize`` filter recognizes ``mailto:`` links and takes
-    ``extra_schemes`` (or ``env.policies["urlize.extra_schemes"]``) to
-    recognize other schemes. It tries to balance parentheses within a
-    URL instead of ignoring trailing characters. The parsing in general
-    has been updated to be more efficient and match more cases. URLs
-    without a scheme are linked as ``https://`` instead of ``http://``.
-    :issue:`522, 827, 1172`, :pr:`1195`
--   Filters that get attributes, such as ``map`` and ``groupby``, can
-    use a false or empty value as a default. :issue:`1331`
--   Fix a bug that prevented variables set in blocks or loops from
-    being accessed in custom context functions. :issue:`768`
--   Fix a bug that caused scoped blocks from accessing special loop
-    variables. :issue:`1088`
--   Update the template globals when calling
-    ``Environment.get_template(globals=...)`` even if the template was
-    already loaded. :issue:`295`
--   Do not raise an error for undefined filters in unexecuted
-    if-statements and conditional expressions. :issue:`842`
--   Add ``is filter`` and ``is test`` tests to test if a name is a
-    registered filter or test. This allows checking if a filter is
-    available in a template before using it. Test functions can be
-    decorated with ``@pass_environment``, ``@pass_eval_context``,
-    or ``@pass_context``. :issue:`842`, :pr:`1248`
--   Support ``pgettext`` and ``npgettext`` (message contexts) in i18n
-    extension. :issue:`441`
--   The ``|indent`` filter's ``width`` argument can be a string to
-    indent by. :pr:`1167`
--   The parser understands hex, octal, and binary integer literals.
-    :issue:`1170`
--   ``Undefined.__contains__`` (``in``) raises an ``UndefinedError``
-    instead of a ``TypeError``. :issue:`1198`
--   ``Undefined`` is iterable in an async environment. :issue:`1294`
--   ``NativeEnvironment`` supports async mode. :issue:`1362`
--   Template rendering only treats ``\n``, ``\r\n`` and ``\r`` as line
-    breaks. Other characters are left unchanged. :issue:`769, 952, 1313`
--   ``|groupby`` filter takes an optional ``default`` argument.
-    :issue:`1359`
--   The function and filter decorators have been renamed and unified.
-    The old names are deprecated. :issue:`1381`
-
-    -   ``pass_context`` replaces ``contextfunction`` and
-        ``contextfilter``.
-    -   ``pass_eval_context`` replaces ``evalcontextfunction`` and
-        ``evalcontextfilter``
-    -   ``pass_environment`` replaces ``environmentfunction`` and
-        ``environmentfilter``.
-
--   Async support no longer requires Jinja to patch itself. It must
-    still be enabled with ``Environment(enable_async=True)``.
-    :issue:`1390`
--   Overriding ``Context.resolve`` is deprecated, override
-    ``resolve_or_missing`` instead. :issue:`1380`
-
-
-Version 2.11.3
---------------
-
-Released 2021-01-31
-
--   Improve the speed of the ``urlize`` filter by reducing regex
-    backtracking. Email matching requires a word character at the start
-    of the domain part, and only word characters in the TLD. :pr:`1343`
 
 
 Version 2.11.2
@@ -582,7 +297,7 @@ Released 2017-01-08
     possible. For more information and a discussion see :issue:`641`
 -   Resolved an issue where ``block scoped`` would not take advantage of
     the new scoping rules. In some more exotic cases a variable
-    overridden in a local scope would not make it into a block.
+    overriden in a local scope would not make it into a block.
 -   Change the code generation of the ``with`` statement to be in line
     with the new scoping rules. This resolves some unlikely bugs in edge
     cases. This also introduces a new internal ``With`` node that can be
@@ -1059,7 +774,7 @@ Released 2008-07-17, codename Jinjavitus
     evaluates to ``false``.
 -   Improved error reporting for undefined values by providing a
     position.
--   ``filesizeformat`` filter uses decimal prefixes now by default and
+-   ``filesizeformat`` filter uses decimal prefixes now per default and
     can be set to binary mode with the second parameter.
 -   Fixed bug in finalizer
 

CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+ advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+ address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at report@palletsprojects.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

MANIFEST.in

@@ -0,0 +1,8 @@
+include CHANGES.rst
+include tox.ini
+graft artwork
+graft docs
+prune docs/_build
+graft examples
+graft tests
+global-exclude *.pyc

README.rst

@@ -1,6 +1,5 @@
-<div align="center"><img src="https://raw.githubusercontent.com/pallets/jinja/refs/heads/stable/docs/_static/jinja-name.svg" alt="" height="150"></div>
-
-# Jinja
+Jinja
+=====
 
 Jinja is a fast, expressive, extensible templating engine. Special
 placeholders in the template allow writing code similar to Python
@@ -27,33 +26,41 @@ possible, it shouldn't make the template designer's job difficult by
 restricting functionality too much.
 
 
-## In A Nutshell
+Installing
+----------
 
-```jinja
-{% extends "base.html" %}
-{% block title %}Members{% endblock %}
-{% block content %}
-  <ul>
-  {% for user in users %}
-    <li><a href="{{ user.url }}">{{ user.username }}</a></li>
-  {% endfor %}
-  </ul>
-{% endblock %}
-```
+Install and update using `pip`_:
 
-## Donate
+.. code-block:: text
 
-The Pallets organization develops and supports Jinja and other popular
-packages. In order to grow the community of contributors and users, and
-allow the maintainers to devote more time to the projects, [please
-donate today][].
+    $ pip install -U Jinja2
 
-[please donate today]: https://palletsprojects.com/donate
+.. _pip: https://pip.pypa.io/en/stable/quickstart/
 
-## Contributing
 
-See our [detailed contributing documentation][contrib] for many ways to
-contribute, including reporting issues, requesting features, asking or answering
-questions, and making PRs.
+In A Nutshell
+-------------
 
-[contrib]: https://palletsprojects.com/contributing/
+.. code-block:: jinja
+
+    {% extends "base.html" %}
+    {% block title %}Members{% endblock %}
+    {% block content %}
+      <ul>
+      {% for user in users %}
+        <li><a href="{{ user.url }}">{{ user.username }}</a></li>
+      {% endfor %}
+      </ul>
+    {% endblock %}
+
+
+Links
+-----
+
+-   Website: https://palletsprojects.com/p/jinja/
+-   Documentation: https://jinja.palletsprojects.com/
+-   Releases: https://pypi.org/project/Jinja2/
+-   Code: https://github.com/pallets/jinja
+-   Issue tracker: https://github.com/pallets/jinja/issues
+-   Test status: https://dev.azure.com/pallets/jinja/_build
+-   Official chat: https://discord.gg/t6rrQZH

docs/_static/jinja-icon.svg

@@ -1,11 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<svg width="100%" height="100%" viewBox="0 0 500 500" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xml:space="preserve" xmlns:serif="http://www.serif.com/" style="fill-rule:evenodd;clip-rule:evenodd;stroke-linejoin:round;stroke-miterlimit:2;">
-    <rect id="Icon" x="0" y="0" width="500" height="500" style="fill:none;"/>
-    <clipPath id="_clip1">
-        <rect x="0" y="0" width="500" height="500"/>
-    </clipPath>
-    <g clip-path="url(#_clip1)">
-        <path d="M491.941,72.796l-1.81,-0c-88.724,29.526 -204.909,29.526 -237.199,28.989l-29.877,-0.536c-88.119,-3.222 -133.085,-37.043 -211.849,-72.206c-5.432,-2.416 -11.77,1.61 -11.166,7.247c0.604,19.327 5.734,100.39 66.392,121.596c2.112,0.805 4.526,1.61 6.639,2.147c3.018,0.537 4.225,2.953 4.828,4.563l5.131,15.837c0.905,3.758 3.621,6.979 6.639,6.979l5.13,0c4.527,0 8.148,3.221 8.45,7.248l-0,15.3c-0,1.61 -1.509,2.953 -3.32,2.953l-38.929,-0c-3.622,-0 -6.64,2.684 -6.64,5.905l0,23.89c0,3.221 3.018,5.905 6.64,5.905l38.929,-0c1.811,-0 3.32,1.342 3.32,2.953l-0,6.442c-0,1.61 -1.509,2.952 -3.32,2.952l-38.929,0c-3.622,0.269 -6.338,2.685 -6.338,5.906l0,23.889c0,2.685 2.414,5.637 5.13,5.637l40.439,0c1.811,0 3.32,1.342 3.32,2.953l-0,157.027c-0,8.053 7.544,14.764 16.597,14.764l27.462,-0c9.054,-0 16.598,-6.711 16.598,-14.764l0,-157.027c0,-1.611 1.509,-2.953 3.32,-2.953l169.6,-0.268c1.811,-0 3.32,1.342 3.32,2.952l-0,157.833c-0,8.053 7.544,14.764 16.598,14.764l27.462,-0c9.053,-0 16.598,-6.711 16.598,-14.764l-0,-158.101c-0,-1.611 1.508,-2.953 3.319,-2.953c0,0 42.249,-0.268 42.853,-0.537c1.811,-1.073 3.018,-2.952 3.018,-5.1l-0,-23.621c-0,-3.221 -3.018,-5.905 -6.941,-5.905l-41.948,-0l0,-0.269l-0.301,0l-0,-8.857c-0,-1.611 1.508,-2.953 3.319,-2.953l38.93,-0c3.621,-0 6.639,-2.684 6.639,-5.905l-0,-23.89c-0,-3.221 -3.018,-5.905 -6.639,-5.905l-38.93,-0c-1.811,-0 -3.319,-1.343 -3.319,-2.953l-0,-15.3c-0,-3.758 3.621,-7.248 8.449,-7.248l5.131,0c3.621,0 5.733,-3.489 6.639,-6.979l5.13,-15.837c0.604,-2.147 2.716,-4.026 5.13,-4.831c38.93,-8.59 54.924,-34.09 68.203,-74.085l-0,-0.268c1.508,-10.2 -5.432,-12.079 -7.847,-12.616Zm-150.89,114.08l0,23.352c0,3.221 -2.112,5.637 -4.828,5.637l-54.321,0c-2.716,0 -4.828,-2.416 -4.828,-5.637l-0,-23.352c-0,-2.953 2.112,-5.637 4.828,-5.637l54.321,-0c2.414,0.268 4.828,2.684 4.828,5.637Zm-111.96,-0l-0,23.352c-0,2.953 -2.112,5.637 -4.828,5.637l-54.321,0c-2.716,0 -4.828,-2.416 -4.828,-5.637l-0,-23.352c-0,-2.953 2.112,-5.637 4.828,-5.637l54.321,-0c2.414,0.268 4.828,2.684 4.828,5.637Z" style="fill:#7e0c1b;fill-rule:nonzero;"/>
-    </g>
-</svg>

docs/_static/jinja-logo.svg

@@ -1,11 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<svg width="100%" height="100%" viewBox="0 0 500 500" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xml:space="preserve" xmlns:serif="http://www.serif.com/" style="fill-rule:evenodd;clip-rule:evenodd;stroke-linejoin:round;stroke-miterlimit:2;">
-    <rect id="Logo" x="0" y="0" width="500" height="500" style="fill:none;"/>
-    <path id="Box" d="M500,50l0,400c0,27.596 -22.404,50 -50,50l-400,0c-27.596,0 -50,-22.404 -50,-50l0,-400c0,-27.596 22.404,-50 50,-50l400,0c27.596,0 50,22.404 50,50Z" style="fill:url(#_Linear1);"/>
-    <path id="Shadow" d="M500,246.897l0,203.103c0,27.596 -22.404,50 -50,50l-164.98,0l-119.852,-119.852c1.802,1.562 4.252,2.533 6.921,2.533l16.477,0c5.432,0 9.959,-4.026 9.959,-8.858l-0,-94.216c-0,-0.966 0.905,-1.772 1.992,-1.772l101.76,-0.161c1.086,0 1.992,0.806 1.992,1.772l-0,94.7c-0,4.831 4.526,8.858 9.958,8.858l16.478,-0c5.432,-0 9.958,-4.027 9.958,-8.858l0,-94.861c0,-0.967 0.906,-1.772 1.992,-1.772c0,0 25.35,-0.161 25.712,-0.322c1.086,-0.644 1.81,-1.771 1.81,-3.06l0,-14.173c0,-1.932 -1.81,-3.543 -4.164,-3.543l-25.169,0l0,-0.161l-0.181,0l0,-5.315c0,-0.966 0.906,-1.771 1.992,-1.771l23.358,-0c2.173,-0 3.983,-1.611 3.983,-3.543l0,-14.334c0,-1.933 -1.81,-3.543 -3.983,-3.543l-23.358,-0c-1.086,-0 -1.992,-0.806 -1.992,-1.772l0,-9.18c0,-2.255 2.173,-4.349 5.07,-4.349l3.078,0c2.173,0 3.441,-2.093 3.984,-4.187l3.078,-9.502c0.362,-1.289 1.63,-2.416 3.078,-2.899c23.358,-5.154 32.955,-20.454 40.922,-44.451l-0,-0.161c0.422,-2.854 -0.258,-4.622 -1.252,-5.729l101.379,101.379Zm-375.729,-61.204c4.362,3.788 9.511,6.914 15.588,9.039c1.267,0.483 2.716,0.966 3.983,1.288c1.811,0.322 2.535,1.772 2.898,2.738l3.078,9.502c0.543,2.255 2.173,4.187 3.983,4.187l3.078,0c2.716,0 4.889,1.933 5.07,4.349l0,6.575l-37.678,-37.678Zm9.606,62.5c0.716,0.602 1.677,0.975 2.723,0.975l23.358,-0c1.086,-0 1.991,0.805 1.991,1.771l0,3.866c0,0.966 -0.905,1.771 -1.991,1.771l-17.698,0l-8.383,-8.383Zm0.033,28.751c0.543,0.537 1.236,0.891 1.965,0.891l24.264,0c1.086,0 1.991,0.806 1.991,1.772l0,25.557l-28.22,-28.22Zm170.721,-64.819l-0,14.012c-0,1.933 -1.268,3.382 -2.897,3.382l-32.593,0c-1.629,0 -2.897,-1.449 -2.897,-3.382l0,-14.012c0,-1.771 1.268,-3.382 2.897,-3.382l32.593,0c1.448,0.161 2.897,1.611 2.897,3.382Zm-67.176,0l-0,14.012c-0,1.772 -1.268,3.382 -2.897,3.382l-32.593,0c-1.629,0 -2.897,-1.449 -2.897,-3.382l0,-14.012c0,-1.771 1.268,-3.382 2.897,-3.382l32.593,0c1.448,0.161 2.897,1.611 2.897,3.382Z" style="fill:#630b28;"/>
-    <path id="Icon" d="M395.165,143.677l-1.087,0c-53.234,17.716 -122.945,17.716 -142.319,17.394l-17.926,-0.322c-52.872,-1.933 -79.851,-22.225 -127.109,-43.323c-3.26,-1.45 -7.062,0.966 -6.7,4.348c0.362,11.596 3.44,60.234 39.835,72.958c1.267,0.483 2.716,0.966 3.983,1.288c1.811,0.322 2.535,1.772 2.898,2.738l3.078,9.502c0.543,2.255 2.173,4.187 3.983,4.187l3.078,0c2.716,0 4.889,1.933 5.07,4.349l0,9.18c0,0.966 -0.905,1.772 -1.991,1.772l-23.358,-0c-2.173,-0 -3.984,1.61 -3.984,3.543l0,14.334c0,1.932 1.811,3.543 3.984,3.543l23.358,-0c1.086,-0 1.991,0.805 1.991,1.771l0,3.866c0,0.966 -0.905,1.771 -1.991,1.771l-23.358,0c-2.173,0.161 -3.803,1.611 -3.803,3.543l0,14.334c0,1.611 1.449,3.382 3.078,3.382l24.264,0c1.086,0 1.991,0.806 1.991,1.772l0,94.216c0,4.832 4.527,8.858 9.959,8.858l16.477,0c5.432,0 9.959,-4.026 9.959,-8.858l-0,-94.216c-0,-0.966 0.905,-1.772 1.992,-1.772l101.76,-0.161c1.086,0 1.992,0.806 1.992,1.772l-0,94.7c-0,4.831 4.526,8.858 9.958,8.858l16.478,-0c5.432,-0 9.958,-4.027 9.958,-8.858l0,-94.861c0,-0.967 0.906,-1.772 1.992,-1.772c0,0 25.35,-0.161 25.712,-0.322c1.086,-0.644 1.81,-1.771 1.81,-3.06l0,-14.173c0,-1.932 -1.81,-3.543 -4.164,-3.543l-25.169,0l0,-0.161l-0.181,0l0,-5.315c0,-0.966 0.906,-1.771 1.992,-1.771l23.358,-0c2.173,-0 3.983,-1.611 3.983,-3.543l0,-14.334c0,-1.933 -1.81,-3.543 -3.983,-3.543l-23.358,-0c-1.086,-0 -1.992,-0.806 -1.992,-1.772l0,-9.18c0,-2.255 2.173,-4.349 5.07,-4.349l3.078,0c2.173,0 3.441,-2.093 3.984,-4.187l3.078,-9.502c0.362,-1.289 1.63,-2.416 3.078,-2.899c23.358,-5.154 32.955,-20.454 40.922,-44.451l-0,-0.161c0.905,-6.12 -3.26,-7.247 -4.708,-7.57Zm-90.534,68.448l-0,14.012c-0,1.933 -1.268,3.382 -2.897,3.382l-32.593,0c-1.629,0 -2.897,-1.449 -2.897,-3.382l0,-14.012c0,-1.771 1.268,-3.382 2.897,-3.382l32.593,0c1.448,0.161 2.897,1.611 2.897,3.382Zm-67.176,0l-0,14.012c-0,1.772 -1.268,3.382 -2.897,3.382l-32.593,0c-1.629,0 -2.897,-1.449 -2.897,-3.382l0,-14.012c0,-1.771 1.268,-3.382 2.897,-3.382l32.593,0c1.448,0.161 2.897,1.611 2.897,3.382Z" style="fill:#fff;fill-rule:nonzero;"/>
-    <defs>
-        <linearGradient id="_Linear1" x1="0" y1="0" x2="1" y2="0" gradientUnits="userSpaceOnUse" gradientTransform="matrix(3.06162e-14,500,-500,3.06162e-14,267.59,0)"><stop offset="0" style="stop-color:#f6cadc;stop-opacity:1"/><stop offset="1" style="stop-color:#7f0d18;stop-opacity:1"/></linearGradient>
-    </defs>
-</svg>

docs/_static/jinja-name.svg

@@ -1,19 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<svg width="100%" height="100%" viewBox="0 0 664 300" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xml:space="preserve" xmlns:serif="http://www.serif.com/" style="fill-rule:evenodd;clip-rule:evenodd;stroke-linejoin:round;stroke-miterlimit:2;">
-    <g>
-        <path id="Name" d="M416,89.625l0,68.7c0,14.3 -2.95,24.375 -8.85,30.225c-5.9,5.85 -14.45,8.775 -25.65,8.775c-7.2,-0 -13.225,-0.95 -18.075,-2.85c-4.85,-1.9 -9.325,-4.9 -13.425,-9l11.7,-12.75c1.7,1.7 4.275,3.125 7.725,4.275c3.45,1.15 7.4,1.725 11.85,1.725c4.45,-0 7.875,-1.075 10.275,-3.225c2.4,-2.15 3.6,-6.175 3.6,-12.075l0,-73.8l20.85,-0Z" style="fill-rule:nonzero;"/>
-        <path d="M446.45,78.075c4.7,-0 7.75,0.725 9.15,2.175c1.4,1.45 2.1,4.5 2.1,9.15c0,4.65 -0.725,7.7 -2.175,9.15c-1.45,1.45 -4.5,2.175 -9.15,2.175c-4.65,-0 -7.7,-0.75 -9.15,-2.25c-1.45,-1.5 -2.175,-4.55 -2.175,-9.15c0,-4.6 0.725,-7.625 2.175,-9.075c1.45,-1.45 4.525,-2.175 9.225,-2.175Zm9.6,118.35l-19.5,-0l0,-82.5l19.5,-0l0,82.5Z" style="fill-rule:nonzero;"/>
-        <path d="M544.25,196.425l-19.5,-0l0,-54.75c0,-4.9 -1,-8.25 -3,-10.05c-2,-1.8 -4.8,-2.7 -8.4,-2.7l-17.55,7.35l0,60.15l-19.5,-0l0,-85.65l14.55,-0l4.5,10.2l24,-11.1c6.7,-0 12.525,2.475 17.475,7.425c4.95,4.95 7.425,12.175 7.425,21.675l0,57.45Z" style="fill-rule:nonzero;"/>
-        <path d="M583.25,113.925l0,85.65c0,6.1 -1.925,11.35 -5.775,15.75c-3.85,4.4 -9.525,6.6 -17.025,6.6l-11.1,-0l0,-16.5l7.5,-0c4.6,-0 6.9,-2.35 6.9,-7.05l0,-84.45l19.5,-0Zm-9.6,-35.85c4.7,-0 7.75,0.725 9.15,2.175c1.4,1.45 2.1,4.5 2.1,9.15c0,4.65 -0.725,7.7 -2.175,9.15c-1.45,1.45 -4.5,2.175 -9.15,2.175c-4.65,-0 -7.7,-0.75 -9.15,-2.25c-1.45,-1.5 -2.175,-4.55 -2.175,-9.15c0,-4.6 0.725,-7.625 2.175,-9.075c1.45,-1.45 4.525,-2.175 9.225,-2.175Z" style="fill-rule:nonzero;"/>
-        <path d="M663.95,196.425l-13.05,-0l-6,-10.35l-20.85,11.25c-11.6,-0 -19.4,-4.2 -23.4,-12.6c-2,-4.1 -3.375,-8.525 -4.125,-13.275c-0.75,-4.75 -1.125,-9.7 -1.125,-14.85c0,-5.15 0.05,-8.95 0.15,-11.4c0.1,-2.45 0.35,-5.3 0.75,-8.55c0.4,-3.25 0.975,-5.975 1.725,-8.175c0.75,-2.2 1.825,-4.475 3.225,-6.825c1.4,-2.35 3.1,-4.225 5.1,-5.625c4.5,-3.1 10.35,-4.65 17.55,-4.65l20.55,-0l19.5,-1.2l0,86.25Zm-19.5,-27.3l0,-40.2l-14.85,-0c-5.5,-0 -9.325,2.1 -11.475,6.3c-2.15,4.2 -3.225,10.475 -3.225,18.825c0,8.35 1.025,14.225 3.075,17.625c2.05,3.4 5.925,5.1 11.625,5.1l14.85,-7.65Z" style="fill-rule:nonzero;"/>
-        <g id="Logo">
-            <path id="Box" d="M300,30l-0,240c-0,16.557 -13.443,30 -30,30l-240,-0c-16.557,-0 -30,-13.443 -30,-30l0,-240c0,-16.557 13.443,-30 30,-30l240,0c16.557,0 30,13.443 30,30Z" style="fill:url(#_Linear1);"/>
-            <path id="Shadow" d="M300,148.138l0,121.862c0,16.557 -13.443,30 -30,30l-98.988,-0l-71.911,-71.911c1.081,0.937 2.551,1.52 4.152,1.52l9.887,-0c3.259,-0 5.975,-2.416 5.975,-5.315l-0,-56.53c-0,-0.58 0.543,-1.063 1.195,-1.063l61.056,-0.096c0.652,-0 1.195,0.483 1.195,1.063l0,56.819c0,2.899 2.716,5.315 5.975,5.315l9.887,0c3.259,0 5.975,-2.416 5.975,-5.315l-0,-56.916c-0,-0.58 0.543,-1.063 1.195,-1.063c0,-0 15.21,-0.097 15.427,-0.193c0.652,-0.387 1.086,-1.063 1.086,-1.836l0,-8.504c0,-1.16 -1.086,-2.126 -2.498,-2.126l-15.101,0l-0,-0.097l-0.109,0l-0,-3.188c-0,-0.58 0.543,-1.063 1.195,-1.063l14.015,-0c1.303,-0 2.39,-0.967 2.39,-2.126l-0,-8.601c-0,-1.159 -1.087,-2.125 -2.39,-2.125l-14.015,-0c-0.652,-0 -1.195,-0.484 -1.195,-1.063l-0,-5.508c-0,-1.353 1.304,-2.61 3.042,-2.61l1.847,0c1.304,0 2.064,-1.256 2.39,-2.512l1.847,-5.701c0.217,-0.773 0.978,-1.45 1.847,-1.74c14.014,-3.092 19.772,-12.272 24.553,-26.67l-0,-0.097c0.253,-1.712 -0.155,-2.773 -0.751,-3.437l60.827,60.827Zm-225.437,-36.722c2.617,2.273 5.706,4.148 9.352,5.423c0.761,0.29 1.63,0.58 2.39,0.773c1.087,0.193 1.521,1.063 1.739,1.643l1.847,5.701c0.326,1.353 1.303,2.512 2.39,2.512l1.847,0c1.629,0 2.933,1.16 3.042,2.61l-0,3.945l-22.607,-22.607Zm5.763,37.5c0.43,0.361 1.007,0.585 1.634,0.585l14.015,-0c0.651,-0 1.195,0.483 1.195,1.063l-0,2.319c-0,0.58 -0.544,1.063 -1.195,1.063l-10.619,-0l-5.03,-5.03Zm0.02,17.251c0.326,0.321 0.742,0.534 1.179,0.534l14.558,0c0.652,0 1.195,0.483 1.195,1.063l0,15.335l-16.932,-16.932Zm102.432,-38.892l0,8.407c0,1.16 -0.76,2.029 -1.738,2.029l-19.555,0c-0.978,0 -1.738,-0.869 -1.738,-2.029l-0,-8.407c-0,-1.063 0.76,-2.029 1.738,-2.029l19.555,-0c0.869,0.097 1.738,0.966 1.738,2.029Zm-40.305,0l-0,8.407c-0,1.063 -0.761,2.029 -1.738,2.029l-19.556,0c-0.978,0 -1.738,-0.869 -1.738,-2.029l-0,-8.407c-0,-1.063 0.76,-2.029 1.738,-2.029l19.556,-0c0.869,0.097 1.738,0.966 1.738,2.029Z" style="fill:#630b28;"/>
-            <path id="Icon" d="M237.099,86.206l-0.652,0c-31.94,10.63 -73.767,10.63 -85.392,10.437l-10.755,-0.194c-31.723,-1.159 -47.911,-13.335 -76.266,-25.994c-1.955,-0.869 -4.237,0.58 -4.02,2.609c0.218,6.958 2.065,36.141 23.901,43.775c0.761,0.29 1.63,0.58 2.39,0.773c1.087,0.193 1.521,1.063 1.739,1.643l1.847,5.701c0.326,1.353 1.303,2.512 2.39,2.512l1.847,0c1.629,0 2.933,1.16 3.042,2.61l-0,5.508c-0,0.579 -0.544,1.063 -1.195,1.063l-14.015,-0c-1.304,-0 -2.39,0.966 -2.39,2.125l-0,8.601c-0,1.159 1.086,2.126 2.39,2.126l14.015,-0c0.651,-0 1.195,0.483 1.195,1.063l-0,2.319c-0,0.58 -0.544,1.063 -1.195,1.063l-14.015,-0c-1.304,0.096 -2.282,0.966 -2.282,2.126l0,8.6c0,0.966 0.87,2.029 1.847,2.029l14.558,0c0.652,0 1.195,0.483 1.195,1.063l0,56.53c0,2.899 2.716,5.315 5.975,5.315l9.887,-0c3.259,-0 5.975,-2.416 5.975,-5.315l-0,-56.53c-0,-0.58 0.543,-1.063 1.195,-1.063l61.056,-0.096c0.652,-0 1.195,0.483 1.195,1.063l0,56.819c0,2.899 2.716,5.315 5.975,5.315l9.887,0c3.259,0 5.975,-2.416 5.975,-5.315l-0,-56.916c-0,-0.58 0.543,-1.063 1.195,-1.063c0,-0 15.21,-0.097 15.427,-0.193c0.652,-0.387 1.086,-1.063 1.086,-1.836l0,-8.504c0,-1.16 -1.086,-2.126 -2.498,-2.126l-15.101,0l-0,-0.097l-0.109,0l-0,-3.188c-0,-0.58 0.543,-1.063 1.195,-1.063l14.015,-0c1.303,-0 2.39,-0.967 2.39,-2.126l-0,-8.601c-0,-1.159 -1.087,-2.125 -2.39,-2.125l-14.015,-0c-0.652,-0 -1.195,-0.484 -1.195,-1.063l-0,-5.508c-0,-1.353 1.304,-2.61 3.042,-2.61l1.847,0c1.304,0 2.064,-1.256 2.39,-2.512l1.847,-5.701c0.217,-0.773 0.978,-1.45 1.847,-1.74c14.014,-3.092 19.772,-12.272 24.553,-26.67l-0,-0.097c0.543,-3.672 -1.956,-4.348 -2.825,-4.542Zm-54.321,41.069l0,8.407c0,1.16 -0.76,2.029 -1.738,2.029l-19.555,0c-0.978,0 -1.738,-0.869 -1.738,-2.029l-0,-8.407c-0,-1.063 0.76,-2.029 1.738,-2.029l19.555,-0c0.869,0.097 1.738,0.966 1.738,2.029Zm-40.305,0l-0,8.407c-0,1.063 -0.761,2.029 -1.738,2.029l-19.556,0c-0.978,0 -1.738,-0.869 -1.738,-2.029l-0,-8.407c-0,-1.063 0.76,-2.029 1.738,-2.029l19.556,-0c0.869,0.097 1.738,0.966 1.738,2.029Z" style="fill:#fff;fill-rule:nonzero;"/>
-        </g>
-    </g>
-    <defs>
-        <linearGradient id="_Linear1" x1="0" y1="0" x2="1" y2="0" gradientUnits="userSpaceOnUse" gradientTransform="matrix(1.83697e-14,300,-300,1.83697e-14,160.554,0)"><stop offset="0" style="stop-color:#f6cadc;stop-opacity:1"/><stop offset="1" style="stop-color:#7f0d18;stop-opacity:1"/></linearGradient>
-    </defs>
-</svg>

docs/api.rst

@@ -25,40 +25,30 @@ initialization and use that to load templates.  In some cases however, it's
 useful to have multiple environments side by side, if different configurations
 are in use.
 
-The simplest way to configure Jinja to load templates for your
-application is to use :class:`~loaders.PackageLoader`.
-
-.. code-block:: python
+The simplest way to configure Jinja to load templates for your application
+looks roughly like this::
 
     from jinja2 import Environment, PackageLoader, select_autoescape
     env = Environment(
-        loader=PackageLoader("yourapp"),
-        autoescape=select_autoescape()
+        loader=PackageLoader('yourapplication', 'templates'),
+        autoescape=select_autoescape(['html', 'xml'])
     )
 
-This will create a template environment with a loader that looks up
-templates in the ``templates`` folder inside the ``yourapp`` Python
-package (or next to the ``yourapp.py`` Python module). It also enables
-autoescaping for HTML files. This loader only requires that ``yourapp``
-is importable, it figures out the absolute path to the folder for you.
+This will create a template environment with the default settings and a
+loader that looks up the templates in the `templates` folder inside the
+`yourapplication` python package.  Different loaders are available
+and you can also write your own if you want to load templates from a
+database or other resources.  This also enables autoescaping for HTML and
+XML files.
 
-Different loaders are available to load templates in other ways or from
-other locations. They're listed in the `Loaders`_ section below. You can
-also write your own if you want to load templates from a source that's
-more specialized to your project.
+To load a template from this environment you just have to call the
+:meth:`get_template` method which then returns the loaded :class:`Template`::
 
-To load a template from this environment, call the :meth:`get_template`
-method, which returns the loaded :class:`Template`.
+    template = env.get_template('mytemplate.html')
 
-.. code-block:: python
+To render it with some variables, just call the :meth:`render` method::
 
-    template = env.get_template("mytemplate.html")
-
-To render it with some variables, call the :meth:`render` method.
-
-.. code-block:: python
-
-    print(template.render(the="variables", go="here"))
+    print(template.render(the='variables', go='here'))
 
 Using a template loader rather than passing strings to :class:`Template`
 or :meth:`Environment.from_string` has multiple advantages.  Besides being
@@ -114,10 +104,10 @@ useful if you want to dig deeper into Jinja or :ref:`develop extensions
 
     .. attribute:: globals
 
-        A dict of variables that are available in every template loaded
-        by the environment. As long as no template was loaded it's safe
-        to modify this. For more details see :ref:`global-namespace`.
-        For valid object names see :ref:`identifier-naming`.
+        A dict of global variables.  These variables are always available
+        in a template.  As long as no template was loaded it's safe
+        to modify this dict.  For more details see :ref:`global-namespace`.
+        For valid object names have a look at :ref:`identifier-naming`.
 
     .. attribute:: policies
 
@@ -180,20 +170,9 @@ useful if you want to dig deeper into Jinja or :ref:`develop extensions
 
     .. attribute:: globals
 
-        A dict of variables that are available every time the template
-        is rendered, without needing to pass them during render. This
-        should not be modified, as depending on how the template was
-        loaded it may be shared with the environment and other
-        templates.
-
-        Defaults to :attr:`Environment.globals` unless extra values are
-        passed to :meth:`Environment.get_template`.
-
-        Globals are only intended for data that is common to every
-        render of the template. Specific data should be passed to
-        :meth:`render`.
-
-        See :ref:`global-namespace`.
+        The dict with the globals of that template.  It's unsafe to modify
+        this dict as it may be shared with other templates or the environment
+        that loaded the template.
 
     .. attribute:: name
 
@@ -239,7 +218,7 @@ in ``'.html'``, ``'.htm'`` and ``'.xml'`` and disabling it by default
 for all other extensions.  You can use the :func:`~jinja2.select_autoescape`
 function for this::
 
-    from jinja2 import Environment, PackageLoader, select_autoescape
+    from jinja2 import Environment, select_autoescape
     env = Environment(autoescape=select_autoescape(['html', 'htm', 'xml']),
                       loader=PackageLoader('mypackage'))
 
@@ -273,7 +252,7 @@ modified identifier syntax.  Filters and tests may contain dots to group
 filters and tests by topic.  For example it's perfectly valid to add a
 function into the filter dict and call it `to.str`.  The regular
 expression for filter and test identifiers is
-``[a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)*``.
+``[a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)*```.
 
 
 Undefined Types
@@ -364,7 +343,7 @@ The Context
 -----------
 
 .. autoclass:: jinja2.runtime.Context()
-    :members: get, resolve, resolve_or_missing, get_exported, get_all
+    :members: resolve, get_exported, get_all
 
     .. attribute:: parent
 
@@ -410,19 +389,16 @@ The Context
     .. automethod:: jinja2.runtime.Context.call(callable, \*args, \**kwargs)
 
 
-The context is immutable, it prevents modifications, and if it is
-modified somehow despite that those changes may not show up. For
-performance, Jinja does not use the context as data storage for, only as
-a primary data source. Variables that the template does not define are
-looked up in the context, but variables the template does define are
-stored locally.
+.. admonition:: Implementation
 
-Instead of modifying the context directly, a function should return
-a value that can be assigned to a variable within the template itself.
+    Context is immutable for the same reason Python's frame locals are
+    immutable inside functions.  Both Jinja and Python are not using the
+    context / frame locals as data storage for variables but only as primary
+    data source.
 
-.. code-block:: jinja
-
-    {% set comments = get_latest_comments() %}
+    When a template accesses a variable the template does not define, Jinja
+    looks up the variable in the context, after that the variable is treated
+    as if it was defined in the template.
 
 
 .. _loaders:
@@ -515,10 +491,13 @@ environment to compile different code behind the scenes in order to
 handle async and sync code in an asyncio event loop. This has the
 following implications:
 
+-   Template rendering requires an event loop to be available to the
+    current thread. :func:`asyncio.get_event_loop` must return an event
+    loop.
 -   The compiled code uses ``await`` for functions and attributes, and
     uses ``async for`` loops. In order to support using both async and
     sync functions in this context, a small wrapper is placed around
-    all calls and access, which adds overhead compared to purely async
+    all calls and access, which add overhead compared to purely async
     code.
 -   Sync methods and filters become wrappers around their corresponding
     async implementations where needed. For example, ``render`` invokes
@@ -561,10 +540,6 @@ Example::
     The default target that is issued for links from the `urlize` filter
     if no other target is defined by the call explicitly.
 
-``urlize.extra_schemes``:
-    Recognize URLs that start with these schemes in addition to the
-    default ``http://``, ``https://``, and ``mailto:``.
-
 ``json.dumps_function``:
     If this is set to a value other than `None` then the `tojson` filter
     will dump with this function instead of the default one.  Note that
@@ -591,16 +566,40 @@ Utilities
 These helper functions and classes are useful if you add custom filters or
 functions to a Jinja environment.
 
-.. autofunction:: jinja2.pass_context
+.. autofunction:: jinja2.environmentfilter
 
-.. autofunction:: jinja2.pass_eval_context
+.. autofunction:: jinja2.contextfilter
 
-.. autofunction:: jinja2.pass_environment
+.. autofunction:: jinja2.evalcontextfilter
+
+.. autofunction:: jinja2.environmentfunction
+
+.. autofunction:: jinja2.contextfunction
+
+.. autofunction:: jinja2.evalcontextfunction
+
+.. function:: escape(s)
+
+    Convert the characters ``&``, ``<``, ``>``, ``'``, and ``"`` in string `s`
+    to HTML-safe sequences.  Use this if you need to display text that might
+    contain such characters in HTML.  This function will not escaped objects
+    that do have an HTML representation such as already escaped data.
+
+    The return value is a :class:`Markup` string.
 
 .. autofunction:: jinja2.clear_caches
 
 .. autofunction:: jinja2.is_undefined
 
+.. autoclass:: jinja2.Markup([string])
+    :members: escape, unescape, striptags
+
+.. admonition:: Note
+
+    The Jinja :class:`Markup` class is compatible with at least Pylons and
+    Genshi.  It's expected that more template engines and framework will pick
+    up the `__html__` concept soon.
+
 
 Exceptions
 ----------
@@ -642,119 +641,56 @@ Exceptions
 Custom Filters
 --------------
 
-Filters are Python functions that take the value to the left of the
-filter as the first argument and produce a new value. Arguments passed
-to the filter are passed after the value.
+Custom filters are just regular Python functions that take the left side of
+the filter as first argument and the arguments passed to the filter as
+extra arguments or keyword arguments.
 
-For example, the filter ``{{ 42|myfilter(23) }}`` is called behind the
-scenes as ``myfilter(42, 23)``.
+For example in the filter ``{{ 42|myfilter(23) }}`` the function would be
+called with ``myfilter(42, 23)``.  Here for example a simple filter that can
+be applied to datetime objects to format them::
 
-Jinja comes with some :ref:`built-in filters <builtin-filters>`. To use
-a custom filter, write a function that takes at least a ``value``
-argument, then register it in :attr:`Environment.filters`.
-
-Here's a filter that formats datetime objects:
-
-.. code-block:: python
-
-    def datetime_format(value, format="%H:%M %d-%m-%y"):
+    def datetimeformat(value, format='%H:%M / %d-%m-%Y'):
         return value.strftime(format)
 
-    environment.filters["datetime_format"] = datetime_format
+You can register it on the template environment by updating the
+:attr:`~Environment.filters` dict on the environment::
 
-Now it can be used in templates:
+    environment.filters['datetimeformat'] = datetimeformat
+
+Inside the template it can then be used as follows:
 
 .. sourcecode:: jinja
 
-    {{ article.pub_date|datetime_format }}
-    {{ article.pub_date|datetime_format("%B %Y") }}
+    written on: {{ article.pub_date|datetimeformat }}
+    publication date: {{ article.pub_date|datetimeformat('%d-%m-%Y') }}
 
-Some decorators are available to tell Jinja to pass extra information to
-the filter. The object is passed as the first argument, making the value
-being filtered the second argument.
+Filters can also be passed the current template context or environment.  This
+is useful if a filter wants to return an undefined value or check the current
+:attr:`~Environment.autoescape` setting.  For this purpose three decorators
+exist: :func:`environmentfilter`, :func:`contextfilter` and
+:func:`evalcontextfilter`.
 
--   :func:`pass_environment` passes the :class:`Environment`.
--   :func:`pass_eval_context` passes the :ref:`eval-context`.
--   :func:`pass_context` passes the current
-    :class:`~jinja2.runtime.Context`.
-
-Here's a filter that converts line breaks into HTML ``<br>`` and ``<p>``
-tags. It uses the eval context to check if autoescape is currently
-enabled before escaping the input and marking the output safe.
-
-.. code-block:: python
+Here a small example filter that breaks a text into HTML line breaks and
+paragraphs and marks the return value as safe HTML string if autoescaping is
+enabled::
 
     import re
-    from jinja2 import pass_eval_context
-    from markupsafe import Markup, escape
+    from jinja2 import evalcontextfilter, Markup, escape
 
-    @pass_eval_context
+    _paragraph_re = re.compile(r"(?:\r\n|\r(?!\n)|\n){2,}")
+
+    @evalcontextfilter
     def nl2br(eval_ctx, value):
-        br = "<br>\n"
-
-        if eval_ctx.autoescape:
-            value = escape(value)
-            br = Markup(br)
-
         result = "\n\n".join(
-            f"<p>{br.join(p.splitlines())}</p>"
-            for p in re.split(r"(?:\r\n|\r(?!\n)|\n){2,}", value)
+            f"<p>{p.replace('\n', Markup('<br>\n'))}</p>"
+            for p in _paragraph_re.split(escape(value))
         )
-        return Markup(result) if eval_ctx.autoescape else result
+        if eval_ctx.autoescape:
+            result = Markup(result)
+        return result
 
-
-.. _writing-tests:
-
-Custom Tests
-------------
-
-Test are Python functions that take the value to the left of the test as
-the first argument, and return ``True`` or ``False``. Arguments passed
-to the test are passed after the value.
-
-For example, the test ``{{ 42 is even }}`` is called behind the scenes
-as ``is_even(42)``.
-
-Jinja comes with some :ref:`built-in tests <builtin-tests>`. To use a
-custom tests, write a function that takes at least a ``value`` argument,
-then register it in :attr:`Environment.tests`.
-
-Here's a test that checks if a value is a prime number:
-
-.. code-block:: python
-
-    import math
-
-    def is_prime(n):
-        if n == 2:
-            return True
-
-        for i in range(2, int(math.ceil(math.sqrt(n))) + 1):
-            if n % i == 0:
-                return False
-
-        return True
-
-    environment.tests["prime"] = is_prime
-
-Now it can be used in templates:
-
-.. sourcecode:: jinja
-
-    {% if value is prime %}
-        {{ value }} is a prime number
-    {% else %}
-        {{ value }} is not a prime number
-    {% endif %}
-
-Some decorators are available to tell Jinja to pass extra information to
-the test. The object is passed as the first argument, making the value
-being tested the second argument.
-
--   :func:`pass_environment` passes the :class:`Environment`.
--   :func:`pass_eval_context` passes the :ref:`eval-context`.
--   :func:`pass_context` passes the current
-    :class:`~jinja2.runtime.Context`.
+Context filters work the same just that the first argument is the current
+active :class:`Context` rather than the environment.
 
 
 .. _eval-context:
@@ -762,53 +698,44 @@ being tested the second argument.
 Evaluation Context
 ------------------
 
-The evaluation context (short eval context or eval ctx) makes it
-possible to activate and deactivate compiled features at runtime.
+The evaluation context (short eval context or eval ctx) is a new object
+introduced in Jinja 2.4 that makes it possible to activate and deactivate
+compiled features at runtime.
 
-Currently it is only used to enable and disable automatic escaping, but
-it can be used by extensions as well.
+Currently it is only used to enable and disable the automatic escaping but
+can be used for extensions as well.
 
-The ``autoescape`` setting should be checked on the evaluation context,
-not the environment. The evaluation context will have the computed value
-for the current template.
+In previous Jinja versions filters and functions were marked as
+environment callables in order to check for the autoescape status from the
+environment.  In new versions it's encouraged to check the setting from the
+evaluation context instead.
 
-Instead of ``pass_environment``:
+Previous versions::
 
-.. code-block:: python
-
-    @pass_environment
+    @environmentfilter
     def filter(env, value):
         result = do_something(value)
-
         if env.autoescape:
             result = Markup(result)
-
         return result
 
-Use ``pass_eval_context`` if you only need the setting:
+In new versions you can either use a :func:`contextfilter` and access the
+evaluation context from the actual context, or use a
+:func:`evalcontextfilter` which directly passes the evaluation context to
+the function::
 
-.. code-block:: python
-
-    @pass_eval_context
-    def filter(eval_ctx, value):
-        result = do_something(value)
-
-        if eval_ctx.autoescape:
-            result = Markup(result)
-
-        return result
-
-Or use ``pass_context`` if you need other context behavior as well:
-
-.. code-block:: python
-
-    @pass_context
+    @contextfilter
     def filter(context, value):
         result = do_something(value)
-
         if context.eval_ctx.autoescape:
             result = Markup(result)
+        return result
 
+    @evalcontextfilter
+    def filter(eval_ctx, value):
+        result = do_something(value)
+        if eval_ctx.autoescape:
+            result = Markup(result)
         return result
 
 The evaluation context must not be modified at runtime.  Modifications
@@ -828,32 +755,57 @@ eval context object itself.
       time.  At runtime this should always be `False`.
 
 
+.. _writing-tests:
+
+Custom Tests
+------------
+
+Tests work like filters just that there is no way for a test to get access
+to the environment or context and that they can't be chained.  The return
+value of a test should be `True` or `False`.  The purpose of a test is to
+give the template designers the possibility to perform type and conformability
+checks.
+
+Here a simple test that checks if a variable is a prime number::
+
+    import math
+
+    def is_prime(n):
+        if n == 2:
+            return True
+        for i in range(2, int(math.ceil(math.sqrt(n))) + 1):
+            if n % i == 0:
+                return False
+        return True
+
+
+You can register it on the template environment by updating the
+:attr:`~Environment.tests` dict on the environment::
+
+    environment.tests['prime'] = is_prime
+
+A template designer can then use the test like this:
+
+.. sourcecode:: jinja
+
+    {% if 42 is prime %}
+        42 is a prime number
+    {% else %}
+        42 is not a prime number
+    {% endif %}
+
+
 .. _global-namespace:
 
 The Global Namespace
 --------------------
 
-The global namespace stores variables and functions that should be
-available without needing to pass them to :meth:`Template.render`. They
-are also available to templates that are imported or included without
-context. Most applications should only use :attr:`Environment.globals`.
-
-:attr:`Environment.globals` are intended for data that is common to all
-templates loaded by that environment. :attr:`Template.globals` are
-intended for data that is common to all renders of that template, and
-default to :attr:`Environment.globals` unless they're given in
-:meth:`Environment.get_template`, etc. Data that is specific to a
-render should be passed as context to :meth:`Template.render`.
-
-Only one set of globals is used during any specific rendering. If
-templates A and B both have template globals, and B extends A, then
-only B's globals are used for both when using ``b.render()``.
-
-Environment globals should not be changed after loading any templates,
-and template globals should not be changed at any time after loading the
-template. Changing globals after loading a template will result in
-unexpected behavior as they may be shared between the environment and
-other templates.
+Variables stored in the :attr:`Environment.globals` dict are special as they
+are available for imported templates too, even if they are imported without
+context.  This is the place where you can put variables and functions
+that should be available all the time.  Additionally :attr:`Template.globals`
+exist that are variables available to a specific template that are available
+to all :meth:`~Template.render` calls.
 
 
 .. _low-level-api:

docs/changelog.rst

@@ -1,4 +1,4 @@
-Changes
-=======
+Changelog
+=========
 
 .. include:: ../CHANGES.rst

docs/conf.py

@@ -10,25 +10,16 @@ release, version = get_version("Jinja2")
 
 # General --------------------------------------------------------------
 
-default_role = "code"
+master_doc = "index"
 extensions = [
     "sphinx.ext.autodoc",
-    "sphinx.ext.extlinks",
     "sphinx.ext.intersphinx",
-    "sphinxcontrib.log_cabinet",
     "pallets_sphinx_themes",
+    "sphinxcontrib.log_cabinet",
+    "sphinx_issues",
 ]
-autodoc_member_order = "bysource"
-autodoc_typehints = "description"
-autodoc_preserve_defaults = True
-extlinks = {
-    "issue": ("https://github.com/pallets/jinja/issues/%s", "#%s"),
-    "pr": ("https://github.com/pallets/jinja/pull/%s", "#%s"),
-    "ghsa": ("https://github.com/pallets/jinja/security/advisories/GHSA-%s", "GHSA-%s"),
-}
-intersphinx_mapping = {
-    "python": ("https://docs.python.org/3/", None),
-}
+intersphinx_mapping = {"python": ("https://docs.python.org/3/", None)}
+issues_github_path = "pallets/jinja"
 
 # HTML -----------------------------------------------------------------
 
@@ -36,20 +27,24 @@ html_theme = "jinja"
 html_theme_options = {"index_sidebar_logo": False}
 html_context = {
     "project_links": [
-        ProjectLink("Donate", "https://palletsprojects.com/donate"),
-        ProjectLink("PyPI Releases", "https://pypi.org/project/Jinja2/"),
+        ProjectLink("Donate to Pallets", "https://palletsprojects.com/donate"),
+        ProjectLink("Jinja Website", "https://palletsprojects.com/p/jinja/"),
+        ProjectLink("PyPI releases", "https://pypi.org/project/Jinja2/"),
         ProjectLink("Source Code", "https://github.com/pallets/jinja/"),
         ProjectLink("Issue Tracker", "https://github.com/pallets/jinja/issues/"),
-        ProjectLink("Chat", "https://discord.gg/pallets"),
     ]
 }
 html_sidebars = {
-    "index": ["project.html", "localtoc.html", "searchbox.html", "ethicalads.html"],
-    "**": ["localtoc.html", "relations.html", "searchbox.html", "ethicalads.html"],
+    "index": ["project.html", "localtoc.html", "searchbox.html"],
+    "**": ["localtoc.html", "relations.html", "searchbox.html"],
 }
-singlehtml_sidebars = {"index": ["project.html", "localtoc.html", "ethicalads.html"]}
+singlehtml_sidebars = {"index": ["project.html", "localtoc.html"]}
 html_static_path = ["_static"]
-html_favicon = "_static/jinja-icon.svg"
-html_logo = "_static/jinja-logo.svg"
+html_favicon = "_static/jinja-logo-sidebar.png"
+html_logo = "_static/jinja-logo-sidebar.png"
 html_title = f"Jinja Documentation ({version})"
 html_show_sourcelink = False
+
+# LaTeX ----------------------------------------------------------------
+
+latex_documents = [(master_doc, f"Jinja-{version}.tex", html_title, author, "manual")]

docs/examples/inline_gettext_extension.py

@@ -5,6 +5,7 @@ from jinja2.ext import Extension
 from jinja2.lexer import count_newlines
 from jinja2.lexer import Token
 
+
 _outside_re = re.compile(r"\\?(gettext|_)\(")
 _inside_re = re.compile(r"\\?[()]")
 
@@ -29,7 +30,7 @@ class InlineGettext(Extension):
             pos = 0
             lineno = token.lineno
 
-            while True:
+            while 1:
                 if not paren_stack:
                     match = _outside_re.search(token.value, pos)
                 else:

docs/extensions.rst

@@ -11,17 +11,14 @@ code into a reusable class like adding support for internationalization.
 Adding Extensions
 -----------------
 
-Extensions are added to the Jinja environment at creation time.  To add an
+Extensions are added to the Jinja environment at creation time.  Once the
+environment is created additional extensions cannot be added.  To add an
 extension pass a list of extension classes or import paths to the
 ``extensions`` parameter of the :class:`~jinja2.Environment` constructor.  The following
 example creates a Jinja environment with the i18n extension loaded::
 
     jinja_env = Environment(extensions=['jinja2.ext.i18n'])
 
-To add extensions after creation time, use the :meth:`~jinja2.Environment.add_extension` method::
-
-    jinja_env.add_extension('jinja2.ext.debug')
-
 
 .. _i18n-extension:
 
@@ -34,15 +31,9 @@ The i18n extension can be used in combination with `gettext`_ or
 `Babel`_.  When it's enabled, Jinja provides a ``trans`` statement that
 marks a block as translatable and calls ``gettext``.
 
-After enabling, an application has to provide functions for ``gettext``,
-``ngettext``, and optionally ``pgettext`` and ``npgettext``, either
-globally or when rendering. A ``_()`` function is added as an alias to
-the ``gettext`` function.
-
-A convenient way to provide these functions is to call one of the below
-methods depending on the translation system in use. If you do not require
-actual translation, use ``Environment.install_null_translations`` to
-install no-op functions.
+After enabling, an application has to provide ``gettext`` and
+``ngettext`` functions, either globally or when rendering. A ``_()``
+function is added as an alias to the ``gettext`` function.
 
 Environment Methods
 ~~~~~~~~~~~~~~~~~~~
@@ -53,16 +44,11 @@ additional methods:
 .. method:: jinja2.Environment.install_gettext_translations(translations, newstyle=False)
 
     Installs a translation globally for the environment. The
-    ``translations`` object must implement ``gettext``, ``ngettext``,
-    and optionally ``pgettext`` and ``npgettext``.
+    ``translations`` object must implement ``gettext`` and ``ngettext``.
     :class:`gettext.NullTranslations`, :class:`gettext.GNUTranslations`,
     and `Babel`_\s ``Translations`` are supported.
 
-    .. versionchanged:: 3.0
-        Added ``pgettext`` and ``npgettext``.
-
-    .. versionchanged:: 2.5
-        Added new-style gettext support.
+    .. versionchanged:: 2.5 Added new-style gettext support.
 
 .. method:: jinja2.Environment.install_null_translations(newstyle=False)
 
@@ -72,21 +58,16 @@ additional methods:
 
     .. versionchanged:: 2.5 Added new-style gettext support.
 
-.. method:: jinja2.Environment.install_gettext_callables(gettext, ngettext, newstyle=False, pgettext=None, npgettext=None)
+.. method:: jinja2.Environment.install_gettext_callables(gettext, ngettext, newstyle=False)
 
-    Install the given ``gettext``, ``ngettext``, ``pgettext``, and
-    ``npgettext`` callables into the environment. They should behave
-    exactly like :func:`gettext.gettext`, :func:`gettext.ngettext`,
-    :func:`gettext.pgettext` and :func:`gettext.npgettext`.
+    Install the given ``gettext`` and ``ngettext`` callables into the
+    environment. They should behave exactly like
+    :func:`gettext.gettext` and :func:`gettext.ngettext`.
 
     If ``newstyle`` is activated, the callables are wrapped to work like
     newstyle callables.  See :ref:`newstyle-gettext` for more information.
 
-    .. versionchanged:: 3.0
-        Added ``pgettext`` and ``npgettext``.
-
-    .. versionadded:: 2.5
-        Added new-style gettext support.
+    .. versionadded:: 2.5 Added new-style gettext support.
 
 .. method:: jinja2.Environment.uninstall_gettext_translations()
 
@@ -127,7 +108,7 @@ The usage of the ``i18n`` extension for template designers is covered in
 :ref:`the template documentation <i18n-in-templates>`.
 
 .. _gettext: https://docs.python.org/3/library/gettext.html
-.. _Babel: https://babel.pocoo.org/
+.. _Babel: http://babel.pocoo.org/
 
 
 Whitespace Trimming
@@ -170,10 +151,6 @@ done with the ``|format`` filter. This requires duplicating work for
     {{ ngettext(
            "%(num)d apple", "%(num)d apples", apples|count
        )|format(num=apples|count) }}
-    {{ pgettext("greeting", "Hello, World!") }}
-    {{ npgettext(
-           "fruit", "%(num)d apple", "%(num)d apples", apples|count
-       )|format(num=apples|count) }}
 
 New style ``gettext`` make formatting part of the call, and behind the
 scenes enforce more consistency.
@@ -183,8 +160,6 @@ scenes enforce more consistency.
     {{ gettext("Hello, World!") }}
     {{ gettext("Hello, %(name)s!", name=name) }}
     {{ ngettext("%(num)d apple", "%(num)d apples", apples|count) }}
-    {{ pgettext("greeting", "Hello, World!") }}
-    {{ npgettext("fruit", "%(num)d apple", "%(num)d apples", apples|count) }}
 
 The advantages of newstyle gettext are:
 

docs/faq.rst

@@ -1,77 +1,175 @@
 Frequently Asked Questions
 ==========================
 
+This page answers some of the often asked questions about Jinja.
+
+.. highlight:: html+jinja
 
 Why is it called Jinja?
 -----------------------
 
-"Jinja" is a Japanese `Shinto shrine`_, or temple, and temple and
-template share a similar English pronunciation. It is not named after
-the `city in Uganda`_.
+The name Jinja was chosen because it's the name of a Japanese temple and
+temple and template share a similar pronunciation.  It is not named after
+the city in Uganda.
 
-.. _Shinto shrine: https://en.wikipedia.org/wiki/Shinto_shrine
-.. _city in Uganda: https://en.wikipedia.org/wiki/Jinja%2C_Uganda
+How fast is it?
+---------------
 
+We really hate benchmarks especially since they don't reflect much.  The
+performance of a template depends on many factors and you would have to
+benchmark different engines in different situations.  The benchmarks from the
+testsuite show that Jinja has a similar performance to `Mako`_ and is between
+10 and 20 times faster than Django's template engine or Genshi.  These numbers
+should be taken with tons of salt as the benchmarks that took these numbers
+only test a few performance related situations such as looping.  Generally
+speaking the performance of a template engine doesn't matter much as the
+usual bottleneck in a web application is either the database or the application
+code.
 
-How fast is Jinja?
-------------------
+.. _Mako: https://www.makotemplates.org/
 
-Jinja is relatively fast among template engines because it compiles and
-caches template code to Python code, so that the template does not need
-to be parsed and interpreted each time. Rendering a template becomes as
-close to executing a Python function as possible.
+How Compatible is Jinja with Django?
+------------------------------------
 
-Jinja also makes extensive use of caching. Templates are cached by name
-after loading, so future uses of the template avoid loading. The
-template loading itself uses a bytecode cache to avoid repeated
-compiling. The caches can be external to persist across restarts.
-Templates can also be precompiled and loaded as fast Python imports.
+The default syntax of Jinja matches Django syntax in many ways.  However
+this similarity doesn't mean that you can use a Django template unmodified
+in Jinja.  For example filter arguments use a function call syntax rather
+than a colon to separate filter name and arguments.  Additionally the
+extension interface in Jinja is fundamentally different from the Django one
+which means that your custom tags won't work any longer.
 
-We dislike benchmarks because they don't reflect real use. Performance
-depends on many factors. Different engines have different default
-configurations and tradeoffs that make it unclear how to set up a useful
-comparison. Often, database access, API calls, and data processing have
-a much larger effect on performance than the template engine.
+Generally speaking you will use much less custom extensions as the Jinja
+template system allows you to use a certain subset of Python expressions
+which can replace most Django extensions.  For example instead of using
+something like this::
 
+    {% load comments %}
+    {% get_latest_comments 10 as latest_comments %}
+    {% for comment in latest_comments %}
+        ...
+    {% endfor %}
 
-Isn't it a bad idea to put logic in templates?
-----------------------------------------------
+You will most likely provide an object with attributes to retrieve
+comments from the database::
+
+    {% for comment in models.comments.latest(10) %}
+        ...
+    {% endfor %}
+
+Or directly provide the model for quick testing::
+
+    {% for comment in Comment.objects.order_by('-pub_date')[:10] %}
+        ...
+    {% endfor %}
+
+Please keep in mind that even though you may put such things into templates
+it still isn't a good idea.  Queries should go into the view code and not
+the template!
+
+Isn't it a terrible idea to put Logic into Templates?
+-----------------------------------------------------
 
 Without a doubt you should try to remove as much logic from templates as
-possible. With less logic, the template is easier to understand, has
-fewer potential side effects, and is faster to compile and render. But a
-template without any logic means processing must be done in code before
-rendering. A template engine that does that is shipped with Python,
-called :class:`string.Template`, and while it's definitely fast it's not
-convenient.
+possible.  But templates without any logic mean that you have to do all
+the processing in the code which is boring and stupid.  A template engine
+that does that is shipped with Python and called `string.Template`.  Comes
+without loops and if conditions and is by far the fastest template engine
+you can get for Python.
 
-Jinja's features such as blocks, statements, filters, and function calls
-make it much easier to write expressive templates, with very few
-restrictions. Jinja doesn't allow arbitrary Python code in templates, or
-every feature available in the Python language. This keeps the engine
-easier to maintain, and keeps templates more readable.
+So some amount of logic is required in templates to keep everyone happy.
+And Jinja leaves it pretty much to you how much logic you want to put into
+templates.  There are some restrictions in what you can do and what not.
 
-Some amount of logic is required in templates to keep everyone happy.
-Too much logic in the template can make it complex to reason about and
-maintain. It's up to you to decide how your application will work and
-balance how much logic you want to put in the template.
+Jinja neither allows you to put arbitrary Python code into templates nor
+does it allow all Python expressions.  The operators are limited to the
+most common ones and more advanced expressions such as list comprehensions
+and generator expressions are not supported.  This keeps the template engine
+easier to maintain and templates more readable.
 
+Why is Autoescaping not the Default?
+------------------------------------
 
-Why is HTML escaping not the default?
+There are multiple reasons why automatic escaping is not the default mode
+and also not the recommended one.  While automatic escaping of variables
+means that you will less likely have an XSS problem it also causes a huge
+amount of extra processing in the template engine which can cause serious
+performance problems.  As Python doesn't provide a way to mark strings as
+unsafe Jinja has to hack around that limitation by providing a custom
+string class (the :class:`Markup` string) that safely interacts with safe
+and unsafe strings.
+
+With explicit escaping however the template engine doesn't have to perform
+any safety checks on variables.  Also a human knows not to escape integers
+or strings that may never contain characters one has to escape or already
+HTML markup.  For example when iterating over a list over a table of
+integers and floats for a table of statistics the template designer can
+omit the escaping because he knows that integers or floats don't contain
+any unsafe parameters.
+
+Additionally Jinja is a general purpose template engine and not only used
+for HTML/XML generation.  For example you may generate LaTeX, emails,
+CSS, JavaScript, or configuration files.
+
+Why is the Context immutable?
+-----------------------------
+
+When writing a :func:`contextfunction` or something similar you may have
+noticed that the context tries to stop you from modifying it.  If you have
+managed to modify the context by using an internal context API you may
+have noticed that changes in the context don't seem to be visible in the
+template.  The reason for this is that Jinja uses the context only as
+primary data source for template variables for performance reasons.
+
+If you want to modify the context write a function that returns a variable
+instead that one can assign to a variable by using set::
+
+    {% set comments = get_latest_comments() %}
+
+My tracebacks look weird. What's happening?
+-------------------------------------------
+
+Jinja can rewrite tracebacks so they show the template lines numbers and
+source rather than the underlying compiled code, but this requires
+special Python support. CPython <3.7 requires ``ctypes``, and PyPy
+requires transparent proxy support.
+
+If you are using Google App Engine, ``ctypes`` is not available. You can
+make it available in development, but not in production.
+
+.. code-block:: python
+
+    import os
+    if os.environ.get('SERVER_SOFTWARE', '').startswith('Dev'):
+        from google.appengine.tools.devappserver2.python import sandbox
+        sandbox._WHITE_LIST_C_MODULES += ['_ctypes', 'gestalt']
+
+Credit for this snippet goes to `Thomas Johansson
+<https://stackoverflow.com/questions/3086091/debug-jinja2-in-google-app-engine/3694434#3694434>`_
+
+My Macros are overridden by something
 -------------------------------------
 
-Jinja provides a feature that can be enabled to escape HTML syntax in
-rendered templates. However, it is disabled by default.
+In some situations the Jinja scoping appears arbitrary:
 
-Jinja is a general purpose template engine, it is not only used for HTML
-documents. You can generate plain text, LaTeX, emails, CSS, JavaScript,
-configuration files, etc. HTML escaping wouldn't make sense for any of
-these document types.
+layout.tmpl:
 
-While automatic escaping means that you are less likely have an XSS
-problem, it also requires significant extra processing during compiling
-and rendering, which can reduce performance. Jinja uses `MarkupSafe`_ for
-escaping, which provides optimized C code for speed, but it still
-introduces overhead to track escaping across methods and formatting.
+.. sourcecode:: jinja
 
-.. _MarkupSafe: https://markupsafe.palletsprojects.com/
+    {% macro foo() %}LAYOUT{% endmacro %}
+    {% block body %}{% endblock %}
+
+child.tmpl:
+
+.. sourcecode:: jinja
+
+    {% extends 'layout.tmpl' %}
+    {% macro foo() %}CHILD{% endmacro %}
+    {% block body %}{{ foo() }}{% endblock %}
+
+This will print ``LAYOUT`` in Jinja.  This is a side effect of having
+the parent template evaluated after the child one.  This allows child
+templates passing information to the parent template.  To avoid this
+issue rename the macro or variable in the parent template to have an
+uncommon prefix.
+
+.. _Jinja 1: https://pypi.org/project/Jinja/

docs/index.rst

@@ -3,9 +3,9 @@
 Jinja
 =====
 
-.. image:: _static/jinja-name.svg
+.. image:: _static/jinja-logo.png
     :align: center
-    :height: 200px
+    :target: https://palletsprojects.com/p/jinja/
 
 Jinja is a fast, expressive, extensible templating engine. Special
 placeholders in the template allow writing code similar to Python
@@ -25,5 +25,4 @@ syntax. Then the template is passed data to render the final document.
     switching
     tricks
     faq
-    license
-    changes
+    changelog

docs/integration.rst

@@ -1,25 +1,6 @@
 Integration
 ===========
 
-
-Flask
------
-
-The `Flask`_ web application framework, also maintained by Pallets, uses
-Jinja templates by default. Flask sets up a Jinja environment and
-template loader for you, and provides functions to easily render
-templates from view functions.
-
-.. _Flask: https://flask.palletsprojects.com
-
-
-Django
-------
-
-Django supports using Jinja as its template engine, see
-https://docs.djangoproject.com/en/stable/topics/templates/#support-for-template-engines.
-
-
 .. _babel-integration:
 
 Babel
@@ -91,4 +72,4 @@ this add this to ``config/environment.py``:
 
     config['pylons.strict_c'] = True
 
-.. _Pylons: https://pylonsproject.org/
+.. _Pylons: https://pylonshq.com/

docs/intro.rst

@@ -12,8 +12,8 @@ It includes:
 -   HTML templates can use autoescaping to prevent XSS from untrusted
     user input.
 -   A sandboxed environment can safely render untrusted templates.
--   Async support for generating templates that automatically handle
-    sync and async functions without extra syntax.
+-   AsyncIO support for generating templates and calling async
+    functions.
 -   I18N support with Babel.
 -   Templates are compiled to optimized Python code just-in-time and
     cached, or can be compiled ahead-of-time.
@@ -30,7 +30,7 @@ Installation
 ------------
 
 We recommend using the latest version of Python. Jinja supports Python
-3.10 and newer. We also recommend using a `virtual environment`_ in order
+3.6 and newer. We also recommend using a `virtual environment`_ in order
 to isolate your project dependencies from other projects and the system.
 
 .. _virtual environment: https://packaging.python.org/tutorials/installing-packages/#creating-virtual-environments
@@ -60,4 +60,4 @@ These distributions will not be installed automatically.
 
 -   `Babel`_ provides translation support in templates.
 
-.. _Babel: https://babel.pocoo.org/
+.. _Babel: http://babel.pocoo.org/

docs/license.rst

@@ -1,5 +0,0 @@
-BSD-3-Clause License
-====================
-
-.. literalinclude:: ../LICENSE.txt
-    :language: text

docs/make.bat

@@ -21,7 +21,7 @@ if errorlevel 9009 (
 	echo.may add the Sphinx directory to PATH.
 	echo.
 	echo.If you don't have Sphinx installed, grab it from
-	echo.https://www.sphinx-doc.org/
+	echo.http://sphinx-doc.org/
 	exit /b 1
 )
 

docs/nativetypes.rst

@@ -55,17 +55,6 @@ Foo
 >>> print(result.value)
 15
 
-Sandboxed Native Environment
-----------------------------
-
-You can combine :class:`.SandboxedEnvironment` and :class:`NativeEnvironment` to
-get both behaviors.
-
-.. code-block:: python
-
-    class SandboxedNativeEnvironment(SandboxedEnvironment, NativeEnvironment):
-        pass
-
 API
 ---
 

docs/requirements.txt

@@ -0,0 +1,4 @@
+Sphinx~=2.1.2
+Pallets-Sphinx-Themes~=1.2.0
+sphinxcontrib-log-cabinet~=1.0.1
+sphinx-issues~=1.2.0

docs/sandbox.rst

@@ -1,56 +1,18 @@
 Sandbox
 =======
 
-The Jinja sandbox can be used to render untrusted templates. Access to
-attributes, method calls, operators, mutating data structures, and
-string formatting can be intercepted and prohibited.
+The Jinja sandbox can be used to evaluate untrusted code.  Access to unsafe
+attributes and methods is prohibited.
 
-.. code-block:: pycon
-
-    >>> from jinja2.sandbox import SandboxedEnvironment
-    >>> env = SandboxedEnvironment()
-    >>> func = lambda: "Hello, Sandbox!"
-    >>> env.from_string("{{ func() }}").render(func=func)
-    'Hello, Sandbox!'
-    >>> env.from_string("{{ func.__code__.co_code }}").render(func=func)
-    Traceback (most recent call last):
-      ...
-    SecurityError: access to attribute '__code__' of 'function' object is unsafe.
-
-A sandboxed environment can be useful, for example, to allow users of an
-internal reporting system to create custom emails. You would document
-what data is available in the templates, then the user would write a
-template using that information. Your code would generate the report
-data and pass it to the user's sandboxed template to render.
-
-
-Security Considerations
------------------------
-
-The sandbox alone is not a solution for perfect security. Keep these
-things in mind when using the sandbox.
-
-Templates can still raise errors when compiled or rendered. Your code
-should attempt to catch errors instead of crashing.
-
-It is possible to construct a relatively small template that renders to
-a very large amount of output, which could correspond to a high use of
-CPU or memory. You should run your application with limits on resources
-such as CPU and memory to mitigate this.
-
-Jinja only renders text, it does not understand, for example, JavaScript
-code. Depending on how the rendered template will be used, you may need
-to do other postprocessing to restrict the output.
-
-Pass only the data that is relevant to the template. Avoid passing
-global data, or objects with methods that have side effects. By default
-the sandbox prevents private and internal attribute access. You can
-override :meth:`~SandboxedEnvironment.is_safe_attribute` to further
-restrict attributes access. Decorate methods with :func:`unsafe` to
-prevent calling them from templates when passing objects as data. Use
-:class:`ImmutableSandboxedEnvironment` to prevent modifying lists and
-dictionaries.
+Assuming `env` is a :class:`SandboxedEnvironment` in the default configuration
+the following piece of code shows how it works:
 
+>>> env.from_string("{{ func.func_code }}").render(func=lambda:None)
+u''
+>>> env.from_string("{{ func.func_code.do_something }}").render(func=lambda:None)
+Traceback (most recent call last):
+  ...
+SecurityError: access to attribute 'func_code' of 'function' object is unsafe.
 
 API
 ---
@@ -72,40 +34,61 @@ API
 
 .. autofunction:: modifies_known_mutable
 
+.. admonition:: Note
+
+    The Jinja sandbox alone is no solution for perfect security.  Especially
+    for web applications you have to keep in mind that users may create
+    templates with arbitrary HTML in so it's crucial to ensure that (if you
+    are running multiple users on the same server) they can't harm each other
+    via JavaScript insertions and much more.
+
+    Also the sandbox is only as good as the configuration.  We strongly
+    recommend only passing non-shared resources to the template and use
+    some sort of whitelisting for attributes.
+
+    Also keep in mind that templates may raise runtime or compile time errors,
+    so make sure to catch them.
 
 Operator Intercepting
 ---------------------
 
-For performance, Jinja outputs operators directly when compiling. This
-means it's not possible to intercept operator behavior by overriding
-:meth:`SandboxEnvironment.call <Environment.call>` by default, because
-operator special methods are handled by the Python interpreter, and
-might not correspond with exactly one method depending on the operator's
-use.
+.. versionadded:: 2.6
 
-The sandbox can instruct the compiler to output a function to intercept
-certain operators instead. Override
-:attr:`SandboxedEnvironment.intercepted_binops` and
-:attr:`SandboxedEnvironment.intercepted_unops` with the operator symbols
-you want to intercept. The compiler will replace the symbols with calls
-to :meth:`SandboxedEnvironment.call_binop` and
-:meth:`SandboxedEnvironment.call_unop` instead. The default
-implementation of those methods will use
-:attr:`SandboxedEnvironment.binop_table` and
-:attr:`SandboxedEnvironment.unop_table` to translate operator symbols
-into :mod:`operator` functions.
+For maximum performance Jinja will let operators call directly the type
+specific callback methods.  This means that it's not possible to have this
+intercepted by overriding :meth:`Environment.call`.  Furthermore a
+conversion from operator to special method is not always directly possible
+due to how operators work.  For instance for divisions more than one
+special method exist.
 
-For example, the power (``**``) operator can be disabled:
+With Jinja 2.6 there is now support for explicit operator intercepting.
+This can be used to customize specific operators as necessary.  In order
+to intercept an operator one has to override the
+:attr:`SandboxedEnvironment.intercepted_binops` attribute.  Once the
+operator that needs to be intercepted is added to that set Jinja will
+generate bytecode that calls the :meth:`SandboxedEnvironment.call_binop`
+function.  For unary operators the `unary` attributes and methods have to
+be used instead.
 
-.. code-block:: python
+The default implementation of :attr:`SandboxedEnvironment.call_binop`
+will use the :attr:`SandboxedEnvironment.binop_table` to translate
+operator symbols into callbacks performing the default operator behavior.
+
+This example shows how the power (``**``) operator can be disabled in
+Jinja::
 
     from jinja2.sandbox import SandboxedEnvironment
 
+
     class MyEnvironment(SandboxedEnvironment):
-        intercepted_binops = frozenset(["**"])
+        intercepted_binops = frozenset(['**'])
 
         def call_binop(self, context, operator, left, right):
-            if operator == "**":
-                return self.undefined("The power (**) operator is unavailable.")
+            if operator == '**':
+                return self.undefined('the power operator is unavailable')
+            return SandboxedEnvironment.call_binop(self, context,
+                                                   operator, left, right)
 
-            return super().call_binop(self, context, operator, left, right)
+Make sure to always call into the super method, even if you are not
+intercepting the call.  Jinja might internally call the method to
+evaluate expressions.

docs/switching.rst

@@ -1,73 +1,141 @@
-Switching From Other Template Engines
+Switching from other Template Engines
 =====================================
 
-This is a brief guide on some of the differences between Jinja syntax
-and other template languages. See :doc:`/templates` for a comprehensive
-guide to Jinja syntax and features.
+.. highlight:: html+jinja
+
+If you have used a different template engine in the past and want to switch
+to Jinja here is a small guide that shows the basic syntactic and semantic
+changes between some common, similar text template engines for Python.
+
+Jinja 1
+-------
+
+Jinja 2 is mostly compatible with Jinja 1 in terms of API usage and template
+syntax.  The differences between Jinja 1 and 2 are explained in the following
+list.
+
+API
+~~~
+
+Loaders
+    Jinja 2 uses a different loader API.  Because the internal representation
+    of templates changed there is no longer support for external caching
+    systems such as memcached.  The memory consumed by templates is comparable
+    with regular Python modules now and external caching doesn't give any
+    advantage.  If you have used a custom loader in the past have a look at
+    the new :ref:`loader API <loaders>`.
+
+Loading templates from strings
+    In the past it was possible to generate templates from a string with the
+    default environment configuration by using `jinja.from_string`.  Jinja 2
+    provides a :class:`Template` class that can be used to do the same, but
+    with optional additional configuration.
+
+Automatic unicode conversion
+    Jinja 1 performed automatic conversion of bytes in a given encoding
+    into unicode objects. This conversion is no longer implemented as it
+    was inconsistent as most libraries are using the regular Python
+    ASCII bytes to Unicode conversion. An application powered by Jinja 2
+    *has to* use unicode internally everywhere or make sure that Jinja 2
+    only gets unicode strings passed.
+
+i18n
+    Jinja 1 used custom translators for internationalization.  i18n is now
+    available as Jinja 2 extension and uses a simpler, more gettext friendly
+    interface and has support for babel.  For more details see
+    :ref:`i18n-extension`.
+
+Internal methods
+    Jinja 1 exposed a few internal methods on the environment object such
+    as `call_function`, `get_attribute` and others.  While they were marked
+    as being an internal method it was possible to override them.  Jinja 2
+    doesn't have equivalent methods.
+
+Sandbox
+    Jinja 1 was running sandbox mode by default.  Few applications actually
+    used that feature so it became optional in Jinja 2.  For more details
+    about the sandboxed execution see :class:`SandboxedEnvironment`.
+
+Context
+    Jinja 1 had a stacked context as storage for variables passed to the
+    environment.  In Jinja 2 a similar object exists but it doesn't allow
+    modifications nor is it a singleton.  As inheritance is dynamic now
+    multiple context objects may exist during template evaluation.
+
+Filters and Tests
+    Filters and tests are regular functions now.  It's no longer necessary
+    and allowed to use factory functions.
+
+
+Templates
+~~~~~~~~~
+
+Jinja 2 has mostly the same syntax as Jinja 1.  What's different is that
+macros require parentheses around the argument list now.
+
+Additionally Jinja 2 allows dynamic inheritance now and dynamic includes.
+The old helper function `rendertemplate` is gone now, `include` can be used
+instead.  Includes no longer import macros and variable assignments, for
+that the new `import` tag is used.  This concept is explained in the
+:ref:`import` documentation.
+
+Another small change happened in the `for`-tag.  The special loop variable
+doesn't have a `parent` attribute, instead you have to alias the loop
+yourself.  See :ref:`accessing-the-parent-loop` for more details.
 
 
 Django
 ------
 
 If you have previously worked with Django templates, you should find
-Jinja very familiar. Many of the syntax elements look and work the same.
-However, Jinja provides some more syntax elements, and some work a bit
-differently.
+Jinja very familiar.  In fact, most of the syntax elements look and
+work the same.
 
-This section covers the template changes. The API, including extension
-support, is fundamentally different so it won't be covered here.
-
-Django supports using Jinja as its template engine, see
-https://docs.djangoproject.com/en/stable/topics/templates/#support-for-template-engines.
+However, Jinja provides some more syntax elements covered in the
+documentation and some work a bit different.
 
+This section covers the template changes.  As the API is fundamentally
+different we won't cover it here.
 
 Method Calls
 ~~~~~~~~~~~~
 
-In Django, methods are called implicitly, without parentheses.
-
-.. code-block:: django
+In Django method calls work implicitly, while Jinja requires the explicit
+Python syntax. Thus this Django code::
 
     {% for page in user.get_created_pages %}
         ...
     {% endfor %}
 
-In Jinja, using parentheses is required for calls, like in Python. This
-allows you to pass variables to the method, which is not possible
-in Django. This syntax is also used for calling macros.
-
-.. code-block:: jinja
+...looks like this in Jinja::
 
     {% for page in user.get_created_pages() %}
         ...
     {% endfor %}
 
+This allows you to pass variables to the method, which is not possible in
+Django. This syntax is also used for macros.
 
 Filter Arguments
 ~~~~~~~~~~~~~~~~
 
-In Django, one literal value can be passed to a filter after a colon.
-
-.. code-block:: django
+Jinja provides more than one argument for filters.  Also the syntax for
+argument passing is different.  A template that looks like this in Django::
 
     {{ items|join:", " }}
 
-In Jinja, filters can take any number of positional and keyword
-arguments in parentheses, like function calls. Arguments can also be
-variables instead of literal values.
+looks like this in Jinja::
 
-.. code-block:: jinja
-
-    {{ items|join(", ") }}
+    {{ items|join(', ') }}
 
+It is a bit more verbose, but it allows different types of arguments -
+including variables - and more than one of them.
 
 Tests
 ~~~~~
 
-In addition to filters, Jinja also has "tests" used with the ``is``
-operator. This operator is not the same as the Python operator.
-
-.. code-block:: jinja
+In addition to filters there also are tests you can perform using the is
+operator.  Here are some examples::
 
     {% if user.user_id is odd %}
         {{ user.username|e }} is odd
@@ -78,85 +146,64 @@ operator. This operator is not the same as the Python operator.
 Loops
 ~~~~~
 
-In Django, the special variable for the loop context is called
-``forloop``, and the ``empty`` is used for no loop items.
+For loops work very similarly to Django, but notably the Jinja special
+variable for the loop context is called `loop`, not `forloop` as in Django.
 
-.. code-block:: django
+In addition, the Django `empty` argument is called `else` in Jinja. For
+example, the Django template::
 
     {% for item in items %}
-        {{ forloop.counter }}. {{ item }}
+        {{ item }}
     {% empty %}
         No items!
     {% endfor %}
 
-In Jinja, the special variable for the loop context is called ``loop``,
-and the ``else`` block is used for no loop items.
-
-.. code-block:: jinja
+...looks like this in Jinja::
 
     {% for item in items %}
-        {{ loop.index }}. {{ item }}
+        {{ item }}
     {% else %}
         No items!
     {% endfor %}
 
-
 Cycle
 ~~~~~
 
-In Django, the ``{% cycle %}`` can be used in a for loop to alternate
-between values per loop.
+The ``{% cycle %}`` tag does not exist in Jinja; however, you can achieve the
+same output by using the `cycle` method on the loop context special variable.
 
-.. code-block:: django
+The following Django template::
 
     {% for user in users %}
         <li class="{% cycle 'odd' 'even' %}">{{ user }}</li>
     {% endfor %}
 
-In Jinja, the ``loop`` context has a ``cycle`` method.
-
-.. code-block:: jinja
+...looks like this in Jinja::
 
     {% for user in users %}
         <li class="{{ loop.cycle('odd', 'even') }}">{{ user }}</li>
     {% endfor %}
 
-A cycler can also be assigned to a variable and used outside or across
-loops with the ``cycle()`` global function.
+There is no equivalent of ``{% cycle ... as variable %}``.
 
 
 Mako
 ----
 
-You can configure Jinja to look more like Mako:
+.. highlight:: html+mako
 
-.. code-block:: python
+If you have used Mako so far and want to switch to Jinja you can configure
+Jinja to look more like Mako:
 
-    env = Environment(
-        block_start_string="<%",
-        block_end_string="%>",
-        variable_start_string="${",
-        variable_end_string="}",
-        comment_start_string="<%doc>",
-        commend_end_string="</%doc>",
-        line_statement_prefix="%",
-        line_comment_prefix="##",
-    )
+.. sourcecode:: python
 
-With an environment configured like that, Jinja should be able to
-interpret a small subset of Mako templates without any changes.
+    env = Environment('<%', '%>', '${', '}', '<%doc>', '</%doc>', '%', '##')
 
-Jinja does not support embedded Python code, so you would have to move
-that out of the template. You could either process the data with the
-same code before rendering, or add a global function or filter to the
-Jinja environment.
-
-The syntax for defs (which are called macros in Jinja) and template
-inheritance is different too.
-
-The following Mako template:
-
-.. code-block:: mako
+With an environment configured like that, Jinja should be able to interpret
+a small subset of Mako templates.  Jinja does not support embedded Python
+code, so you would have to move that out of the template.  The syntax for defs
+(which are called macros in Jinja) and template inheritance is different too.
+The following Mako template::
 
     <%inherit file="layout.html" />
     <%def name="title()">Page Title</%def>
@@ -166,9 +213,7 @@ The following Mako template:
     % endfor
     </ul>
 
-Looks like this in Jinja with the above configuration:
-
-.. code-block:: jinja
+Looks like this in Jinja with the above configuration::
 
     <% extends "layout.html" %>
     <% block title %>Page Title<% endblock %>

docs/templates.rst

@@ -1,9 +1,8 @@
-.. py:currentmodule:: jinja2
-.. highlight:: html+jinja
-
 Template Designer Documentation
 ===============================
 
+.. highlight:: html+jinja
+
 This document describes the syntax and semantics of the template engine and
 will be most useful as reference to those creating Jinja templates.  As the
 template engine is very flexible, the configuration from the application can
@@ -55,11 +54,7 @@ configured as follows:
 * ``{% ... %}`` for :ref:`Statements <list-of-control-structures>`
 * ``{{ ... }}`` for :ref:`Expressions` to print to the template output
 * ``{# ... #}`` for :ref:`Comments` not included in the template output
-
-:ref:`Line Statements and Comments <line-statements>` are also possible,
-though they don't have default prefix characters. To use them, set
-``line_statement_prefix`` and ``line_comment_prefix`` when creating the
-:class:`~jinja2.Environment`.
+* ``#  ... ##`` for :ref:`Line Statements <line-statements>`
 
 
 Template File Extension
@@ -202,11 +197,10 @@ option can also be set to strip tabs and spaces from the beginning of a
 line to the start of a block. (Nothing will be stripped if there are
 other characters before the start of the block.)
 
-With both ``trim_blocks`` and ``lstrip_blocks`` disabled (the default), block
-tags on their own lines will be removed, but a blank line will remain and the
-spaces in the content will be preserved. For example, this template:
-
-.. code-block:: jinja
+With both `trim_blocks` and `lstrip_blocks` enabled, you can put block tags
+on their own lines, and the entire block line will be removed when
+rendered, preserving the whitespace of the contents.  For example,
+without the `trim_blocks` and `lstrip_blocks` options, this template::
 
     <div>
         {% if True %}
@@ -214,10 +208,7 @@ spaces in the content will be preserved. For example, this template:
         {% endif %}
     </div>
 
-With both ``trim_blocks`` and ``lstrip_blocks`` disabled, the template is
-rendered with blank lines inside the div:
-
-.. code-block:: text
+gets rendered with blank lines inside the div::
 
     <div>
 
@@ -225,10 +216,8 @@ rendered with blank lines inside the div:
 
     </div>
 
-With both ``trim_blocks`` and ``lstrip_blocks`` enabled, the template block
-lines are completely removed:
-
-.. code-block:: text
+But with both `trim_blocks` and `lstrip_blocks` enabled, the template block
+lines are removed and other whitespace is preserved::
 
     <div>
             yay
@@ -241,15 +230,6 @@ plus sign (``+``) at the start of a block::
             {%+ if something %}yay{% endif %}
     </div>
 
-Similarly, you can manually disable the ``trim_blocks`` behavior by
-putting a plus sign (``+``) at the end of a block::
-
-    <div>
-        {% if something +%}
-            yay
-        {% endif %}
-    </div>
-
 You can also strip whitespace in templates by hand.  If you add a minus
 sign (``-``) to the start or end of a block (e.g. a :ref:`for-loop` tag), a
 comment, or a variable expression, the whitespaces before or after
@@ -433,7 +413,7 @@ this template "extends" another template.  When the template system evaluates
 this template, it first locates the parent.  The extends tag should be the
 first tag in the template.  Everything before it is printed out normally and
 may cause confusion.  For details about this behavior and how to take
-advantage of it, see :ref:`null-default-fallback`. Also a block will always be
+advantage of it, see :ref:`null-master-fallback`. Also a block will always be
 filled in regardless of whether the surrounding condition is evaluated to be true
 or false.
 
@@ -528,8 +508,8 @@ However, the name after the `endblock` word must match the block name.
 Block Nesting and Scope
 ~~~~~~~~~~~~~~~~~~~~~~~
 
-Blocks can be nested for more complex layouts. By default, a block may not
-access variables from outside the block (outer scopes)::
+Blocks can be nested for more complex layouts.  However, per default blocks
+may not access variables from outer scopes::
 
     {% for item in seq %}
         <li>{% block loop_item %}{{ item }}{% endblock %}</li>
@@ -551,69 +531,20 @@ modifier to a block declaration::
 When overriding a block, the `scoped` modifier does not have to be provided.
 
 
-Required Blocks
-~~~~~~~~~~~~~~~
-
-Blocks can be marked as ``required``. They must be overridden at some
-point, but not necessarily by the direct child template. Required blocks
-may only contain space and comments, and they cannot be rendered
-directly.
-
-.. code-block:: jinja
-    :caption: ``page.txt``
-
-    {% block body required %}{% endblock %}
-
-.. code-block:: jinja
-    :caption: ``issue.txt``
-
-    {% extends "page.txt" %}
-
-.. code-block:: jinja
-    :caption: ``bug_report.txt``
-
-    {% extends "issue.txt" %}
-    {% block body %}Provide steps to demonstrate the bug.{% endblock %}
-
-Rendering ``page.txt`` or ``issue.txt`` will raise
-``TemplateRuntimeError`` because they don't override the ``body`` block.
-Rendering ``bug_report.txt`` will succeed because it does override the
-block.
-
-When combined with ``scoped``, the ``required`` modifier must be placed
-*after* the scoped modifier. Here are some valid examples:
-
-.. code-block:: jinja
-
-    {% block body scoped %}{% endblock %}
-    {% block body required %}{% endblock %}
-    {% block body scoped required %}{% endblock %}
-
-
 Template Objects
 ~~~~~~~~~~~~~~~~
 
-``extends``, ``include``, and ``import`` can take a template object
-instead of the name of a template to load. This could be useful in some
-advanced situations, since you can use Python code to load a template
-first and pass it in to ``render``.
+.. versionchanged:: 2.4
 
-.. code-block:: python
+If a template object was passed in the template context, you can
+extend from that object as well.  Assuming the calling code passes
+a layout template as `layout_template` to the environment, this
+code works::
 
-    if debug_mode:
-        layout = env.get_template("debug_layout.html")
-    else:
-        layout = env.get_template("layout.html")
+    {% extends layout_template %}
 
-    user_detail = env.get_template("user/detail.html")
-    return user_detail.render(layout=layout)
-
-.. code-block:: jinja
-
-    {% extends layout %}
-
-Note how ``extends`` is passed the variable with the template object
-that was passed to ``render``, instead of a string.
+Previously, the `layout_template` variable had to be a string with
+the layout template's filename for this to work.
 
 
 HTML Escaping
@@ -709,17 +640,9 @@ iterate over containers like `dict`::
     {% endfor %}
     </dl>
 
-Python dicts may not be in the order you want to display them in. If
-order matters, use the ``|dictsort`` filter.
-
-.. code-block:: jinja
-
-    <dl>
-    {% for key, value in my_dict | dictsort %}
-        <dt>{{ key|e }}</dt>
-        <dd>{{ value|e }}</dd>
-    {% endfor %}
-    </dl>
+Note, however, that **Python dicts are not ordered**; so you might want to
+either pass a sorted ``list`` of ``tuple`` s -- or a
+``collections.OrderedDict`` -- to the template, or use the `dictsort` filter.
 
 Inside of a for-loop block, you can access some special variables:
 
@@ -930,6 +853,9 @@ are available on a macro object:
 `arguments`
     A tuple of the names of arguments the macro accepts.
 
+`defaults`
+    A tuple of default values.
+
 `catch_kwargs`
     This is `true` if the macro accepts extra keyword arguments (i.e.: accesses
     the special `kwargs` variable).
@@ -945,23 +871,6 @@ are available on a macro object:
 If a macro name starts with an underscore, it's not exported and can't
 be imported.
 
-Due to how scopes work in Jinja, a macro in a child template does not
-override a macro in a parent template. The following will output
-"LAYOUT", not "CHILD".
-
-.. code-block:: jinja
-    :caption: ``layout.txt``
-
-    {% macro foo() %}LAYOUT{% endmacro %}
-    {% block body %}{% endblock %}
-
-.. code-block:: jinja
-    :caption: ``child.txt``
-
-    {% extends 'layout.txt' %}
-    {% macro foo() %}CHILD{% endmacro %}
-    {% block body %}{{ foo() }}{% endblock %}
-
 
 .. _call:
 
@@ -1021,9 +930,6 @@ template data.  Just wrap the code in the special `filter` section::
         This text becomes uppercase
     {% endfilter %}
 
-Filters that accept arguments can be called like this::
-
-    {% filter center(100) %}Center this{% endfilter %}
 
 .. _assignments:
 
@@ -1086,34 +992,34 @@ Assignments use the `set` tag and can have multiple targets::
 Block Assignments
 ~~~~~~~~~~~~~~~~~
 
-It's possible to use `set` as a block to assign the content of the block to a
-variable. This can be used to create multi-line strings, since Jinja doesn't
-support Python's triple quotes (``"""``, ``'''``).
+.. versionadded:: 2.8
 
-Instead of using an equals sign and a value, you only write the variable name,
-and everything until ``{% endset %}`` is captured.
+Starting with Jinja 2.8, it's possible to also use block assignments to
+capture the contents of a block into a variable name.  This can be useful
+in some situations as an alternative for macros.  In that case, instead of
+using an equals sign and a value, you just write the variable name and then
+everything until ``{% endset %}`` is captured.
 
-.. code-block:: jinja
+Example::
 
     {% set navigation %}
         <li><a href="/">Index</a>
         <li><a href="/downloads">Downloads</a>
     {% endset %}
 
-Filters applied to the variable name will be applied to the block's content.
+The `navigation` variable then contains the navigation HTML source.
 
-.. code-block:: jinja
+.. versionchanged:: 2.10
+
+Starting with Jinja 2.10, the block assignment supports filters.
+
+Example::
 
     {% set reply | wordwrap %}
         You wrote:
         {{ message }}
     {% endset %}
 
-.. versionadded:: 2.8
-
-.. versionchanged:: 2.10
-
-    Block assignment supports filters.
 
 .. _extends:
 
@@ -1140,45 +1046,42 @@ at the same time.  They are documented in detail in the
 Include
 ~~~~~~~
 
-The ``include`` tag renders another template and outputs the result into
-the current template.
-
-.. code-block:: jinja
+The `include` tag is useful to include a template and return the
+rendered contents of that file into the current namespace::
 
     {% include 'header.html' %}
-    Body goes here.
+        Body
     {% include 'footer.html' %}
 
-The included template has access to context of the current template by
-default. Use ``without context`` to use a separate context instead.
-``with context`` is also valid, but is the default behavior. See
-:ref:`import-visibility`.
+Included templates have access to the variables of the active context by
+default.  For more details about context behavior of imports and includes,
+see :ref:`import-visibility`.
 
-The included template can ``extend`` another template and override
-blocks in that template. However, the current template cannot override
-any blocks that the included template outputs.
+From Jinja 2.2 onwards, you can mark an include with ``ignore missing``; in
+which case Jinja will ignore the statement if the template to be included
+does not exist.  When combined with ``with`` or ``without context``, it must
+be placed *before* the context visibility statement.  Here are some valid
+examples::
 
-Use ``ignore missing`` to ignore the statement if the template does not
-exist. It must be placed *before* a context visibility statement.
-
-.. code-block:: jinja
-
-    {% include "sidebar.html" without context %}
     {% include "sidebar.html" ignore missing %}
     {% include "sidebar.html" ignore missing with context %}
     {% include "sidebar.html" ignore missing without context %}
 
-If a list of templates is given, each will be tried in order until one
-is not missing. This can be used with ``ignore missing`` to ignore if
-none of the templates exist.
+.. versionadded:: 2.2
 
-.. code-block:: jinja
+You can also provide a list of templates that are checked for existence
+before inclusion.  The first template that exists will be included.  If
+`ignore missing` is given, it will fall back to rendering nothing if
+none of the templates exist, otherwise it will raise an exception.
+
+Example::
 
     {% include ['page_detailed.html', 'page.html'] %}
     {% include ['special_sidebar.html', 'sidebar.html'] ignore missing %}
 
-A variable, with either a template name or template object, can also be
-passed to the statement.
+.. versionchanged:: 2.4
+   If a template object was passed to the template context, you can
+   include that object using `include`.
 
 .. _import:
 
@@ -1374,19 +1277,8 @@ but exists for completeness' sake.  The following operators are supported:
     ``{{ '=' * 80 }}`` would print a bar of 80 equal signs.
 
 ``**``
-    Raise the left operand to the power of the right operand.
-    ``{{ 2**3 }}`` would return ``8``.
-
-    Unlike Python, chained pow is evaluated left to right.
-    ``{{ 3**3**3 }}`` is evaluated as ``(3**3)**3`` in Jinja, but would
-    be evaluated as ``3**(3**3)`` in Python. Use parentheses in Jinja
-    to be explicit about what order you want. It is usually preferable
-    to do extended math in Python and pass the results to ``render``
-    rather than doing it in the template.
-
-    This behavior may be changed in the future to match Python, if it's
-    possible to introduce an upgrade path.
-
+    Raise the left operand to the power of the right operand.  ``{{ 2**3 }}``
+    would return ``8``.
 
 Comparisons
 ~~~~~~~~~~~
@@ -1412,31 +1304,27 @@ Comparisons
 Logic
 ~~~~~
 
-For ``if`` statements, ``for`` filtering, and ``if`` expressions, it can be
-useful to combine multiple expressions.
+For ``if`` statements, ``for`` filtering, and ``if`` expressions, it can be useful to
+combine multiple expressions:
 
 ``and``
-    For ``x and y``, if ``x`` is false, then the value is ``x``, else ``y``. In
-    a boolean context, this will be treated as ``True`` if both operands are
-    truthy.
+    Return true if the left and the right operand are true.
 
 ``or``
-    For ``x or y``, if ``x`` is true, then the value is ``x``, else ``y``. In a
-    boolean context, this will be treated as ``True`` if at least one operand is
-    truthy.
+    Return true if the left or the right operand are true.
 
 ``not``
-    For ``not x``, if ``x`` is false, then the value is ``True``, else
-    ``False``.
-
-    Prefer negating ``is`` and ``in`` using their infix notation:
-    ``foo is not bar`` instead of ``not foo is bar``; ``foo not in bar`` instead
-    of ``not foo in bar``. All other expressions require prefix notation:
-    ``not (foo and bar).``
+    negate a statement (see below).
 
 ``(expr)``
-    Parentheses group an expression. This is used to change evaluation order, or
-    to make a long expression easier to read or less ambiguous.
+    Parentheses group an expression.
+
+.. admonition:: Note
+
+    The ``is`` and ``in`` operators support negation using an infix notation,
+    too: ``foo is not bar`` and ``foo not in bar`` instead of ``not foo is bar``
+    and ``not foo in bar``.  All other expressions require a prefix notation:
+    ``not (foo and bar).``
 
 
 Other Operators
@@ -1453,10 +1341,10 @@ two categories:
 ``is``
     Performs a :ref:`test <tests>`.
 
-``|`` (pipe, vertical bar)
+``|``
     Applies a :ref:`filter <filters>`.
 
-``~`` (tilde)
+``~``
     Converts all operands into strings and concatenates them.
 
     ``{{ "Hello " ~ name ~ "!" }}`` would return (assuming `name` is set
@@ -1481,7 +1369,7 @@ It is also possible to use inline `if` expressions.  These are useful in some
 situations.  For example, you can use this to extend from one template if a
 variable is defined, otherwise from the default layout template::
 
-    {% extends layout_template if layout_template is defined else 'default.html' %}
+    {% extends layout_template if layout_template is defined else 'master.html' %}
 
 The general syntax is ``<do something> if <something is true> else <do
 something else>``.
@@ -1490,7 +1378,7 @@ The `else` part is optional.  If not provided, the else block implicitly
 evaluates into an :class:`Undefined` object (regardless of what ``undefined``
 in the environment is set to):
 
-.. code-block:: jinja
+.. sourcecode:: jinja
 
     {{ "[{}]".format(page.title) if page.title }}
 
@@ -1500,7 +1388,7 @@ in the environment is set to):
 Python Methods
 ~~~~~~~~~~~~~~
 
-You can also use any of the methods defined on a variable's type.
+You can also use any of the methods of defined on a variable's type.
 The value returned from the method invocation is used as the value of the expression.
 Here is an example that uses methods defined on strings (where ``page.title`` is a string):
 
@@ -1536,8 +1424,6 @@ is a bit contrived in the context of rendering a template):
 List of Builtin Filters
 -----------------------
 
-.. py:currentmodule:: jinja-filters
-
 .. jinja:filters:: jinja2.defaults.DEFAULT_FILTERS
 
 
@@ -1546,8 +1432,6 @@ List of Builtin Filters
 List of Builtin Tests
 ---------------------
 
-.. py:currentmodule:: jinja-tests
-
 .. jinja:tests:: jinja2.defaults.DEFAULT_TESTS
 
 
@@ -1558,8 +1442,6 @@ List of Global Functions
 
 The following functions are available in the global scope by default:
 
-.. py:currentmodule:: jinja-globals
-
 .. function:: range([start,] stop[, step])
 
     Return a list containing an arithmetic progression of integers.
@@ -1621,7 +1503,8 @@ The following functions are available in the global scope by default:
 
     .. versionadded:: 2.1
 
-    .. property:: current
+    .. method:: current
+        :property:
 
         Return the current item. Equivalent to the item that will be
         returned next time :meth:`next` is called.
@@ -1678,15 +1561,10 @@ The following functions are available in the global scope by default:
 
     .. versionadded:: 2.10
 
-    .. versionchanged:: 3.2
-        Namespace attributes can be assigned to in multiple assignment.
-
 
 Extensions
 ----------
 
-.. py:currentmodule:: jinja2
-
 The following sections cover the built-in Jinja extensions that may be
 enabled by an application.  An application could also provide further
 extensions not covered by this documentation; in which case there should
@@ -1768,35 +1646,11 @@ to disable it for a block.
 .. versionadded:: 2.10
    The ``trimmed`` and ``notrimmed`` modifiers have been added.
 
-If the translation depends on the context that the message appears in,
-the ``pgettext`` and ``npgettext`` functions take a ``context`` string
-as the first argument, which is used to select the appropriate
-translation. To specify a context with the ``{% trans %}`` tag, provide
-a string as the first token after ``trans``.
-
-.. code-block:: jinja
-
-    {% trans "fruit" %}apple{% endtrans %}
-    {% trans "fruit" trimmed count -%}
-        1 apple
-    {%- pluralize -%}
-        {{ count }} apples
-    {%- endtrans %}
-
-.. versionadded:: 3.1
-    A context can be passed to the ``trans`` tag to use ``pgettext`` and
-    ``npgettext``.
-
 It's possible to translate strings in expressions with these functions:
 
--   ``_(message)``: Alias for ``gettext``.
--   ``gettext(message)``: Translate a message.
--   ``ngettext(singular, plural, n)``: Translate a singular or plural
-    message based on a count variable.
--   ``pgettext(context, message)``: Like ``gettext()``, but picks the
-    translation based on the context string.
--   ``npgettext(context, singular, plural, n)``: Like ``npgettext()``,
-    but picks the translation based on the context string.
+-   ``gettext``: translate a single string
+-   ``ngettext``: translate a pluralizable string
+-   ``_``: alias for ``gettext``
 
 You can print a translated string like this:
 

docs/tricks.rst

@@ -7,10 +7,10 @@ This part of the documentation shows some tips and tricks for Jinja
 templates.
 
 
-.. _null-default-fallback:
+.. _null-master-fallback:
 
-Null-Default Fallback
----------------------
+Null-Master Fallback
+--------------------
 
 Jinja supports dynamic inheritance and does not distinguish between parent
 and child template as long as no `extends` tag is visited.  While this leads
@@ -21,12 +21,12 @@ for a neat trick.
 Usually child templates extend from one template that adds a basic HTML
 skeleton.  However it's possible to put the `extends` tag into an `if` tag to
 only extend from the layout template if the `standalone` variable evaluates
-to false, which it does by default if it's not defined.  Additionally a very
+to false which it does per default if it's not defined.  Additionally a very
 basic skeleton is added to the file so that if it's indeed rendered with
 `standalone` set to `True` a very basic HTML skeleton is added::
 
-    {% if not standalone %}{% extends 'default.html' %}{% endif -%}
-    <!DOCTYPE html>
+    {% if not standalone %}{% extends 'master.html' %}{% endif -%}
+    <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
     <title>{% block title %}The Page Title{% endblock %}</title>
     <link rel="stylesheet" href="style.css" type="text/css">
     {% block body %}
@@ -46,7 +46,7 @@ list you can use the `cycle` method on the `loop` object::
     {% endfor %}
     </ul>
 
-`cycle` can take an unlimited number of strings.  Each time this
+`cycle` can take an unlimited amount of strings.  Each time this
 tag is encountered the next item from the list is rendered.
 
 
@@ -74,8 +74,8 @@ sense to define a default for that variable::
     ...
     <ul id="navigation">
     {% for href, id, caption in navigation_bar %}
-      <li{% if id == active_page %} class="active"{% endif %}>
-      <a href="{{ href|e }}">{{ caption|e }}</a></li>
+      <li{% if id == active_page %} class="active"{% endif
+      %}><a href="{{ href|e }}">{{ caption|e }}</a></li>
     {% endfor %}
     </ul>
     ...

examples/basic/test.py

@@ -5,16 +5,16 @@ env = Environment(
     loader=DictLoader(
         {
             "child.html": """\
-{% extends default_layout or 'default.html' %}
-{% import 'helpers.html' as helpers %}
+{% extends master_layout or 'master.html' %}
+{% include helpers = 'helpers.html' %}
 {% macro get_the_answer() %}42{% endmacro %}
-{% set title = 'Hello World' %}
+{% title = 'Hello World' %}
 {% block body %}
     {{ get_the_answer() }}
     {{ helpers.conspirate() }}
 {% endblock %}
 """,
-            "default.html": """\
+            "master.html": """\
 <!doctype html>
 <title>{{ title }}</title>
 {% block body %}{% endblock %}

grammar.ebnf

@@ -0,0 +1,664 @@
+start
+    =
+    expressions
+    $
+    ;
+
+expressions
+    =
+    {expression}*
+    ;
+
+expression
+    =
+    | content
+    | raw_block_expression
+    | block_expression
+    | line_block_expression
+    | variable_expression
+    | comment_expression
+    | line_comment_expression
+    ;
+
+raw_block_expression
+    =
+    raw_block_start
+    raw:{ !raw_block_end CHAR }*
+    raw_block_end
+    ;
+
+raw_block_start
+    =
+    block_open "raw" {SP}* block_close
+    ;
+
+raw_block_end
+    =
+    block_open "endraw" {SP}* block_close
+    ;
+
+block_expression
+    =
+    | block_expression_pair
+    | block_expression_single
+    ;
+
+block_expression_pair
+    =
+    start:block_start contents:expressions end:block_end
+    ;
+
+block_expression_single
+    =
+    block:block_start
+    ;
+
+block_start
+    =
+    block_open !("end") name:IDENTIFIER [ "(" name_call_parameters:variable_accessor_call_parameters ")" ]
+    [ {SP}+ parameters:block_parameters ]
+    {SP}* block_close
+    ;
+
+block_end
+    =
+    block_open "end" name:IDENTIFIER {SP}* block_close
+    ;
+
+block_open
+    =
+    | ( {SP}* block_open_symbol "-" {SP}* )
+    | block_open_symbol {SP}*
+    ;
+
+block_open_symbol
+    =
+    "{%"
+    ;
+
+block_close
+    =
+    | ( "-" block_close_symbol {SP}* )
+    | block_close_symbol
+    ;
+
+block_close_symbol
+    =
+    "%}"
+    ;
+
+line_block_expression
+    =
+    | line_block_expression_pair
+    | line_block_expression_single
+    ;
+
+line_block_expression_pair
+    =
+    start:line_block_start contents:expressions end:line_block_end
+    ;
+
+line_block_expression_single
+    =
+    block:line_block_start
+    ;
+
+line_block_start
+    =
+    line_block_open !("end") name:IDENTIFIER { !"\n" SP}* parameters:[ line_block_parameters ] [ { !"\n" SP }* ":" ] line_block_close
+    ;
+
+line_block_end
+    =
+    line_block_open "end" name:IDENTIFIER line_block_close
+    ;
+
+line_block_open
+    =
+    { !"\n" SP }* line_block_open_symbol { !"\n" SP }*
+    ;
+
+line_block_open_symbol
+    =
+    "#"
+    ;
+
+line_block_close
+    =
+    | ( {SP}* $ )
+    | ( { !"\n" SP }* "\n" )
+    ;
+
+line_block_parameters
+    =
+    @+:block_parameter { { !"\n" SP }+ @+:block_parameter }*
+    ;
+
+block_parameters
+    =
+    @+:block_parameter
+    {
+        block_parameter_separator
+        @+:block_parameter
+     }*
+    ;
+
+block_parameter_separator
+    =
+    | ( {SP}* "," {SP}* )
+    | ( {SP}+ )
+    ;
+
+block_parameter
+    =
+    | block_parameter_key_value
+    | block_parameter_value_only
+    ;
+
+block_parameter_key_value
+    =
+    key:block_parameter_key {SP}* "=" {SP}* value:variable_accessor_call_parameter_value
+    ;
+
+block_parameter_key
+    =
+    variable_accessor_call_parameter_key
+    ;
+
+block_parameter_value_only
+    =
+    | value:variable_identifier_with_alias
+    | value:variable_accessor_call_parameter_value
+    | value:conditional_expression
+    ;
+
+variable_expression
+    =
+    variable_open type:`variable` name:variable_expression_name variable_close
+    ;
+
+variable_open
+    =
+    | ( {SP}* variable_open_symbol "-" {SP}* )
+    | ( variable_open_symbol {SP}* )
+    ;
+
+variable_open_symbol
+    =
+    "{{"
+    ;
+
+variable_close
+    =
+    | ( {SP}* "-" variable_close_symbol {SP}* )
+    | ( {SP}* variable_close_symbol )
+    ;
+
+variable_close_symbol
+    =
+    "}}"
+    ;
+
+variable_expression_name
+    =
+    | TUPLE_LITERAL
+    | conditional_expression
+    ;
+
+variable_identifier
+    =
+    | variable_identifier_parentheses
+    | variable_identifier_raw
+    ;
+
+variable_identifier_parentheses
+    =
+    "(" @:conditional_expression ")"
+    ;
+
+variable_identifier_raw
+    =
+    [ signed:( "-" | "+" ) ]
+    variable:( LITERAL | IDENTIFIER )
+    accessors:{ variable_accessor }*
+    { {SP}* filters+:variable_filter }*
+    ;
+
+variable_identifier_with_alias
+    =
+    variable:IDENTIFIER
+    {SP}* "as" {SP}*
+    alias:IDENTIFIER
+    ;
+
+variable_accessor
+    =
+    | variable_accessor_brackets
+    | variable_accessor_call
+    | variable_accessor_dot
+    ;
+
+variable_accessor_brackets
+    =
+    accessor_type:`brackets`
+    "[" parameter:variable_identifier "]"
+    ;
+
+variable_accessor_call
+    =
+    accessor_type:`call`
+    "(" parameters:[variable_accessor_call_parameters] ")"
+    ;
+
+variable_accessor_dot
+    =
+    accessor_type:`dot`
+    "." parameter:( IDENTIFIER | NUMBER_LITERAL )
+    ;
+
+variable_accessor_call_parameters
+    =
+    @+:variable_accessor_call_parameter
+    { {SP}* "," {SP}* @+:variable_accessor_call_parameter }*
+    ;
+
+variable_accessor_call_parameter
+    =
+    | variable_accessor_call_parameter_key_value
+    | variable_accessor_call_parameter_value_only
+    | variable_accessor_call_parameter_vararg
+    | variable_accessor_call_parameter_varkwarg
+    ;
+
+variable_accessor_call_parameter_vararg
+    =
+    "*" dynamic_argument:variable_identifier
+    ;
+
+variable_accessor_call_parameter_varkwarg
+    =
+    "**" dynamic_keyword_argument:variable_identifier
+    ;
+
+variable_accessor_call_parameter_key_value
+    =
+    key:variable_accessor_call_parameter_key {SP}* "=" {SP}* value:variable_accessor_call_parameter_value
+    ;
+
+variable_accessor_call_parameter_value_only
+    =
+    value:variable_accessor_call_parameter_value
+    ;
+
+variable_accessor_call_parameter_key
+    =
+    IDENTIFIER
+    ;
+
+variable_accessor_call_parameter_value
+    =
+    conditional_expression
+    ;
+
+conditional_expression
+    =
+    | conditional_expression_not
+    | conditional_expression_if
+    | conditional_expression_logical
+    | conditional_expression_operator
+    | conditional_expression_test
+    | complex_expression
+    | variable_identifier
+    | conditional_expression_parentheses
+    ;
+
+complex_expression
+    =
+    | complex_expression_powers
+    | complex_expression_math2
+    | concatenate_expression
+    | complex_expression_math1
+    | complex_expression_parentheses
+    | variable_identifier
+    ;
+
+complex_expression_powers
+    =
+    left:variable_identifier {SP}* math_operator:"**" {SP}* right:variable_identifier
+    ;
+
+complex_expression_math2
+    =
+    left:variable_identifier
+    {SP}* math_operator:complex_expression_math2_operations {SP}*
+    right:variable_identifier
+    ;
+
+complex_expression_math2_operations
+    =
+    | "*"
+    | "/"
+    | "//"
+    | "%"
+    ;
+
+complex_expression_math1
+    =
+    left:variable_identifier
+    {SP}* math_operator:complex_expression_math1_operations {SP}*
+    right:complex_expression
+    ;
+
+complex_expression_math1_operations
+    =
+    | "+"
+    | "-"
+    ;
+
+complex_expression_parentheses
+    =
+    "(" {SP}*
+    complex_expression
+    {SP}* ")"
+    ;
+
+conditional_expression_parentheses
+    =
+    "(" {SP}* @:conditional_expression {SP}* ")"
+    ;
+
+conditional_expression_not
+    =
+    "not" {SP}+ not:conditional_expression
+    ;
+
+conditional_expression_if
+    =
+    true_value:variable_identifier
+    {SP}* "if" {SP}*
+    test_expression:conditional_expression
+    [ {SP}* "else" {SP}* false_value:conditional_expression ]
+    ;
+
+conditional_expression_logical
+    =
+    left:conditional_expression
+    {SP}* logical_operator:variable_tests_logical_operator {SP}*
+    right:conditional_expression
+    ;
+
+conditional_expression_operator
+    =
+    conditional_expression_operator_in
+    | (
+        left:complex_expression
+        {SP}* operator:conditional_expression_operator_operations {SP}*
+        right:conditional_expression
+    )
+    ;
+
+conditional_expression_operator_in
+    =
+    | (
+        "not"
+        left:variable_identifier
+        {SP}* operator:`"notin"` "in" {SP}*
+        right:variable_identifier
+    )
+    | (
+        left:variable_identifier
+        {SP}+
+        (
+            | ( "not" {SP}* "in" operator:`"notin"` )
+            | operator:"in"
+        )
+        {SP}+
+        right:variable_identifier
+    )
+    ;
+
+conditional_expression_test
+    =
+    test_variable:variable_identifier
+    {SP}* "is"
+    [ {SP}+ "not" {SP} negated:`True` ]
+    {SP}*
+    test_function:variable_identifier
+    [
+        {SP}*
+        !( (variable_tests_logical_operator | "is" | "else" ) {SP}* )
+        test_function_parameter:variable_identifier
+    ]
+    ;
+
+conditional_expression_operator_operations
+    =
+    | "=="
+    | "!="
+    | ">"
+    | ">="
+    | "<"
+    | "<="
+    ;
+
+variable_tests_logical_operator
+    =
+    | "and"
+    | "or"
+    ;
+
+concatenate_expression
+    =
+    concatenate+:variable_identifier
+    { {SP}* "~" {SP}* concatenate+:variable_identifier }+
+    ;
+
+variable_filter
+    =
+    "|" {SP}* @:filter
+    ;
+filter =
+    variable:IDENTIFIER
+    accessors:{ variable_accessor_call }*
+    ;
+
+comment_expression
+    =
+    comment_open comment:comment_content comment_close
+    ;
+
+comment_open =
+    comment_open_symbol
+    ;
+
+comment_open_symbol
+    =
+    "{#"
+    ;
+
+comment_close
+    =
+    comment_close_symbol
+    ;
+
+comment_close_symbol
+    =
+    "#}"
+    ;
+
+comment_content
+    =
+    { !comment_close CHAR }*
+    ;
+
+line_comment_expression
+    =
+    line_comment_open comment:line_comment_content &"\n"
+    ;
+
+line_comment_open
+    =
+    {SP}* line_comment_open_symbol
+    ;
+
+line_comment_open_symbol
+    =
+    '##'
+    ;
+
+line_comment_content
+    =
+    { !"\n" CHAR }*
+    ;
+
+content
+    =
+    !(
+        | line_block_open
+        | block_open
+        | variable_open
+        | comment_open
+        | line_comment_open
+    ) CHAR ;
+
+LITERAL
+    =
+    | NONE_LITERAL
+    | STRING_LITERAL
+    | NUMBER_LITERAL
+    | BOOLEAN_LITERAL
+    | DICTIONARY_LITERAL
+    | LIST_LITERAL
+    | EXPLICIT_TUPLE_LITERAL
+    ;
+
+DICTIONARY_LITERAL
+    =
+    literal_type:`dictionary`
+    (
+        | ( "{" {SP}* value+:dictionary_key_value { {SP}* "," {SP}* value+:dictionary_key_value } {SP}* "}" )
+        | ( "{" {SP}* "}" )
+    )
+    ;
+
+dictionary_key_value
+    =
+    key:STRING_LITERAL {SP}* ":" {SP}* value:variable_identifier
+    ;
+
+LIST_LITERAL
+    =
+    literal_type:`list`
+    (
+        | ( "[" {SP}* value+:variable_identifier {SP}* { "," {SP}* value+:variable_identifier }* {SP}* "]" )
+        | ( "[" {SP}* "]" )
+    )
+    ;
+
+TUPLE_LITERAL
+    =
+    | EXPLICIT_TUPLE_LITERAL
+    | IMPLICIT_TUPLE_LITERAL
+    | EMPTY_TUPLE_LITERAL
+    ;
+
+EXPLICIT_TUPLE_LITERAL
+    =
+    literal_type:`tuple`
+    "(" value:TUPLE_LITERAL_CONTENTS ")"
+    ;
+
+IMPLICIT_TUPLE_LITERAL
+    =
+    literal_type:`tuple`
+    value:TUPLE_LITERAL_CONTENTS
+    ;
+
+TUPLE_LITERAL_CONTENTS
+    =
+    | ( @+:variable_identifier {SP}* { "," {SP}* @+:variable_identifier {SP}* }+ )
+    | ( @+:variable_identifier {SP}* "," {SP}* )
+    ;
+
+EMPTY_TUPLE_LITERAL
+    =
+    literal_type:`tuple`
+    "(" {SP}* ")"
+    ;
+
+INTEGER_LITERAL
+    =
+    /[\d_]*\d+/
+    ;
+
+SIGNED_INTEGER_LITERAL
+    =
+    /[+-]?[\d_]*\d+/
+    ;
+
+NUMBER_LITERAL
+    =
+    literal_type:`number`
+    whole:INTEGER_LITERAL
+    [ "." fractional:INTEGER_LITERAL ]
+    [ ( "e" | "E" ) exponent:SIGNED_INTEGER_LITERAL ]
+    ;
+
+STRING_LITERAL
+    =
+    | STRING_LITERAL_SINGLE_QUOTE
+    | STRING_LITERAL_DOUBLE_QUOTE
+    ;
+
+STRING_LITERAL_SINGLE_QUOTE
+    =
+    literal_type:`string`
+    "'" value:{ !"'" /./ }* "'"
+    ;
+
+STRING_LITERAL_DOUBLE_QUOTE
+    =
+    literal_type:`string`
+    '"' value:{ !'"' /./ }* '"'
+    ;
+
+BOOLEAN_LITERAL
+    =
+    literal_type:`boolean`
+    (
+        | ( ("true" | "True") value:`True`)
+        | ( ("false" | "False") value:`False`)
+    )
+    ;
+
+NONE_LITERAL
+    =
+    literal_type:`none`
+    ( "none" | "None" ) value:`None`
+    ;
+
+IDENTIFIER
+    =
+    /[a-zA-Z_][a-zA-Z0-9_]*/
+    ;
+
+ALPHA
+    =
+    /[a-zA-Z]/
+    ;
+
+DIGIT
+    =
+    /[0-9]/
+    ;
+
+SP
+    =
+    /\s/
+    ;
+
+CHAR
+    =
+    | ?'.'
+    | ?'\s'
+    ;

pyproject.toml

@@ -1,211 +0,0 @@
-[project]
-name = "Jinja2"
-version = "3.2.0.dev"
-description = "A very fast and expressive template engine."
-readme = "README.md"
-license = "BSD-3-Clause"
-license-files = ["LICENSE.txt"]
-maintainers = [{name = "Pallets", email = "contact@palletsprojects.com"}]
-classifiers = [
-    "Development Status :: 5 - Production/Stable",
-    "Environment :: Web Environment",
-    "Intended Audience :: Developers",
-    "Operating System :: OS Independent",
-    "Programming Language :: Python",
-    "Topic :: Internet :: WWW/HTTP :: Dynamic Content",
-    "Topic :: Text Processing :: Markup :: HTML",
-    "Typing :: Typed",
-]
-requires-python = ">=3.10"
-dependencies = ["MarkupSafe>=3.0"]
-
-[project.urls]
-Donate = "https://palletsprojects.com/donate"
-Documentation = "https://jinja.palletsprojects.com/"
-Changes = "https://jinja.palletsprojects.com/page/changes/"
-Source = "https://github.com/pallets/jinja/"
-Chat = "https://discord.gg/pallets"
-
-[project.optional-dependencies]
-i18n = ["Babel>=2.17"]
-
-[dependency-groups]
-dev = [
-    "ruff",
-    "tox",
-    "tox-uv",
-]
-docs = [
-    "pallets-sphinx-themes",
-    "sphinx",
-    "sphinxcontrib-log-cabinet",
-]
-docs-auto = [
-    "sphinx-autobuild",
-]
-gha-update = [
-    "gha-update ; python_full_version >= '3.12'",
-]
-pre-commit = [
-    "pre-commit",
-    "pre-commit-uv",
-]
-tests = [
-    "pytest",
-    "pytest-timeout",
-    "trio"
-]
-typing = [
-    "mypy",
-    "pyright",
-    "pytest",
-]
-
-[build-system]
-requires = ["flit_core<4"]
-build-backend = "flit_core.buildapi"
-
-[tool.flit.module]
-name = "jinja2"
-
-[tool.flit.sdist]
-include = [
-    "docs/",
-    "examples/",
-    "tests/",
-    "CHANGES.rst",
-    "uv.lock"
-]
-exclude = [
-    "docs/_build/",
-]
-
-[tool.uv]
-default-groups = ["dev", "pre-commit", "tests", "typing"]
-
-[tool.pytest.ini_options]
-testpaths = ["tests"]
-filterwarnings = [
-    "error",
-]
-
-[tool.coverage.run]
-branch = true
-source = ["jinja2", "tests"]
-
-[tool.coverage.paths]
-source = ["src", "*/site-packages"]
-
-[tool.coverage.report]
-exclude_also = [
-    "if t.TYPE_CHECKING",
-    "raise NotImplementedError",
-    ": \\.{3}",
-]
-
-[tool.mypy]
-python_version = "3.10"
-files = ["src"]
-show_error_codes = true
-pretty = true
-strict = true
-
-[tool.pyright]
-pythonVersion = "3.10"
-include = ["src"]
-typeCheckingMode = "standard"
-
-[tool.ruff]
-src = ["src"]
-fix = true
-show-fixes = true
-output-format = "full"
-
-[tool.ruff.lint]
-select = [
-    "B",  # flake8-bugbear
-    "E",  # pycodestyle error
-    "F",  # pyflakes
-    "I",  # isort
-    "UP",  # pyupgrade
-    "W",  # pycodestyle warning
-]
-ignore = [
-    "UP038",  # keep isinstance tuple
-]
-
-[tool.ruff.lint.isort]
-force-single-line = true
-order-by-type = false
-
-[tool.gha-update]
-tag-only = [
-    "slsa-framework/slsa-github-generator",
-]
-
-[tool.tox]
-env_list = [
-    "py3.13", "py3.12", "py3.11", "py3.10",
-    "pypy3.11",
-    "style",
-    "typing",
-    "docs",
-]
-
-[tool.tox.env_run_base]
-description = "pytest on latest dependency versions"
-runner = "uv-venv-lock-runner"
-package = "wheel"
-wheel_build_env = ".pkg"
-constrain_package_deps = true
-use_frozen_constraints = true
-dependency_groups = ["tests"]
-commands = [[
-    "pytest", "-v", "--tb=short", "--basetemp={env_tmp_dir}",
-    {replace = "posargs", default = [], extend = true},
-]]
-
-[tool.tox.env.style]
-description = "run all pre-commit hooks on all files"
-dependency_groups = ["pre-commit"]
-skip_install = true
-commands = [["pre-commit", "run", "--all-files"]]
-
-[tool.tox.env.typing]
-description = "run static type checkers"
-dependency_groups = ["typing"]
-commands = [
-    ["mypy"],
-]
-
-[tool.tox.env.docs]
-description = "build docs"
-dependency_groups = ["docs"]
-commands = [["sphinx-build", "-E", "-W", "-b", "dirhtml", "docs", "docs/_build/dirhtml"]]
-
-[tool.tox.env.docs-auto]
-description = "continuously rebuild docs and start a local server"
-dependency_groups = ["docs", "docs-auto"]
-commands = [["sphinx-autobuild", "-W", "-b", "dirhtml", "--watch", "src", "docs", "docs/_build/dirhtml"]]
-
-[tool.tox.env.update-actions]
-description = "update GitHub Actions pins"
-labels = ["update"]
-dependency_groups = ["gha-update"]
-skip_install = true
-commands = [["gha-update"]]
-
-[tool.tox.env.update-pre_commit]
-description = "update pre-commit pins"
-labels = ["update"]
-dependency_groups = ["pre-commit"]
-skip_install = true
-commands = [["pre-commit", "autoupdate", "--freeze", "-j4"]]
-
-[tool.tox.env.update-requirements]
-description = "update uv lock"
-labels = ["update"]
-dependency_groups = []
-no_default_groups = true
-skip_install = true
-commands = [["uv", "lock", {replace = "posargs", default = ["-U"], extend = true}]]

scripts/generate_identifier_pattern.py

@@ -29,9 +29,9 @@ def collapse_ranges(data):
 
     Source: https://stackoverflow.com/a/4629241/400617
     """
-    for _, g in itertools.groupby(enumerate(data), lambda x: ord(x[1]) - x[0]):
-        lb = list(g)
-        yield lb[0][1], lb[-1][1]
+    for _, b in itertools.groupby(enumerate(data), lambda x: ord(x[1]) - x[0]):
+        b = list(b)
+        yield b[0][1], b[-1][1]
 
 
 def build_pattern(ranges):
@@ -54,16 +54,17 @@ def build_pattern(ranges):
 
 
 def main():
-    """Build the regex pattern and write it to ``jinja2/_identifier.py``."""
+    """Build the regex pattern and write it to
+    ``jinja2/_identifier.py``.
+    """
     pattern = build_pattern(collapse_ranges(get_characters()))
     filename = os.path.abspath(
         os.path.join(os.path.dirname(__file__), "..", "src", "jinja2", "_identifier.py")
     )
 
     with open(filename, "w", encoding="utf8") as f:
-        f.write("# generated by scripts/generate_identifier_pattern.py")
-        f.write(f"# Python {sys.version_info[0]}.{sys.version_info[1]}\n")
         f.write("import re\n\n")
+        f.write("# generated by scripts/generate_identifier_pattern.py\n")
         f.write("pattern = re.compile(\n")
         f.write(f'    r"[\\w{pattern}]+"  # noqa: B950\n')
         f.write(")\n")

setup.cfg

@@ -0,0 +1,42 @@
+[metadata]
+license_file = LICENSE.rst
+long_description = file:README.rst
+long_description_content_type = text/x-rst
+
+[tool:pytest]
+testpaths = tests
+filterwarnings =
+    error
+
+[coverage:run]
+branch = True
+source =
+    jinja2
+    tests
+
+[coverage:paths]
+source =
+    src
+    */site-packages
+
+[flake8]
+# B = bugbear
+# E = pycodestyle errors
+# F = flake8 pyflakes
+# W = pycodestyle warnings
+# B9 = bugbear opinions
+select = B, E, F, W, B9
+ignore =
+    # slice notation whitespace, invalid
+    E203
+    # line length, handled by bugbear B950
+    E501
+    # bare except, handled by bugbear B001
+    E722
+    # bin op line break, invalid
+    W503
+# up to 88 allowed by bugbear B950
+max-line-length = 80
+per-file-ignores =
+    # __init__ module exports names
+    src/jinja2/__init__.py: F401

setup.py

@@ -0,0 +1,40 @@
+import re
+
+from setuptools import find_packages
+from setuptools import setup
+
+with open("src/jinja2/__init__.py", "rt", encoding="utf8") as f:
+    version = re.search(r'__version__ = "(.*?)"', f.read(), re.M).group(1)
+
+setup(
+    name="Jinja2",
+    version=version,
+    url="https://palletsprojects.com/p/jinja/",
+    project_urls={
+        "Documentation": "https://jinja.palletsprojects.com/",
+        "Code": "https://github.com/pallets/jinja",
+        "Issue tracker": "https://github.com/pallets/jinja/issues",
+    },
+    license="BSD-3-Clause",
+    maintainer="Pallets",
+    maintainer_email="contact@palletsprojects.com",
+    description="A very fast and expressive template engine.",
+    classifiers=[
+        "Development Status :: 5 - Production/Stable",
+        "Environment :: Web Environment",
+        "Intended Audience :: Developers",
+        "License :: OSI Approved :: BSD License",
+        "Operating System :: OS Independent",
+        "Programming Language :: Python",
+        "Topic :: Internet :: WWW/HTTP :: Dynamic Content",
+        "Topic :: Software Development :: Libraries :: Python Modules",
+        "Topic :: Text Processing :: Markup :: HTML",
+    ],
+    packages=find_packages("src"),
+    package_dir={"": "src"},
+    include_package_data=True,
+    python_requires=">=3.6",
+    install_requires=["MarkupSafe>=1.1", "TatSu"],
+    extras_require={"i18n": ["Babel>=2.1"]},
+    entry_points={"babel.extractors": ["jinja2 = jinja2.ext:babel_extract[i18n]"]},
+)

src/jinja2/__init__.py

@@ -2,56 +2,42 @@
 non-XML syntax that supports inline expressions and an optional
 sandboxed environment.
 """
+from markupsafe import escape
+from markupsafe import Markup
 
-from __future__ import annotations
+from .bccache import BytecodeCache
+from .bccache import FileSystemBytecodeCache
+from .bccache import MemcachedBytecodeCache
+from .environment import Environment
+from .environment import Template
+from .exceptions import TemplateAssertionError
+from .exceptions import TemplateError
+from .exceptions import TemplateNotFound
+from .exceptions import TemplateRuntimeError
+from .exceptions import TemplatesNotFound
+from .exceptions import TemplateSyntaxError
+from .exceptions import UndefinedError
+from .filters import contextfilter
+from .filters import environmentfilter
+from .filters import evalcontextfilter
+from .loaders import BaseLoader
+from .loaders import ChoiceLoader
+from .loaders import DictLoader
+from .loaders import FileSystemLoader
+from .loaders import FunctionLoader
+from .loaders import ModuleLoader
+from .loaders import PackageLoader
+from .loaders import PrefixLoader
+from .runtime import ChainableUndefined
+from .runtime import DebugUndefined
+from .runtime import make_logging_undefined
+from .runtime import StrictUndefined
+from .runtime import Undefined
+from .utils import clear_caches
+from .utils import contextfunction
+from .utils import environmentfunction
+from .utils import evalcontextfunction
+from .utils import is_undefined
+from .utils import select_autoescape
 
-import typing as t
-
-from .bccache import BytecodeCache as BytecodeCache
-from .bccache import FileSystemBytecodeCache as FileSystemBytecodeCache
-from .bccache import MemcachedBytecodeCache as MemcachedBytecodeCache
-from .environment import Environment as Environment
-from .environment import Template as Template
-from .exceptions import TemplateAssertionError as TemplateAssertionError
-from .exceptions import TemplateError as TemplateError
-from .exceptions import TemplateNotFound as TemplateNotFound
-from .exceptions import TemplateRuntimeError as TemplateRuntimeError
-from .exceptions import TemplatesNotFound as TemplatesNotFound
-from .exceptions import TemplateSyntaxError as TemplateSyntaxError
-from .exceptions import UndefinedError as UndefinedError
-from .loaders import BaseLoader as BaseLoader
-from .loaders import ChoiceLoader as ChoiceLoader
-from .loaders import DictLoader as DictLoader
-from .loaders import FileSystemLoader as FileSystemLoader
-from .loaders import FunctionLoader as FunctionLoader
-from .loaders import ModuleLoader as ModuleLoader
-from .loaders import PackageLoader as PackageLoader
-from .loaders import PrefixLoader as PrefixLoader
-from .runtime import ChainableUndefined as ChainableUndefined
-from .runtime import DebugUndefined as DebugUndefined
-from .runtime import make_logging_undefined as make_logging_undefined
-from .runtime import StrictUndefined as StrictUndefined
-from .runtime import Undefined as Undefined
-from .utils import clear_caches as clear_caches
-from .utils import is_undefined as is_undefined
-from .utils import pass_context as pass_context
-from .utils import pass_environment as pass_environment
-from .utils import pass_eval_context as pass_eval_context
-from .utils import select_autoescape as select_autoescape
-
-
-def __getattr__(name: str) -> t.Any:
-    if name == "__version__":
-        import importlib.metadata
-        import warnings
-
-        warnings.warn(
-            "The `__version__` attribute is deprecated and will be removed in"
-            " Jinja 3.3. Use feature detection or"
-            ' `importlib.metadata.version("jinja2")` instead.',
-            DeprecationWarning,
-            stacklevel=2,
-        )
-        return importlib.metadata.version("jinja2")
-
-    raise AttributeError(name)
+__version__ = "3.0.0a1"

src/jinja2/_identifier.py

@@ -1,6 +1,6 @@
-# generated by scripts/generate_identifier_pattern.py for Python 3.10
 import re
 
+# generated by scripts/generate_identifier_pattern.py
 pattern = re.compile(
-    r"[\w·̀-ͯ·҃-֑҇-ׇֽֿׁׂׅׄؐ-ًؚ-ٰٟۖ-ۜ۟-۪ۤۧۨ-ܑۭܰ-݊ަ-ް߫-߽߳ࠖ-࠙ࠛ-ࠣࠥ-ࠧࠩ-࡙࠭-࡛࣓-ࣣ࣡-ःऺ-़ा-ॏ॑-ॗॢॣঁ-ঃ়া-ৄেৈো-্ৗৢৣ৾ਁ-ਃ਼ਾ-ੂੇੈੋ-੍ੑੰੱੵઁ-ઃ઼ા-ૅે-ૉો-્ૢૣૺ-૿ଁ-ଃ଼ା-ୄେୈୋ-୍୕-ୗୢୣஂா-ூெ-ைொ-்ௗఀ-ఄా-ౄె-ైొ-్ౕౖౢౣಁ-ಃ಼ಾ-ೄೆ-ೈೊ-್ೕೖೢೣഀ-ഃ഻഼ാ-ൄെ-ൈൊ-്ൗൢൣඁ-ඃ්ා-ුූෘ-ෟෲෳัิ-ฺ็-๎ັິ-ຼ່-ໍ༹༘༙༵༷༾༿ཱ-྄྆྇ྍ-ྗྙ-ྼ࿆ါ-ှၖ-ၙၞ-ၠၢ-ၤၧ-ၭၱ-ၴႂ-ႍႏႚ-ႝ፝-፟ᜒ-᜔ᜲ-᜴ᝒᝓᝲᝳ឴-៓៝᠋-᠍ᢅᢆᢩᤠ-ᤫᤰ-᤻ᨗ-ᨛᩕ-ᩞ᩠-᩿᩼᪰-᪽ᪿᫀᬀ-ᬄ᬴-᭄᭫-᭳ᮀ-ᮂᮡ-ᮭ᯦-᯳ᰤ-᰷᳐-᳔᳒-᳨᳭᳴᳷-᳹᷀-᷹᷻-᷿‿⁀⁔⃐-⃥⃜⃡-⃰℘℮⳯-⵿⳱ⷠ-〪ⷿ-゙゚〯꙯ꙴ-꙽ꚞꚟ꛰꛱ꠂ꠆ꠋꠣ-ꠧ꠬ꢀꢁꢴ-ꣅ꣠-꣱ꣿꤦ-꤭ꥇ-꥓ꦀ-ꦃ꦳-꧀ꧥꨩ-ꨶꩃꩌꩍꩻ-ꩽꪰꪲ-ꪴꪷꪸꪾ꪿꫁ꫫ-ꫯꫵ꫶ꯣ-ꯪ꯬꯭ﬞ︀-️︠-︯︳︴﹍-﹏＿𐇽𐋠𐍶-𐍺𐨁-𐨃𐨅𐨆𐨌-𐨏𐨸-𐨿𐨺𐫦𐫥𐴤-𐽆𐴧𐺫𐺬-𐽐𑀀-𑀂𑀸-𑁆𑁿-𑂂𑂰-𑂺𑄀-𑄂𑄧-𑄴𑅅𑅆𑅳𑆀-𑆂𑆳-𑇀𑇉-𑇌𑇎𑇏𑈬-𑈷𑈾𑋟-𑋪𑌀-𑌃𑌻𑌼𑌾-𑍄𑍇𑍈𑍋-𑍍𑍗𑍢𑍣𑍦-𑍬𑍰-𑍴𑐵-𑑆𑑞𑒰-𑓃𑖯-𑖵𑖸-𑗀𑗜𑗝𑘰-𑙀𑚫-𑚷𑜝-𑜫𑠬-𑠺𑤰-𑤵𑤷𑤸𑤻-𑤾𑥀𑥂𑥃𑧑-𑧗𑧚-𑧠𑧤𑨁-𑨊𑨳-𑨹𑨻-𑨾𑩇𑩑-𑩛𑪊-𑪙𑰯-𑰶𑰸-𑰿𑲒-𑲧𑲩-𑲶𑴱-𑴶𑴺𑴼𑴽𑴿-𑵅𑵇𑶊-𑶎𑶐𑶑𑶓-𑶗𑻳-𑻶𖫰-𖫴𖬰-𖬶𖽏𖽑-𖾇𖾏-𖾒𖿤𖿰𖿱𛲝𛲞𝅥-𝅩𝅭-𝅲𝅻-𝆂𝆅-𝆋𝆪-𝆭𝉂-𝉄𝨀-𝨶𝨻-𝩬𝩵𝪄𝪛-𝪟𝪡-𝪯𞀀-𞀆𞀈-𞀘𞀛-𞀡𞀣𞀤𞀦-𞀪𞄰-𞄶𞋬-𞣐𞋯-𞣖𞥄-𞥊󠄀-󠇯]+"  # noqa: B950
+    r"[\w·̀-ͯ·҃-֑҇-ׇֽֿׁׂׅׄؐ-ًؚ-ٰٟۖ-ۜ۟-۪ۤۧۨ-ܑۭܰ-݊ަ-ް߫-߳ࠖ-࠙ࠛ-ࠣࠥ-ࠧࠩ-࡙࠭-࡛ࣔ-ࣣ࣡-ःऺ-़ा-ॏ॑-ॗॢॣঁ-ঃ়া-ৄেৈো-্ৗৢৣਁ-ਃ਼ਾ-ੂੇੈੋ-੍ੑੰੱੵઁ-ઃ઼ા-ૅે-ૉો-્ૢૣଁ-ଃ଼ା-ୄେୈୋ-୍ୖୗୢୣஂா-ூெ-ைொ-்ௗఀ-ఃా-ౄె-ైొ-్ౕౖౢౣಁ-ಃ಼ಾ-ೄೆ-ೈೊ-್ೕೖೢೣഁ-ഃാ-ൄെ-ൈൊ-്ൗൢൣංඃ්ා-ුූෘ-ෟෲෳัิ-ฺ็-๎ັິ-ູົຼ່-ໍ༹༘༙༵༷༾༿ཱ-྄྆྇ྍ-ྗྙ-ྼ࿆ါ-ှၖ-ၙၞ-ၠၢ-ၤၧ-ၭၱ-ၴႂ-ႍႏႚ-ႝ፝-፟ᜒ-᜔ᜲ-᜴ᝒᝓᝲᝳ឴-៓៝᠋-᠍ᢅᢆᢩᤠ-ᤫᤰ-᤻ᨗ-ᨛᩕ-ᩞ᩠-᩿᩼᪰-᪽ᬀ-ᬄ᬴-᭄᭫-᭳ᮀ-ᮂᮡ-ᮭ᯦-᯳ᰤ-᰷᳐-᳔᳒-᳨᳭ᳲ-᳴᳸᳹᷀-᷵᷻-᷿‿⁀⁔⃐-⃥⃜⃡-⃰℘℮⳯-⵿⳱ⷠ-〪ⷿ-゙゚〯꙯ꙴ-꙽ꚞꚟ꛰꛱ꠂ꠆ꠋꠣ-ꠧꢀꢁꢴ-ꣅ꣠-꣱ꤦ-꤭ꥇ-꥓ꦀ-ꦃ꦳-꧀ꧥꨩ-ꨶꩃꩌꩍꩻ-ꩽꪰꪲ-ꪴꪷꪸꪾ꪿꫁ꫫ-ꫯꫵ꫶ꯣ-ꯪ꯬꯭ﬞ︀-️︠-︯︳︴﹍-﹏＿𐇽𐋠𐍶-𐍺𐨁-𐨃𐨅𐨆𐨌-𐨏𐨸-𐨿𐨺𐫦𐫥𑀀-𑀂𑀸-𑁆𑁿-𑂂𑂰-𑂺𑄀-𑄂𑄧-𑅳𑄴𑆀-𑆂𑆳-𑇊𑇀-𑇌𑈬-𑈷𑈾𑋟-𑋪𑌀-𑌃𑌼𑌾-𑍄𑍇𑍈𑍋-𑍍𑍗𑍢𑍣𑍦-𑍬𑍰-𑍴𑐵-𑑆𑒰-𑓃𑖯-𑖵𑖸-𑗀𑗜𑗝𑘰-𑙀𑚫-𑚷𑜝-𑜫𑰯-𑰶𑰸-𑰿𑲒-𑲧𑲩-𑲶𖫰-𖫴𖬰-𖬶𖽑-𖽾𖾏-𖾒𛲝𛲞𝅥-𝅩𝅭-𝅲𝅻-𝆂𝆅-𝆋𝆪-𝆭𝉂-𝉄𝨀-𝨶𝨻-𝩬𝩵𝪄𝪛-𝪟𝪡-𝪯𞀀-𞀆𞀈-𞀘𞀛-𞀡𞀣𞀤𞀦-𞣐𞀪-𞣖𞥄-𞥊󠄀-󠇯]+"  # noqa: B950
 )

src/jinja2/async_utils.py

@@ -1,99 +0,0 @@
-import inspect
-import typing as t
-from functools import WRAPPER_ASSIGNMENTS
-from functools import wraps
-
-from .utils import _PassArg
-from .utils import pass_eval_context
-
-if t.TYPE_CHECKING:
-    import typing_extensions as te
-
-V = t.TypeVar("V")
-
-
-def async_variant(normal_func):  # type: ignore
-    def decorator(async_func):  # type: ignore
-        pass_arg = _PassArg.from_obj(normal_func)
-        need_eval_context = pass_arg is None
-
-        if pass_arg is _PassArg.environment:
-
-            def is_async(args: t.Any) -> bool:
-                return t.cast(bool, args[0].is_async)
-
-        else:
-
-            def is_async(args: t.Any) -> bool:
-                return t.cast(bool, args[0].environment.is_async)
-
-        # Take the doc and annotations from the sync function, but the
-        # name from the async function. Pallets-Sphinx-Themes
-        # build_function_directive expects __wrapped__ to point to the
-        # sync function.
-        async_func_attrs = ("__module__", "__name__", "__qualname__")
-        normal_func_attrs = tuple(set(WRAPPER_ASSIGNMENTS).difference(async_func_attrs))
-
-        @wraps(normal_func, assigned=normal_func_attrs)
-        @wraps(async_func, assigned=async_func_attrs, updated=())
-        def wrapper(*args, **kwargs):  # type: ignore
-            b = is_async(args)
-
-            if need_eval_context:
-                args = args[1:]
-
-            if b:
-                return async_func(*args, **kwargs)
-
-            return normal_func(*args, **kwargs)
-
-        if need_eval_context:
-            wrapper = pass_eval_context(wrapper)
-
-        wrapper.jinja_async_variant = True  # type: ignore[attr-defined]
-        return wrapper
-
-    return decorator
-
-
-_common_primitives = {int, float, bool, str, list, dict, tuple, type(None)}
-
-
-async def auto_await(value: t.Union[t.Awaitable["V"], "V"]) -> "V":
-    # Avoid a costly call to isawaitable
-    if type(value) in _common_primitives:
-        return t.cast("V", value)
-
-    if inspect.isawaitable(value):
-        return await t.cast("t.Awaitable[V]", value)
-
-    return value
-
-
-class _IteratorToAsyncIterator(t.Generic[V]):
-    def __init__(self, iterator: "t.Iterator[V]"):
-        self._iterator = iterator
-
-    def __aiter__(self) -> "te.Self":
-        return self
-
-    async def __anext__(self) -> V:
-        try:
-            return next(self._iterator)
-        except StopIteration as e:
-            raise StopAsyncIteration(e.value) from e
-
-
-def auto_aiter(
-    iterable: "t.AsyncIterable[V] | t.Iterable[V]",
-) -> "t.AsyncIterator[V]":
-    if hasattr(iterable, "__aiter__"):
-        return iterable.__aiter__()
-    else:
-        return _IteratorToAsyncIterator(iter(iterable))
-
-
-async def auto_to_list(
-    value: "t.AsyncIterable[V] | t.Iterable[V]",
-) -> list["V"]:
-    return [x async for x in auto_aiter(value)]

src/jinja2/asyncfilters.py

@@ -0,0 +1,157 @@
+from functools import wraps
+
+from . import filters
+from .asyncsupport import auto_aiter
+from .asyncsupport import auto_await
+
+
+async def auto_to_seq(value):
+    seq = []
+    if hasattr(value, "__aiter__"):
+        async for item in value:
+            seq.append(item)
+    else:
+        for item in value:
+            seq.append(item)
+    return seq
+
+
+async def async_select_or_reject(args, kwargs, modfunc, lookup_attr):
+    seq, func = filters.prepare_select_or_reject(args, kwargs, modfunc, lookup_attr)
+    if seq:
+        async for item in auto_aiter(seq):
+            if func(item):
+                yield item
+
+
+def dualfilter(normal_filter, async_filter):
+    wrap_evalctx = False
+    if getattr(normal_filter, "environmentfilter", False) is True:
+
+        def is_async(args):
+            return args[0].is_async
+
+        wrap_evalctx = False
+    else:
+        has_evalctxfilter = getattr(normal_filter, "evalcontextfilter", False) is True
+        has_ctxfilter = getattr(normal_filter, "contextfilter", False) is True
+        wrap_evalctx = not has_evalctxfilter and not has_ctxfilter
+
+        def is_async(args):
+            return args[0].environment.is_async
+
+    @wraps(normal_filter)
+    def wrapper(*args, **kwargs):
+        b = is_async(args)
+        if wrap_evalctx:
+            args = args[1:]
+        if b:
+            return async_filter(*args, **kwargs)
+        return normal_filter(*args, **kwargs)
+
+    if wrap_evalctx:
+        wrapper.evalcontextfilter = True
+
+    wrapper.asyncfiltervariant = True
+
+    return wrapper
+
+
+def asyncfiltervariant(original):
+    def decorator(f):
+        return dualfilter(original, f)
+
+    return decorator
+
+
+@asyncfiltervariant(filters.do_first)
+async def do_first(environment, seq):
+    try:
+        return await auto_aiter(seq).__anext__()
+    except StopAsyncIteration:
+        return environment.undefined("No first item, sequence was empty.")
+
+
+@asyncfiltervariant(filters.do_groupby)
+async def do_groupby(environment, value, attribute):
+    expr = filters.make_attrgetter(environment, attribute)
+    return [
+        filters._GroupTuple(key, await auto_to_seq(values))
+        for key, values in filters.groupby(
+            sorted(await auto_to_seq(value), key=expr), expr
+        )
+    ]
+
+
+@asyncfiltervariant(filters.do_join)
+async def do_join(eval_ctx, value, d="", attribute=None):
+    return filters.do_join(eval_ctx, await auto_to_seq(value), d, attribute)
+
+
+@asyncfiltervariant(filters.do_list)
+async def do_list(value):
+    return await auto_to_seq(value)
+
+
+@asyncfiltervariant(filters.do_reject)
+async def do_reject(*args, **kwargs):
+    return async_select_or_reject(args, kwargs, lambda x: not x, False)
+
+
+@asyncfiltervariant(filters.do_rejectattr)
+async def do_rejectattr(*args, **kwargs):
+    return async_select_or_reject(args, kwargs, lambda x: not x, True)
+
+
+@asyncfiltervariant(filters.do_select)
+async def do_select(*args, **kwargs):
+    return async_select_or_reject(args, kwargs, lambda x: x, False)
+
+
+@asyncfiltervariant(filters.do_selectattr)
+async def do_selectattr(*args, **kwargs):
+    return async_select_or_reject(args, kwargs, lambda x: x, True)
+
+
+@asyncfiltervariant(filters.do_map)
+async def do_map(*args, **kwargs):
+    seq, func = filters.prepare_map(args, kwargs)
+    if seq:
+        async for item in auto_aiter(seq):
+            yield await auto_await(func(item))
+
+
+@asyncfiltervariant(filters.do_sum)
+async def do_sum(environment, iterable, attribute=None, start=0):
+    rv = start
+    if attribute is not None:
+        func = filters.make_attrgetter(environment, attribute)
+    else:
+
+        def func(x):
+            return x
+
+    async for item in auto_aiter(iterable):
+        rv += func(item)
+    return rv
+
+
+@asyncfiltervariant(filters.do_slice)
+async def do_slice(value, slices, fill_with=None):
+    return filters.do_slice(await auto_to_seq(value), slices, fill_with)
+
+
+ASYNC_FILTERS = {
+    "first": do_first,
+    "groupby": do_groupby,
+    "join": do_join,
+    "list": do_list,
+    # we intentionally do not support do_last because it may not be safe in async
+    "reject": do_reject,
+    "rejectattr": do_rejectattr,
+    "map": do_map,
+    "select": do_select,
+    "selectattr": do_selectattr,
+    "sum": do_sum,
+    "slice": do_slice,
+}

src/jinja2/asyncsupport.py

@@ -0,0 +1,249 @@
+"""The code for async support. Importing this patches Jinja."""
+import asyncio
+import inspect
+from functools import update_wrapper
+
+from markupsafe import Markup
+
+from .environment import TemplateModule
+from .runtime import LoopContext
+from .utils import concat
+from .utils import internalcode
+from .utils import missing
+
+
+async def concat_async(async_gen):
+    rv = []
+
+    async def collect():
+        async for event in async_gen:
+            rv.append(event)
+
+    await collect()
+    return concat(rv)
+
+
+async def generate_async(self, *args, **kwargs):
+    vars = dict(*args, **kwargs)
+    try:
+        async for event in self.root_render_func(self.new_context(vars)):
+            yield event
+    except Exception:
+        yield self.environment.handle_exception()
+
+
+def wrap_generate_func(original_generate):
+    def _convert_generator(self, loop, args, kwargs):
+        async_gen = self.generate_async(*args, **kwargs)
+        try:
+            while 1:
+                yield loop.run_until_complete(async_gen.__anext__())
+        except StopAsyncIteration:
+            pass
+
+    def generate(self, *args, **kwargs):
+        if not self.environment.is_async:
+            return original_generate(self, *args, **kwargs)
+        return _convert_generator(self, asyncio.get_event_loop(), args, kwargs)
+
+    return update_wrapper(generate, original_generate)
+
+
+async def render_async(self, *args, **kwargs):
+    if not self.environment.is_async:
+        raise RuntimeError("The environment was not created with async mode enabled.")
+
+    vars = dict(*args, **kwargs)
+    ctx = self.new_context(vars)
+
+    try:
+        return await concat_async(self.root_render_func(ctx))
+    except Exception:
+        return self.environment.handle_exception()
+
+
+def wrap_render_func(original_render):
+    def render(self, *args, **kwargs):
+        if not self.environment.is_async:
+            return original_render(self, *args, **kwargs)
+        loop = asyncio.get_event_loop()
+        return loop.run_until_complete(self.render_async(*args, **kwargs))
+
+    return update_wrapper(render, original_render)
+
+
+def wrap_block_reference_call(original_call):
+    @internalcode
+    async def async_call(self):
+        rv = await concat_async(self._stack[self._depth](self._context))
+        if self._context.eval_ctx.autoescape:
+            rv = Markup(rv)
+        return rv
+
+    @internalcode
+    def __call__(self):
+        if not self._context.environment.is_async:
+            return original_call(self)
+        return async_call(self)
+
+    return update_wrapper(__call__, original_call)
+
+
+def wrap_macro_invoke(original_invoke):
+    @internalcode
+    async def async_invoke(self, arguments, autoescape):
+        rv = await self._func(*arguments)
+        if autoescape:
+            rv = Markup(rv)
+        return rv
+
+    @internalcode
+    def _invoke(self, arguments, autoescape):
+        if not self._environment.is_async:
+            return original_invoke(self, arguments, autoescape)
+        return async_invoke(self, arguments, autoescape)
+
+    return update_wrapper(_invoke, original_invoke)
+
+
+@internalcode
+async def get_default_module_async(self):
+    if self._module is not None:
+        return self._module
+    self._module = rv = await self.make_module_async()
+    return rv
+
+
+def wrap_default_module(original_default_module):
+    @internalcode
+    def _get_default_module(self):
+        if self.environment.is_async:
+            raise RuntimeError("Template module attribute is unavailable in async mode")
+        return original_default_module(self)
+
+    return _get_default_module
+
+
+async def make_module_async(self, vars=None, shared=False, locals=None):
+    context = self.new_context(vars, shared, locals)
+    body_stream = []
+    async for item in self.root_render_func(context):
+        body_stream.append(item)
+    return TemplateModule(self, context, body_stream)
+
+
+def patch_template():
+    from . import Template
+
+    Template.generate = wrap_generate_func(Template.generate)
+    Template.generate_async = update_wrapper(generate_async, Template.generate_async)
+    Template.render_async = update_wrapper(render_async, Template.render_async)
+    Template.render = wrap_render_func(Template.render)
+    Template._get_default_module = wrap_default_module(Template._get_default_module)
+    Template._get_default_module_async = get_default_module_async
+    Template.make_module_async = update_wrapper(
+        make_module_async, Template.make_module_async
+    )
+
+
+def patch_runtime():
+    from .runtime import BlockReference, Macro
+
+    BlockReference.__call__ = wrap_block_reference_call(BlockReference.__call__)
+    Macro._invoke = wrap_macro_invoke(Macro._invoke)
+
+
+def patch_filters():
+    from .filters import FILTERS
+    from .asyncfilters import ASYNC_FILTERS
+
+    FILTERS.update(ASYNC_FILTERS)
+
+
+def patch_all():
+    patch_template()
+    patch_runtime()
+    patch_filters()
+
+
+async def auto_await(value):
+    if inspect.isawaitable(value):
+        return await value
+    return value
+
+
+async def auto_aiter(iterable):
+    if hasattr(iterable, "__aiter__"):
+        async for item in iterable:
+            yield item
+        return
+    for item in iterable:
+        yield item
+
+
+class AsyncLoopContext(LoopContext):
+    _to_iterator = staticmethod(auto_aiter)
+
+    @property
+    async def length(self):
+        if self._length is not None:
+            return self._length
+
+        try:
+            self._length = len(self._iterable)
+        except TypeError:
+            iterable = [x async for x in self._iterator]
+            self._iterator = self._to_iterator(iterable)
+            self._length = len(iterable) + self.index + (self._after is not missing)
+
+        return self._length
+
+    @property
+    async def revindex0(self):
+        return await self.length - self.index
+
+    @property
+    async def revindex(self):
+        return await self.length - self.index0
+
+    async def _peek_next(self):
+        if self._after is not missing:
+            return self._after
+
+        try:
+            self._after = await self._iterator.__anext__()
+        except StopAsyncIteration:
+            self._after = missing
+
+        return self._after
+
+    @property
+    async def last(self):
+        return await self._peek_next() is missing
+
+    @property
+    async def nextitem(self):
+        rv = await self._peek_next()
+
+        if rv is missing:
+            return self._undefined("there is no next item")
+
+        return rv
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self):
+        if self._after is not missing:
+            rv = self._after
+            self._after = missing
+        else:
+            rv = await self._iterator.__anext__()
+
+        self.index0 += 1
+        self._before = self._current
+        self._current = rv
+        return rv, self
+
+
+patch_all()

src/jinja2/bccache.py

@@ -5,7 +5,6 @@ slows down your application too much.
 Situations where this is useful are often forking web applications that
 are initialized on the first request.
 """
-
 import errno
 import fnmatch
 import marshal
@@ -14,21 +13,10 @@ import pickle
 import stat
 import sys
 import tempfile
-import typing as t
 from hashlib import sha1
 from io import BytesIO
-from types import CodeType
-
-if t.TYPE_CHECKING:
-    import typing_extensions as te
-
-    from .environment import Environment
-
-    class _MemcachedClient(te.Protocol):
-        def get(self, key: str) -> bytes: ...
-
-        def set(self, key: str, value: bytes, timeout: int | None = None) -> None: ...
 
+from .utils import open_if_exists
 
 bc_version = 5
 # Magic bytes to identify Jinja bytecode cache files. Contains the
@@ -50,17 +38,17 @@ class Bucket:
     cache subclasses don't have to care about cache invalidation.
     """
 
-    def __init__(self, environment: "Environment", key: str, checksum: str) -> None:
+    def __init__(self, environment, key, checksum):
         self.environment = environment
         self.key = key
         self.checksum = checksum
         self.reset()
 
-    def reset(self) -> None:
+    def reset(self):
         """Resets the bucket (unloads the bytecode)."""
-        self.code: CodeType | None = None
+        self.code = None
 
-    def load_bytecode(self, f: t.BinaryIO) -> None:
+    def load_bytecode(self, f):
         """Loads bytecode from a file or file like object."""
         # make sure the magic header is correct
         magic = f.read(len(bc_magic))
@@ -79,7 +67,7 @@ class Bucket:
             self.reset()
             return
 
-    def write_bytecode(self, f: t.IO[bytes]) -> None:
+    def write_bytecode(self, f):
         """Dump the bytecode into the file or file like object passed."""
         if self.code is None:
             raise TypeError("can't write empty bucket")
@@ -87,12 +75,12 @@ class Bucket:
         pickle.dump(self.checksum, f, 2)
         marshal.dump(self.code, f)
 
-    def bytecode_from_string(self, string: bytes) -> None:
-        """Load bytecode from bytes."""
+    def bytecode_from_string(self, string):
+        """Load bytecode from a string."""
         self.load_bytecode(BytesIO(string))
 
-    def bytecode_to_string(self) -> bytes:
-        """Return the bytecode as bytes."""
+    def bytecode_to_string(self):
+        """Return the bytecode as string."""
         out = BytesIO()
         self.write_bytecode(out)
         return out.getvalue()
@@ -127,46 +115,41 @@ class BytecodeCache:
     Jinja.
     """
 
-    def load_bytecode(self, bucket: Bucket) -> None:
+    def load_bytecode(self, bucket):
         """Subclasses have to override this method to load bytecode into a
         bucket.  If they are not able to find code in the cache for the
         bucket, it must not do anything.
         """
         raise NotImplementedError()
 
-    def dump_bytecode(self, bucket: Bucket) -> None:
+    def dump_bytecode(self, bucket):
         """Subclasses have to override this method to write the bytecode
         from a bucket back to the cache.  If it unable to do so it must not
         fail silently but raise an exception.
         """
         raise NotImplementedError()
 
-    def clear(self) -> None:
+    def clear(self):
         """Clears the cache.  This method is not used by Jinja but should be
         implemented to allow applications to clear the bytecode cache used
         by a particular environment.
         """
 
-    def get_cache_key(self, name: str, filename: str | None = None) -> str:
+    def get_cache_key(self, name, filename=None):
         """Returns the unique hash key for this template name."""
         hash = sha1(name.encode("utf-8"))
-
         if filename is not None:
-            hash.update(f"|{filename}".encode())
-
+            filename = "|" + filename
+            if isinstance(filename, str):
+                filename = filename.encode("utf-8")
+            hash.update(filename)
         return hash.hexdigest()
 
-    def get_source_checksum(self, source: str) -> str:
+    def get_source_checksum(self, source):
         """Returns a checksum for the source."""
         return sha1(source.encode("utf-8")).hexdigest()
 
-    def get_bucket(
-        self,
-        environment: "Environment",
-        name: str,
-        filename: str | None,
-        source: str,
-    ) -> Bucket:
+    def get_bucket(self, environment, name, filename, source):
         """Return a cache bucket for the given template.  All arguments are
         mandatory but filename may be `None`.
         """
@@ -176,7 +159,7 @@ class BytecodeCache:
         self.load_bytecode(bucket)
         return bucket
 
-    def set_bucket(self, bucket: Bucket) -> None:
+    def set_bucket(self, bucket):
         """Put the bucket into the cache."""
         self.dump_bytecode(bucket)
 
@@ -199,16 +182,14 @@ class FileSystemBytecodeCache(BytecodeCache):
     This bytecode cache supports clearing of the cache using the clear method.
     """
 
-    def __init__(
-        self, directory: str | None = None, pattern: str = "__jinja2_%s.cache"
-    ) -> None:
+    def __init__(self, directory=None, pattern="__jinja2_%s.cache"):
         if directory is None:
             directory = self._get_default_cache_dir()
         self.directory = directory
         self.pattern = pattern
 
-    def _get_default_cache_dir(self) -> str:
-        def _unsafe_dir() -> "te.NoReturn":
+    def _get_default_cache_dir(self):
+        def _unsafe_dir():
             raise RuntimeError(
                 "Cannot determine safe temp directory.  You "
                 "need to explicitly provide one."
@@ -254,63 +235,25 @@ class FileSystemBytecodeCache(BytecodeCache):
 
         return actual_dir
 
-    def _get_cache_filename(self, bucket: Bucket) -> str:
+    def _get_cache_filename(self, bucket):
         return os.path.join(self.directory, self.pattern % (bucket.key,))
 
-    def load_bytecode(self, bucket: Bucket) -> None:
-        filename = self._get_cache_filename(bucket)
-
-        # Don't test for existence before opening the file, since the
-        # file could disappear after the test before the open.
-        try:
-            f = open(filename, "rb")
-        except (FileNotFoundError, IsADirectoryError, PermissionError):
-            # PermissionError can occur on Windows when an operation is
-            # in progress, such as calling clear().
-            return
-
-        with f:
-            bucket.load_bytecode(f)
-
-    def dump_bytecode(self, bucket: Bucket) -> None:
-        # Write to a temporary file, then rename to the real name after
-        # writing. This avoids another process reading the file before
-        # it is fully written.
-        name = self._get_cache_filename(bucket)
-        f = tempfile.NamedTemporaryFile(
-            mode="wb",
-            dir=os.path.dirname(name),
-            prefix=os.path.basename(name),
-            suffix=".tmp",
-            delete=False,
-        )
-
-        def remove_silent() -> None:
+    def load_bytecode(self, bucket):
+        f = open_if_exists(self._get_cache_filename(bucket), "rb")
+        if f is not None:
             try:
-                os.remove(f.name)
-            except OSError:
-                # Another process may have called clear(). On Windows,
-                # another program may be holding the file open.
-                pass
+                bucket.load_bytecode(f)
+            finally:
+                f.close()
 
+    def dump_bytecode(self, bucket):
+        f = open(self._get_cache_filename(bucket), "wb")
         try:
-            with f:
-                bucket.write_bytecode(f)
-        except BaseException:
-            remove_silent()
-            raise
+            bucket.write_bytecode(f)
+        finally:
+            f.close()
 
-        try:
-            os.replace(f.name, name)
-        except OSError:
-            # Another process may have called clear(). On Windows,
-            # another program may be holding the file open.
-            remove_silent()
-        except BaseException:
-            remove_silent()
-            raise
-
-    def clear(self) -> None:
+    def clear(self):
         # imported lazily here because google app-engine doesn't support
         # write access on the file system and the function does not exist
         # normally.
@@ -371,34 +314,32 @@ class MemcachedBytecodeCache(BytecodeCache):
 
     def __init__(
         self,
-        client: "_MemcachedClient",
-        prefix: str = "jinja2/bytecode/",
-        timeout: int | None = None,
-        ignore_memcache_errors: bool = True,
+        client,
+        prefix="jinja2/bytecode/",
+        timeout=None,
+        ignore_memcache_errors=True,
     ):
         self.client = client
         self.prefix = prefix
         self.timeout = timeout
         self.ignore_memcache_errors = ignore_memcache_errors
 
-    def load_bytecode(self, bucket: Bucket) -> None:
+    def load_bytecode(self, bucket):
         try:
             code = self.client.get(self.prefix + bucket.key)
         except Exception:
             if not self.ignore_memcache_errors:
                 raise
-        else:
+            code = None
+        if code is not None:
             bucket.bytecode_from_string(code)
 
-    def dump_bytecode(self, bucket: Bucket) -> None:
-        key = self.prefix + bucket.key
-        value = bucket.bytecode_to_string()
-
+    def dump_bytecode(self, bucket):
+        args = (self.prefix + bucket.key, bucket.bytecode_to_string())
+        if self.timeout is not None:
+            args += (self.timeout,)
         try:
-            if self.timeout is not None:
-                self.client.set(key, value, self.timeout)
-            else:
-                self.client.set(key, value)
+            self.client.set(*args)
         except Exception:
             if not self.ignore_memcache_errors:
                 raise

src/jinja2/debug.py

@@ -1,17 +1,13 @@
+import platform
 import sys
-import typing as t
 from types import CodeType
-from types import TracebackType
 
-from .exceptions import TemplateSyntaxError
+from . import TemplateSyntaxError
 from .utils import internal_code
 from .utils import missing
 
-if t.TYPE_CHECKING:
-    from .runtime import Context
 
-
-def rewrite_traceback_stack(source: str | None = None) -> BaseException:
+def rewrite_traceback_stack(source=None):
     """Rewrite the current exception to replace any tracebacks from
     within compiled template code with tracebacks that look like they
     came from the template source.
@@ -23,8 +19,6 @@ def rewrite_traceback_stack(source: str | None = None) -> BaseException:
     :return: The original exception with the rewritten traceback.
     """
     _, exc_value, tb = sys.exc_info()
-    exc_value = t.cast(BaseException, exc_value)
-    tb = t.cast(TracebackType, tb)
 
     if isinstance(exc_value, TemplateSyntaxError) and not exc_value.translated:
         exc_value.translated = True
@@ -67,15 +61,12 @@ def rewrite_traceback_stack(source: str | None = None) -> BaseException:
 
     # Assign tb_next in reverse to avoid circular references.
     for tb in reversed(stack):
-        tb.tb_next = tb_next
-        tb_next = tb
+        tb_next = tb_set_next(tb, tb_next)
 
     return exc_value.with_traceback(tb_next)
 
 
-def fake_traceback(  # type: ignore
-    exc_value: BaseException, tb: TracebackType | None, filename: str, lineno: int
-) -> TracebackType:
+def fake_traceback(exc_value, tb, filename, lineno):
     """Produce a new traceback object that looks like it came from the
     template source instead of the compiled code. The filename, line
     number, and location name will point to the template, and the local
@@ -102,41 +93,79 @@ def fake_traceback(  # type: ignore
         "__jinja_exception__": exc_value,
     }
     # Raise an exception at the correct line number.
-    code: CodeType = compile(
-        "\n" * (lineno - 1) + "raise __jinja_exception__", filename, "exec"
-    )
+    code = compile("\n" * (lineno - 1) + "raise __jinja_exception__", filename, "exec")
 
     # Build a new code object that points to the template file and
     # replaces the location with a block name.
-    location = "template"
+    try:
+        location = "template"
 
-    if tb is not None:
-        function = tb.tb_frame.f_code.co_name
+        if tb is not None:
+            function = tb.tb_frame.f_code.co_name
 
-        if function == "root":
-            location = "top-level template code"
-        elif function.startswith("block_"):
-            location = f"block {function[6:]!r}"
+            if function == "root":
+                location = "top-level template code"
+            elif function.startswith("block_"):
+                location = f"block {function[6:]!r}"
 
-    code = code.replace(co_name=location)
+        # Collect arguments for the new code object. CodeType only
+        # accepts positional arguments, and arguments were inserted in
+        # new Python versions.
+        code_args = []
+
+        for attr in (
+            "argcount",
+            "posonlyargcount",  # Python 3.8
+            "kwonlyargcount",
+            "nlocals",
+            "stacksize",
+            "flags",
+            "code",  # codestring
+            "consts",  # constants
+            "names",
+            "varnames",
+            ("filename", filename),
+            ("name", location),
+            "firstlineno",
+            "lnotab",
+            "freevars",
+            "cellvars",
+        ):
+            if isinstance(attr, tuple):
+                # Replace with given value.
+                code_args.append(attr[1])
+                continue
+
+            try:
+                # Copy original value if it exists.
+                code_args.append(getattr(code, "co_" + attr))
+            except AttributeError:
+                # Some arguments were added later.
+                continue
+
+        code = CodeType(*code_args)
+    except Exception:
+        # Some environments such as Google App Engine don't support
+        # modifying code objects.
+        pass
 
     # Execute the new code, which is guaranteed to raise, and return
     # the new traceback without this frame.
     try:
         exec(code, globals, locals)
     except BaseException:
-        return sys.exc_info()[2].tb_next  # type: ignore
+        return sys.exc_info()[2].tb_next
 
 
-def get_template_locals(real_locals: t.Mapping[str, t.Any]) -> dict[str, t.Any]:
+def get_template_locals(real_locals):
     """Based on the runtime locals, get the context that would be
     available at that point in the template.
     """
     # Start with the current template context.
-    ctx: Context | None = real_locals.get("context")
+    ctx = real_locals.get("context")
 
-    if ctx is not None:
-        data: dict[str, t.Any] = ctx.get_all().copy()
+    if ctx:
+        data = ctx.get_all().copy()
     else:
         data = {}
 
@@ -144,7 +173,7 @@ def get_template_locals(real_locals: t.Mapping[str, t.Any]) -> dict[str, t.Any]:
     # rather than pushing a context. Local variables follow the scheme
     # l_depth_name. Find the highest-depth local that has a value for
     # each name.
-    local_overrides: dict[str, tuple[int, t.Any]] = {}
+    local_overrides = {}
 
     for name, value in real_locals.items():
         if not name.startswith("l_") or value is missing:
@@ -152,8 +181,8 @@ def get_template_locals(real_locals: t.Mapping[str, t.Any]) -> dict[str, t.Any]:
             continue
 
         try:
-            _, depth_str, name = name.split("_", 2)
-            depth = int(depth_str)
+            _, depth, name = name.split("_", 2)
+            depth = int(depth)
         except ValueError:
             continue
 
@@ -170,3 +199,63 @@ def get_template_locals(real_locals: t.Mapping[str, t.Any]) -> dict[str, t.Any]:
             data[name] = value
 
     return data
+
+
+if sys.version_info >= (3, 7):
+    # tb_next is directly assignable as of Python 3.7
+    def tb_set_next(tb, tb_next):
+        tb.tb_next = tb_next
+        return tb
+
+
+elif platform.python_implementation() == "PyPy":
+    # PyPy might have special support, and won't work with ctypes.
+    try:
+        import tputil
+    except ImportError:
+        # Without tproxy support, use the original traceback.
+        def tb_set_next(tb, tb_next):
+            return tb
+
+    else:
+        # With tproxy support, create a proxy around the traceback that
+        # returns the new tb_next.
+        def tb_set_next(tb, tb_next):
+            def controller(op):
+                if op.opname == "__getattribute__" and op.args[0] == "tb_next":
+                    return tb_next
+
+                return op.delegate()
+
+            return tputil.make_proxy(controller, obj=tb)
+
+
+else:
+    # Use ctypes to assign tb_next at the C level since it's read-only
+    # from Python.
+    import ctypes
+
+    class _CTraceback(ctypes.Structure):
+        _fields_ = [
+            # Extra PyObject slots when compiled with Py_TRACE_REFS.
+            ("PyObject_HEAD", ctypes.c_byte * object().__sizeof__()),
+            # Only care about tb_next as an object, not a traceback.
+            ("tb_next", ctypes.py_object),
+        ]
+
+    def tb_set_next(tb, tb_next):
+        c_tb = _CTraceback.from_address(id(tb))
+
+        # Clear out the old tb_next.
+        if tb.tb_next is not None:
+            c_tb_next = ctypes.py_object(tb.tb_next)
+            c_tb.tb_next = ctypes.py_object()
+            ctypes.pythonapi.Py_DecRef(c_tb_next)
+
+        # Assign the new tb_next.
+        if tb_next is not None:
+            c_tb_next = ctypes.py_object(tb_next)
+            ctypes.pythonapi.Py_IncRef(c_tb_next)
+            c_tb.tb_next = c_tb_next
+
+        return tb

src/jinja2/defaults.py

@@ -1,5 +1,3 @@
-import typing as t
-
 from .filters import FILTERS as DEFAULT_FILTERS  # noqa: F401
 from .tests import TESTS as DEFAULT_TESTS  # noqa: F401
 from .utils import Cycler
@@ -7,9 +5,6 @@ from .utils import generate_lorem_ipsum
 from .utils import Joiner
 from .utils import Namespace
 
-if t.TYPE_CHECKING:
-    import typing_extensions as te
-
 # defaults for the parser / lexer
 BLOCK_START_STRING = "{%"
 BLOCK_END_STRING = "%}"
@@ -17,11 +12,11 @@ VARIABLE_START_STRING = "{{"
 VARIABLE_END_STRING = "}}"
 COMMENT_START_STRING = "{#"
 COMMENT_END_STRING = "#}"
-LINE_STATEMENT_PREFIX: str | None = None
-LINE_COMMENT_PREFIX: str | None = None
+LINE_STATEMENT_PREFIX = None
+LINE_COMMENT_PREFIX = None
 TRIM_BLOCKS = False
 LSTRIP_BLOCKS = False
-NEWLINE_SEQUENCE: "te.Literal['\\n', '\\r\\n', '\\r']" = "\n"
+NEWLINE_SEQUENCE = "\n"
 KEEP_TRAILING_NEWLINE = False
 
 # default filters, tests and namespace
@@ -36,11 +31,10 @@ DEFAULT_NAMESPACE = {
 }
 
 # default policies
-DEFAULT_POLICIES: dict[str, t.Any] = {
+DEFAULT_POLICIES = {
     "compiler.ascii_str": True,
     "urlize.rel": "noopener",
     "urlize.target": None,
-    "urlize.extra_schemes": None,
     "truncate.leeway": 5,
     "json.dumps_function": None,
     "json.dumps_kwargs": {"sort_keys": True},

src/jinja2/exceptions.py

@@ -1,18 +1,13 @@
-import typing as t
-
-if t.TYPE_CHECKING:
-    from .runtime import Undefined
-
-
 class TemplateError(Exception):
     """Baseclass for all template errors."""
 
-    def __init__(self, message: str | None = None) -> None:
+    def __init__(self, message=None):
         super().__init__(message)
 
     @property
-    def message(self) -> str | None:
-        return self.args[0] if self.args else None
+    def message(self):
+        if self.args:
+            return self.args[0]
 
 
 class TemplateNotFound(IOError, LookupError, TemplateError):
@@ -25,13 +20,9 @@ class TemplateNotFound(IOError, LookupError, TemplateError):
 
     # Silence the Python warning about message being deprecated since
     # it's not valid here.
-    message: str | None = None
+    message = None
 
-    def __init__(
-        self,
-        name: t.Union[str, "Undefined"] | None,
-        message: str | None = None,
-    ) -> None:
+    def __init__(self, name, message=None):
         IOError.__init__(self, name)
 
         if message is None:
@@ -46,8 +37,8 @@ class TemplateNotFound(IOError, LookupError, TemplateError):
         self.name = name
         self.templates = [name]
 
-    def __str__(self) -> str:
-        return str(self.message)
+    def __str__(self):
+        return self.message
 
 
 class TemplatesNotFound(TemplateNotFound):
@@ -62,11 +53,7 @@ class TemplatesNotFound(TemplateNotFound):
     .. versionadded:: 2.2
     """
 
-    def __init__(
-        self,
-        names: t.Sequence[t.Union[str, "Undefined"]] = (),
-        message: str | None = None,
-    ) -> None:
+    def __init__(self, names=(), message=None):
         if message is None:
             from .runtime import Undefined
 
@@ -78,57 +65,51 @@ class TemplatesNotFound(TemplateNotFound):
                 else:
                     parts.append(name)
 
-            parts_str = ", ".join(map(str, parts))
-            message = f"none of the templates given were found: {parts_str}"
-
-        super().__init__(names[-1] if names else None, message)
+            message = "none of the templates given were found: " + ", ".join(
+                map(str, parts)
+            )
+        TemplateNotFound.__init__(self, names[-1] if names else None, message)
         self.templates = list(names)
 
 
 class TemplateSyntaxError(TemplateError):
     """Raised to tell the user that there is a problem with the template."""
 
-    def __init__(
-        self,
-        message: str,
-        lineno: int,
-        name: str | None = None,
-        filename: str | None = None,
-    ) -> None:
-        super().__init__(message)
+    def __init__(self, message, lineno, name=None, filename=None):
+        TemplateError.__init__(self, message)
         self.lineno = lineno
         self.name = name
         self.filename = filename
-        self.source: str | None = None
+        self.source = None
 
         # this is set to True if the debug.translate_syntax_error
         # function translated the syntax error into a new traceback
         self.translated = False
 
-    def __str__(self) -> str:
+    def __str__(self):
         # for translated errors we only return the message
         if self.translated:
-            return t.cast(str, self.message)
+            return self.message
 
         # otherwise attach some stuff
         location = f"line {self.lineno}"
         name = self.filename or self.name
         if name:
             location = f'File "{name}", {location}'
-        lines = [t.cast(str, self.message), "  " + location]
+        lines = [self.message, "  " + location]
 
         # if the source is set, add the line to the output
         if self.source is not None:
             try:
                 line = self.source.splitlines()[self.lineno - 1]
             except IndexError:
-                pass
-            else:
+                line = None
+            if line:
                 lines.append("    " + line.strip())
 
         return "\n".join(lines)
 
-    def __reduce__(self):  # type: ignore
+    def __reduce__(self):
         # https://bugs.python.org/issue1692335 Exceptions that take
         # multiple required arguments have problems with pickling.
         # Without this, raises TypeError: __init__() missing 1 required

src/jinja2/ext.py

@@ -1,58 +1,47 @@
 """Extension API for adding custom tags and behavior."""
-
 import pprint
 import re
-import typing as t
+from sys import version_info
 
 from markupsafe import Markup
 
-from . import defaults
 from . import nodes
+from .defaults import BLOCK_END_STRING
+from .defaults import BLOCK_START_STRING
+from .defaults import COMMENT_END_STRING
+from .defaults import COMMENT_START_STRING
+from .defaults import KEEP_TRAILING_NEWLINE
+from .defaults import LINE_COMMENT_PREFIX
+from .defaults import LINE_STATEMENT_PREFIX
+from .defaults import LSTRIP_BLOCKS
+from .defaults import NEWLINE_SEQUENCE
+from .defaults import TRIM_BLOCKS
+from .defaults import VARIABLE_END_STRING
+from .defaults import VARIABLE_START_STRING
 from .environment import Environment
 from .exceptions import TemplateAssertionError
 from .exceptions import TemplateSyntaxError
-from .runtime import concat  # type: ignore
-from .runtime import Context
-from .runtime import Undefined
+from .nodes import ContextReference
+from .runtime import concat
+from .utils import contextfunction
 from .utils import import_string
-from .utils import pass_context
-
-if t.TYPE_CHECKING:
-    import typing_extensions as te
-
-    from .lexer import Token
-    from .lexer import TokenStream
-    from .parser import Parser
-
-    class _TranslationsBasic(te.Protocol):
-        def gettext(self, message: str) -> str: ...
-
-        def ngettext(self, singular: str, plural: str, n: int) -> str:
-            pass
-
-    class _TranslationsContext(_TranslationsBasic):
-        def pgettext(self, context: str, message: str) -> str: ...
-
-        def npgettext(
-            self, context: str, singular: str, plural: str, n: int
-        ) -> str: ...
-
-    _SupportedTranslations = _TranslationsBasic | _TranslationsContext
-
 
 # I18N functions available in Jinja templates. If the I18N library
 # provides ugettext, it will be assigned to gettext.
-GETTEXT_FUNCTIONS: tuple[str, ...] = (
-    "_",
-    "gettext",
-    "ngettext",
-    "pgettext",
-    "npgettext",
-)
+GETTEXT_FUNCTIONS = ("_", "gettext", "ngettext")
 _ws_re = re.compile(r"\s*\n\s*")
 
 
-class Extension:
+class ExtensionRegistry(type):
+    """Gives the extension an unique identifier."""
+
+    def __new__(mcs, name, bases, d):
+        rv = type.__new__(mcs, name, bases, d)
+        rv.identifier = f"{rv.__module__}.{rv.__name__}"
+        return rv
+
+
+class Extension(metaclass=ExtensionRegistry):
     """Extensions can be used to add extra functionality to the Jinja template
     system at the parser level.  Custom extensions are bound to an environment
     but may not store environment specific data on `self`.  The reason for
@@ -71,13 +60,8 @@ class Extension:
     name as includes the name of the extension (fragment cache).
     """
 
-    identifier: t.ClassVar[str]
-
-    def __init_subclass__(cls) -> None:
-        cls.identifier = f"{cls.__module__}.{cls.__name__}"
-
     #: if this extension parses this is the list of tags it's listening to.
-    tags: set[str] = set()
+    tags = set()
 
     #: the priority of that extension.  This is especially useful for
     #: extensions that preprocess values.  A lower value means higher
@@ -86,28 +70,24 @@ class Extension:
     #: .. versionadded:: 2.4
     priority = 100
 
-    def __init__(self, environment: Environment) -> None:
+    def __init__(self, environment):
         self.environment = environment
 
-    def bind(self, environment: Environment) -> "te.Self":
+    def bind(self, environment):
         """Create a copy of this extension bound to another environment."""
         rv = object.__new__(self.__class__)
         rv.__dict__.update(self.__dict__)
         rv.environment = environment
         return rv
 
-    def preprocess(
-        self, source: str, name: str | None, filename: str | None = None
-    ) -> str:
+    def preprocess(self, source, name, filename=None):
         """This method is called before the actual lexing and can be used to
         preprocess the source.  The `filename` is optional.  The return value
         must be the preprocessed source.
         """
         return source
 
-    def filter_stream(
-        self, stream: "TokenStream"
-    ) -> t.Union["TokenStream", t.Iterable["Token"]]:
+    def filter_stream(self, stream):
         """It's passed a :class:`~jinja2.lexer.TokenStream` that can be used
         to filter tokens returned.  This method has to return an iterable of
         :class:`~jinja2.lexer.Token`\\s, but it doesn't have to return a
@@ -115,7 +95,7 @@ class Extension:
         """
         return stream
 
-    def parse(self, parser: "Parser") -> nodes.Node | list[nodes.Node]:
+    def parse(self, parser):
         """If any of the :attr:`tags` matched this method is called with the
         parser as first argument.  The token the parser stream is pointing at
         is the name token that matched.  This method has to return one or a
@@ -123,7 +103,7 @@ class Extension:
         """
         raise NotImplementedError()
 
-    def attr(self, name: str, lineno: int | None = None) -> nodes.ExtensionAttribute:
+    def attr(self, name, lineno=None):
         """Return an attribute node for the current extension.  This is useful
         to pass constants on extensions to generated template code.
 
@@ -134,14 +114,8 @@ class Extension:
         return nodes.ExtensionAttribute(self.identifier, name, lineno=lineno)
 
     def call_method(
-        self,
-        name: str,
-        args: list[nodes.Expr] | None = None,
-        kwargs: list[nodes.Keyword] | None = None,
-        dyn_args: nodes.Expr | None = None,
-        dyn_kwargs: nodes.Expr | None = None,
-        lineno: int | None = None,
-    ) -> nodes.Call:
+        self, name, args=None, kwargs=None, dyn_args=None, dyn_kwargs=None, lineno=None
+    ):
         """Call a method of the extension.  This is a shortcut for
         :meth:`attr` + :class:`jinja2.nodes.Call`.
         """
@@ -159,88 +133,38 @@ class Extension:
         )
 
 
-@pass_context
-def _gettext_alias(
-    __context: Context, *args: t.Any, **kwargs: t.Any
-) -> t.Any | Undefined:
+@contextfunction
+def _gettext_alias(__context, *args, **kwargs):
     return __context.call(__context.resolve("gettext"), *args, **kwargs)
 
 
-def _make_new_gettext(func: t.Callable[[str], str]) -> t.Callable[..., str]:
-    @pass_context
-    def gettext(__context: Context, __string: str, **variables: t.Any) -> str:
+def _make_new_gettext(func):
+    @contextfunction
+    def gettext(__context, __string, **variables):
         rv = __context.call(func, __string)
         if __context.eval_ctx.autoescape:
             rv = Markup(rv)
         # Always treat as a format string, even if there are no
         # variables. This makes translation strings more consistent
         # and predictable. This requires escaping
-        return rv % variables  # type: ignore
+        return rv % variables
 
     return gettext
 
 
-def _make_new_ngettext(func: t.Callable[[str, str, int], str]) -> t.Callable[..., str]:
-    @pass_context
-    def ngettext(
-        __context: Context,
-        __singular: str,
-        __plural: str,
-        __num: int,
-        **variables: t.Any,
-    ) -> str:
+def _make_new_ngettext(func):
+    @contextfunction
+    def ngettext(__context, __singular, __plural, __num, **variables):
         variables.setdefault("num", __num)
         rv = __context.call(func, __singular, __plural, __num)
         if __context.eval_ctx.autoescape:
             rv = Markup(rv)
         # Always treat as a format string, see gettext comment above.
-        return rv % variables  # type: ignore
+        return rv % variables
 
     return ngettext
 
 
-def _make_new_pgettext(func: t.Callable[[str, str], str]) -> t.Callable[..., str]:
-    @pass_context
-    def pgettext(
-        __context: Context, __string_ctx: str, __string: str, **variables: t.Any
-    ) -> str:
-        variables.setdefault("context", __string_ctx)
-        rv = __context.call(func, __string_ctx, __string)
-
-        if __context.eval_ctx.autoescape:
-            rv = Markup(rv)
-
-        # Always treat as a format string, see gettext comment above.
-        return rv % variables  # type: ignore
-
-    return pgettext
-
-
-def _make_new_npgettext(
-    func: t.Callable[[str, str, str, int], str],
-) -> t.Callable[..., str]:
-    @pass_context
-    def npgettext(
-        __context: Context,
-        __string_ctx: str,
-        __singular: str,
-        __plural: str,
-        __num: int,
-        **variables: t.Any,
-    ) -> str:
-        variables.setdefault("context", __string_ctx)
-        variables.setdefault("num", __num)
-        rv = __context.call(func, __string_ctx, __singular, __plural, __num)
-
-        if __context.eval_ctx.autoescape:
-            rv = Markup(rv)
-
-        # Always treat as a format string, see gettext comment above.
-        return rv % variables  # type: ignore
-
-    return npgettext
-
-
 class InternationalizationExtension(Extension):
     """This extension adds gettext support to Jinja."""
 
@@ -253,8 +177,8 @@ class InternationalizationExtension(Extension):
     # something is called twice here.  One time for the gettext value and
     # the other time for the n-parameter of the ngettext function.
 
-    def __init__(self, environment: Environment) -> None:
-        super().__init__(environment)
+    def __init__(self, environment):
+        Extension.__init__(self, environment)
         environment.globals["_"] = _gettext_alias
         environment.extend(
             install_gettext_translations=self._install,
@@ -265,9 +189,7 @@ class InternationalizationExtension(Extension):
             newstyle_gettext=False,
         )
 
-    def _install(
-        self, translations: "_SupportedTranslations", newstyle: bool | None = None
-    ) -> None:
+    def _install(self, translations, newstyle=None):
         # ugettext and ungettext are preferred in case the I18N library
         # is providing compatibility with older Python versions.
         gettext = getattr(translations, "ugettext", None)
@@ -276,79 +198,41 @@ class InternationalizationExtension(Extension):
         ngettext = getattr(translations, "ungettext", None)
         if ngettext is None:
             ngettext = translations.ngettext
+        self._install_callables(gettext, ngettext, newstyle)
 
-        pgettext = getattr(translations, "pgettext", None)
-        npgettext = getattr(translations, "npgettext", None)
+    def _install_null(self, newstyle=None):
         self._install_callables(
-            gettext, ngettext, newstyle=newstyle, pgettext=pgettext, npgettext=npgettext
+            lambda x: x, lambda s, p, n: s if n == 1 else p, newstyle
         )
 
-    def _install_null(self, newstyle: bool | None = None) -> None:
-        import gettext
-
-        translations = gettext.NullTranslations()
-        self._install_callables(
-            gettext=translations.gettext,
-            ngettext=translations.ngettext,
-            newstyle=newstyle,
-            pgettext=translations.pgettext,
-            npgettext=translations.npgettext,
-        )
-
-    def _install_callables(
-        self,
-        gettext: t.Callable[[str], str],
-        ngettext: t.Callable[[str, str, int], str],
-        newstyle: bool | None = None,
-        pgettext: t.Callable[[str, str], str] | None = None,
-        npgettext: t.Callable[[str, str, str, int], str] | None = None,
-    ) -> None:
+    def _install_callables(self, gettext, ngettext, newstyle=None):
         if newstyle is not None:
-            self.environment.newstyle_gettext = newstyle  # type: ignore
-        if self.environment.newstyle_gettext:  # type: ignore
+            self.environment.newstyle_gettext = newstyle
+        if self.environment.newstyle_gettext:
             gettext = _make_new_gettext(gettext)
             ngettext = _make_new_ngettext(ngettext)
+        self.environment.globals.update(gettext=gettext, ngettext=ngettext)
 
-            if pgettext is not None:
-                pgettext = _make_new_pgettext(pgettext)
-
-            if npgettext is not None:
-                npgettext = _make_new_npgettext(npgettext)
-
-        self.environment.globals.update(
-            gettext=gettext, ngettext=ngettext, pgettext=pgettext, npgettext=npgettext
-        )
-
-    def _uninstall(self, translations: "_SupportedTranslations") -> None:
-        for key in ("gettext", "ngettext", "pgettext", "npgettext"):
+    def _uninstall(self, translations):
+        for key in "gettext", "ngettext":
             self.environment.globals.pop(key, None)
 
-    def _extract(
-        self,
-        source: str | nodes.Template,
-        gettext_functions: t.Sequence[str] = GETTEXT_FUNCTIONS,
-    ) -> t.Iterator[tuple[int, str, str | None | tuple[str | None, ...]]]:
+    def _extract(self, source, gettext_functions=GETTEXT_FUNCTIONS):
         if isinstance(source, str):
             source = self.environment.parse(source)
         return extract_from_ast(source, gettext_functions)
 
-    def parse(self, parser: "Parser") -> nodes.Node | list[nodes.Node]:
+    def parse(self, parser):
         """Parse a translatable tag."""
         lineno = next(parser.stream).lineno
-
-        context = None
-        context_token = parser.stream.next_if("string")
-
-        if context_token is not None:
-            context = context_token.value
+        num_called_num = False
 
         # find all the variables referenced.  Additionally a variable can be
         # defined in the body of the trans block too, but this is checked at
         # a later state.
-        plural_expr: nodes.Expr | None = None
-        plural_expr_assignment: nodes.Assign | None = None
-        num_called_num = False
-        variables: dict[str, nodes.Expr] = {}
+        plural_expr = None
+        plural_expr_assignment = None
+        variables = {}
         trimmed = None
         while parser.stream.current.type != "block_end":
             if variables:
@@ -358,34 +242,34 @@ class InternationalizationExtension(Extension):
             if parser.stream.skip_if("colon"):
                 break
 
-            token = parser.stream.expect("name")
-            if token.value in variables:
+            name = parser.stream.expect("name")
+            if name.value in variables:
                 parser.fail(
-                    f"translatable variable {token.value!r} defined twice.",
-                    token.lineno,
+                    f"translatable variable {name.value!r} defined twice.",
+                    name.lineno,
                     exc=TemplateAssertionError,
                 )
 
             # expressions
             if parser.stream.current.type == "assign":
                 next(parser.stream)
-                variables[token.value] = var = parser.parse_expression()
-            elif trimmed is None and token.value in ("trimmed", "notrimmed"):
-                trimmed = token.value == "trimmed"
+                variables[name.value] = var = parser.parse_expression()
+            elif trimmed is None and name.value in ("trimmed", "notrimmed"):
+                trimmed = name.value == "trimmed"
                 continue
             else:
-                variables[token.value] = var = nodes.Name(token.value, "load")
+                variables[name.value] = var = nodes.Name(name.value, "load")
 
             if plural_expr is None:
                 if isinstance(var, nodes.Call):
                     plural_expr = nodes.Name("_trans", "load")
-                    variables[token.value] = plural_expr
+                    variables[name.value] = plural_expr
                     plural_expr_assignment = nodes.Assign(
                         nodes.Name("_trans", "store"), var
                     )
                 else:
                     plural_expr = var
-                num_called_num = token.value == "num"
+                num_called_num = name.value == "num"
 
         parser.stream.expect("block_end")
 
@@ -406,15 +290,15 @@ class InternationalizationExtension(Extension):
             have_plural = True
             next(parser.stream)
             if parser.stream.current.type != "block_end":
-                token = parser.stream.expect("name")
-                if token.value not in variables:
+                name = parser.stream.expect("name")
+                if name.value not in variables:
                     parser.fail(
-                        f"unknown variable {token.value!r} for pluralization",
-                        token.lineno,
+                        f"unknown variable {name.value!r} for pluralization",
+                        name.lineno,
                         exc=TemplateAssertionError,
                     )
-                plural_expr = variables[token.value]
-                num_called_num = token.value == "num"
+                plural_expr = variables[name.value]
+                num_called_num = name.value == "num"
             parser.stream.expect("block_end")
             plural_names, plural = self._parse_block(parser, False)
             next(parser.stream)
@@ -423,9 +307,9 @@ class InternationalizationExtension(Extension):
             next(parser.stream)
 
         # register free names as simple name expressions
-        for name in referenced:
-            if name not in variables:
-                variables[name] = nodes.Name(name, "load")
+        for var in referenced:
+            if var not in variables:
+                variables[var] = nodes.Name(var, "load")
 
         if not have_plural:
             plural_expr = None
@@ -442,7 +326,6 @@ class InternationalizationExtension(Extension):
         node = self._make_node(
             singular,
             plural,
-            context,
             variables,
             plural_expr,
             bool(referenced),
@@ -454,17 +337,14 @@ class InternationalizationExtension(Extension):
         else:
             return node
 
-    def _trim_whitespace(self, string: str, _ws_re: t.Pattern[str] = _ws_re) -> str:
+    def _trim_whitespace(self, string, _ws_re=_ws_re):
         return _ws_re.sub(" ", string.strip())
 
-    def _parse_block(
-        self, parser: "Parser", allow_pluralize: bool
-    ) -> tuple[list[str], str]:
+    def _parse_block(self, parser, allow_pluralize):
         """Parse until the next block tag with a given name."""
         referenced = []
         buf = []
-
-        while True:
+        while 1:
             if parser.stream.current.type == "data":
                 buf.append(parser.stream.current.value.replace("%", "%%"))
                 next(parser.stream)
@@ -476,26 +356,16 @@ class InternationalizationExtension(Extension):
                 parser.stream.expect("variable_end")
             elif parser.stream.current.type == "block_begin":
                 next(parser.stream)
-                block_name = (
-                    parser.stream.current.value
-                    if parser.stream.current.type == "name"
-                    else None
-                )
-                if block_name == "endtrans":
+                if parser.stream.current.test("name:endtrans"):
                     break
-                elif block_name == "pluralize":
+                elif parser.stream.current.test("name:pluralize"):
                     if allow_pluralize:
                         break
                     parser.fail(
                         "a translatable section can have only one pluralize section"
                     )
-                elif block_name == "trans":
-                    parser.fail(
-                        "trans blocks can't be nested; did you mean `endtrans`?"
-                    )
                 parser.fail(
-                    f"control structures in translatable sections are not allowed; "
-                    f"saw `{block_name}`"
+                    "control structures in translatable sections are not allowed"
                 )
             elif parser.stream.eos:
                 parser.fail("unclosed translation block")
@@ -505,43 +375,36 @@ class InternationalizationExtension(Extension):
         return referenced, concat(buf)
 
     def _make_node(
-        self,
-        singular: str,
-        plural: str | None,
-        context: str | None,
-        variables: dict[str, nodes.Expr],
-        plural_expr: nodes.Expr | None,
-        vars_referenced: bool,
-        num_called_num: bool,
-    ) -> nodes.Output:
+        self, singular, plural, variables, plural_expr, vars_referenced, num_called_num
+    ):
         """Generates a useful node from the data provided."""
-        newstyle = self.environment.newstyle_gettext  # type: ignore
-        node: nodes.Expr
-
         # no variables referenced?  no need to escape for old style
         # gettext invocations only if there are vars.
-        if not vars_referenced and not newstyle:
+        if not vars_referenced and not self.environment.newstyle_gettext:
             singular = singular.replace("%%", "%")
             if plural:
                 plural = plural.replace("%%", "%")
 
-        func_name = "gettext"
-        func_args: list[nodes.Expr] = [nodes.Const(singular)]
+        # singular only:
+        if plural_expr is None:
+            gettext = nodes.Name("gettext", "load")
+            node = nodes.Call(gettext, [nodes.Const(singular)], [], None, None)
 
-        if context is not None:
-            func_args.insert(0, nodes.Const(context))
-            func_name = f"p{func_name}"
-
-        if plural_expr is not None:
-            func_name = f"n{func_name}"
-            func_args.extend((nodes.Const(plural), plural_expr))
-
-        node = nodes.Call(nodes.Name(func_name, "load"), func_args, [], None, None)
+        # singular and plural
+        else:
+            ngettext = nodes.Name("ngettext", "load")
+            node = nodes.Call(
+                ngettext,
+                [nodes.Const(singular), nodes.Const(plural), plural_expr],
+                [],
+                None,
+                None,
+            )
 
         # in case newstyle gettext is used, the method is powerful
         # enough to handle the variable expansion and autoescape
         # handling itself
-        if newstyle:
+        if self.environment.newstyle_gettext:
             for key, value in variables.items():
                 # the function adds that later anyways in case num was
                 # called num, so just skip it.
@@ -574,7 +437,7 @@ class ExprStmtExtension(Extension):
 
     tags = {"do"}
 
-    def parse(self, parser: "Parser") -> nodes.ExprStmt:
+    def parse(self, parser):
         node = nodes.ExprStmt(lineno=next(parser.stream).lineno)
         node.node = parser.parse_tuple()
         return node
@@ -585,13 +448,21 @@ class LoopControlExtension(Extension):
 
     tags = {"break", "continue"}
 
-    def parse(self, parser: "Parser") -> nodes.Break | nodes.Continue:
+    def parse(self, parser):
         token = next(parser.stream)
         if token.value == "break":
             return nodes.Break(lineno=token.lineno)
         return nodes.Continue(lineno=token.lineno)
 
 
+class WithExtension(Extension):
+    pass
+
+
+class AutoEscapeExtension(Extension):
+    pass
+
+
 class DebugExtension(Extension):
     """A ``{% debug %}`` tag that dumps the available variables,
     filters, and tests.
@@ -615,13 +486,13 @@ class DebugExtension(Extension):
 
     tags = {"debug"}
 
-    def parse(self, parser: "Parser") -> nodes.Output:
+    def parse(self, parser):
         lineno = parser.stream.expect("name:debug").lineno
-        context = nodes.ContextReference()
+        context = ContextReference()
         result = self.call_method("_render", [context], lineno=lineno)
         return nodes.Output([result], lineno=lineno)
 
-    def _render(self, context: Context) -> str:
+    def _render(self, context):
         result = {
             "context": context.get_all(),
             "filters": sorted(self.environment.filters.keys()),
@@ -629,14 +500,13 @@ class DebugExtension(Extension):
         }
 
         # Set the depth since the intent is to show the top few names.
-        return pprint.pformat(result, depth=3, compact=True)
+        if version_info[:2] >= (3, 4):
+            return pprint.pformat(result, depth=3, compact=True)
+        else:
+            return pprint.pformat(result, depth=3)
 
 
-def extract_from_ast(
-    ast: nodes.Template,
-    gettext_functions: t.Sequence[str] = GETTEXT_FUNCTIONS,
-    babel_style: bool = True,
-) -> t.Iterator[tuple[int, str, str | None | tuple[str | None, ...]]]:
+def extract_from_ast(node, gettext_functions=GETTEXT_FUNCTIONS, babel_style=True):
     """Extract localizable strings from the given template node.  Per
     default this function returns matches in babel style that means non string
     parameters as well as keyword arguments are returned as `None`.  This
@@ -671,17 +541,14 @@ def extract_from_ast(
     to extract any comments.  For comment support you have to use the babel
     extraction interface or extract comments yourself.
     """
-    out: str | None | tuple[str | None, ...]
-
-    for node in ast.find_all(nodes.Call):
+    for node in node.find_all(nodes.Call):
         if (
             not isinstance(node.node, nodes.Name)
             or node.node.name not in gettext_functions
         ):
             continue
 
-        strings: list[str | None] = []
-
+        strings = []
         for arg in node.args:
             if isinstance(arg, nodes.Const) and isinstance(arg.value, str):
                 strings.append(arg.value)
@@ -696,17 +563,15 @@ def extract_from_ast(
             strings.append(None)
 
         if not babel_style:
-            out = tuple(x for x in strings if x is not None)
-
-            if not out:
+            strings = tuple(x for x in strings if x is not None)
+            if not strings:
                 continue
         else:
             if len(strings) == 1:
-                out = strings[0]
+                strings = strings[0]
             else:
-                out = tuple(strings)
-
-        yield node.lineno, node.node.name, out
+                strings = tuple(strings)
+        yield node.lineno, node.node.name, strings
 
 
 class _CommentFinder:
@@ -716,15 +581,13 @@ class _CommentFinder:
     usable value.
     """
 
-    def __init__(
-        self, tokens: t.Sequence[tuple[int, str, str]], comment_tags: t.Sequence[str]
-    ) -> None:
+    def __init__(self, tokens, comment_tags):
         self.tokens = tokens
         self.comment_tags = comment_tags
         self.offset = 0
         self.last_lineno = 0
 
-    def find_backwards(self, offset: int) -> list[str]:
+    def find_backwards(self, offset):
         try:
             for _, token_type, token_value in reversed(
                 self.tokens[self.offset : offset]
@@ -740,7 +603,7 @@ class _CommentFinder:
         finally:
             self.offset = offset
 
-    def find_comments(self, lineno: int) -> list[str]:
+    def find_comments(self, lineno):
         if not self.comment_tags or self.last_lineno > lineno:
             return []
         for idx, (token_lineno, _, _) in enumerate(self.tokens[self.offset :]):
@@ -749,12 +612,7 @@ class _CommentFinder:
         return self.find_backwards(len(self.tokens))
 
 
-def babel_extract(
-    fileobj: t.BinaryIO,
-    keywords: t.Sequence[str],
-    comment_tags: t.Sequence[str],
-    options: dict[str, t.Any],
-) -> t.Iterator[tuple[int, str, str | None | tuple[str | None, ...], list[str]]]:
+def babel_extract(fileobj, keywords, comment_tags, options):
     """Babel extraction method for Jinja templates.
 
     .. versionchanged:: 2.3
@@ -782,37 +640,33 @@ def babel_extract(
     :return: an iterator over ``(lineno, funcname, message, comments)`` tuples.
              (comments will be empty currently)
     """
-    extensions: dict[type[Extension], None] = {}
-
-    for extension_name in options.get("extensions", "").split(","):
-        extension_name = extension_name.strip()
-
-        if not extension_name:
+    extensions = set()
+    for extension in options.get("extensions", "").split(","):
+        extension = extension.strip()
+        if not extension:
             continue
-
-        extensions[import_string(extension_name)] = None
-
+        extensions.add(import_string(extension))
     if InternationalizationExtension not in extensions:
-        extensions[InternationalizationExtension] = None
+        extensions.add(InternationalizationExtension)
 
-    def getbool(options: t.Mapping[str, str], key: str, default: bool = False) -> bool:
-        return options.get(key, str(default)).lower() in {"1", "on", "yes", "true"}
+    def getbool(options, key, default=False):
+        return options.get(key, str(default)).lower() in ("1", "on", "yes", "true")
 
     silent = getbool(options, "silent", True)
     environment = Environment(
-        options.get("block_start_string", defaults.BLOCK_START_STRING),
-        options.get("block_end_string", defaults.BLOCK_END_STRING),
-        options.get("variable_start_string", defaults.VARIABLE_START_STRING),
-        options.get("variable_end_string", defaults.VARIABLE_END_STRING),
-        options.get("comment_start_string", defaults.COMMENT_START_STRING),
-        options.get("comment_end_string", defaults.COMMENT_END_STRING),
-        options.get("line_statement_prefix") or defaults.LINE_STATEMENT_PREFIX,
-        options.get("line_comment_prefix") or defaults.LINE_COMMENT_PREFIX,
-        getbool(options, "trim_blocks", defaults.TRIM_BLOCKS),
-        getbool(options, "lstrip_blocks", defaults.LSTRIP_BLOCKS),
-        defaults.NEWLINE_SEQUENCE,
-        getbool(options, "keep_trailing_newline", defaults.KEEP_TRAILING_NEWLINE),
-        tuple(extensions),
+        options.get("block_start_string", BLOCK_START_STRING),
+        options.get("block_end_string", BLOCK_END_STRING),
+        options.get("variable_start_string", VARIABLE_START_STRING),
+        options.get("variable_end_string", VARIABLE_END_STRING),
+        options.get("comment_start_string", COMMENT_START_STRING),
+        options.get("comment_end_string", COMMENT_END_STRING),
+        options.get("line_statement_prefix") or LINE_STATEMENT_PREFIX,
+        options.get("line_comment_prefix") or LINE_COMMENT_PREFIX,
+        getbool(options, "trim_blocks", TRIM_BLOCKS),
+        getbool(options, "lstrip_blocks", LSTRIP_BLOCKS),
+        NEWLINE_SEQUENCE,
+        getbool(options, "keep_trailing_newline", KEEP_TRAILING_NEWLINE),
+        frozenset(extensions),
         cache_size=0,
         auto_reload=False,
     )
@@ -820,7 +674,7 @@ def babel_extract(
     if getbool(options, "trimmed"):
         environment.policies["ext.i18n.trimmed"] = True
     if getbool(options, "newstyle_gettext"):
-        environment.newstyle_gettext = True  # type: ignore
+        environment.newstyle_gettext = True
 
     source = fileobj.read().decode(options.get("encoding", "utf-8"))
     try:
@@ -841,4 +695,6 @@ def babel_extract(
 i18n = InternationalizationExtension
 do = ExprStmtExtension
 loopcontrols = LoopControlExtension
+with_ = WithExtension
+autoescape = AutoEscapeExtension
 debug = DebugExtension

src/jinja2/idtracking.py

@@ -1,20 +1,12 @@
-import typing as t
-
-from . import nodes
 from .visitor import NodeVisitor
 
-if t.TYPE_CHECKING:
-    import typing_extensions as te
-
 VAR_LOAD_PARAMETER = "param"
 VAR_LOAD_RESOLVE = "resolve"
 VAR_LOAD_ALIAS = "alias"
 VAR_LOAD_UNDEFINED = "undefined"
 
 
-def find_symbols(
-    nodes: t.Iterable[nodes.Node], parent_symbols: t.Optional["Symbols"] = None
-) -> "Symbols":
+def find_symbols(nodes, parent_symbols=None):
     sym = Symbols(parent=parent_symbols)
     visitor = FrameSymbolVisitor(sym)
     for node in nodes:
@@ -22,60 +14,49 @@ def find_symbols(
     return sym
 
 
-def symbols_for_node(
-    node: nodes.Node, parent_symbols: t.Optional["Symbols"] = None
-) -> "Symbols":
+def symbols_for_node(node, parent_symbols=None):
     sym = Symbols(parent=parent_symbols)
     sym.analyze_node(node)
     return sym
 
 
 class Symbols:
-    def __init__(
-        self, parent: t.Optional["Symbols"] = None, level: int | None = None
-    ) -> None:
+    def __init__(self, parent=None, level=None):
         if level is None:
             if parent is None:
                 level = 0
             else:
                 level = parent.level + 1
-
-        self.level: int = level
+        self.level = level
         self.parent = parent
-        self.refs: dict[str, str] = {}
-        self.loads: dict[str, t.Any] = {}
-        self.stores: set[str] = set()
+        self.refs = {}
+        self.loads = {}
+        self.stores = set()
 
-    def analyze_node(self, node: nodes.Node, **kwargs: t.Any) -> None:
+    def analyze_node(self, node, **kwargs):
         visitor = RootVisitor(self)
         visitor.visit(node, **kwargs)
 
-    def _define_ref(self, name: str, load: tuple[str, str | None] | None = None) -> str:
+    def _define_ref(self, name, load=None):
         ident = f"l_{self.level}_{name}"
         self.refs[name] = ident
         if load is not None:
             self.loads[ident] = load
         return ident
 
-    def find_load(self, target: str) -> t.Any | None:
+    def find_load(self, target):
         if target in self.loads:
             return self.loads[target]
-
         if self.parent is not None:
             return self.parent.find_load(target)
 
-        return None
-
-    def find_ref(self, name: str) -> str | None:
+    def find_ref(self, name):
         if name in self.refs:
             return self.refs[name]
-
         if self.parent is not None:
             return self.parent.find_ref(name)
 
-        return None
-
-    def ref(self, name: str) -> str:
+    def ref(self, name):
         rv = self.find_ref(name)
         if rv is None:
             raise AssertionError(
@@ -84,7 +65,7 @@ class Symbols:
             )
         return rv
 
-    def copy(self) -> "te.Self":
+    def copy(self):
         rv = object.__new__(self.__class__)
         rv.__dict__.update(self.__dict__)
         rv.refs = self.refs.copy()
@@ -92,7 +73,7 @@ class Symbols:
         rv.stores = self.stores.copy()
         return rv
 
-    def store(self, name: str) -> None:
+    def store(self, name):
         self.stores.add(name)
 
         # If we have not see the name referenced yet, we need to figure
@@ -110,28 +91,31 @@ class Symbols:
             # Otherwise we can just set it to undefined.
             self._define_ref(name, load=(VAR_LOAD_UNDEFINED, None))
 
-    def declare_parameter(self, name: str) -> str:
+    def declare_parameter(self, name):
         self.stores.add(name)
         return self._define_ref(name, load=(VAR_LOAD_PARAMETER, None))
 
-    def load(self, name: str) -> None:
-        if self.find_ref(name) is None:
+    def load(self, name):
+        target = self.find_ref(name)
+        if target is None:
             self._define_ref(name, load=(VAR_LOAD_RESOLVE, name))
 
-    def branch_update(self, branch_symbols: t.Sequence["Symbols"]) -> None:
-        stores: set[str] = set()
-
+    def branch_update(self, branch_symbols):
+        stores = {}
         for branch in branch_symbols:
-            stores.update(branch.stores)
-
-        stores.difference_update(self.stores)
+            for target in branch.stores:
+                if target in self.stores:
+                    continue
+                stores[target] = stores.get(target, 0) + 1
 
         for sym in branch_symbols:
             self.refs.update(sym.refs)
             self.loads.update(sym.loads)
             self.stores.update(sym.stores)
 
-        for name in stores:
+        for name, branch_count in stores.items():
+            if branch_count == len(branch_symbols):
+                continue
             target = self.find_ref(name)
             assert target is not None, "should not happen"
 
@@ -142,64 +126,56 @@ class Symbols:
                     continue
             self.loads[target] = (VAR_LOAD_RESOLVE, name)
 
-    def dump_stores(self) -> dict[str, str]:
-        rv: dict[str, str] = {}
-        node: Symbols | None = self
-
+    def dump_stores(self):
+        rv = {}
+        node = self
         while node is not None:
-            for name in sorted(node.stores):
+            for name in node.stores:
                 if name not in rv:
-                    rv[name] = self.find_ref(name)  # type: ignore
-
+                    rv[name] = self.find_ref(name)
             node = node.parent
-
         return rv
 
-    def dump_param_targets(self) -> set[str]:
+    def dump_param_targets(self):
         rv = set()
-        node: Symbols | None = self
-
+        node = self
         while node is not None:
             for target, (instr, _) in self.loads.items():
                 if instr == VAR_LOAD_PARAMETER:
                     rv.add(target)
-
             node = node.parent
-
         return rv
 
 
 class RootVisitor(NodeVisitor):
-    def __init__(self, symbols: "Symbols") -> None:
+    def __init__(self, symbols):
         self.sym_visitor = FrameSymbolVisitor(symbols)
 
-    def _simple_visit(self, node: nodes.Node, **kwargs: t.Any) -> None:
+    def _simple_visit(self, node, **kwargs):
         for child in node.iter_child_nodes():
             self.sym_visitor.visit(child)
 
-    visit_Template = _simple_visit
-    visit_Block = _simple_visit
-    visit_Macro = _simple_visit
-    visit_FilterBlock = _simple_visit
-    visit_Scope = _simple_visit
-    visit_If = _simple_visit
-    visit_ScopedEvalContextModifier = _simple_visit
+    visit_Template = (
+        visit_Block
+    ) = (
+        visit_Macro
+    ) = (
+        visit_FilterBlock
+    ) = visit_Scope = visit_If = visit_ScopedEvalContextModifier = _simple_visit
 
-    def visit_AssignBlock(self, node: nodes.AssignBlock, **kwargs: t.Any) -> None:
+    def visit_AssignBlock(self, node, **kwargs):
         for child in node.body:
             self.sym_visitor.visit(child)
 
-    def visit_CallBlock(self, node: nodes.CallBlock, **kwargs: t.Any) -> None:
+    def visit_CallBlock(self, node, **kwargs):
         for child in node.iter_child_nodes(exclude=("call",)):
             self.sym_visitor.visit(child)
 
-    def visit_OverlayScope(self, node: nodes.OverlayScope, **kwargs: t.Any) -> None:
+    def visit_OverlayScope(self, node, **kwargs):
         for child in node.body:
             self.sym_visitor.visit(child)
 
-    def visit_For(
-        self, node: nodes.For, for_branch: str = "body", **kwargs: t.Any
-    ) -> None:
+    def visit_For(self, node, for_branch="body", **kwargs):
         if for_branch == "body":
             self.sym_visitor.visit(node.target, store_as_param=True)
             branch = node.body
@@ -212,30 +188,28 @@ class RootVisitor(NodeVisitor):
             return
         else:
             raise RuntimeError("Unknown for branch")
+        for item in branch or ():
+            self.sym_visitor.visit(item)
 
-        if branch:
-            for item in branch:
-                self.sym_visitor.visit(item)
-
-    def visit_With(self, node: nodes.With, **kwargs: t.Any) -> None:
+    def visit_With(self, node, **kwargs):
         for target in node.targets:
             self.sym_visitor.visit(target)
         for child in node.body:
             self.sym_visitor.visit(child)
 
-    def generic_visit(self, node: nodes.Node, *args: t.Any, **kwargs: t.Any) -> None:
-        raise NotImplementedError(f"Cannot find symbols for {type(node).__name__!r}")
+    def generic_visit(self, node, *args, **kwargs):
+        raise NotImplementedError(
+            f"Cannot find symbols for {node.__class__.__name__!r}"
+        )
 
 
 class FrameSymbolVisitor(NodeVisitor):
     """A visitor for `Frame.inspect`."""
 
-    def __init__(self, symbols: "Symbols") -> None:
+    def __init__(self, symbols):
         self.symbols = symbols
 
-    def visit_Name(
-        self, node: nodes.Name, store_as_param: bool = False, **kwargs: t.Any
-    ) -> None:
+    def visit_Name(self, node, store_as_param=False, **kwargs):
         """All assignments to names go through this function."""
         if store_as_param or node.ctx == "param":
             self.symbols.declare_parameter(node.name)
@@ -244,73 +218,72 @@ class FrameSymbolVisitor(NodeVisitor):
         elif node.ctx == "load":
             self.symbols.load(node.name)
 
-    def visit_NSRef(self, node: nodes.NSRef, **kwargs: t.Any) -> None:
+    def visit_NSRef(self, node, **kwargs):
         self.symbols.load(node.name)
 
-    def visit_If(self, node: nodes.If, **kwargs: t.Any) -> None:
+    def visit_If(self, node, **kwargs):
         self.visit(node.test, **kwargs)
+
         original_symbols = self.symbols
 
-        def inner_visit(nodes: t.Iterable[nodes.Node]) -> "Symbols":
+        def inner_visit(nodes):
             self.symbols = rv = original_symbols.copy()
-
             for subnode in nodes:
                 self.visit(subnode, **kwargs)
-
             self.symbols = original_symbols
             return rv
 
         body_symbols = inner_visit(node.body)
         elif_symbols = inner_visit(node.elif_)
         else_symbols = inner_visit(node.else_ or ())
+
         self.symbols.branch_update([body_symbols, elif_symbols, else_symbols])
 
-    def visit_Macro(self, node: nodes.Macro, **kwargs: t.Any) -> None:
+    def visit_Macro(self, node, **kwargs):
         self.symbols.store(node.name)
 
-    def visit_Import(self, node: nodes.Import, **kwargs: t.Any) -> None:
+    def visit_Import(self, node, **kwargs):
         self.generic_visit(node, **kwargs)
         self.symbols.store(node.target)
 
-    def visit_FromImport(self, node: nodes.FromImport, **kwargs: t.Any) -> None:
+    def visit_FromImport(self, node, **kwargs):
         self.generic_visit(node, **kwargs)
-
         for name in node.names:
             if isinstance(name, tuple):
                 self.symbols.store(name[1])
             else:
                 self.symbols.store(name)
 
-    def visit_Assign(self, node: nodes.Assign, **kwargs: t.Any) -> None:
+    def visit_Assign(self, node, **kwargs):
         """Visit assignments in the correct order."""
         self.visit(node.node, **kwargs)
         self.visit(node.target, **kwargs)
 
-    def visit_For(self, node: nodes.For, **kwargs: t.Any) -> None:
+    def visit_For(self, node, **kwargs):
         """Visiting stops at for blocks.  However the block sequence
         is visited as part of the outer scope.
         """
         self.visit(node.iter, **kwargs)
 
-    def visit_CallBlock(self, node: nodes.CallBlock, **kwargs: t.Any) -> None:
+    def visit_CallBlock(self, node, **kwargs):
         self.visit(node.call, **kwargs)
 
-    def visit_FilterBlock(self, node: nodes.FilterBlock, **kwargs: t.Any) -> None:
+    def visit_FilterBlock(self, node, **kwargs):
         self.visit(node.filter, **kwargs)
 
-    def visit_With(self, node: nodes.With, **kwargs: t.Any) -> None:
+    def visit_With(self, node, **kwargs):
         for target in node.values:
             self.visit(target)
 
-    def visit_AssignBlock(self, node: nodes.AssignBlock, **kwargs: t.Any) -> None:
+    def visit_AssignBlock(self, node, **kwargs):
         """Stop visiting at block assigns."""
         self.visit(node.target, **kwargs)
 
-    def visit_Scope(self, node: nodes.Scope, **kwargs: t.Any) -> None:
+    def visit_Scope(self, node, **kwargs):
         """Stop visiting at scopes."""
 
-    def visit_Block(self, node: nodes.Block, **kwargs: t.Any) -> None:
+    def visit_Block(self, node, **kwargs):
         """Stop visiting at blocks."""
 
-    def visit_OverlayScope(self, node: nodes.OverlayScope, **kwargs: t.Any) -> None:
+    def visit_OverlayScope(self, node, **kwargs):
         """Do not visit into overlay scopes."""

src/jinja2/lexer.py

@@ -3,25 +3,19 @@ is used to do some preprocessing. It filters out invalid operators like
 the bitshift operators we don't allow in templates. It separates
 template code and python code in expressions.
 """
-
 import re
-import typing as t
 from ast import literal_eval
 from collections import deque
+from operator import itemgetter
 from sys import intern
 
 from ._identifier import pattern as name_re
 from .exceptions import TemplateSyntaxError
 from .utils import LRUCache
 
-if t.TYPE_CHECKING:
-    import typing_extensions as te
-
-    from .environment import Environment
-
 # cache for the lexers. Exists in order to be able to have multiple
 # environments with the same lexer
-_lexer_cache: t.MutableMapping[tuple, "Lexer"] = LRUCache(50)  # type: ignore
+_lexer_cache = LRUCache(50)
 
 # static regular expressions
 whitespace_re = re.compile(r"\s+")
@@ -29,22 +23,7 @@ newline_re = re.compile(r"(\r\n|\r|\n)")
 string_re = re.compile(
     r"('([^'\\]*(?:\\.[^'\\]*)*)'" r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S
 )
-integer_re = re.compile(
-    r"""
-    (
-        0b(_?[0-1])+ # binary
-    |
-        0o(_?[0-7])+ # octal
-    |
-        0x(_?[\da-f])+ # hex
-    |
-        [1-9](_?\d)* # decimal
-    |
-        0(_?0)* # decimal zero
-    )
-    """,
-    re.IGNORECASE | re.VERBOSE,
-)
+integer_re = re.compile(r"(\d+_)*\d+")
 float_re = re.compile(
     r"""
     (?<!\.)  # doesn't start with a .
@@ -162,10 +141,9 @@ ignore_if_empty = frozenset(
 )
 
 
-def _describe_token_type(token_type: str) -> str:
+def _describe_token_type(token_type):
     if token_type in reverse_operators:
         return reverse_operators[token_type]
-
     return {
         TOKEN_COMMENT_BEGIN: "begin of comment",
         TOKEN_COMMENT_END: "end of comment",
@@ -182,35 +160,32 @@ def _describe_token_type(token_type: str) -> str:
     }.get(token_type, token_type)
 
 
-def describe_token(token: "Token") -> str:
+def describe_token(token):
     """Returns a description of the token."""
     if token.type == TOKEN_NAME:
         return token.value
-
     return _describe_token_type(token.type)
 
 
-def describe_token_expr(expr: str) -> str:
+def describe_token_expr(expr):
     """Like `describe_token` but for token expressions."""
     if ":" in expr:
         type, value = expr.split(":", 1)
-
         if type == TOKEN_NAME:
             return value
     else:
         type = expr
-
     return _describe_token_type(type)
 
 
-def count_newlines(value: str) -> int:
+def count_newlines(value):
     """Count the number of newline characters in the string.  This is
     useful for extensions that filter a stream.
     """
     return len(newline_re.findall(value))
 
 
-def compile_rules(environment: "Environment") -> list[tuple[str, str]]:
+def compile_rules(environment):
     """Compiles all the rules from the environment into a list of rules."""
     e = re.escape
     rules = [
@@ -256,25 +231,31 @@ class Failure:
     Used by the `Lexer` to specify known errors.
     """
 
-    def __init__(
-        self, message: str, cls: type[TemplateSyntaxError] = TemplateSyntaxError
-    ) -> None:
+    def __init__(self, message, cls=TemplateSyntaxError):
         self.message = message
         self.error_class = cls
 
-    def __call__(self, lineno: int, filename: str | None) -> "te.NoReturn":
+    def __call__(self, lineno, filename):
         raise self.error_class(self.message, lineno, filename)
 
 
-class Token(t.NamedTuple):
-    lineno: int
-    type: str
-    value: str
+class Token(tuple):
+    """Token class."""
 
-    def __str__(self) -> str:
-        return describe_token(self)
+    __slots__ = ()
+    lineno, type, value = (property(itemgetter(x)) for x in range(3))
 
-    def test(self, expr: str) -> bool:
+    def __new__(cls, lineno, type, value):
+        return tuple.__new__(cls, (lineno, intern(str(type)), value))
+
+    def __str__(self):
+        if self.type in reverse_operators:
+            return reverse_operators[self.type]
+        elif self.type == "name":
+            return self.value
+        return self.type
+
+    def test(self, expr):
         """Test a token against a token expression.  This can either be a
         token type or ``'token_type:token_value'``.  This can only test
         against string values and types.
@@ -283,15 +264,19 @@ class Token(t.NamedTuple):
         # passed an iterable of not interned strings.
         if self.type == expr:
             return True
-
-        if ":" in expr:
+        elif ":" in expr:
             return expr.split(":", 1) == [self.type, self.value]
-
         return False
 
-    def test_any(self, *iterable: str) -> bool:
+    def test_any(self, *iterable):
         """Test against multiple token expressions."""
-        return any(self.test(expr) for expr in iterable)
+        for expr in iterable:
+            if self.test(expr):
+                return True
+        return False
+
+    def __repr__(self):
+        return f"Token({self.lineno!r}, {self.type!r}, {self.value!r})"
 
 
 class TokenStreamIterator:
@@ -299,19 +284,17 @@ class TokenStreamIterator:
     until the eof token is reached.
     """
 
-    def __init__(self, stream: "TokenStream") -> None:
+    def __init__(self, stream):
         self.stream = stream
 
-    def __iter__(self) -> "TokenStreamIterator":
+    def __iter__(self):
         return self
 
-    def __next__(self) -> Token:
+    def __next__(self):
         token = self.stream.current
-
         if token.type is TOKEN_EOF:
             self.stream.close()
-            raise StopIteration
-
+            raise StopIteration()
         next(self.stream)
         return token
 
@@ -322,36 +305,33 @@ class TokenStream:
     one token ahead.  The current active token is stored as :attr:`current`.
     """
 
-    def __init__(
-        self,
-        generator: t.Iterable[Token],
-        name: str | None,
-        filename: str | None,
-    ):
+    def __init__(self, generator, name, filename):
         self._iter = iter(generator)
-        self._pushed: deque[Token] = deque()
+        self._pushed = deque()
         self.name = name
         self.filename = filename
         self.closed = False
         self.current = Token(1, TOKEN_INITIAL, "")
         next(self)
 
-    def __iter__(self) -> TokenStreamIterator:
+    def __iter__(self):
         return TokenStreamIterator(self)
 
-    def __bool__(self) -> bool:
+    def __bool__(self):
         return bool(self._pushed) or self.current.type is not TOKEN_EOF
 
+    __nonzero__ = __bool__  # py2
+
     @property
-    def eos(self) -> bool:
+    def eos(self):
         """Are we at the end of the stream?"""
         return not self
 
-    def push(self, token: Token) -> None:
+    def push(self, token):
         """Push a token back to the stream."""
         self._pushed.append(token)
 
-    def look(self) -> Token:
+    def look(self):
         """Look at the next token."""
         old_token = next(self)
         result = self.current
@@ -359,31 +339,28 @@ class TokenStream:
         self.current = old_token
         return result
 
-    def skip(self, n: int = 1) -> None:
+    def skip(self, n=1):
         """Got n tokens ahead."""
         for _ in range(n):
             next(self)
 
-    def next_if(self, expr: str) -> Token | None:
+    def next_if(self, expr):
         """Perform the token test and return the token if it matched.
         Otherwise the return value is `None`.
         """
         if self.current.test(expr):
             return next(self)
 
-        return None
-
-    def skip_if(self, expr: str) -> bool:
+    def skip_if(self, expr):
         """Like :meth:`next_if` but only returns `True` or `False`."""
         return self.next_if(expr) is not None
 
-    def __next__(self) -> Token:
+    def __next__(self):
         """Go one token ahead and return the old one.
 
         Use the built-in :func:`next` instead of calling this directly.
         """
         rv = self.current
-
         if self._pushed:
             self.current = self._pushed.popleft()
         elif self.current.type is not TOKEN_EOF:
@@ -391,22 +368,20 @@ class TokenStream:
                 self.current = next(self._iter)
             except StopIteration:
                 self.close()
-
         return rv
 
-    def close(self) -> None:
+    def close(self):
         """Close the stream."""
         self.current = Token(self.current.lineno, TOKEN_EOF, "")
-        self._iter = iter(())
+        self._iter = None
         self.closed = True
 
-    def expect(self, expr: str) -> Token:
+    def expect(self, expr):
         """Expect a given token type and return it.  This accepts the same
         argument as :meth:`jinja2.lexer.Token.test`.
         """
         if not self.current.test(expr):
             expr = describe_token_expr(expr)
-
             if self.current.type is TOKEN_EOF:
                 raise TemplateSyntaxError(
                     f"unexpected end of template, expected {expr!r}.",
@@ -414,18 +389,19 @@ class TokenStream:
                     self.name,
                     self.filename,
                 )
-
             raise TemplateSyntaxError(
                 f"expected token {expr!r}, got {describe_token(self.current)!r}",
                 self.current.lineno,
                 self.name,
                 self.filename,
             )
-
-        return next(self)
+        try:
+            return self.current
+        finally:
+            next(self)
 
 
-def get_lexer(environment: "Environment") -> "Lexer":
+def get_lexer(environment):
     """Return a lexer which is probably cached."""
     key = (
         environment.block_start_string,
@@ -442,14 +418,13 @@ def get_lexer(environment: "Environment") -> "Lexer":
         environment.keep_trailing_newline,
     )
     lexer = _lexer_cache.get(key)
-
     if lexer is None:
-        _lexer_cache[key] = lexer = Lexer(environment)
-
+        lexer = Lexer(environment)
+        _lexer_cache[key] = lexer
     return lexer
 
 
-class OptionalLStrip(tuple):  # type: ignore[type-arg]
+class OptionalLStrip(tuple):
     """A special tuple for marking a point in the state that can have
     lstrip applied.
     """
@@ -458,16 +433,10 @@ class OptionalLStrip(tuple):  # type: ignore[type-arg]
 
     # Even though it looks like a no-op, creating instances fails
     # without this.
-    def __new__(cls, *members, **kwargs):  # type: ignore
+    def __new__(cls, *members, **kwargs):
         return super().__new__(cls, members)
 
 
-class _Rule(t.NamedTuple):
-    pattern: t.Pattern[str]
-    tokens: str | tuple[str, ...] | tuple[Failure]
-    command: str | None
-
-
 class Lexer:
     """Class that implements a lexer for a given environment. Automatically
     created by the environment class, usually you don't have to do that.
@@ -476,21 +445,21 @@ class Lexer:
     Multiple environments can share the same lexer.
     """
 
-    def __init__(self, environment: "Environment") -> None:
+    def __init__(self, environment):
         # shortcuts
         e = re.escape
 
-        def c(x: str) -> t.Pattern[str]:
+        def c(x):
             return re.compile(x, re.M | re.S)
 
         # lexing rules for tags
-        tag_rules: list[_Rule] = [
-            _Rule(whitespace_re, TOKEN_WHITESPACE, None),
-            _Rule(float_re, TOKEN_FLOAT, None),
-            _Rule(integer_re, TOKEN_INTEGER, None),
-            _Rule(name_re, TOKEN_NAME, None),
-            _Rule(string_re, TOKEN_STRING, None),
-            _Rule(operator_re, TOKEN_OPERATOR, None),
+        tag_rules = [
+            (whitespace_re, TOKEN_WHITESPACE, None),
+            (float_re, TOKEN_FLOAT, None),
+            (integer_re, TOKEN_INTEGER, None),
+            (name_re, TOKEN_NAME, None),
+            (string_re, TOKEN_STRING, None),
+            (operator_re, TOKEN_OPERATOR, None),
         ]
 
         # assemble the root lexing rule. because "|" is ungreedy
@@ -509,50 +478,49 @@ class Lexer:
         # block suffix if trimming is enabled
         block_suffix_re = "\\n?" if environment.trim_blocks else ""
 
-        self.lstrip_blocks = environment.lstrip_blocks
+        # If lstrip is enabled, it should not be applied if there is any
+        # non-whitespace between the newline and block.
+        self.lstrip_unless_re = c(r"[^ \t]") if environment.lstrip_blocks else None
 
         self.newline_sequence = environment.newline_sequence
         self.keep_trailing_newline = environment.keep_trailing_newline
 
         root_raw_re = (
-            rf"(?P<raw_begin>{block_start_re}(\-|\+|)\s*raw\s*"
-            rf"(?:\-{block_end_re}\s*|{block_end_re}))"
+            fr"(?P<raw_begin>{block_start_re}(\-|\+|)\s*raw\s*"
+            fr"(?:\-{block_end_re}\s*|{block_end_re}))"
         )
         root_parts_re = "|".join(
-            [root_raw_re] + [rf"(?P<{n}>{r}(\-|\+|))" for n, r in root_tag_rules]
+            [root_raw_re] + [fr"(?P<{n}>{r}(\-|\+|))" for n, r in root_tag_rules]
         )
 
         # global lexing rules
-        self.rules: dict[str, list[_Rule]] = {
+        self.rules = {
             "root": [
                 # directives
-                _Rule(
-                    c(rf"(.*?)(?:{root_parts_re})"),
-                    OptionalLStrip(TOKEN_DATA, "#bygroup"),  # type: ignore
+                (
+                    c(fr"(.*?)(?:{root_parts_re})"),
+                    OptionalLStrip(TOKEN_DATA, "#bygroup"),
                     "#bygroup",
                 ),
                 # data
-                _Rule(c(".+"), TOKEN_DATA, None),
+                (c(".+"), TOKEN_DATA, None),
             ],
             # comments
             TOKEN_COMMENT_BEGIN: [
-                _Rule(
+                (
                     c(
-                        rf"(.*?)((?:\+{comment_end_re}|\-{comment_end_re}\s*"
-                        rf"|{comment_end_re}{block_suffix_re}))"
+                        fr"(.*?)((?:\-{comment_end_re}\s*"
+                        fr"|{comment_end_re}){block_suffix_re})"
                     ),
                     (TOKEN_COMMENT, TOKEN_COMMENT_END),
                     "#pop",
                 ),
-                _Rule(c(r"(.)"), (Failure("Missing end of comment tag"),), None),
+                (c(r"(.)"), (Failure("Missing end of comment tag"),), None),
             ],
             # blocks
             TOKEN_BLOCK_BEGIN: [
-                _Rule(
-                    c(
-                        rf"(?:\+{block_end_re}|\-{block_end_re}\s*"
-                        rf"|{block_end_re}{block_suffix_re})"
-                    ),
+                (
+                    c(fr"(?:\-{block_end_re}\s*|{block_end_re}){block_suffix_re}"),
                     TOKEN_BLOCK_END,
                     "#pop",
                 ),
@@ -560,8 +528,8 @@ class Lexer:
             + tag_rules,
             # variables
             TOKEN_VARIABLE_BEGIN: [
-                _Rule(
-                    c(rf"\-{variable_end_re}\s*|{variable_end_re}"),
+                (
+                    c(fr"\-{variable_end_re}\s*|{variable_end_re}"),
                     TOKEN_VARIABLE_END,
                     "#pop",
                 )
@@ -569,25 +537,24 @@ class Lexer:
             + tag_rules,
             # raw block
             TOKEN_RAW_BEGIN: [
-                _Rule(
+                (
                     c(
-                        rf"(.*?)((?:{block_start_re}(\-|\+|))\s*endraw\s*"
-                        rf"(?:\+{block_end_re}|\-{block_end_re}\s*"
-                        rf"|{block_end_re}{block_suffix_re}))"
+                        fr"(.*?)((?:{block_start_re}(\-|\+|))\s*endraw\s*"
+                        fr"(?:\-{block_end_re}\s*|{block_end_re}{block_suffix_re}))"
                     ),
-                    OptionalLStrip(TOKEN_DATA, TOKEN_RAW_END),  # type: ignore
+                    OptionalLStrip(TOKEN_DATA, TOKEN_RAW_END),
                     "#pop",
                 ),
-                _Rule(c(r"(.)"), (Failure("Missing end of raw directive"),), None),
+                (c(r"(.)"), (Failure("Missing end of raw directive"),), None),
             ],
             # line statements
             TOKEN_LINESTATEMENT_BEGIN: [
-                _Rule(c(r"\s*(\n|$)"), TOKEN_LINESTATEMENT_END, "#pop")
+                (c(r"\s*(\n|$)"), TOKEN_LINESTATEMENT_END, "#pop")
             ]
             + tag_rules,
             # line comments
             TOKEN_LINECOMMENT_BEGIN: [
-                _Rule(
+                (
                     c(r"(.*?)()(?=\n|$)"),
                     (TOKEN_LINECOMMENT, TOKEN_LINECOMMENT_END),
                     "#pop",
@@ -595,39 +562,25 @@ class Lexer:
             ],
         }
 
-    def _normalize_newlines(self, value: str) -> str:
+    def _normalize_newlines(self, value):
         """Replace all newlines with the configured sequence in strings
         and template data.
         """
         return newline_re.sub(self.newline_sequence, value)
 
-    def tokenize(
-        self,
-        source: str,
-        name: str | None = None,
-        filename: str | None = None,
-        state: str | None = None,
-    ) -> TokenStream:
+    def tokenize(self, source, name=None, filename=None, state=None):
         """Calls tokeniter + tokenize and wraps it in a token stream."""
         stream = self.tokeniter(source, name, filename, state)
         return TokenStream(self.wrap(stream, name, filename), name, filename)
 
-    def wrap(
-        self,
-        stream: t.Iterable[tuple[int, str, str]],
-        name: str | None = None,
-        filename: str | None = None,
-    ) -> t.Iterator[Token]:
+    def wrap(self, stream, name=None, filename=None):
         """This is called with the stream as returned by `tokenize` and wraps
         every token in a :class:`Token` and converts the value.
         """
-        for lineno, token, value_str in stream:
+        for lineno, token, value in stream:
             if token in ignored_tokens:
                 continue
-
-            value: t.Any = value_str
-
-            if token == TOKEN_LINESTATEMENT_BEGIN:
+            elif token == TOKEN_LINESTATEMENT_BEGIN:
                 token = TOKEN_BLOCK_BEGIN
             elif token == TOKEN_LINESTATEMENT_END:
                 token = TOKEN_BLOCK_END
@@ -635,12 +588,11 @@ class Lexer:
             elif token in (TOKEN_RAW_BEGIN, TOKEN_RAW_END):
                 continue
             elif token == TOKEN_DATA:
-                value = self._normalize_newlines(value_str)
+                value = self._normalize_newlines(value)
             elif token == "keyword":
-                token = value_str
+                token = value
             elif token == TOKEN_NAME:
-                value = value_str
-
+                value = str(value)
                 if not value.isidentifier():
                     raise TemplateSyntaxError(
                         "Invalid character in identifier", lineno, name, filename
@@ -649,62 +601,48 @@ class Lexer:
                 # try to unescape string
                 try:
                     value = (
-                        self._normalize_newlines(value_str[1:-1])
+                        self._normalize_newlines(value[1:-1])
                         .encode("ascii", "backslashreplace")
                         .decode("unicode-escape")
                     )
                 except Exception as e:
                     msg = str(e).split(":")[-1].strip()
-                    raise TemplateSyntaxError(msg, lineno, name, filename) from e
+                    raise TemplateSyntaxError(msg, lineno, name, filename)
             elif token == TOKEN_INTEGER:
-                value = int(value_str.replace("_", ""), 0)
+                value = int(value.replace("_", ""))
             elif token == TOKEN_FLOAT:
                 # remove all "_" first to support more Python versions
-                value = literal_eval(value_str.replace("_", ""))
+                value = literal_eval(value.replace("_", ""))
             elif token == TOKEN_OPERATOR:
-                token = operators[value_str]
-
+                token = operators[value]
             yield Token(lineno, token, value)
 
-    def tokeniter(
-        self,
-        source: str,
-        name: str | None,
-        filename: str | None = None,
-        state: str | None = None,
-    ) -> t.Iterator[tuple[int, str, str]]:
+    def tokeniter(self, source, name, filename=None, state=None):
         """This method tokenizes the text and returns the tokens in a
-        generator. Use this method if you just want to tokenize a template.
-
-        .. versionchanged:: 3.0
-            Only ``\\n``, ``\\r\\n`` and ``\\r`` are treated as line
-            breaks.
+        generator.  Use this method if you just want to tokenize a template.
         """
-        lines = newline_re.split(source)[::2]
-
-        if not self.keep_trailing_newline and lines[-1] == "":
-            del lines[-1]
-
+        lines = source.splitlines()
+        if self.keep_trailing_newline and source:
+            if source.endswith(("\r\n", "\r", "\n")):
+                lines.append("")
         source = "\n".join(lines)
         pos = 0
         lineno = 1
         stack = ["root"]
-
         if state is not None and state != "root":
             assert state in ("variable", "block"), "invalid state"
             stack.append(state + "_begin")
-
         statetokens = self.rules[stack[-1]]
         source_length = len(source)
-        balancing_stack: list[str] = []
+        balancing_stack = []
+        lstrip_unless_re = self.lstrip_unless_re
         newlines_stripped = 0
         line_starting = True
 
-        while True:
+        while 1:
             # tokenizer loop
             for regex, tokens, new_state in statetokens:
                 m = regex.match(source, pos)
-
                 # if no match we try again with the next rule
                 if m is None:
                     continue
@@ -722,12 +660,13 @@ class Lexer:
 
                 # tuples support more options
                 if isinstance(tokens, tuple):
-                    groups: t.Sequence[str] = m.groups()
+                    groups = m.groups()
 
                     if isinstance(tokens, OptionalLStrip):
                         # Rule supports lstrip. Match will look like
                         # text, block type, whitespace control, type, control, ...
                         text = groups[0]
+
                         # Skipping the text and first type, every other group is the
                         # whitespace control for each type. One of the groups will be
                         # -, +, or empty string instead of None.
@@ -737,27 +676,26 @@ class Lexer:
                             # Strip all whitespace between the text and the tag.
                             stripped = text.rstrip()
                             newlines_stripped = text[len(stripped) :].count("\n")
-                            groups = [stripped, *groups[1:]]
+                            groups = (stripped,) + groups[1:]
                         elif (
                             # Not marked for preserving whitespace.
                             strip_sign != "+"
                             # lstrip is enabled.
-                            and self.lstrip_blocks
+                            and lstrip_unless_re is not None
                             # Not a variable expression.
                             and not m.groupdict().get(TOKEN_VARIABLE_BEGIN)
                         ):
                             # The start of text between the last newline and the tag.
                             l_pos = text.rfind("\n") + 1
-
                             if l_pos > 0 or line_starting:
                                 # If there's only whitespace between the newline and the
                                 # tag, strip it.
-                                if whitespace_re.fullmatch(text, l_pos):
-                                    groups = [text[:l_pos], *groups[1:]]
+                                if not lstrip_unless_re.search(text, l_pos):
+                                    groups = (text[:l_pos],) + groups[1:]
 
                     for idx, token in enumerate(tokens):
                         # failure group
-                        if isinstance(token, Failure):
+                        if token.__class__ is Failure:
                             raise token(lineno, filename)
                         # bygroup is a bit more complex, in that case we
                         # yield for the current token the first named
@@ -776,17 +714,14 @@ class Lexer:
                         # normal group
                         else:
                             data = groups[idx]
-
                             if data or token not in ignore_if_empty:
-                                yield lineno, token, data  # type: ignore[misc]
-
+                                yield lineno, token, data
                             lineno += data.count("\n") + newlines_stripped
                             newlines_stripped = 0
 
                 # strings as token just are yielded as it.
                 else:
                     data = m.group()
-
                     # update brace/parentheses balance
                     if tokens == TOKEN_OPERATOR:
                         if data == "{":
@@ -800,9 +735,7 @@ class Lexer:
                                 raise TemplateSyntaxError(
                                     f"unexpected '{data}'", lineno, name, filename
                                 )
-
                             expected_op = balancing_stack.pop()
-
                             if expected_op != data:
                                 raise TemplateSyntaxError(
                                     f"unexpected '{data}', expected '{expected_op}'",
@@ -810,14 +743,13 @@ class Lexer:
                                     name,
                                     filename,
                                 )
-
                     # yield items
                     if data or tokens not in ignore_if_empty:
                         yield lineno, tokens, data
-
                     lineno += data.count("\n")
 
                 line_starting = m.group()[-1:] == "\n"
+
                 # fetch new position into new variable so that we can check
                 # if there is a internal parsing error which would result
                 # in an infinite loop
@@ -842,7 +774,6 @@ class Lexer:
                     # direct state name given
                     else:
                         stack.append(new_state)
-
                     statetokens = self.rules[stack[-1]]
                 # we are still at the same position and no stack change.
                 # this means a loop without break condition, avoid that and
@@ -851,7 +782,6 @@ class Lexer:
                     raise RuntimeError(
                         f"{regex!r} yielded empty string without stack change"
                     )
-
                 # publish new function and start again
                 pos = pos2
                 break
@@ -861,7 +791,6 @@ class Lexer:
                 # end of text
                 if pos >= source_length:
                     return
-
                 # something went wrong
                 raise TemplateSyntaxError(
                     f"unexpected char {source[pos]!r} at {pos}", lineno, name, filename

src/jinja2/loaders.py

@@ -1,12 +1,9 @@
 """API and implementations for loading templates from different data
 sources.
 """
-
 import importlib.util
 import os
-import posixpath
 import sys
-import typing as t
 import weakref
 import zipimport
 from collections import abc
@@ -16,20 +13,17 @@ from types import ModuleType
 
 from .exceptions import TemplateNotFound
 from .utils import internalcode
-
-if t.TYPE_CHECKING:
-    from .environment import Environment
-    from .environment import Template
+from .utils import open_if_exists
 
 
-def split_template_path(template: str) -> list[str]:
+def split_template_path(template):
     """Split a path into segments and perform a sanity check.  If it detects
     '..' in the path it will raise a `TemplateNotFound` error.
     """
     pieces = []
     for piece in template.split("/"):
         if (
-            os.sep in piece
+            os.path.sep in piece
             or (os.path.altsep and os.path.altsep in piece)
             or piece == os.path.pardir
         ):
@@ -72,9 +66,7 @@ class BaseLoader:
     #: .. versionadded:: 2.4
     has_source_access = True
 
-    def get_source(
-        self, environment: "Environment", template: str
-    ) -> tuple[str, str | None, t.Callable[[], bool] | None]:
+    def get_source(self, environment, template):
         """Get the template source, filename and reload helper for a template.
         It's passed the environment and template name and has to return a
         tuple in the form ``(source, filename, uptodate)`` or raise a
@@ -94,23 +86,18 @@ class BaseLoader:
         """
         if not self.has_source_access:
             raise RuntimeError(
-                f"{type(self).__name__} cannot provide access to the source"
+                f"{self.__class__.__name__} cannot provide access to the source"
             )
         raise TemplateNotFound(template)
 
-    def list_templates(self) -> list[str]:
+    def list_templates(self):
         """Iterates over all templates.  If the loader does not support that
         it should raise a :exc:`TypeError` which is the default behavior.
         """
         raise TypeError("this loader cannot iterate over all templates")
 
     @internalcode
-    def load(
-        self,
-        environment: "Environment",
-        name: str,
-        globals: t.MutableMapping[str, t.Any] | None = None,
-    ) -> "Template":
+    def load(self, environment, name, globals=None):
         """Loads a template.  This method looks up the template in the cache
         or loads one by calling :meth:`get_source`.  Subclasses should not
         override this method as loaders working on collections of other
@@ -150,82 +137,59 @@ class BaseLoader:
 
 
 class FileSystemLoader(BaseLoader):
-    """Load templates from a directory in the file system.
+    """Loads templates from the file system.  This loader can find templates
+    in folders on the file system and is the preferred way to load them.
 
-    The path can be relative or absolute. Relative paths are relative to
-    the current working directory.
+    The loader takes the path to the templates as string, or if multiple
+    locations are wanted a list of them which is then looked up in the
+    given order::
 
-    .. code-block:: python
+    >>> loader = FileSystemLoader('/path/to/templates')
+    >>> loader = FileSystemLoader(['/path/to/templates', '/other/path'])
 
-        loader = FileSystemLoader("templates")
+    Per default the template encoding is ``'utf-8'`` which can be changed
+    by setting the `encoding` parameter to something else.
 
-    A list of paths can be given. The directories will be searched in
-    order, stopping at the first matching template.
+    To follow symbolic links, set the *followlinks* parameter to ``True``::
 
-    .. code-block:: python
-
-        loader = FileSystemLoader(["/override/templates", "/default/templates"])
-
-    :param searchpath: A path, or list of paths, to the directory that
-        contains the templates.
-    :param encoding: Use this encoding to read the text from template
-        files.
-    :param followlinks: Follow symbolic links in the path.
+    >>> loader = FileSystemLoader('/path/to/templates', followlinks=True)
 
     .. versionchanged:: 2.8
-        Added the ``followlinks`` parameter.
+       The ``followlinks`` parameter was added.
     """
 
-    def __init__(
-        self,
-        searchpath: t.Union[
-            str, "os.PathLike[str]", t.Sequence[t.Union[str, "os.PathLike[str]"]]
-        ],
-        encoding: str = "utf-8",
-        followlinks: bool = False,
-    ) -> None:
+    def __init__(self, searchpath, encoding="utf-8", followlinks=False):
         if not isinstance(searchpath, abc.Iterable) or isinstance(searchpath, str):
             searchpath = [searchpath]
 
-        self.searchpath = [os.fspath(p) for p in searchpath]
+        self.searchpath = list(searchpath)
         self.encoding = encoding
         self.followlinks = followlinks
 
-    def get_source(
-        self, environment: "Environment", template: str
-    ) -> tuple[str, str, t.Callable[[], bool]]:
+    def get_source(self, environment, template):
         pieces = split_template_path(template)
-
         for searchpath in self.searchpath:
-            # Use posixpath even on Windows to avoid "drive:" or UNC
-            # segments breaking out of the search directory.
-            filename = posixpath.join(searchpath, *pieces)
-
-            if os.path.isfile(filename):
-                break
-        else:
-            plural = "path" if len(self.searchpath) == 1 else "paths"
-            paths_str = ", ".join(repr(p) for p in self.searchpath)
-            raise TemplateNotFound(
-                template,
-                f"{template!r} not found in search {plural}: {paths_str}",
-            )
-
-        with open(filename, encoding=self.encoding) as f:
-            contents = f.read()
-
-        mtime = os.path.getmtime(filename)
-
-        def uptodate() -> bool:
+            filename = os.path.join(searchpath, *pieces)
+            f = open_if_exists(filename)
+            if f is None:
+                continue
             try:
-                return os.path.getmtime(filename) == mtime
-            except OSError:
-                return False
+                contents = f.read().decode(self.encoding)
+            finally:
+                f.close()
 
-        # Use normpath to convert Windows altsep to sep.
-        return contents, os.path.normpath(filename), uptodate
+            mtime = os.path.getmtime(filename)
 
-    def list_templates(self) -> list[str]:
+            def uptodate():
+                try:
+                    return os.path.getmtime(filename) == mtime
+                except OSError:
+                    return False
+
+            return contents, filename, uptodate
+        raise TemplateNotFound(template)
+
+    def list_templates(self):
         found = set()
         for searchpath in self.searchpath:
             walk_dir = os.walk(searchpath, followlinks=self.followlinks)
@@ -233,8 +197,8 @@ class FileSystemLoader(BaseLoader):
                 for filename in filenames:
                     template = (
                         os.path.join(dirpath, filename)[len(searchpath) :]
-                        .strip(os.sep)
-                        .replace(os.sep, "/")
+                        .strip(os.path.sep)
+                        .replace(os.path.sep, "/")
                     )
                     if template[:2] == "./":
                         template = template[2:]
@@ -243,29 +207,6 @@ class FileSystemLoader(BaseLoader):
         return sorted(found)
 
 
-if sys.version_info >= (3, 13):
-
-    def _get_zipimporter_files(z: t.Any) -> dict[str, object]:
-        try:
-            get_files = z._get_files
-        except AttributeError as e:
-            raise TypeError(
-                "This zip import does not have the required metadata to list templates."
-            ) from e
-        return get_files()
-
-else:
-
-    def _get_zipimporter_files(z: t.Any) -> dict[str, object]:
-        try:
-            files = z._files
-        except AttributeError as e:
-            raise TypeError(
-                "This zip import does not have the required metadata to list templates."
-            ) from e
-        return files  # type: ignore[no-any-return]
-
-
 class PackageLoader(BaseLoader):
     """Load templates from a directory in a Python package.
 
@@ -299,20 +240,13 @@ class PackageLoader(BaseLoader):
         Limited PEP 420 namespace package support.
     """
 
-    def __init__(
-        self,
-        package_name: str,
-        package_path: "str" = "templates",
-        encoding: str = "utf-8",
-    ) -> None:
-        package_path = os.path.normpath(package_path).rstrip(os.sep)
-
-        # normpath preserves ".", which isn't valid in zip paths.
+    def __init__(self, package_name, package_path="templates", encoding="utf-8"):
         if package_path == os.path.curdir:
             package_path = ""
-        elif package_path[:2] == os.path.curdir + os.sep:
+        elif package_path[:2] == os.path.curdir + os.path.sep:
             package_path = package_path[2:]
 
+        package_path = os.path.normpath(package_path).rstrip(os.path.sep)
         self.package_path = package_path
         self.package_name = package_name
         self.encoding = encoding
@@ -321,57 +255,32 @@ class PackageLoader(BaseLoader):
         # packages work, otherwise get_loader returns None.
         import_module(package_name)
         spec = importlib.util.find_spec(package_name)
-        assert spec is not None, "An import spec was not found for the package."
-        loader = spec.loader
-        assert loader is not None, "A loader was not found for the package."
-        self._loader = loader
+        self._loader = loader = spec.loader
         self._archive = None
+        self._template_root = None
 
         if isinstance(loader, zipimport.zipimporter):
             self._archive = loader.archive
-            pkgdir = next(iter(spec.submodule_search_locations))  # type: ignore
-            template_root = os.path.join(pkgdir, package_path).rstrip(os.sep)
-        else:
-            roots: list[str] = []
-
-            # One element for regular packages, multiple for namespace
-            # packages, or None for single module file.
-            if spec.submodule_search_locations:
-                roots.extend(spec.submodule_search_locations)
-            # A single module file, use the parent directory instead.
-            elif spec.origin is not None:
-                roots.append(os.path.dirname(spec.origin))
-
-            if not roots:
-                raise ValueError(
-                    f"The {package_name!r} package was not installed in a"
-                    " way that PackageLoader understands."
-                )
-
-            for root in roots:
+            pkgdir = next(iter(spec.submodule_search_locations))
+            self._template_root = os.path.join(pkgdir, package_path)
+        elif spec.submodule_search_locations:
+            # This will be one element for regular packages and multiple
+            # for namespace packages.
+            for root in spec.submodule_search_locations:
                 root = os.path.join(root, package_path)
 
                 if os.path.isdir(root):
-                    template_root = root
+                    self._template_root = root
                     break
-            else:
-                raise ValueError(
-                    f"PackageLoader could not find a {package_path!r} directory"
-                    f" in the {package_name!r} package."
-                )
 
-        self._template_root = template_root
+        if self._template_root is None:
+            raise ValueError(
+                f"The {package_name!r} package was not installed in a"
+                " way that PackageLoader understands."
+            )
 
-    def get_source(
-        self, environment: "Environment", template: str
-    ) -> tuple[str, str, t.Callable[[], bool] | None]:
-        # Use posixpath even on Windows to avoid "drive:" or UNC
-        # segments breaking out of the search directory. Use normpath to
-        # convert Windows altsep to sep.
-        p = os.path.normpath(
-            posixpath.join(self._template_root, *split_template_path(template))
-        )
-        up_to_date: t.Callable[[], bool] | None
+    def get_source(self, environment, template):
+        p = os.path.join(self._template_root, *split_template_path(template))
 
         if self._archive is None:
             # Package is a directory.
@@ -383,15 +292,15 @@ class PackageLoader(BaseLoader):
 
             mtime = os.path.getmtime(p)
 
-            def up_to_date() -> bool:
+            def up_to_date():
                 return os.path.isfile(p) and os.path.getmtime(p) == mtime
 
         else:
             # Package is a zip file.
             try:
-                source = self._loader.get_data(p)  # type: ignore
-            except OSError as e:
-                raise TemplateNotFound(template) from e
+                source = self._loader.get_data(p)
+            except OSError:
+                raise TemplateNotFound(template)
 
             # Could use the zip's mtime for all template mtimes, but
             # would need to safely reload the module if it's out of
@@ -400,30 +309,37 @@ class PackageLoader(BaseLoader):
 
         return source.decode(self.encoding), p, up_to_date
 
-    def list_templates(self) -> list[str]:
-        results: list[str] = []
+    def list_templates(self):
+        results = []
 
         if self._archive is None:
             # Package is a directory.
             offset = len(self._template_root)
 
             for dirpath, _, filenames in os.walk(self._template_root):
-                dirpath = dirpath[offset:].lstrip(os.sep)
+                dirpath = dirpath[offset:].lstrip(os.path.sep)
                 results.extend(
-                    os.path.join(dirpath, name).replace(os.sep, "/")
+                    os.path.join(dirpath, name).replace(os.path.sep, "/")
                     for name in filenames
                 )
         else:
-            files = _get_zipimporter_files(self._loader)
+            if not hasattr(self._loader, "_files"):
+                raise TypeError(
+                    "This zip import does not have the required"
+                    " metadata to list templates."
+                )
 
             # Package is a zip file.
-            prefix = self._template_root[len(self._archive) :].lstrip(os.sep) + os.sep
+            prefix = (
+                self._template_root[len(self._archive) :].lstrip(os.path.sep)
+                + os.path.sep
+            )
             offset = len(prefix)
 
-            for name in files:
+            for name in self._loader._files.keys():
                 # Find names under the templates directory that aren't directories.
-                if name.startswith(prefix) and name[-1] != os.sep:
-                    results.append(name[offset:].replace(os.sep, "/"))
+                if name.startswith(prefix) and name[-1] != os.path.sep:
+                    results.append(name[offset:].replace(os.path.sep, "/"))
 
         results.sort()
         return results
@@ -435,21 +351,19 @@ class DictLoader(BaseLoader):
 
     >>> loader = DictLoader({'index.html': 'source here'})
 
-    Because auto reloading is rarely useful this is disabled by default.
+    Because auto reloading is rarely useful this is disabled per default.
     """
 
-    def __init__(self, mapping: t.Mapping[str, str]) -> None:
+    def __init__(self, mapping):
         self.mapping = mapping
 
-    def get_source(
-        self, environment: "Environment", template: str
-    ) -> tuple[str, None, t.Callable[[], bool]]:
+    def get_source(self, environment, template):
         if template in self.mapping:
             source = self.mapping[template]
             return source, None, lambda: source == self.mapping.get(template)
         raise TemplateNotFound(template)
 
-    def list_templates(self) -> list[str]:
+    def list_templates(self):
         return sorted(self.mapping)
 
 
@@ -471,26 +385,15 @@ class FunctionLoader(BaseLoader):
     return value.
     """
 
-    def __init__(
-        self,
-        load_func: t.Callable[
-            [str],
-            str | tuple[str, str | None, t.Callable[[], bool] | None] | None,
-        ],
-    ) -> None:
+    def __init__(self, load_func):
         self.load_func = load_func
 
-    def get_source(
-        self, environment: "Environment", template: str
-    ) -> tuple[str, str | None, t.Callable[[], bool] | None]:
+    def get_source(self, environment, template):
         rv = self.load_func(template)
-
         if rv is None:
             raise TemplateNotFound(template)
-
-        if isinstance(rv, str):
+        elif isinstance(rv, str):
             return rv, None, None
-
         return rv
 
 
@@ -509,47 +412,38 @@ class PrefixLoader(BaseLoader):
     by loading ``'app2/index.html'`` the file from the second.
     """
 
-    def __init__(
-        self, mapping: t.Mapping[str, BaseLoader], delimiter: str = "/"
-    ) -> None:
+    def __init__(self, mapping, delimiter="/"):
         self.mapping = mapping
         self.delimiter = delimiter
 
-    def get_loader(self, template: str) -> tuple[BaseLoader, str]:
+    def get_loader(self, template):
         try:
             prefix, name = template.split(self.delimiter, 1)
             loader = self.mapping[prefix]
-        except (ValueError, KeyError) as e:
-            raise TemplateNotFound(template) from e
+        except (ValueError, KeyError):
+            raise TemplateNotFound(template)
         return loader, name
 
-    def get_source(
-        self, environment: "Environment", template: str
-    ) -> tuple[str, str | None, t.Callable[[], bool] | None]:
+    def get_source(self, environment, template):
         loader, name = self.get_loader(template)
         try:
             return loader.get_source(environment, name)
-        except TemplateNotFound as e:
+        except TemplateNotFound:
             # re-raise the exception with the correct filename here.
             # (the one that includes the prefix)
-            raise TemplateNotFound(template) from e
+            raise TemplateNotFound(template)
 
     @internalcode
-    def load(
-        self,
-        environment: "Environment",
-        name: str,
-        globals: t.MutableMapping[str, t.Any] | None = None,
-    ) -> "Template":
+    def load(self, environment, name, globals=None):
         loader, local_name = self.get_loader(name)
         try:
             return loader.load(environment, local_name, globals)
-        except TemplateNotFound as e:
+        except TemplateNotFound:
             # re-raise the exception with the correct filename here.
             # (the one that includes the prefix)
-            raise TemplateNotFound(name) from e
+            raise TemplateNotFound(name)
 
-    def list_templates(self) -> list[str]:
+    def list_templates(self):
         result = []
         for prefix, loader in self.mapping.items():
             for template in loader.list_templates():
@@ -571,12 +465,10 @@ class ChoiceLoader(BaseLoader):
     from a different location.
     """
 
-    def __init__(self, loaders: t.Sequence[BaseLoader]) -> None:
+    def __init__(self, loaders):
         self.loaders = loaders
 
-    def get_source(
-        self, environment: "Environment", template: str
-    ) -> tuple[str, str | None, t.Callable[[], bool] | None]:
+    def get_source(self, environment, template):
         for loader in self.loaders:
             try:
                 return loader.get_source(environment, template)
@@ -585,12 +477,7 @@ class ChoiceLoader(BaseLoader):
         raise TemplateNotFound(template)
 
     @internalcode
-    def load(
-        self,
-        environment: "Environment",
-        name: str,
-        globals: t.MutableMapping[str, t.Any] | None = None,
-    ) -> "Template":
+    def load(self, environment, name, globals=None):
         for loader in self.loaders:
             try:
                 return loader.load(environment, name, globals)
@@ -598,7 +485,7 @@ class ChoiceLoader(BaseLoader):
                 pass
         raise TemplateNotFound(name)
 
-    def list_templates(self) -> list[str]:
+    def list_templates(self):
         found = set()
         for loader in self.loaders:
             found.update(loader.list_templates())
@@ -614,19 +501,17 @@ class ModuleLoader(BaseLoader):
 
     Example usage:
 
-    >>> loader = ModuleLoader('/path/to/compiled/templates')
+    >>> loader = ChoiceLoader([
+    ...     ModuleLoader('/path/to/compiled/templates'),
+    ...     FileSystemLoader('/path/to/templates')
+    ... ])
 
     Templates can be precompiled with :meth:`Environment.compile_templates`.
     """
 
     has_source_access = False
 
-    def __init__(
-        self,
-        path: t.Union[
-            str, "os.PathLike[str]", t.Sequence[t.Union[str, "os.PathLike[str]"]]
-        ],
-    ) -> None:
+    def __init__(self, path):
         package_name = f"_jinja2_module_templates_{id(self):x}"
 
         # create a fake module that looks for the templates in the
@@ -649,37 +534,28 @@ class ModuleLoader(BaseLoader):
         self.package_name = package_name
 
     @staticmethod
-    def get_template_key(name: str) -> str:
+    def get_template_key(name):
         return "tmpl_" + sha1(name.encode("utf-8")).hexdigest()
 
     @staticmethod
-    def get_module_filename(name: str) -> str:
+    def get_module_filename(name):
         return ModuleLoader.get_template_key(name) + ".py"
 
     @internalcode
-    def load(
-        self,
-        environment: "Environment",
-        name: str,
-        globals: t.MutableMapping[str, t.Any] | None = None,
-    ) -> "Template":
+    def load(self, environment, name, globals=None):
         key = self.get_template_key(name)
         module = f"{self.package_name}.{key}"
         mod = getattr(self.module, module, None)
-
         if mod is None:
             try:
                 mod = __import__(module, None, None, ["root"])
-            except ImportError as e:
-                raise TemplateNotFound(name) from e
+            except ImportError:
+                raise TemplateNotFound(name)
 
             # remove the entry from sys.modules, we only want the attribute
             # on the module object we have stored on the loader.
             sys.modules.pop(module, None)
 
-        if globals is None:
-            globals = {}
-
         return environment.template_class.from_module_dict(
             environment, mod.__dict__, globals
         )

src/jinja2/meta.py

@@ -1,37 +1,29 @@
 """Functions that expose information about templates that might be
 interesting for introspection.
 """
-
-import typing as t
-
 from . import nodes
 from .compiler import CodeGenerator
-from .compiler import Frame
-
-if t.TYPE_CHECKING:
-    from .environment import Environment
 
 
 class TrackingCodeGenerator(CodeGenerator):
     """We abuse the code generator for introspection."""
 
-    def __init__(self, environment: "Environment") -> None:
-        super().__init__(environment, "<introspection>", "<introspection>")
-        self.undeclared_identifiers: set[str] = set()
+    def __init__(self, environment):
+        CodeGenerator.__init__(self, environment, "<introspection>", "<introspection>")
+        self.undeclared_identifiers = set()
 
-    def write(self, x: str) -> None:
+    def write(self, x):
         """Don't write."""
 
-    def enter_frame(self, frame: Frame) -> None:
+    def enter_frame(self, frame):
         """Remember all undeclared identifiers."""
-        super().enter_frame(frame)
-
+        CodeGenerator.enter_frame(self, frame)
         for _, (action, param) in frame.symbols.loads.items():
             if action == "resolve" and param not in self.environment.globals:
                 self.undeclared_identifiers.add(param)
 
 
-def find_undeclared_variables(ast: nodes.Template) -> set[str]:
+def find_undeclared_variables(ast):
     """Returns a set of all variables in the AST that will be looked up from
     the context at runtime.  Because at compile time it's not known which
     variables will be used depending on the path the execution takes at
@@ -50,16 +42,12 @@ def find_undeclared_variables(ast: nodes.Template) -> set[str]:
        :exc:`TemplateAssertionError` during compilation and as a matter of
        fact this function can currently raise that exception as well.
     """
-    codegen = TrackingCodeGenerator(ast.environment)  # type: ignore
+    codegen = TrackingCodeGenerator(ast.environment)
     codegen.visit(ast)
     return codegen.undeclared_identifiers
 
 
-_ref_types = (nodes.Extends, nodes.FromImport, nodes.Import, nodes.Include)
-_RefType = nodes.Extends | nodes.FromImport | nodes.Import | nodes.Include
-
-
-def find_referenced_templates(ast: nodes.Template) -> t.Iterator[str | None]:
+def find_referenced_templates(ast):
     """Finds all the referenced templates from the AST.  This will return an
     iterator over all the hardcoded template extensions, inclusions and
     imports.  If dynamic inheritance or inclusion is used, `None` will be
@@ -74,15 +62,13 @@ def find_referenced_templates(ast: nodes.Template) -> t.Iterator[str | None]:
     This function is useful for dependency tracking.  For example if you want
     to rebuild parts of the website after a layout template has changed.
     """
-    template_name: t.Any
-
-    for node in ast.find_all(_ref_types):
-        template: nodes.Expr = node.template  # type: ignore
-
-        if not isinstance(template, nodes.Const):
+    for node in ast.find_all(
+        (nodes.Extends, nodes.FromImport, nodes.Import, nodes.Include)
+    ):
+        if not isinstance(node.template, nodes.Const):
             # a tuple with some non consts in there
-            if isinstance(template, (nodes.Tuple, nodes.List)):
-                for template_name in template.items:
+            if isinstance(node.template, (nodes.Tuple, nodes.List)):
+                for template_name in node.template.items:
                     # something const, only yield the strings and ignore
                     # non-string consts that really just make no sense
                     if isinstance(template_name, nodes.Const):
@@ -96,15 +82,15 @@ def find_referenced_templates(ast: nodes.Template) -> t.Iterator[str | None]:
                 yield None
             continue
         # constant is a basestring, direct template name
-        if isinstance(template.value, str):
-            yield template.value
+        if isinstance(node.template.value, str):
+            yield node.template.value
         # a tuple or list (latter *should* not happen) made of consts,
         # yield the consts that are strings.  We could warn here for
         # non string values
         elif isinstance(node, nodes.Include) and isinstance(
-            template.value, (tuple, list)
+            node.template.value, (tuple, list)
         ):
-            for template_name in template.value:
+            for template_name in node.template.value:
                 if isinstance(template_name, str):
                     yield template_name
         # something else we don't care about, we could warn here

src/jinja2/nativetypes.py

@@ -1,48 +1,35 @@
-import typing as t
 from ast import literal_eval
-from ast import parse
 from itertools import chain
 from itertools import islice
-from types import GeneratorType
 
 from . import nodes
 from .compiler import CodeGenerator
-from .compiler import Frame
 from .compiler import has_safe_repr
 from .environment import Environment
 from .environment import Template
 
 
-def native_concat(values: t.Iterable[t.Any]) -> t.Any | None:
+def native_concat(nodes):
     """Return a native Python type from the list of compiled nodes. If
     the result is a single node, its value is returned. Otherwise, the
     nodes are concatenated as strings. If the result can be parsed with
     :func:`ast.literal_eval`, the parsed value is returned. Otherwise,
     the string is returned.
 
-    :param values: Iterable of outputs to concatenate.
+    :param nodes: Iterable of nodes to concatenate.
     """
-    head = list(islice(values, 2))
+    head = list(islice(nodes, 2))
 
     if not head:
         return None
 
     if len(head) == 1:
         raw = head[0]
-        if not isinstance(raw, str):
-            return raw
     else:
-        if isinstance(values, GeneratorType):
-            values = chain(head, values)
-        raw = "".join([str(v) for v in values])
+        raw = "".join([str(v) for v in chain(head, nodes)])
 
     try:
-        return literal_eval(
-            # In Python 3.10+ ast.literal_eval removes leading spaces/tabs
-            # from the given string. For backwards compatibility we need to
-            # parse the string ourselves without removing leading spaces/tabs.
-            parse(raw, mode="eval")
-        )
+        return literal_eval(raw)
     except (ValueError, SyntaxError, MemoryError):
         return raw
 
@@ -53,15 +40,13 @@ class NativeCodeGenerator(CodeGenerator):
     """
 
     @staticmethod
-    def _default_finalize(value: t.Any) -> t.Any:
+    def _default_finalize(value):
         return value
 
-    def _output_const_repr(self, group: t.Iterable[t.Any]) -> str:
+    def _output_const_repr(self, group):
         return repr("".join([str(v) for v in group]))
 
-    def _output_child_to_const(
-        self, node: nodes.Expr, frame: Frame, finalize: CodeGenerator._FinalizeInfo
-    ) -> t.Any:
+    def _output_child_to_const(self, node, frame, finalize):
         const = node.as_const(frame.eval_ctx)
 
         if not has_safe_repr(const):
@@ -70,17 +55,13 @@ class NativeCodeGenerator(CodeGenerator):
         if isinstance(node, nodes.TemplateData):
             return const
 
-        return finalize.const(const)  # type: ignore
+        return finalize.const(const)
 
-    def _output_child_pre(
-        self, node: nodes.Expr, frame: Frame, finalize: CodeGenerator._FinalizeInfo
-    ) -> None:
+    def _output_child_pre(self, node, frame, finalize):
         if finalize.src is not None:
             self.write(finalize.src)
 
-    def _output_child_post(
-        self, node: nodes.Expr, frame: Frame, finalize: CodeGenerator._FinalizeInfo
-    ) -> None:
+    def _output_child_post(self, node, frame, finalize):
         if finalize.src is not None:
             self.write(")")
 
@@ -89,40 +70,22 @@ class NativeEnvironment(Environment):
     """An environment that renders templates to native Python types."""
 
     code_generator_class = NativeCodeGenerator
-    concat = staticmethod(native_concat)  # type: ignore
 
 
 class NativeTemplate(Template):
     environment_class = NativeEnvironment
 
-    def render(self, *args: t.Any, **kwargs: t.Any) -> t.Any:
+    def render(self, *args, **kwargs):
         """Render the template to produce a native Python type. If the
         result is a single node, its value is returned. Otherwise, the
         nodes are concatenated as strings. If the result can be parsed
         with :func:`ast.literal_eval`, the parsed value is returned.
         Otherwise, the string is returned.
         """
-        ctx = self.new_context(dict(*args, **kwargs))
+        vars = dict(*args, **kwargs)
 
         try:
-            return self.environment_class.concat(  # type: ignore
-                self.root_render_func(ctx)
-            )
-        except Exception:
-            return self.environment.handle_exception()
-
-    async def render_async(self, *args: t.Any, **kwargs: t.Any) -> t.Any:
-        if not self.environment.is_async:
-            raise RuntimeError(
-                "The environment was not created with async mode enabled."
-            )
-
-        ctx = self.new_context(dict(*args, **kwargs))
-
-        try:
-            return self.environment_class.concat(  # type: ignore
-                [n async for n in self.root_render_func(ctx)]  # type: ignore
-            )
+            return native_concat(self.root_render_func(self.new_context(vars)))
         except Exception:
             return self.environment.handle_exception()
 

src/jinja2/new_parser.py

@@ -0,0 +1,998 @@
+from tatsu.exceptions import FailedSemantics
+from . import nodes
+from .exceptions import TemplateSyntaxError
+
+
+class JinjaSemantics(object):
+
+    def block_expression_pair(self, ast):
+        start_block = ast['start']
+        end_block = ast['end']
+
+        if start_block['name'] != end_block['name']:
+            raise FailedSemantics()
+
+        return ast
+
+    def line_block_expression_pair(self, ast):
+        return self.block_expression_pair(ast)
+
+
+def lineno_from_parseinfo(parseinfo):
+    return parseinfo.line + 1
+
+def parse(ast):
+    def merge_output(blocks):
+        if len(blocks) < 2:
+            return blocks
+
+        for idx in range(len(blocks) - 1, 0, -1):
+            block = blocks[idx]
+            previous_block = blocks[idx - 1]
+
+            if isinstance(block, nodes.Output) and isinstance(previous_block, nodes.Output):
+                previous_block.nodes += block.nodes
+                del blocks[idx]
+
+        return blocks
+
+    def merge_template_data(blocks):
+        for block in blocks:
+            if isinstance(block, nodes.Output):
+                if len(block.nodes) < 2:
+                    continue
+
+                outputs = block.nodes
+
+                for idx in range(len(outputs) - 1, 0, -1):
+                    output = outputs[idx]
+                    previous_output = outputs[idx - 1]
+
+                    if isinstance(output, nodes.TemplateData) and isinstance(previous_output, nodes.TemplateData):
+                        previous_output.data += output.data
+                        del outputs[idx]
+
+        return blocks
+
+    def remove_none(blocks):
+        return [block for block in blocks if block is not None]
+
+    if isinstance(ast, list):
+        blocks = [parse(item) for item in ast]
+        return merge_template_data(merge_output(remove_none(blocks)))
+
+    if isinstance(ast, str):
+        return parse_output(ast)
+
+    if 'type' in ast and ast['type'] == 'variable':
+        return parse_print(ast)
+
+    if 'block' in ast:
+        return parse_block(ast)
+
+    if 'start' in ast and 'end' in ast:
+        return parse_block_pair(ast)
+
+    if 'raw' in ast:
+        return parse_raw(ast)
+
+    if 'comment' in ast:
+        return parse_comment(ast)
+
+    return None
+
+def parse_block(ast):
+    block_name = ast['block']['name']
+
+    if block_name == 'extends':
+        return parse_block_extends(ast)
+
+    if block_name == 'from':
+        return parse_block_from(ast)
+
+    if block_name == 'import':
+        return parse_block_import(ast)
+
+    if block_name == 'include':
+        return parse_block_include(ast)
+
+    if block_name == 'print':
+        return parse_block_print(ast)
+
+    if block_name == 'set':
+        return parse_block_set(ast)
+
+    return None
+
+def parse_block_pair(ast):
+    block_name = ast['start']['name']
+
+    if block_name == 'autoescape':
+        return parse_block_autoescape(ast)
+
+    if block_name == 'block':
+        return parse_block_block(ast)
+
+    if block_name == 'call':
+        return parse_block_call(ast)
+
+    if block_name == 'filter':
+        return parse_block_filter(ast)
+
+    if block_name == 'for':
+        return parse_block_for(ast)
+
+    if block_name == 'if':
+        return parse_block_if(ast)
+
+    if block_name == 'macro':
+        return parse_block_macro(ast)
+
+    if block_name == 'set':
+        return parse_block_set(ast)
+
+    if block_name == 'with':
+        return parse_block_with(ast)
+
+    return None
+
+def parse_block_autoescape(ast):
+    return nodes.Scope(
+        [nodes.ScopedEvalContextModifier(
+            [nodes.Keyword(
+                'autoescape',
+                parse_variable(ast['start']['parameters'][0]['value']),
+                lineno=lineno_from_parseinfo(ast['start']['parameters'][0]['parseinfo'])
+            )],
+            parse(ast['contents']),
+            lineno=lineno_from_parseinfo(ast['parseinfo'])
+        )],
+        lineno=lineno_from_parseinfo(ast['parseinfo'])
+    )
+
+def parse_block_block(ast):
+    name = parse_variable(ast['start']['parameters'][0]['value']).name
+    scoped = False
+
+    if len(ast['start']['parameters']) > 1:
+        scoped = ast['start']['parameters'][-1]['value']['variable'] == 'scoped'
+
+    return nodes.Block(
+        name,
+        parse(ast['contents']),
+        scoped,
+        lineno=lineno_from_parseinfo(ast['parseinfo'])
+    )
+
+def parse_block_call(ast):
+    parameters = ast['start']['parameters']
+
+    call = parse_variable(parameters[0]['value'])
+    args = []
+    defaults = []
+    body = parse(ast['contents'])
+
+    if 'name_call_parameters' in ast['start']:
+        for arg in ast['start']['name_call_parameters']:
+            args.append(parse_variable(arg['value'], variable_context='param'))
+
+    return nodes.CallBlock(
+        call,
+        args,
+        defaults,
+        body,
+        lineno=lineno_from_parseinfo(ast['parseinfo'])
+    )
+
+def parse_block_extends(ast):
+    return nodes.Extends(
+        parse_conditional_expression(ast['block']['parameters'][0]['value'])
+    )
+
+def parse_block_filter(ast):
+    body = parse(ast['contents'])
+    filter_parameter = ast['start']['parameters'][0]['value']
+
+    filter_base = parse_variable(filter_parameter)
+
+    if isinstance(filter_base, nodes.Filter):
+        filter = filter_base
+        while isinstance(filter.node, nodes.Filter):
+            filter = filter.node
+
+        args = []
+        kwargs = []
+        dynamic_args = None
+        dynamic_kwargs = None
+
+        inner_filter = filter.node
+
+        if isinstance(inner_filter, nodes.Call):
+            args = inner_filter.args
+            kwargs = inner_filter.kwargs
+            dynamic_args = inner_filter.dyn_args
+            dynamic_kwargs = inner_filter.dyn_kwargs
+
+            inner_filter = inner_filter.node
+
+        inner_filter = nodes.Filter(
+            None,
+            inner_filter.name,
+            args,
+            kwargs,
+            dynamic_args,
+            dynamic_kwargs,
+            lineno=inner_filter.lineno
+        )
+        filter.node = inner_filter
+
+    return nodes.FilterBlock(
+        body,
+        filter_base,
+        lineno=lineno_from_parseinfo(ast['parseinfo'])
+    )
+
+def parse_block_for(ast):
+    iter = None
+    body = ast['contents']
+    else_ = []
+    test = None
+    recursive = False
+
+    block_parameters = ast['start']['parameters']
+
+    target = []
+    for param_number, param in enumerate(block_parameters):
+        if param['value']['variable'] == 'in':
+            break
+
+        if param['value']['operator'] == 'in':
+            block_parameters[param_number:param_number + 1] = [
+                {
+                    "value": param['value']['left']
+                },
+                {
+                    "value": {
+                        "variable": "in"
+                    }
+                },
+                {
+                    "value": param['value']['right']
+                },
+            ]
+
+            target.append(
+                parse_variable(
+                    param['value']['left'],
+                    variable_context='store'
+                )
+            )
+
+            break
+
+        target.append(parse_variable(param['value'], variable_context='store'))
+
+    if len(target) == 0:
+        raise TemplateSyntaxError(
+            "expected token 'in'",
+            lineno=lineno_from_parseinfo(ast['start']['parseinfo'])
+        )
+
+    if len(target) == len(block_parameters):
+        raise TemplateSyntaxError(
+            "expected token 'in'",
+            lineno=target[1].lineno
+        )
+
+    if len(target) == 1:
+        target = target[0]
+    else:
+        target = nodes.Tuple(
+            target,
+            'store',
+            lineno=target[0].lineno
+        )
+    param_number += 2
+
+    iter = parse_variable(block_parameters[param_number]['value'])
+    param_number += 1
+
+    if len(block_parameters) > param_number + 1:
+        if block_parameters[param_number]['value']['variable'] == 'if':
+            param_number += 1
+
+            test = parse_conditional_expression(
+                block_parameters[param_number]['value']
+            )
+            param_number += 1
+
+    if len(block_parameters) > param_number + 2:
+        raise
+
+    if len(block_parameters) == param_number + 1:
+        recursive = block_parameters[param_number]['value']['variable'] == 'recursive'
+
+    else_ = _split_contents_at_block(ast['contents'], 'else')
+
+    if else_ is not None:
+        body, _, else_ = else_
+    else:
+        else_ = []
+
+    return nodes.For(
+        target, iter, parse(body), parse(else_), test, recursive,
+        lineno=lineno_from_parseinfo(ast['parseinfo'])
+    )
+
+def parse_block_from(ast):
+    parameters = ast['block']['parameters']
+
+    template = parse_variable(parameters[0]['value'])
+    names = []
+    with_context = _parse_import_context(parameters)
+
+    if with_context is None:
+        with_context = False
+    else:
+        del parameters[-2:]
+
+    if len(parameters) > 1 and parameters[1]['value']['variable'] != 'import':
+        raise TemplateSyntaxError(
+            "Expecting 'import' but did not find it",
+            lineno=lineno_from_parseinfo(parameters[1]['parseinfo'])
+        )
+
+    if len(parameters) == 2:
+        raise TemplateSyntaxError(
+            "expected token 'name', got 'end of statement block'",
+            lineno=lineno_from_parseinfo(parameters[1]['parseinfo'])
+        )
+
+    def _variable_to_name(variable):
+        if isinstance(variable, str):
+            return variable
+
+        if 'alias' in variable:
+            return (
+                variable['variable'],
+                variable['alias']
+            )
+
+        return variable['variable']
+
+    for parameter in parameters[2:]:
+        if 'tuple' in parameter['value']:
+            for variable in parameter['value']['tuple']:
+                names.append(_variable_to_name(variable))
+        else:
+            names.append(_variable_to_name(parameter['value']))
+
+    from_import = nodes.FromImport(
+        template,
+        names,
+        with_context,
+        lineno=lineno_from_parseinfo(ast['parseinfo'])
+    )
+
+    return from_import
+
+def parse_block_if(ast):
+    test = parse_conditional_expression(ast['start']['parameters'][0]['value'])
+    body = ast['contents']
+    elif_ = []
+
+    else_ = _split_contents_at_block(body, 'else')
+
+    if else_ is not None:
+        body, _, else_ = else_
+    else:
+        else_ = []
+
+    elif_contents = _split_contents_at_block(body, 'elif')
+
+    if elif_contents is not None:
+        body, _, _ = elif_contents
+
+    while elif_contents is not None:
+        _, elif_condition, elif_contents = elif_contents
+
+        elif_parsed = _split_contents_at_block(elif_contents, 'elif')
+
+        if elif_parsed is not None:
+            elif_body, _, _ = elif_parsed
+        else:
+            elif_body = elif_contents
+
+        elif_.append(
+            nodes.If(
+                parse_conditional_expression(elif_condition['block']['parameters'][0]['value']),
+                parse(elif_body),
+                [],
+                [],
+                lineno=lineno_from_parseinfo(elif_condition['parseinfo'])
+            )
+        )
+
+        elif_contents = elif_parsed
+
+    return nodes.If(
+        test,
+        parse(body),
+        elif_,
+        parse(else_),
+        lineno=lineno_from_parseinfo(ast['parseinfo'])
+    )
+
+def parse_block_import(ast):
+    block_parameters = ast['block']['parameters']
+
+    template = parse_variable(block_parameters[0]['value'])
+    target = None
+    with_context = _parse_import_context(block_parameters) or False
+
+    if len(block_parameters) > 2 and block_parameters[1]['value']['variable'] == 'as':
+        target = parse_variable(block_parameters[2]['value']).name
+
+    return nodes.Import(
+        template,
+        target,
+        with_context,
+        lineno=lineno_from_parseinfo(ast['parseinfo'])
+    )
+
+def parse_block_include(ast):
+    block_parameters = ast['block']['parameters']
+
+    template = parse_conditional_expression(block_parameters[0]['value'])
+    with_context = _parse_import_context(block_parameters)
+    ignore_missing = False
+
+    if with_context is None:
+        with_context = True
+    else:
+        del block_parameters[-2:]
+
+    if len(block_parameters) == 3:
+        ignore_missing = True
+
+        if block_parameters[1]['value']['variable'] != 'ignore' and block_parameters[2]['value']['variable'] != 'missing':
+            raise
+
+    return nodes.Include(
+        template,
+        with_context,
+        ignore_missing,
+        lineno=lineno_from_parseinfo(ast['parseinfo'])
+    )
+
+def parse_block_macro(ast):
+    definition = parse_variable(ast['start']['parameters'][0]['value'])
+    name = definition.node.name
+    params = []
+    defaults = []
+    body = parse(ast['contents'])
+
+    for arg in definition.args:
+        arg.set_ctx('param')
+        params.append(arg)
+
+    for kwarg in definition.kwargs:
+        params.append(
+            nodes.Name(kwarg.key, 'param')
+        )
+        defaults.append(kwarg.value)
+
+    return nodes.Macro(
+        name,
+        params,
+        defaults,
+        body,
+        lineno=lineno_from_parseinfo(ast['parseinfo'])
+    )
+
+def parse_block_print(ast):
+    node = parse_variable(ast['block']['parameters'][0]['value'])
+
+    return nodes.Output([node])
+
+def parse_block_set(ast):
+    if 'block' in ast:
+        assignment = ast['block']['parameters'][0]
+
+        if isinstance(assignment['key'], str):
+            key = nodes.Name(assignment['key'], 'store')
+        else:
+            key = parse_variable(assignment['key'], variable_context="store")
+
+        return nodes.Assign(
+            key,
+            parse_conditional_expression(assignment['value']),
+            lineno=lineno_from_parseinfo(ast['parseinfo'])
+        )
+    elif 'start' in ast:
+        key = parse_variable(ast['start']['parameters'][0]['value'], variable_context="store")
+        filter = None
+
+        if isinstance(key, nodes.Filter):
+            filter = key
+            key = key.node
+            filter.node = None
+
+        return nodes.AssignBlock(
+            key,
+            filter,
+            parse(ast['contents']),
+            lineno=lineno_from_parseinfo(ast['parseinfo'])
+        )
+    return None
+
+def parse_block_with(ast):
+    with_node = nodes.With(
+        lineno=lineno_from_parseinfo(ast['parseinfo'])
+    )
+
+    targets = []
+    values = []
+
+    for parameter in ast['start']['parameters']:
+        if 'key' not in parameter:
+            raise
+
+        targets.append(nodes.Name(parameter['key'], 'param'))
+        values.append(parse_variable(parameter['value']))
+
+    with_node.targets = targets
+    with_node.values = values
+    with_node.body = parse(ast['contents'])
+
+    return with_node
+
+def parse_comment(ast):
+    return
+
+def parse_concatenate_expression(ast):
+    vars = [
+        parse_variable(var) for var in ast['concatenate']
+    ]
+
+    return nodes.Concat(
+        vars,
+        lineno=lineno_from_parseinfo(ast['parseinfo'])
+    )
+
+def parse_conditional_expression(ast):
+    if 'variable' in ast:
+        return parse_variable(ast)
+
+    if 'literal_type' in ast:
+        return parse_literal(ast)
+
+    if 'concatenate' in ast:
+        return parse_concatenate_expression(ast)
+
+    if 'logical_operator' in ast:
+        return parse_conditional_expression_logical(ast)
+
+    if 'math_operator' in ast:
+        return parse_conditional_expression_math(ast)
+
+    if 'not' in ast:
+        return parse_conditional_expression_not(ast)
+
+    if 'operator' in ast:
+        return parse_conditional_expression_operator(ast)
+
+    if 'test_expression' in ast:
+        return parse_conditional_expression_if(ast)
+
+    if 'test_function' in ast:
+        return parse_conditional_expression_test(ast)
+
+    return None
+
+def parse_conditional_expression_if(ast):
+    test = parse_conditional_expression(ast['test_expression'])
+    expr1 = parse_variable(ast['true_value'])
+    expr2 = None
+
+    if 'false_value' in ast:
+        expr2 = parse_variable(ast['false_value'])
+
+    return nodes.CondExpr(
+        test,
+        expr1,
+        expr2,
+        lineno=lineno_from_parseinfo(ast['parseinfo'])
+    )
+
+def parse_conditional_expression_logical(ast):
+    node_class_map = {
+        'and': nodes.And,
+        'or': nodes.Or,
+    }
+
+    node_class = node_class_map[ast['logical_operator']]
+
+    return node_class(
+        parse_conditional_expression(ast['left']),
+        parse_conditional_expression(ast['right']),
+        lineno=lineno_from_parseinfo(ast['parseinfo'])
+    )
+
+def parse_conditional_expression_math(ast):
+    node_class_map = {
+        '+': nodes.Add,
+        '-': nodes.Sub,
+        '*': nodes.Mul,
+        '**': nodes.Pow,
+        '/': nodes.Div,
+        '//': nodes.FloorDiv,
+        '%': nodes.Mod,
+    }
+
+    node_class = node_class_map[ast['math_operator']]
+
+    return node_class(
+        parse_conditional_expression(ast['left']),
+        parse_conditional_expression(ast['right']),
+        lineno=lineno_from_parseinfo(ast['parseinfo'])
+    )
+
+def parse_conditional_expression_not(ast):
+    return nodes.Not(
+        parse_conditional_expression(ast['not']),
+        lineno=lineno_from_parseinfo(ast['parseinfo'])
+    )
+
+def parse_conditional_expression_operator(ast):
+    operand_map = {
+        '>': 'gt',
+        '>=': 'gteq',
+        '==': 'eq',
+        '!=': 'ne',
+        '<': 'lt',
+        '<=': 'lteq',
+    }
+
+    expr = parse_conditional_expression(ast['left'])
+    operator = operand_map.get(ast['operator'], ast['operator'])
+    operands = []
+
+    right = parse_conditional_expression(ast['right'])
+
+    if isinstance(right, nodes.Compare):
+        operands.append(
+            nodes.Operand(
+                operator,
+                right.expr
+            )
+        )
+        operands.extend(right.ops)
+    else:
+
+        operands.append(
+            nodes.Operand(
+                operator,
+                right
+            )
+        )
+
+    return nodes.Compare(
+        expr,
+        operands,
+        lineno=lineno_from_parseinfo(ast['parseinfo'])
+    )
+
+def parse_conditional_expression_test(ast):
+    node = parse_conditional_expression(ast['test_variable'])
+    test_function = parse_variable(ast['test_function'])
+
+    args = []
+    kwargs = []
+    dynamic_args = None
+    dynamic_kwargs = None
+
+    if isinstance(test_function, nodes.Call):
+        call = test_function
+
+        name = call.node.name
+        args = call.args
+        kwargs = call.kwargs
+        dynamic_args = call.dyn_args
+        dynamic_kwargs = call.dyn_kwargs
+    elif isinstance(test_function, nodes.Const):
+        const_map = {
+            None: 'none',
+            True: 'true',
+            False: 'false',
+        }
+
+        name = const_map[test_function.value]
+    else:
+        name = test_function.name
+
+
+    if ast['test_function_parameter']:
+        args = [
+            parse_conditional_expression(ast['test_function_parameter'])
+        ]
+
+    test_node = nodes.Test(
+        node,
+        name,
+        args,
+        kwargs,
+        dynamic_args,
+        dynamic_kwargs,
+        lineno=lineno_from_parseinfo(ast['parseinfo'])
+    )
+
+    if 'negated' in ast and ast['negated']:
+        test_node = nodes.Not(
+            test_node,
+            lineno=lineno_from_parseinfo(ast['parseinfo'])
+        )
+
+    return test_node
+
+def parse_literal(ast):
+    if 'literal_type' not in ast:
+        raise
+
+    literal_type = ast['literal_type']
+
+    if literal_type == 'boolean':
+        return nodes.Const(
+            ast['value'],
+            lineno=lineno_from_parseinfo(ast['parseinfo'])
+        )
+    elif literal_type == 'string':
+        return nodes.Const(
+            ''.join(ast['value']),
+            lineno=lineno_from_parseinfo(ast['parseinfo'])
+        )
+    elif literal_type == 'number':
+        if 'fractional' not in ast and 'exponent' not in ast:
+            const = int(ast['whole'])
+        else:
+            number = ast['whole']
+
+            if 'fractional' in ast:
+                number += '.' + ast['fractional']
+
+            if 'exponent' in ast:
+                number += 'e' + ast['exponent']
+
+            const = float(number)
+
+        return nodes.Const(
+            const,
+            lineno=lineno_from_parseinfo(ast['parseinfo'])
+        )
+    elif literal_type == 'dictionary':
+        if not ast['value']:
+            ast['value'] = []
+
+
+        items = [
+            nodes.Pair(
+                parse_literal(item['key']),
+                parse_variable(item['value']),
+                lineno=lineno_from_parseinfo(item['parseinfo'])
+            )
+            for item in ast['value']
+        ]
+
+        return nodes.Dict(
+            items,
+            lineno=lineno_from_parseinfo(ast['parseinfo'])
+        )
+    elif literal_type == 'none':
+        return nodes.Const(
+            None,
+            lineno=lineno_from_parseinfo(ast['parseinfo'])
+        )
+    elif literal_type == 'list':
+        if not ast['value']:
+            ast['value'] = []
+
+        items = [
+            parse_variable(item) for item in ast['value']
+        ]
+
+        return nodes.List(
+            items,
+            lineno=lineno_from_parseinfo(ast['parseinfo'])
+        )
+    elif literal_type == 'tuple':
+        if not ast['value']:
+            ast['value'] = []
+
+        items = [
+            parse_variable(item) for item in ast['value']
+        ]
+
+        return nodes.Tuple(
+            items,
+            'load',
+            lineno=lineno_from_parseinfo(ast['parseinfo'])
+        )
+    return None
+
+def parse_output(ast):
+    return nodes.Output(
+        [nodes.TemplateData(ast)]
+    )
+
+def parse_print(ast):
+    variable = ast['name']
+
+    node = parse_conditional_expression(variable)
+
+    return nodes.Output([node])
+
+def parse_raw(ast):
+    return parse_output(
+        ''.join(ast['raw'])
+    )
+
+def parse_template(ast):
+    return nodes.Template(parse(ast), lineno=1)
+
+def parse_variable(ast, variable_context='load'):
+    name = ast['variable']
+
+    if 'literal_type' in name:
+        node = parse_literal(name)
+    else:
+        node = nodes.Name(
+            name,
+            variable_context,
+            lineno=lineno_from_parseinfo(ast['parseinfo'])
+        )
+
+    for accessor_ast in ast['accessors']:
+        node = parse_variable_accessor(node, accessor_ast)
+
+    signed_node_map = {
+        '-': nodes.Neg,
+        '+': nodes.Pos,
+    }
+
+    if 'signed' in ast:
+        node_class = signed_node_map[ast['signed']]
+
+        node = node_class(node)
+
+    if ast['filters']:
+        for filter_ast in ast['filters']:
+            node = parse_variable_filter(node, filter_ast)
+
+    return node
+
+def parse_variable_accessor(node, ast):
+    accessor_type = ast['accessor_type']
+
+    if accessor_type == 'brackets':
+        accessor_node = nodes.Getitem()
+        accessor_node.arg = parse_variable(ast['parameter'])
+    elif accessor_type == 'dot':
+        if isinstance(ast['parameter'], str):
+            accessor_node = nodes.Getattr()
+            accessor_node.attr = ast['parameter']
+        else:
+            accessor_node = nodes.Getitem()
+            accessor_node.arg = parse_literal(ast['parameter'])
+    elif accessor_type == 'call':
+        accessor_node = parse_variable_accessor_call(ast)
+
+    accessor_node.node = node
+    accessor_node.ctx = "load"
+    accessor_node.lineno = lineno_from_parseinfo(ast['parseinfo'])
+
+    return accessor_node
+
+def parse_variable_accessor_call(ast):
+    args = []
+    kwargs = []
+    dynamic_args = None
+    dynamic_kwargs = None
+
+    if ast['parameters']:
+        for argument in ast['parameters']:
+            if dynamic_kwargs is not None:
+                raise TemplateSyntaxError(
+                    'invalid syntax for function call expression',
+                    lineno=lineno_from_parseinfo(argument['parseinfo'])
+                )
+
+            if 'dynamic_keyword_argument' in argument:
+
+                dynamic_kwargs = parse_variable(argument['dynamic_keyword_argument'])
+
+                continue
+
+            if dynamic_args is not None:
+                raise TemplateSyntaxError(
+                    'invalid syntax for function call expression',
+                    lineno=lineno_from_parseinfo(argument['parseinfo'])
+                )
+
+            if 'dynamic_argument' in argument:
+                dynamic_args = parse_variable(argument['dynamic_argument'])
+
+                continue
+
+            value = parse_variable(argument['value'])
+
+            if 'key' in argument:
+                kwargs.append(
+                    nodes.Keyword(argument['key'], value)
+                )
+            else:
+                args.append(value)
+
+    node = nodes.Call()
+    node.args = args
+    node.kwargs = kwargs
+    node.dyn_args = dynamic_args
+    node.dyn_kwargs = dynamic_kwargs
+
+    return node
+
+def parse_variable_filter(node, ast):
+    args = []
+    kwargs = []
+    dynamic_args = None
+    dynamic_kwargs = None
+
+    variable = parse_variable(ast)
+
+    filter_node = None
+    last_filter = None
+    start_variable = variable
+
+    while not isinstance(variable, nodes.Name):
+        if isinstance(variable, nodes.Call):
+            last_filter = filter_node
+            filter_node = variable
+
+        variable = variable.node
+
+    new_filter = nodes.Filter(
+        node,
+        variable.name,
+        args,
+        kwargs,
+        dynamic_args,
+        dynamic_kwargs,
+        lineno=lineno_from_parseinfo(ast['parseinfo'])
+    )
+
+    if filter_node is not None:
+        new_filter.args = filter_node.args
+        new_filter.kwargs = filter_node.kwargs
+
+    if last_filter is None:
+        return new_filter
+
+    last_filter.node = new_filter
+
+    return last_filter
+
+def _parse_import_context(block_parameters):
+    if block_parameters[-1]['value']['variable'] != 'context':
+        return None
+
+    if block_parameters[-2]['value']['variable'] not in ['with', 'without']:
+        return None
+
+    return block_parameters[-2]['value']['variable'] == 'with'
+
+def _split_contents_at_block(contents, block_name):
+    for index, expression in enumerate(contents):
+        if 'block' in expression:
+            if expression['block']['name'] == block_name:
+                return (contents[:index], expression, contents[index + 1:])
+
+    return None

src/jinja2/nodes.py

@@ -2,24 +2,12 @@
 some node tree helper functions used by the parser and compiler in order
 to normalize nodes.
 """
-
-import inspect
 import operator
-import typing as t
 from collections import deque
 
 from markupsafe import Markup
 
-from .utils import _PassArg
-
-if t.TYPE_CHECKING:
-    import typing_extensions as te
-
-    from .environment import Environment
-
-_NodeBound = t.TypeVar("_NodeBound", bound="Node")
-
-_binop_to_func: dict[str, t.Callable[[t.Any, t.Any], t.Any]] = {
+_binop_to_func = {
     "*": operator.mul,
     "/": operator.truediv,
     "//": operator.floordiv,
@@ -29,13 +17,13 @@ _binop_to_func: dict[str, t.Callable[[t.Any, t.Any], t.Any]] = {
     "-": operator.sub,
 }
 
-_uaop_to_func: dict[str, t.Callable[[t.Any], t.Any]] = {
+_uaop_to_func = {
     "not": operator.not_,
     "+": operator.pos,
     "-": operator.neg,
 }
 
-_cmpop_to_func: dict[str, t.Callable[[t.Any, t.Any], t.Any]] = {
+_cmpop_to_func = {
     "eq": operator.eq,
     "ne": operator.ne,
     "gt": operator.gt,
@@ -56,9 +44,9 @@ class NodeType(type):
     inheritance.  fields and attributes from the parent class are
     automatically forwarded to the child."""
 
-    def __new__(mcs, name, bases, d):  # type: ignore
+    def __new__(mcs, name, bases, d):
         for attr in "fields", "attributes":
-            storage: list[tuple[str, ...]] = []
+            storage = []
             storage.extend(getattr(bases[0] if bases else object, attr, ()))
             storage.extend(d.get(attr, ()))
             assert len(bases) <= 1, "multiple inheritance not allowed"
@@ -73,9 +61,7 @@ class EvalContext:
     to it in extensions.
     """
 
-    def __init__(
-        self, environment: "Environment", template_name: str | None = None
-    ) -> None:
+    def __init__(self, environment, template_name=None):
         self.environment = environment
         if callable(environment.autoescape):
             self.autoescape = environment.autoescape(template_name)
@@ -83,15 +69,15 @@ class EvalContext:
             self.autoescape = environment.autoescape
         self.volatile = False
 
-    def save(self) -> t.Mapping[str, t.Any]:
+    def save(self):
         return self.__dict__.copy()
 
-    def revert(self, old: t.Mapping[str, t.Any]) -> None:
+    def revert(self, old):
         self.__dict__.clear()
         self.__dict__.update(old)
 
 
-def get_eval_context(node: "Node", ctx: EvalContext | None) -> EvalContext:
+def get_eval_context(node, ctx):
     if ctx is None:
         if node.environment is None:
             raise RuntimeError(
@@ -119,36 +105,29 @@ class Node(metaclass=NodeType):
     all nodes automatically.
     """
 
-    fields: tuple[str, ...] = ()
-    attributes: tuple[str, ...] = ("lineno", "environment")
+    fields = ()
+    attributes = ("lineno", "environment")
     abstract = True
 
-    lineno: int
-    environment: t.Optional["Environment"]
-
-    def __init__(self, *fields: t.Any, **attributes: t.Any) -> None:
+    def __init__(self, *fields, **attributes):
         if self.abstract:
             raise TypeError("abstract nodes are not instantiable")
         if fields:
             if len(fields) != len(self.fields):
                 if not self.fields:
-                    raise TypeError(f"{type(self).__name__!r} takes 0 arguments")
+                    raise TypeError(f"{self.__class__.__name__!r} takes 0 arguments")
                 raise TypeError(
-                    f"{type(self).__name__!r} takes 0 or {len(self.fields)}"
+                    f"{self.__class__.__name__!r} takes 0 or {len(self.fields)}"
                     f" argument{'s' if len(self.fields) != 1 else ''}"
                 )
-            for name, arg in zip(self.fields, fields, strict=False):
+            for name, arg in zip(self.fields, fields):
                 setattr(self, name, arg)
         for attr in self.attributes:
             setattr(self, attr, attributes.pop(attr, None))
         if attributes:
             raise TypeError(f"unknown attribute {next(iter(attributes))!r}")
 
-    def iter_fields(
-        self,
-        exclude: t.Container[str] | None = None,
-        only: t.Container[str] | None = None,
-    ) -> t.Iterator[tuple[str, t.Any]]:
+    def iter_fields(self, exclude=None, only=None):
         """This method iterates over all fields that are defined and yields
         ``(key, value)`` tuples.  Per default all fields are returned, but
         it's possible to limit that to some fields by providing the `only`
@@ -157,7 +136,7 @@ class Node(metaclass=NodeType):
         """
         for name in self.fields:
             if (
-                (exclude is None and only is None)
+                (exclude is only is None)
                 or (exclude is not None and name not in exclude)
                 or (only is not None and name in only)
             ):
@@ -166,11 +145,7 @@ class Node(metaclass=NodeType):
                 except AttributeError:
                     pass
 
-    def iter_child_nodes(
-        self,
-        exclude: t.Container[str] | None = None,
-        only: t.Container[str] | None = None,
-    ) -> t.Iterator["Node"]:
+    def iter_child_nodes(self, exclude=None, only=None):
         """Iterates over all direct child nodes of the node.  This iterates
         over all fields and yields the values of they are nodes.  If the value
         of a field is a list all the nodes in that list are returned.
@@ -183,27 +158,23 @@ class Node(metaclass=NodeType):
             elif isinstance(item, Node):
                 yield item
 
-    def find(self, node_type: type[_NodeBound]) -> _NodeBound | None:
+    def find(self, node_type):
         """Find the first node of a given type.  If no such node exists the
         return value is `None`.
         """
         for result in self.find_all(node_type):
             return result
 
-        return None
-
-    def find_all(
-        self, node_type: type[_NodeBound] | tuple[type[_NodeBound], ...]
-    ) -> t.Iterator[_NodeBound]:
+    def find_all(self, node_type):
         """Find all the nodes of a given type.  If the type is a tuple,
         the check is performed for any of the tuple items.
         """
         for child in self.iter_child_nodes():
             if isinstance(child, node_type):
-                yield child  # type: ignore
+                yield child
             yield from child.find_all(node_type)
 
-    def set_ctx(self, ctx: str) -> "Node":
+    def set_ctx(self, ctx):
         """Reset the context of a node and all child nodes.  Per default the
         parser will all generate nodes that have a 'load' context as it's the
         most common one.  This method is used in the parser to set assignment
@@ -213,11 +184,11 @@ class Node(metaclass=NodeType):
         while todo:
             node = todo.popleft()
             if "ctx" in node.fields:
-                node.ctx = ctx  # type: ignore
+                node.ctx = ctx
             todo.extend(node.iter_child_nodes())
         return self
 
-    def set_lineno(self, lineno: int, override: bool = False) -> "Node":
+    def set_lineno(self, lineno, override=False):
         """Set the line numbers of the node and children."""
         todo = deque([self])
         while todo:
@@ -228,7 +199,7 @@ class Node(metaclass=NodeType):
             todo.extend(node.iter_child_nodes())
         return self
 
-    def set_environment(self, environment: "Environment") -> "Node":
+    def set_environment(self, environment):
         """Set the environment for all nodes."""
         todo = deque([self])
         while todo:
@@ -237,25 +208,26 @@ class Node(metaclass=NodeType):
             todo.extend(node.iter_child_nodes())
         return self
 
-    def __eq__(self, other: t.Any) -> bool:
+    def __eq__(self, other):
         if type(self) is not type(other):
             return NotImplemented
 
         return tuple(self.iter_fields()) == tuple(other.iter_fields())
 
-    __hash__ = object.__hash__
+    def __hash__(self):
+        return hash(tuple(self.iter_fields()))
 
-    def __repr__(self) -> str:
+    def __repr__(self):
         args_str = ", ".join(f"{a}={getattr(self, a, None)!r}" for a in self.fields)
-        return f"{type(self).__name__}({args_str})"
+        return f"{self.__class__.__name__}({args_str})"
 
-    def dump(self) -> str:
-        def _dump(node: Node | t.Any) -> None:
+    def dump(self):
+        def _dump(node):
             if not isinstance(node, Node):
                 buf.append(repr(node))
                 return
 
-            buf.append(f"nodes.{type(node).__name__}(")
+            buf.append(f"nodes.{node.__class__.__name__}(")
             if not node.fields:
                 buf.append(")")
                 return
@@ -274,7 +246,7 @@ class Node(metaclass=NodeType):
                     _dump(value)
             buf.append(")")
 
-        buf: list[str] = []
+        buf = []
         _dump(self)
         return "".join(buf)
 
@@ -297,7 +269,6 @@ class Template(Node):
     """
 
     fields = ("body",)
-    body: list[Node]
 
 
 class Output(Stmt):
@@ -306,14 +277,12 @@ class Output(Stmt):
     """
 
     fields = ("nodes",)
-    nodes: list["Expr"]
 
 
 class Extends(Stmt):
     """Represents an extends statement."""
 
     fields = ("template",)
-    template: "Expr"
 
 
 class For(Stmt):
@@ -326,22 +295,12 @@ class For(Stmt):
     """
 
     fields = ("target", "iter", "body", "else_", "test", "recursive")
-    target: Node
-    iter: Node
-    body: list[Node]
-    else_: list[Node]
-    test: Node | None
-    recursive: bool
 
 
 class If(Stmt):
     """If `test` is true, `body` is rendered, else `else_`."""
 
     fields = ("test", "body", "elif_", "else_")
-    test: Node
-    body: list[Node]
-    elif_: list["If"]
-    else_: list[Node]
 
 
 class Macro(Stmt):
@@ -351,10 +310,6 @@ class Macro(Stmt):
     """
 
     fields = ("name", "args", "defaults", "body")
-    name: str
-    args: list["Name"]
-    defaults: list["Expr"]
-    body: list[Node]
 
 
 class CallBlock(Stmt):
@@ -363,18 +318,12 @@ class CallBlock(Stmt):
     """
 
     fields = ("call", "args", "defaults", "body")
-    call: "Call"
-    args: list["Name"]
-    defaults: list["Expr"]
-    body: list[Node]
 
 
 class FilterBlock(Stmt):
     """Node for filter sections."""
 
     fields = ("body", "filter")
-    body: list[Node]
-    filter: "Filter"
 
 
 class With(Stmt):
@@ -385,41 +334,24 @@ class With(Stmt):
     """
 
     fields = ("targets", "values", "body")
-    targets: list["Expr"]
-    values: list["Expr"]
-    body: list[Node]
 
 
 class Block(Stmt):
-    """A node that represents a block.
+    """A node that represents a block."""
 
-    .. versionchanged:: 3.0.0
-        the `required` field was added.
-    """
-
-    fields = ("name", "body", "scoped", "required")
-    name: str
-    body: list[Node]
-    scoped: bool
-    required: bool
+    fields = ("name", "body", "scoped")
 
 
 class Include(Stmt):
     """A node that represents the include tag."""
 
     fields = ("template", "with_context", "ignore_missing")
-    template: "Expr"
-    with_context: bool
-    ignore_missing: bool
 
 
 class Import(Stmt):
     """A node that represents the import tag."""
 
     fields = ("template", "target", "with_context")
-    template: "Expr"
-    target: str
-    with_context: bool
 
 
 class FromImport(Stmt):
@@ -435,33 +367,24 @@ class FromImport(Stmt):
     """
 
     fields = ("template", "names", "with_context")
-    template: "Expr"
-    names: list[str | tuple[str, str]]
-    with_context: bool
 
 
 class ExprStmt(Stmt):
     """A statement that evaluates an expression and discards the result."""
 
     fields = ("node",)
-    node: Node
 
 
 class Assign(Stmt):
     """Assigns an expression to a target."""
 
     fields = ("target", "node")
-    target: "Expr"
-    node: Node
 
 
 class AssignBlock(Stmt):
     """Assigns a block to a target."""
 
     fields = ("target", "filter", "body")
-    target: "Expr"
-    filter: t.Optional["Filter"]
-    body: list[Node]
 
 
 class Expr(Node):
@@ -469,7 +392,7 @@ class Expr(Node):
 
     abstract = True
 
-    def as_const(self, eval_ctx: EvalContext | None = None) -> t.Any:
+    def as_const(self, eval_ctx=None):
         """Return the value of the expression as constant or raise
         :exc:`Impossible` if this was not possible.
 
@@ -482,7 +405,7 @@ class Expr(Node):
         """
         raise Impossible()
 
-    def can_assign(self) -> bool:
+    def can_assign(self):
         """Check if it's possible to assign something to this node."""
         return False
 
@@ -491,49 +414,44 @@ class BinExpr(Expr):
     """Baseclass for all binary expressions."""
 
     fields = ("left", "right")
-    left: Expr
-    right: Expr
-    operator: str
+    operator = None
     abstract = True
 
-    def as_const(self, eval_ctx: EvalContext | None = None) -> t.Any:
+    def as_const(self, eval_ctx=None):
         eval_ctx = get_eval_context(self, eval_ctx)
-
         # intercepted operators cannot be folded at compile time
         if (
-            eval_ctx.environment.sandboxed
-            and self.operator in eval_ctx.environment.intercepted_binops  # type: ignore
+            self.environment.sandboxed
+            and self.operator in self.environment.intercepted_binops
         ):
             raise Impossible()
         f = _binop_to_func[self.operator]
         try:
             return f(self.left.as_const(eval_ctx), self.right.as_const(eval_ctx))
-        except Exception as e:
-            raise Impossible() from e
+        except Exception:
+            raise Impossible()
 
 
 class UnaryExpr(Expr):
     """Baseclass for all unary expressions."""
 
     fields = ("node",)
-    node: Expr
-    operator: str
+    operator = None
     abstract = True
 
-    def as_const(self, eval_ctx: EvalContext | None = None) -> t.Any:
+    def as_const(self, eval_ctx=None):
         eval_ctx = get_eval_context(self, eval_ctx)
-
         # intercepted operators cannot be folded at compile time
         if (
-            eval_ctx.environment.sandboxed
-            and self.operator in eval_ctx.environment.intercepted_unops  # type: ignore
+            self.environment.sandboxed
+            and self.operator in self.environment.intercepted_unops
         ):
             raise Impossible()
         f = _uaop_to_func[self.operator]
         try:
             return f(self.node.as_const(eval_ctx))
-        except Exception as e:
-            raise Impossible() from e
+        except Exception:
+            raise Impossible()
 
 
 class Name(Expr):
@@ -546,21 +464,17 @@ class Name(Expr):
     """
 
     fields = ("name", "ctx")
-    name: str
-    ctx: str
 
-    def can_assign(self) -> bool:
-        return self.name not in {"true", "false", "none", "True", "False", "None"}
+    def can_assign(self):
+        return self.name not in ("true", "false", "none", "True", "False", "None")
 
 
 class NSRef(Expr):
     """Reference to a namespace value assignment"""
 
     fields = ("name", "attr")
-    name: str
-    attr: str
 
-    def can_assign(self) -> bool:
+    def can_assign(self):
         # We don't need any special checks here; NSRef assignments have a
         # runtime check to ensure the target is a namespace object which will
         # have been checked already as it is created using a normal assignment
@@ -582,18 +496,12 @@ class Const(Literal):
     """
 
     fields = ("value",)
-    value: t.Any
 
-    def as_const(self, eval_ctx: EvalContext | None = None) -> t.Any:
+    def as_const(self, eval_ctx=None):
         return self.value
 
     @classmethod
-    def from_untrusted(
-        cls,
-        value: t.Any,
-        lineno: int | None = None,
-        environment: "Environment | None" = None,
-    ) -> "Const":
+    def from_untrusted(cls, value, lineno=None, environment=None):
         """Return a const object if the value is representable as
         constant value in the generated code, otherwise it will raise
         an `Impossible` exception.
@@ -609,9 +517,8 @@ class TemplateData(Literal):
     """A constant template string."""
 
     fields = ("data",)
-    data: str
 
-    def as_const(self, eval_ctx: EvalContext | None = None) -> str:
+    def as_const(self, eval_ctx=None):
         eval_ctx = get_eval_context(self, eval_ctx)
         if eval_ctx.volatile:
             raise Impossible()
@@ -627,14 +534,12 @@ class Tuple(Literal):
     """
 
     fields = ("items", "ctx")
-    items: list[Expr]
-    ctx: str
 
-    def as_const(self, eval_ctx: EvalContext | None = None) -> tuple[t.Any, ...]:
+    def as_const(self, eval_ctx=None):
         eval_ctx = get_eval_context(self, eval_ctx)
         return tuple(x.as_const(eval_ctx) for x in self.items)
 
-    def can_assign(self) -> bool:
+    def can_assign(self):
         for item in self.items:
             if not item.can_assign():
                 return False
@@ -645,9 +550,8 @@ class List(Literal):
     """Any list literal such as ``[1, 2, 3]``"""
 
     fields = ("items",)
-    items: list[Expr]
 
-    def as_const(self, eval_ctx: EvalContext | None = None) -> list[t.Any]:
+    def as_const(self, eval_ctx=None):
         eval_ctx = get_eval_context(self, eval_ctx)
         return [x.as_const(eval_ctx) for x in self.items]
 
@@ -658,9 +562,8 @@ class Dict(Literal):
     """
 
     fields = ("items",)
-    items: list["Pair"]
 
-    def as_const(self, eval_ctx: EvalContext | None = None) -> dict[t.Any, t.Any]:
+    def as_const(self, eval_ctx=None):
         eval_ctx = get_eval_context(self, eval_ctx)
         return dict(x.as_const(eval_ctx) for x in self.items)
 
@@ -669,10 +572,8 @@ class Pair(Helper):
     """A key, value pair for dicts."""
 
     fields = ("key", "value")
-    key: Expr
-    value: Expr
 
-    def as_const(self, eval_ctx: EvalContext | None = None) -> tuple[t.Any, t.Any]:
+    def as_const(self, eval_ctx=None):
         eval_ctx = get_eval_context(self, eval_ctx)
         return self.key.as_const(eval_ctx), self.value.as_const(eval_ctx)
 
@@ -681,10 +582,8 @@ class Keyword(Helper):
     """A key, value pair for keyword arguments where key is a string."""
 
     fields = ("key", "value")
-    key: str
-    value: Expr
 
-    def as_const(self, eval_ctx: EvalContext | None = None) -> tuple[str, t.Any]:
+    def as_const(self, eval_ctx=None):
         eval_ctx = get_eval_context(self, eval_ctx)
         return self.key, self.value.as_const(eval_ctx)
 
@@ -695,11 +594,8 @@ class CondExpr(Expr):
     """
 
     fields = ("test", "expr1", "expr2")
-    test: Expr
-    expr1: Expr
-    expr2: Expr | None
 
-    def as_const(self, eval_ctx: EvalContext | None = None) -> t.Any:
+    def as_const(self, eval_ctx=None):
         eval_ctx = get_eval_context(self, eval_ctx)
         if self.test.as_const(eval_ctx):
             return self.expr1.as_const(eval_ctx)
@@ -711,103 +607,88 @@ class CondExpr(Expr):
         return self.expr2.as_const(eval_ctx)
 
 
-def args_as_const(
-    node: t.Union["_FilterTestCommon", "Call"], eval_ctx: EvalContext | None
-) -> tuple[list[t.Any], dict[t.Any, t.Any]]:
+def args_as_const(node, eval_ctx):
     args = [x.as_const(eval_ctx) for x in node.args]
     kwargs = dict(x.as_const(eval_ctx) for x in node.kwargs)
 
     if node.dyn_args is not None:
         try:
             args.extend(node.dyn_args.as_const(eval_ctx))
-        except Exception as e:
-            raise Impossible() from e
+        except Exception:
+            raise Impossible()
 
     if node.dyn_kwargs is not None:
         try:
             kwargs.update(node.dyn_kwargs.as_const(eval_ctx))
-        except Exception as e:
-            raise Impossible() from e
+        except Exception:
+            raise Impossible()
 
     return args, kwargs
 
 
-class _FilterTestCommon(Expr):
-    fields = ("node", "name", "args", "kwargs", "dyn_args", "dyn_kwargs")
-    node: Expr
-    name: str
-    args: list[Expr]
-    kwargs: list[Pair]
-    dyn_args: Expr | None
-    dyn_kwargs: Expr | None
-    abstract = True
-    _is_filter = True
+class Filter(Expr):
+    """This node applies a filter on an expression.  `name` is the name of
+    the filter, the rest of the fields are the same as for :class:`Call`.
 
-    def as_const(self, eval_ctx: EvalContext | None = None) -> t.Any:
+    If the `node` of a filter is `None` the contents of the last buffer are
+    filtered.  Buffers are created by macros and filter blocks.
+    """
+
+    fields = ("node", "name", "args", "kwargs", "dyn_args", "dyn_kwargs")
+
+    def as_const(self, eval_ctx=None):
         eval_ctx = get_eval_context(self, eval_ctx)
 
-        if eval_ctx.volatile:
+        if eval_ctx.volatile or self.node is None:
             raise Impossible()
 
-        if self._is_filter:
-            env_map = eval_ctx.environment.filters
-        else:
-            env_map = eval_ctx.environment.tests
+        filter_ = self.environment.filters.get(self.name)
 
-        func = env_map.get(self.name)
-        pass_arg = _PassArg.from_obj(func)  # type: ignore
-
-        if func is None or pass_arg is _PassArg.context:
+        if filter_ is None or getattr(filter_, "contextfilter", False) is True:
             raise Impossible()
 
-        if eval_ctx.environment.is_async and (
-            getattr(func, "jinja_async_variant", False) is True
-            or inspect.iscoroutinefunction(func)
+        # We cannot constant handle async filters, so we need to make sure
+        # to not go down this path.
+        if eval_ctx.environment.is_async and getattr(
+            filter_, "asyncfiltervariant", False
         ):
             raise Impossible()
 
         args, kwargs = args_as_const(self, eval_ctx)
         args.insert(0, self.node.as_const(eval_ctx))
 
-        if pass_arg is _PassArg.eval_context:
+        if getattr(filter_, "evalcontextfilter", False) is True:
             args.insert(0, eval_ctx)
-        elif pass_arg is _PassArg.environment:
-            args.insert(0, eval_ctx.environment)
+        elif getattr(filter_, "environmentfilter", False) is True:
+            args.insert(0, self.environment)
 
         try:
-            return func(*args, **kwargs)
-        except Exception as e:
-            raise Impossible() from e
-
-
-class Filter(_FilterTestCommon):
-    """Apply a filter to an expression. ``name`` is the name of the
-    filter, the other fields are the same as :class:`Call`.
-
-    If ``node`` is ``None``, the filter is being used in a filter block
-    and is applied to the content of the block.
-    """
-
-    node: Expr | None  # type: ignore
-
-    def as_const(self, eval_ctx: EvalContext | None = None) -> t.Any:
-        if self.node is None:
+            return filter_(*args, **kwargs)
+        except Exception:
             raise Impossible()
 
-        return super().as_const(eval_ctx=eval_ctx)
 
-
-class Test(_FilterTestCommon):
-    """Apply a test to an expression. ``name`` is the name of the test,
-    the other field are the same as :class:`Call`.
-
-    .. versionchanged:: 3.0
-        ``as_const`` shares the same logic for filters and tests. Tests
-        check for volatile, async, and ``@pass_context`` etc.
-        decorators.
+class Test(Expr):
+    """Applies a test on an expression.  `name` is the name of the test, the
+    rest of the fields are the same as for :class:`Call`.
     """
 
-    _is_filter = False
+    fields = ("node", "name", "args", "kwargs", "dyn_args", "dyn_kwargs")
+
+    def as_const(self, eval_ctx=None):
+        test = self.environment.tests.get(self.name)
+
+        if test is None:
+            raise Impossible()
+
+        eval_ctx = get_eval_context(self, eval_ctx)
+        args, kwargs = args_as_const(self, eval_ctx)
+        args.insert(0, self.node.as_const(eval_ctx))
+
+        try:
+            return test(*args, **kwargs)
+        except Exception:
+            raise Impossible()
 
 
 class Call(Expr):
@@ -819,33 +700,26 @@ class Call(Expr):
     """
 
     fields = ("node", "args", "kwargs", "dyn_args", "dyn_kwargs")
-    node: Expr
-    args: list[Expr]
-    kwargs: list[Keyword]
-    dyn_args: Expr | None
-    dyn_kwargs: Expr | None
 
 
 class Getitem(Expr):
     """Get an attribute or item from an expression and prefer the item."""
 
     fields = ("node", "arg", "ctx")
-    node: Expr
-    arg: Expr
-    ctx: str
 
-    def as_const(self, eval_ctx: EvalContext | None = None) -> t.Any:
+    def as_const(self, eval_ctx=None):
+        eval_ctx = get_eval_context(self, eval_ctx)
         if self.ctx != "load":
             raise Impossible()
-
-        eval_ctx = get_eval_context(self, eval_ctx)
-
         try:
-            return eval_ctx.environment.getitem(
+            return self.environment.getitem(
                 self.node.as_const(eval_ctx), self.arg.as_const(eval_ctx)
             )
-        except Exception as e:
-            raise Impossible() from e
+        except Exception:
+            raise Impossible()
+
+    def can_assign(self):
+        return False
 
 
 class Getattr(Expr):
@@ -854,20 +728,18 @@ class Getattr(Expr):
     """
 
     fields = ("node", "attr", "ctx")
-    node: Expr
-    attr: str
-    ctx: str
 
-    def as_const(self, eval_ctx: EvalContext | None = None) -> t.Any:
+    def as_const(self, eval_ctx=None):
         if self.ctx != "load":
             raise Impossible()
-
-        eval_ctx = get_eval_context(self, eval_ctx)
-
         try:
-            return eval_ctx.environment.getattr(self.node.as_const(eval_ctx), self.attr)
-        except Exception as e:
-            raise Impossible() from e
+            eval_ctx = get_eval_context(self, eval_ctx)
+            return self.environment.getattr(self.node.as_const(eval_ctx), self.attr)
+        except Exception:
+            raise Impossible()
+
+    def can_assign(self):
+        return False
 
 
 class Slice(Expr):
@@ -876,14 +748,11 @@ class Slice(Expr):
     """
 
     fields = ("start", "stop", "step")
-    start: Expr | None
-    stop: Expr | None
-    step: Expr | None
 
-    def as_const(self, eval_ctx: EvalContext | None = None) -> slice:
+    def as_const(self, eval_ctx=None):
         eval_ctx = get_eval_context(self, eval_ctx)
 
-        def const(obj: Expr | None) -> t.Any | None:
+        def const(obj):
             if obj is None:
                 return None
             return obj.as_const(eval_ctx)
@@ -897,9 +766,8 @@ class Concat(Expr):
     """
 
     fields = ("nodes",)
-    nodes: list[Expr]
 
-    def as_const(self, eval_ctx: EvalContext | None = None) -> str:
+    def as_const(self, eval_ctx=None):
         eval_ctx = get_eval_context(self, eval_ctx)
         return "".join(str(x.as_const(eval_ctx)) for x in self.nodes)
 
@@ -910,10 +778,8 @@ class Compare(Expr):
     """
 
     fields = ("expr", "ops")
-    expr: Expr
-    ops: list["Operand"]
 
-    def as_const(self, eval_ctx: EvalContext | None = None) -> t.Any:
+    def as_const(self, eval_ctx=None):
         eval_ctx = get_eval_context(self, eval_ctx)
         result = value = self.expr.as_const(eval_ctx)
 
@@ -926,8 +792,8 @@ class Compare(Expr):
                     return False
 
                 value = new_value
-        except Exception as e:
-            raise Impossible() from e
+        except Exception:
+            raise Impossible()
 
         return result
 
@@ -936,8 +802,6 @@ class Operand(Helper):
     """Holds an operator and an expression."""
 
     fields = ("op", "expr")
-    op: str
-    expr: Expr
 
 
 class Mul(BinExpr):
@@ -953,7 +817,7 @@ class Div(BinExpr):
 
 
 class FloorDiv(BinExpr):
-    """Divides the left by the right node and converts the
+    """Divides the left by the right node and truncates conver the
     result into an integer by truncating.
     """
 
@@ -989,7 +853,7 @@ class And(BinExpr):
 
     operator = "and"
 
-    def as_const(self, eval_ctx: EvalContext | None = None) -> t.Any:
+    def as_const(self, eval_ctx=None):
         eval_ctx = get_eval_context(self, eval_ctx)
         return self.left.as_const(eval_ctx) and self.right.as_const(eval_ctx)
 
@@ -999,7 +863,7 @@ class Or(BinExpr):
 
     operator = "or"
 
-    def as_const(self, eval_ctx: EvalContext | None = None) -> t.Any:
+    def as_const(self, eval_ctx=None):
         eval_ctx = get_eval_context(self, eval_ctx)
         return self.left.as_const(eval_ctx) or self.right.as_const(eval_ctx)
 
@@ -1031,7 +895,6 @@ class EnvironmentAttribute(Expr):
     """
 
     fields = ("name",)
-    name: str
 
 
 class ExtensionAttribute(Expr):
@@ -1043,8 +906,6 @@ class ExtensionAttribute(Expr):
     """
 
     fields = ("identifier", "name")
-    identifier: str
-    name: str
 
 
 class ImportedName(Expr):
@@ -1055,7 +916,6 @@ class ImportedName(Expr):
     """
 
     fields = ("importname",)
-    importname: str
 
 
 class InternalName(Expr):
@@ -1063,13 +923,12 @@ class InternalName(Expr):
     yourself but the parser provides a
     :meth:`~jinja2.parser.Parser.free_identifier` method that creates
     a new identifier for you.  This identifier is not available from the
-    template and is not treated specially by the compiler.
+    template and is not threated specially by the compiler.
     """
 
     fields = ("name",)
-    name: str
 
-    def __init__(self) -> None:
+    def __init__(self):
         raise TypeError(
             "Can't create internal names.  Use the "
             "`free_identifier` method on a parser."
@@ -1080,9 +939,8 @@ class MarkSafe(Expr):
     """Mark the wrapped expression as safe (wrap it as `Markup`)."""
 
     fields = ("expr",)
-    expr: Expr
 
-    def as_const(self, eval_ctx: EvalContext | None = None) -> Markup:
+    def as_const(self, eval_ctx=None):
         eval_ctx = get_eval_context(self, eval_ctx)
         return Markup(self.expr.as_const(eval_ctx))
 
@@ -1095,9 +953,8 @@ class MarkSafeIfAutoescape(Expr):
     """
 
     fields = ("expr",)
-    expr: Expr
 
-    def as_const(self, eval_ctx: EvalContext | None = None) -> Markup | t.Any:
+    def as_const(self, eval_ctx=None):
         eval_ctx = get_eval_context(self, eval_ctx)
         if eval_ctx.volatile:
             raise Impossible()
@@ -1119,9 +976,9 @@ class ContextReference(Expr):
                Getattr(ContextReference(), 'name'))
 
     This is basically equivalent to using the
-    :func:`~jinja2.pass_context` decorator when using the high-level
-    API, which causes a reference to the context to be passed as the
-    first argument to a function.
+    :func:`~jinja2.contextfunction` decorator when using the
+    high-level API, which causes a reference to the context to be passed
+    as the first argument to a function.
     """
 
 
@@ -1146,7 +1003,6 @@ class Scope(Stmt):
     """An artificial scope."""
 
     fields = ("body",)
-    body: list[Node]
 
 
 class OverlayScope(Stmt):
@@ -1164,8 +1020,6 @@ class OverlayScope(Stmt):
     """
 
     fields = ("context", "body")
-    context: Expr
-    body: list[Node]
 
 
 class EvalContextModifier(Stmt):
@@ -1178,7 +1032,6 @@ class EvalContextModifier(Stmt):
     """
 
     fields = ("options",)
-    options: list[Keyword]
 
 
 class ScopedEvalContextModifier(EvalContextModifier):
@@ -1188,13 +1041,12 @@ class ScopedEvalContextModifier(EvalContextModifier):
     """
 
     fields = ("body",)
-    body: list[Node]
 
 
 # make sure nobody creates custom nodes
-def _failing_new(*args: t.Any, **kwargs: t.Any) -> "te.NoReturn":
+def _failing_new(*args, **kwargs):
     raise TypeError("can't create custom node types")
 
 
-NodeType.__new__ = staticmethod(_failing_new)  # type: ignore
+NodeType.__new__ = staticmethod(_failing_new)
 del _failing_new

src/jinja2/optimizer.py

@@ -7,30 +7,22 @@ want. For example, loop unrolling doesn't work because unrolled loops
 would have a different scope. The solution would be a second syntax tree
 that stored the scoping rules.
 """
-
-import typing as t
-
 from . import nodes
 from .visitor import NodeTransformer
 
-if t.TYPE_CHECKING:
-    from .environment import Environment
 
-
-def optimize(node: nodes.Node, environment: "Environment") -> nodes.Node:
+def optimize(node, environment):
     """The context hint can be used to perform an static optimization
     based on the context given."""
     optimizer = Optimizer(environment)
-    return t.cast(nodes.Node, optimizer.visit(node))
+    return optimizer.visit(node)
 
 
 class Optimizer(NodeTransformer):
-    def __init__(self, environment: "Environment | None") -> None:
+    def __init__(self, environment):
         self.environment = environment
 
-    def generic_visit(
-        self, node: nodes.Node, *args: t.Any, **kwargs: t.Any
-    ) -> nodes.Node:
+    def generic_visit(self, node, *args, **kwargs):
         node = super().generic_visit(node, *args, **kwargs)
 
         # Do constant folding. Some other nodes besides Expr have

src/jinja2/parser.py

@@ -1,21 +1,10 @@
 """Parse tokens from the lexer into nodes for the compiler."""
-
-import typing
-import typing as t
-
 from . import nodes
 from .exceptions import TemplateAssertionError
 from .exceptions import TemplateSyntaxError
 from .lexer import describe_token
 from .lexer import describe_token_expr
 
-if t.TYPE_CHECKING:
-    import typing_extensions as te
-
-    from .environment import Environment
-
-_ImportInclude = t.TypeVar("_ImportInclude", nodes.Import, nodes.Include)
-_MacroCall = t.TypeVar("_MacroCall", nodes.Macro, nodes.CallBlock)
 
 _statement_keywords = frozenset(
     [
@@ -35,7 +24,7 @@ _statement_keywords = frozenset(
 )
 _compare_operators = frozenset(["eq", "ne", "lt", "lteq", "gt", "gteq"])
 
-_math_nodes: dict[str, type[nodes.Expr]] = {
+_math_nodes = {
     "add": nodes.Add,
     "sub": nodes.Sub,
     "mul": nodes.Mul,
@@ -50,35 +39,23 @@ class Parser:
     extensions and can be used to parse expressions or statements.
     """
 
-    def __init__(
-        self,
-        environment: "Environment",
-        source: str,
-        name: str | None = None,
-        filename: str | None = None,
-        state: str | None = None,
-    ) -> None:
+    def __init__(self, environment, source, name=None, filename=None, state=None):
         self.environment = environment
+        self.source = source
+        self.grammar = environment.get_grammar()
         self.stream = environment._tokenize(source, name, filename, state)
         self.name = name
         self.filename = filename
         self.closed = False
-        self.extensions: dict[
-            str, t.Callable[[Parser], nodes.Node | list[nodes.Node]]
-        ] = {}
+        self.extensions = {}
         for extension in environment.iter_extensions():
             for tag in extension.tags:
                 self.extensions[tag] = extension.parse
         self._last_identifier = 0
-        self._tag_stack: list[str] = []
-        self._end_token_stack: list[tuple[str, ...]] = []
+        self._tag_stack = []
+        self._end_token_stack = []
 
-    def fail(
-        self,
-        msg: str,
-        lineno: int | None = None,
-        exc: type[TemplateSyntaxError] = TemplateSyntaxError,
-    ) -> "te.NoReturn":
+    def fail(self, msg, lineno=None, exc=TemplateSyntaxError):
         """Convenience method that raises `exc` with the message, passed
         line number or last line number as well as the current name and
         filename.
@@ -87,17 +64,12 @@ class Parser:
             lineno = self.stream.current.lineno
         raise exc(msg, lineno, self.name, self.filename)
 
-    def _fail_ut_eof(
-        self,
-        name: str | None,
-        end_token_stack: list[tuple[str, ...]],
-        lineno: int | None,
-    ) -> "te.NoReturn":
-        expected: set[str] = set()
+    def _fail_ut_eof(self, name, end_token_stack, lineno):
+        expected = []
         for exprs in end_token_stack:
-            expected.update(map(describe_token_expr, exprs))
+            expected.extend(map(describe_token_expr, exprs))
         if end_token_stack:
-            currently_looking: str | None = " or ".join(
+            currently_looking = " or ".join(
                 map(repr, map(describe_token_expr, end_token_stack[-1]))
             )
         else:
@@ -127,40 +99,36 @@ class Parser:
 
         self.fail(" ".join(message), lineno)
 
-    def fail_unknown_tag(self, name: str, lineno: int | None = None) -> "te.NoReturn":
+    def fail_unknown_tag(self, name, lineno=None):
         """Called if the parser encounters an unknown tag.  Tries to fail
         with a human readable error message that could help to identify
         the problem.
         """
-        self._fail_ut_eof(name, self._end_token_stack, lineno)
+        return self._fail_ut_eof(name, self._end_token_stack, lineno)
 
-    def fail_eof(
-        self,
-        end_tokens: tuple[str, ...] | None = None,
-        lineno: int | None = None,
-    ) -> "te.NoReturn":
+    def fail_eof(self, end_tokens=None, lineno=None):
         """Like fail_unknown_tag but for end of template situations."""
         stack = list(self._end_token_stack)
         if end_tokens is not None:
             stack.append(end_tokens)
-        self._fail_ut_eof(None, stack, lineno)
+        return self._fail_ut_eof(None, stack, lineno)
 
-    def is_tuple_end(self, extra_end_rules: tuple[str, ...] | None = None) -> bool:
+    def is_tuple_end(self, extra_end_rules=None):
         """Are we at the end of a tuple?"""
         if self.stream.current.type in ("variable_end", "block_end", "rparen"):
             return True
         elif extra_end_rules is not None:
-            return self.stream.current.test_any(extra_end_rules)  # type: ignore
+            return self.stream.current.test_any(extra_end_rules)
         return False
 
-    def free_identifier(self, lineno: int | None = None) -> nodes.InternalName:
+    def free_identifier(self, lineno=None):
         """Return a new free identifier as :class:`~jinja2.nodes.InternalName`."""
         self._last_identifier += 1
         rv = object.__new__(nodes.InternalName)
         nodes.Node.__init__(rv, f"fi{self._last_identifier}", lineno=lineno)
         return rv
 
-    def parse_statement(self) -> nodes.Node | list[nodes.Node]:
+    def parse_statement(self):
         """Parse a single statement."""
         token = self.stream.current
         if token.type != "name":
@@ -169,8 +137,7 @@ class Parser:
         pop_tag = True
         try:
             if token.value in _statement_keywords:
-                f = getattr(self, f"parse_{self.stream.current.value}")
-                return f()  # type: ignore
+                return getattr(self, "parse_" + self.stream.current.value)()
             if token.value == "call":
                 return self.parse_call_block()
             if token.value == "filter":
@@ -189,9 +156,7 @@ class Parser:
             if pop_tag:
                 self._tag_stack.pop()
 
-    def parse_statements(
-        self, end_tokens: tuple[str, ...], drop_needle: bool = False
-    ) -> list[nodes.Node]:
+    def parse_statements(self, end_tokens, drop_needle=False):
         """Parse multiple statements into a list until one of the end tokens
         is reached.  This is used to parse the body of statements as it also
         parses template data if appropriate.  The parser checks first if the
@@ -218,7 +183,7 @@ class Parser:
             next(self.stream)
         return result
 
-    def parse_set(self) -> nodes.Assign | nodes.AssignBlock:
+    def parse_set(self):
         """Parse an assign statement."""
         lineno = next(self.stream).lineno
         target = self.parse_assign_target(with_namespace=True)
@@ -229,7 +194,7 @@ class Parser:
         body = self.parse_statements(("name:endset",), drop_needle=True)
         return nodes.AssignBlock(target, filter_node, body, lineno=lineno)
 
-    def parse_for(self) -> nodes.For:
+    def parse_for(self):
         """Parse a for loop."""
         lineno = self.stream.expect("name:for").lineno
         target = self.parse_assign_target(extra_end_rules=("name:in",))
@@ -248,10 +213,10 @@ class Parser:
             else_ = self.parse_statements(("name:endfor",), drop_needle=True)
         return nodes.For(target, iter, body, else_, test, recursive, lineno=lineno)
 
-    def parse_if(self) -> nodes.If:
+    def parse_if(self):
         """Parse an if construct."""
         node = result = nodes.If(lineno=self.stream.expect("name:if").lineno)
-        while True:
+        while 1:
             node.test = self.parse_tuple(with_condexpr=False)
             node.body = self.parse_statements(("name:elif", "name:else", "name:endif"))
             node.elif_ = []
@@ -266,10 +231,10 @@ class Parser:
             break
         return result
 
-    def parse_with(self) -> nodes.With:
+    def parse_with(self):
         node = nodes.With(lineno=next(self.stream).lineno)
-        targets: list[nodes.Expr] = []
-        values: list[nodes.Expr] = []
+        targets = []
+        values = []
         while self.stream.current.type != "block_end":
             if targets:
                 self.stream.expect("comma")
@@ -283,17 +248,16 @@ class Parser:
         node.body = self.parse_statements(("name:endwith",), drop_needle=True)
         return node
 
-    def parse_autoescape(self) -> nodes.Scope:
+    def parse_autoescape(self):
         node = nodes.ScopedEvalContextModifier(lineno=next(self.stream).lineno)
         node.options = [nodes.Keyword("autoescape", self.parse_expression())]
         node.body = self.parse_statements(("name:endautoescape",), drop_needle=True)
         return nodes.Scope([node])
 
-    def parse_block(self) -> nodes.Block:
+    def parse_block(self):
         node = nodes.Block(lineno=next(self.stream).lineno)
         node.name = self.stream.expect("name").value
         node.scoped = self.stream.skip_if("name:scoped")
-        node.required = self.stream.skip_if("name:required")
 
         # common problem people encounter when switching from django
         # to jinja.  we do not support hyphens in block names, so let's
@@ -305,30 +269,15 @@ class Parser:
             )
 
         node.body = self.parse_statements(("name:endblock",), drop_needle=True)
-
-        # enforce that required blocks only contain whitespace or comments
-        # by asserting that the body, if not empty, is just TemplateData nodes
-        # with whitespace data
-        if node.required:
-            for body_node in node.body:
-                if not isinstance(body_node, nodes.Output) or any(
-                    not isinstance(output_node, nodes.TemplateData)
-                    or not output_node.data.isspace()
-                    for output_node in body_node.nodes
-                ):
-                    self.fail("Required blocks can only contain comments or whitespace")
-
         self.stream.skip_if("name:" + node.name)
         return node
 
-    def parse_extends(self) -> nodes.Extends:
+    def parse_extends(self):
         node = nodes.Extends(lineno=next(self.stream).lineno)
         node.template = self.parse_expression()
         return node
 
-    def parse_import_context(
-        self, node: _ImportInclude, default: bool
-    ) -> _ImportInclude:
+    def parse_import_context(self, node, default):
         if self.stream.current.test_any(
             "name:with", "name:without"
         ) and self.stream.look().test("name:context"):
@@ -338,7 +287,7 @@ class Parser:
             node.with_context = default
         return node
 
-    def parse_include(self) -> nodes.Include:
+    def parse_include(self):
         node = nodes.Include(lineno=next(self.stream).lineno)
         node.template = self.parse_expression()
         if self.stream.current.test("name:ignore") and self.stream.look().test(
@@ -350,30 +299,30 @@ class Parser:
             node.ignore_missing = False
         return self.parse_import_context(node, True)
 
-    def parse_import(self) -> nodes.Import:
+    def parse_import(self):
         node = nodes.Import(lineno=next(self.stream).lineno)
         node.template = self.parse_expression()
         self.stream.expect("name:as")
         node.target = self.parse_assign_target(name_only=True).name
         return self.parse_import_context(node, False)
 
-    def parse_from(self) -> nodes.FromImport:
+    def parse_from(self):
         node = nodes.FromImport(lineno=next(self.stream).lineno)
         node.template = self.parse_expression()
         self.stream.expect("name:import")
         node.names = []
 
-        def parse_context() -> bool:
-            if self.stream.current.value in {
+        def parse_context():
+            if self.stream.current.value in (
                 "with",
                 "without",
-            } and self.stream.look().test("name:context"):
+            ) and self.stream.look().test("name:context"):
                 node.with_context = next(self.stream).value == "with"
                 self.stream.skip()
                 return True
             return False
 
-        while True:
+        while 1:
             if node.names:
                 self.stream.expect("comma")
             if self.stream.current.type == "name":
@@ -399,9 +348,9 @@ class Parser:
             node.with_context = False
         return node
 
-    def parse_signature(self, node: _MacroCall) -> None:
-        args = node.args = []
-        defaults = node.defaults = []
+    def parse_signature(self, node):
+        node.args = args = []
+        node.defaults = defaults = []
         self.stream.expect("lparen")
         while self.stream.current.type != "rparen":
             if args:
@@ -415,7 +364,7 @@ class Parser:
             args.append(arg)
         self.stream.expect("rparen")
 
-    def parse_call_block(self) -> nodes.CallBlock:
+    def parse_call_block(self):
         node = nodes.CallBlock(lineno=next(self.stream).lineno)
         if self.stream.current.type == "lparen":
             self.parse_signature(node)
@@ -423,27 +372,26 @@ class Parser:
             node.args = []
             node.defaults = []
 
-        call_node = self.parse_expression()
-        if not isinstance(call_node, nodes.Call):
+        node.call = self.parse_expression()
+        if not isinstance(node.call, nodes.Call):
             self.fail("expected call", node.lineno)
-        node.call = call_node
         node.body = self.parse_statements(("name:endcall",), drop_needle=True)
         return node
 
-    def parse_filter_block(self) -> nodes.FilterBlock:
+    def parse_filter_block(self):
         node = nodes.FilterBlock(lineno=next(self.stream).lineno)
-        node.filter = self.parse_filter(None, start_inline=True)  # type: ignore
+        node.filter = self.parse_filter(None, start_inline=True)
         node.body = self.parse_statements(("name:endfilter",), drop_needle=True)
         return node
 
-    def parse_macro(self) -> nodes.Macro:
+    def parse_macro(self):
         node = nodes.Macro(lineno=next(self.stream).lineno)
         node.name = self.parse_assign_target(name_only=True).name
         self.parse_signature(node)
         node.body = self.parse_statements(("name:endmacro",), drop_needle=True)
         return node
 
-    def parse_print(self) -> nodes.Output:
+    def parse_print(self):
         node = nodes.Output(lineno=next(self.stream).lineno)
         node.nodes = []
         while self.stream.current.type != "block_end":
@@ -452,27 +400,13 @@ class Parser:
             node.nodes.append(self.parse_expression())
         return node
 
-    @typing.overload
-    def parse_assign_target(
-        self, with_tuple: bool = ..., name_only: "te.Literal[True]" = ...
-    ) -> nodes.Name: ...
-
-    @typing.overload
     def parse_assign_target(
         self,
-        with_tuple: bool = True,
-        name_only: bool = False,
-        extra_end_rules: tuple[str, ...] | None = None,
-        with_namespace: bool = False,
-    ) -> nodes.NSRef | nodes.Name | nodes.Tuple: ...
-
-    def parse_assign_target(
-        self,
-        with_tuple: bool = True,
-        name_only: bool = False,
-        extra_end_rules: tuple[str, ...] | None = None,
-        with_namespace: bool = False,
-    ) -> nodes.NSRef | nodes.Name | nodes.Tuple:
+        with_tuple=True,
+        name_only=False,
+        extra_end_rules=None,
+        with_namespace=False,
+    ):
         """Parse an assignment target.  As Jinja allows assignments to
         tuples, this function can parse all allowed assignment targets.  Per
         default assignments to tuples are parsed, that can be disable however
@@ -481,31 +415,29 @@ class Parser:
         parameter is forwarded to the tuple parsing function.  If
         `with_namespace` is enabled, a namespace assignment may be parsed.
         """
-        target: nodes.Expr
-
-        if name_only:
+        if with_namespace and self.stream.look().type == "dot":
+            token = self.stream.expect("name")
+            next(self.stream)  # dot
+            attr = self.stream.expect("name")
+            target = nodes.NSRef(token.value, attr.value, lineno=token.lineno)
+        elif name_only:
             token = self.stream.expect("name")
             target = nodes.Name(token.value, "store", lineno=token.lineno)
         else:
             if with_tuple:
                 target = self.parse_tuple(
-                    simplified=True,
-                    extra_end_rules=extra_end_rules,
-                    with_namespace=with_namespace,
+                    simplified=True, extra_end_rules=extra_end_rules
                 )
             else:
-                target = self.parse_primary(with_namespace=with_namespace)
-
+                target = self.parse_primary()
             target.set_ctx("store")
-
         if not target.can_assign():
             self.fail(
-                f"can't assign to {type(target).__name__.lower()!r}", target.lineno
+                f"can't assign to {target.__class__.__name__.lower()!r}", target.lineno
             )
+        return target
 
-        return target  # type: ignore
-
-    def parse_expression(self, with_condexpr: bool = True) -> nodes.Expr:
+    def parse_expression(self, with_condexpr=True):
         """Parse an expression.  Per default all expressions are parsed, if
         the optional `with_condexpr` parameter is set to `False` conditional
         expressions are not parsed.
@@ -514,11 +446,9 @@ class Parser:
             return self.parse_condexpr()
         return self.parse_or()
 
-    def parse_condexpr(self) -> nodes.Expr:
+    def parse_condexpr(self):
         lineno = self.stream.current.lineno
         expr1 = self.parse_or()
-        expr3: nodes.Expr | None
-
         while self.stream.skip_if("name:if"):
             expr2 = self.parse_or()
             if self.stream.skip_if("name:else"):
@@ -529,7 +459,7 @@ class Parser:
             lineno = self.stream.current.lineno
         return expr1
 
-    def parse_or(self) -> nodes.Expr:
+    def parse_or(self):
         lineno = self.stream.current.lineno
         left = self.parse_and()
         while self.stream.skip_if("name:or"):
@@ -538,7 +468,7 @@ class Parser:
             lineno = self.stream.current.lineno
         return left
 
-    def parse_and(self) -> nodes.Expr:
+    def parse_and(self):
         lineno = self.stream.current.lineno
         left = self.parse_not()
         while self.stream.skip_if("name:and"):
@@ -547,17 +477,17 @@ class Parser:
             lineno = self.stream.current.lineno
         return left
 
-    def parse_not(self) -> nodes.Expr:
+    def parse_not(self):
         if self.stream.current.test("name:not"):
             lineno = next(self.stream).lineno
             return nodes.Not(self.parse_not(), lineno=lineno)
         return self.parse_compare()
 
-    def parse_compare(self) -> nodes.Expr:
+    def parse_compare(self):
         lineno = self.stream.current.lineno
         expr = self.parse_math1()
         ops = []
-        while True:
+        while 1:
             token_type = self.stream.current.type
             if token_type in _compare_operators:
                 next(self.stream)
@@ -576,7 +506,7 @@ class Parser:
             return expr
         return nodes.Compare(expr, ops, lineno=lineno)
 
-    def parse_math1(self) -> nodes.Expr:
+    def parse_math1(self):
         lineno = self.stream.current.lineno
         left = self.parse_concat()
         while self.stream.current.type in ("add", "sub"):
@@ -587,7 +517,7 @@ class Parser:
             lineno = self.stream.current.lineno
         return left
 
-    def parse_concat(self) -> nodes.Expr:
+    def parse_concat(self):
         lineno = self.stream.current.lineno
         args = [self.parse_math2()]
         while self.stream.current.type == "tilde":
@@ -597,7 +527,7 @@ class Parser:
             return args[0]
         return nodes.Concat(args, lineno=lineno)
 
-    def parse_math2(self) -> nodes.Expr:
+    def parse_math2(self):
         lineno = self.stream.current.lineno
         left = self.parse_pow()
         while self.stream.current.type in ("mul", "div", "floordiv", "mod"):
@@ -608,7 +538,7 @@ class Parser:
             lineno = self.stream.current.lineno
         return left
 
-    def parse_pow(self) -> nodes.Expr:
+    def parse_pow(self):
         lineno = self.stream.current.lineno
         left = self.parse_unary()
         while self.stream.current.type == "pow":
@@ -618,11 +548,9 @@ class Parser:
             lineno = self.stream.current.lineno
         return left
 
-    def parse_unary(self, with_filter: bool = True) -> nodes.Expr:
+    def parse_unary(self, with_filter=True):
         token_type = self.stream.current.type
         lineno = self.stream.current.lineno
-        node: nodes.Expr
-
         if token_type == "sub":
             next(self.stream)
             node = nodes.Neg(self.parse_unary(False), lineno=lineno)
@@ -636,25 +564,16 @@ class Parser:
             node = self.parse_filter_expr(node)
         return node
 
-    def parse_primary(self, with_namespace: bool = False) -> nodes.Expr:
-        """Parse a name or literal value. If ``with_namespace`` is enabled, also
-        parse namespace attr refs, for use in assignments."""
+    def parse_primary(self):
         token = self.stream.current
-        node: nodes.Expr
         if token.type == "name":
-            next(self.stream)
             if token.value in ("true", "false", "True", "False"):
                 node = nodes.Const(token.value in ("true", "True"), lineno=token.lineno)
             elif token.value in ("none", "None"):
                 node = nodes.Const(None, lineno=token.lineno)
-            elif with_namespace and self.stream.current.type == "dot":
-                # If namespace attributes are allowed at this point, and the next
-                # token is a dot, produce a namespace reference.
-                next(self.stream)
-                attr = self.stream.expect("name")
-                node = nodes.NSRef(token.value, attr.value, lineno=token.lineno)
             else:
                 node = nodes.Name(token.value, "load", lineno=token.lineno)
+            next(self.stream)
         elif token.type == "string":
             next(self.stream)
             buf = [token.value]
@@ -680,21 +599,19 @@ class Parser:
 
     def parse_tuple(
         self,
-        simplified: bool = False,
-        with_condexpr: bool = True,
-        extra_end_rules: tuple[str, ...] | None = None,
-        explicit_parentheses: bool = False,
-        with_namespace: bool = False,
-    ) -> nodes.Tuple | nodes.Expr:
+        simplified=False,
+        with_condexpr=True,
+        extra_end_rules=None,
+        explicit_parentheses=False,
+    ):
         """Works like `parse_expression` but if multiple expressions are
         delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created.
         This method could also return a regular expression instead of a tuple
         if no commas where found.
 
         The default parsing mode is a full tuple.  If `simplified` is `True`
-        only names and literals are parsed; ``with_namespace`` allows namespace
-        attr refs as well. The `no_condexpr` parameter is forwarded to
-        :meth:`parse_expression`.
+        only names and literals are parsed.  The `no_condexpr` parameter is
+        forwarded to :meth:`parse_expression`.
 
         Because tuples do not require delimiters and may end in a bogus comma
         an extra hint is needed that marks the end of a tuple.  For example
@@ -707,19 +624,17 @@ class Parser:
         """
         lineno = self.stream.current.lineno
         if simplified:
-
-            def parse() -> nodes.Expr:
-                return self.parse_primary(with_namespace=with_namespace)
-
+            parse = self.parse_primary
+        elif with_condexpr:
+            parse = self.parse_expression
         else:
 
-            def parse() -> nodes.Expr:
-                return self.parse_expression(with_condexpr=with_condexpr)
+            def parse():
+                return self.parse_expression(with_condexpr=False)
 
-        args: list[nodes.Expr] = []
+        args = []
         is_tuple = False
-
-        while True:
+        while 1:
             if args:
                 self.stream.expect("comma")
             if self.is_tuple_end(extra_end_rules):
@@ -747,9 +662,9 @@ class Parser:
 
         return nodes.Tuple(args, "load", lineno=lineno)
 
-    def parse_list(self) -> nodes.List:
+    def parse_list(self):
         token = self.stream.expect("lbracket")
-        items: list[nodes.Expr] = []
+        items = []
         while self.stream.current.type != "rbracket":
             if items:
                 self.stream.expect("comma")
@@ -759,9 +674,9 @@ class Parser:
         self.stream.expect("rbracket")
         return nodes.List(items, lineno=token.lineno)
 
-    def parse_dict(self) -> nodes.Dict:
+    def parse_dict(self):
         token = self.stream.expect("lbrace")
-        items: list[nodes.Pair] = []
+        items = []
         while self.stream.current.type != "rbrace":
             if items:
                 self.stream.expect("comma")
@@ -774,8 +689,8 @@ class Parser:
         self.stream.expect("rbrace")
         return nodes.Dict(items, lineno=token.lineno)
 
-    def parse_postfix(self, node: nodes.Expr) -> nodes.Expr:
-        while True:
+    def parse_postfix(self, node):
+        while 1:
             token_type = self.stream.current.type
             if token_type == "dot" or token_type == "lbracket":
                 node = self.parse_subscript(node)
@@ -787,11 +702,11 @@ class Parser:
                 break
         return node
 
-    def parse_filter_expr(self, node: nodes.Expr) -> nodes.Expr:
-        while True:
+    def parse_filter_expr(self, node):
+        while 1:
             token_type = self.stream.current.type
             if token_type == "pipe":
-                node = self.parse_filter(node)  # type: ignore
+                node = self.parse_filter(node)
             elif token_type == "name" and self.stream.current.value == "is":
                 node = self.parse_test(node)
             # calls are valid both after postfix expressions (getattr
@@ -802,10 +717,8 @@ class Parser:
                 break
         return node
 
-    def parse_subscript(self, node: nodes.Expr) -> nodes.Getattr | nodes.Getitem:
+    def parse_subscript(self, node):
         token = next(self.stream)
-        arg: nodes.Expr
-
         if token.type == "dot":
             attr_token = self.stream.current
             next(self.stream)
@@ -818,7 +731,7 @@ class Parser:
             arg = nodes.Const(attr_token.value, lineno=attr_token.lineno)
             return nodes.Getitem(node, arg, "load", lineno=token.lineno)
         if token.type == "lbracket":
-            args: list[nodes.Expr] = []
+            args = []
             while self.stream.current.type != "rbracket":
                 if args:
                     self.stream.expect("comma")
@@ -831,9 +744,8 @@ class Parser:
             return nodes.Getitem(node, arg, "load", lineno=token.lineno)
         self.fail("expected subscript expression", token.lineno)
 
-    def parse_subscribed(self) -> nodes.Expr:
+    def parse_subscribed(self):
         lineno = self.stream.current.lineno
-        args: list[nodes.Expr | None]
 
         if self.stream.current.type == "colon":
             next(self.stream)
@@ -861,35 +773,25 @@ class Parser:
         else:
             args.append(None)
 
-        return nodes.Slice(lineno=lineno, *args)  # noqa: B026
+        return nodes.Slice(lineno=lineno, *args)
 
-    def parse_call_args(
-        self,
-    ) -> tuple[
-        list[nodes.Expr],
-        list[nodes.Keyword],
-        nodes.Expr | None,
-        nodes.Expr | None,
-    ]:
+    def parse_call(self, node):
         token = self.stream.expect("lparen")
         args = []
         kwargs = []
-        dyn_args = None
-        dyn_kwargs = None
+        dyn_args = dyn_kwargs = None
         require_comma = False
 
-        def ensure(expr: bool) -> None:
+        def ensure(expr):
             if not expr:
                 self.fail("invalid syntax for function call expression", token.lineno)
 
         while self.stream.current.type != "rparen":
             if require_comma:
                 self.stream.expect("comma")
-
                 # support for trailing comma
                 if self.stream.current.type == "rparen":
                     break
-
             if self.stream.current.type == "mul":
                 ensure(dyn_args is None and dyn_kwargs is None)
                 next(self.stream)
@@ -915,20 +817,13 @@ class Parser:
                     args.append(self.parse_expression())
 
             require_comma = True
-
         self.stream.expect("rparen")
-        return args, kwargs, dyn_args, dyn_kwargs
 
-    def parse_call(self, node: nodes.Expr) -> nodes.Call:
-        # The lparen will be expected in parse_call_args, but the lineno
-        # needs to be recorded before the stream is advanced.
-        token = self.stream.current
-        args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
+        if node is None:
+            return args, kwargs, dyn_args, dyn_kwargs
         return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno)
 
-    def parse_filter(
-        self, node: nodes.Expr | None, start_inline: bool = False
-    ) -> nodes.Expr | None:
+    def parse_filter(self, node, start_inline=False):
         while self.stream.current.type == "pipe" or start_inline:
             if not start_inline:
                 next(self.stream)
@@ -938,7 +833,7 @@ class Parser:
                 next(self.stream)
                 name += "." + self.stream.expect("name").value
             if self.stream.current.type == "lparen":
-                args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
+                args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None)
             else:
                 args = []
                 kwargs = []
@@ -949,7 +844,7 @@ class Parser:
             start_inline = False
         return node
 
-    def parse_test(self, node: nodes.Expr) -> nodes.Expr:
+    def parse_test(self, node):
         token = next(self.stream)
         if self.stream.current.test("name:not"):
             next(self.stream)
@@ -961,10 +856,10 @@ class Parser:
             next(self.stream)
             name += "." + self.stream.expect("name").value
         dyn_args = dyn_kwargs = None
-        kwargs: list[nodes.Keyword] = []
+        kwargs = []
         if self.stream.current.type == "lparen":
-            args, kwargs, dyn_args, dyn_kwargs = self.parse_call_args()
-        elif self.stream.current.type in {
+            args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None)
+        elif self.stream.current.type in (
             "name",
             "string",
             "integer",
@@ -972,7 +867,7 @@ class Parser:
             "lparen",
             "lbracket",
             "lbrace",
-        } and not self.stream.current.test_any("name:else", "name:or", "name:and"):
+        ) and not self.stream.current.test_any("name:else", "name:or", "name:and"):
             if self.stream.current.test("name:is"):
                 self.fail("You cannot chain multiple tests with is")
             arg_node = self.parse_primary()
@@ -987,15 +882,15 @@ class Parser:
             node = nodes.Not(node, lineno=token.lineno)
         return node
 
-    def subparse(self, end_tokens: tuple[str, ...] | None = None) -> list[nodes.Node]:
-        body: list[nodes.Node] = []
-        data_buffer: list[nodes.Node] = []
+    def subparse(self, end_tokens=None):
+        body = []
+        data_buffer = []
         add_data = data_buffer.append
 
         if end_tokens is not None:
             self._end_token_stack.append(end_tokens)
 
-        def flush_data() -> None:
+        def flush_data():
             if data_buffer:
                 lineno = data_buffer[0].lineno
                 body.append(nodes.Output(data_buffer[:], lineno=lineno))
@@ -1032,10 +927,26 @@ class Parser:
         finally:
             if end_tokens is not None:
                 self._end_token_stack.pop()
+
         return body
 
-    def parse(self) -> nodes.Template:
-        """Parse the whole template into a `Template` node."""
+    def parse_old(self):
         result = nodes.Template(self.subparse(), lineno=1)
         result.set_environment(self.environment)
         return result
+
+    def parse(self):
+        """Parse the whole template into a `Template` node."""
+        from .new_parser import JinjaSemantics, parse_template
+
+        result = parse_template(
+            self.grammar.parse(
+                self.source.rstrip('\n'),
+                whitespace='',
+                parseinfo=True,
+                semantics=JinjaSemantics(),
+            )
+        )
+        result.set_environment(self.environment)
+
+        return result

src/jinja2/sandbox.py

@@ -1,14 +1,11 @@
 """A sandbox layer that ensures unsafe operations cannot be performed.
 Useful when the template itself comes from an untrusted source.
 """
-
 import operator
 import types
-import typing as t
-from _string import formatter_field_name_split  # type: ignore
+from _string import formatter_field_name_split
 from collections import abc
 from collections import deque
-from functools import update_wrapper
 from string import Formatter
 
 from markupsafe import EscapeFormatter
@@ -16,19 +13,15 @@ from markupsafe import Markup
 
 from .environment import Environment
 from .exceptions import SecurityError
-from .runtime import Context
-from .runtime import Undefined
-
-F = t.TypeVar("F", bound=t.Callable[..., t.Any])
 
 #: maximum number of items a range may produce
 MAX_RANGE = 100000
 
 #: Unsafe function attributes.
-UNSAFE_FUNCTION_ATTRIBUTES: set[str] = set()
+UNSAFE_FUNCTION_ATTRIBUTES = set()
 
 #: Unsafe method attributes. Function attributes are unsafe for methods too.
-UNSAFE_METHOD_ATTRIBUTES: set[str] = set()
+UNSAFE_METHOD_ATTRIBUTES = set()
 
 #: unsafe generator attributes.
 UNSAFE_GENERATOR_ATTRIBUTES = {"gi_frame", "gi_code"}
@@ -39,7 +32,7 @@ UNSAFE_COROUTINE_ATTRIBUTES = {"cr_frame", "cr_code"}
 #: unsafe attributes on async generators
 UNSAFE_ASYNC_GENERATOR_ATTRIBUTES = {"ag_code", "ag_frame"}
 
-_mutable_spec: tuple[tuple[type[t.Any], frozenset[str]], ...] = (
+_mutable_spec = (
     (
         abc.MutableSet,
         frozenset(
@@ -61,9 +54,7 @@ _mutable_spec: tuple[tuple[type[t.Any], frozenset[str]], ...] = (
     ),
     (
         abc.MutableSequence,
-        frozenset(
-            ["append", "clear", "pop", "reverse", "insert", "sort", "extend", "remove"]
-        ),
+        frozenset(["append", "reverse", "insert", "sort", "extend", "remove"]),
     ),
     (
         deque,
@@ -84,7 +75,48 @@ _mutable_spec: tuple[tuple[type[t.Any], frozenset[str]], ...] = (
 )
 
 
-def safe_range(*args: int) -> range:
+class _MagicFormatMapping(abc.Mapping):
+    """This class implements a dummy wrapper to fix a bug in the Python
+    standard library for string formatting.
+
+    See https://bugs.python.org/issue13598 for information about why
+    this is necessary.
+    """
+
+    def __init__(self, args, kwargs):
+        self._args = args
+        self._kwargs = kwargs
+        self._last_index = 0
+
+    def __getitem__(self, key):
+        if key == "":
+            idx = self._last_index
+            self._last_index += 1
+            try:
+                return self._args[idx]
+            except LookupError:
+                pass
+            key = str(idx)
+        return self._kwargs[key]
+
+    def __iter__(self):
+        return iter(self._kwargs)
+
+    def __len__(self):
+        return len(self._kwargs)
+
+
+def inspect_format_method(callable):
+    if not isinstance(
+        callable, (types.MethodType, types.BuiltinMethodType)
+    ) or callable.__name__ not in ("format", "format_map"):
+        return None
+    obj = callable.__self__
+    if isinstance(obj, str):
+        return obj
+
+
+def safe_range(*args):
     """A range that can't generate ranges with a length of more than
     MAX_RANGE items.
     """
@@ -99,7 +131,7 @@ def safe_range(*args: int) -> range:
     return rng
 
 
-def unsafe(f: F) -> F:
+def unsafe(f):
     """Marks a function or method as unsafe.
 
     .. code-block: python
@@ -108,11 +140,11 @@ def unsafe(f: F) -> F:
         def delete(self):
             pass
     """
-    f.unsafe_callable = True  # type: ignore
+    f.unsafe_callable = True
     return f
 
 
-def is_internal_attribute(obj: t.Any, attr: str) -> bool:
+def is_internal_attribute(obj, attr):
     """Test if the attribute given is an internal python attribute.  For
     example this function returns `True` for the `func_code` attribute of
     python objects.  This is useful if the environment method
@@ -149,7 +181,7 @@ def is_internal_attribute(obj: t.Any, attr: str) -> bool:
     return attr.startswith("__")
 
 
-def modifies_known_mutable(obj: t.Any, attr: str) -> bool:
+def modifies_known_mutable(obj, attr):
     """This function checks if an attribute on a builtin mutable object
     (list, dict, set or deque) or the corresponding ABCs would modify it
     if called.
@@ -190,7 +222,7 @@ class SandboxedEnvironment(Environment):
     #: default callback table for the binary operators.  A copy of this is
     #: available on each instance of a sandboxed environment as
     #: :attr:`binop_table`
-    default_binop_table: dict[str, t.Callable[[t.Any, t.Any], t.Any]] = {
+    default_binop_table = {
         "+": operator.add,
         "-": operator.sub,
         "*": operator.mul,
@@ -203,10 +235,7 @@ class SandboxedEnvironment(Environment):
     #: default callback table for the unary operators.  A copy of this is
     #: available on each instance of a sandboxed environment as
     #: :attr:`unop_table`
-    default_unop_table: dict[str, t.Callable[[t.Any], t.Any]] = {
-        "+": operator.pos,
-        "-": operator.neg,
-    }
+    default_unop_table = {"+": operator.pos, "-": operator.neg}
 
     #: a set of binary operators that should be intercepted.  Each operator
     #: that is added to this set (empty by default) is delegated to the
@@ -222,7 +251,7 @@ class SandboxedEnvironment(Environment):
     #: interested in.
     #:
     #: .. versionadded:: 2.6
-    intercepted_binops: frozenset[str] = frozenset()
+    intercepted_binops = frozenset()
 
     #: a set of unary operators that should be intercepted.  Each operator
     #: that is added to this set (empty by default) is delegated to the
@@ -237,15 +266,32 @@ class SandboxedEnvironment(Environment):
     #: interested in.
     #:
     #: .. versionadded:: 2.6
-    intercepted_unops: frozenset[str] = frozenset()
+    intercepted_unops = frozenset()
 
-    def __init__(self, *args: t.Any, **kwargs: t.Any) -> None:
-        super().__init__(*args, **kwargs)
+    def intercept_unop(self, operator):
+        """Called during template compilation with the name of a unary
+        operator to check if it should be intercepted at runtime.  If this
+        method returns `True`, :meth:`call_unop` is executed for this unary
+        operator.  The default implementation of :meth:`call_unop` will use
+        the :attr:`unop_table` dictionary to perform the operator with the
+        same logic as the builtin one.
+
+        The following unary operators are interceptable: ``+`` and ``-``
+
+        Intercepted calls are always slower than the native operator call,
+        so make sure only to intercept the ones you are interested in.
+
+        .. versionadded:: 2.6
+        """
+        return False
+
+    def __init__(self, *args, **kwargs):
+        Environment.__init__(self, *args, **kwargs)
         self.globals["range"] = safe_range
         self.binop_table = self.default_binop_table.copy()
         self.unop_table = self.default_unop_table.copy()
 
-    def is_safe_attribute(self, obj: t.Any, attr: str, value: t.Any) -> bool:
+    def is_safe_attribute(self, obj, attr, value):
         """The sandboxed environment will call this method to check if the
         attribute of an object is safe to access.  Per default all attributes
         starting with an underscore are considered private as well as the
@@ -254,20 +300,17 @@ class SandboxedEnvironment(Environment):
         """
         return not (attr.startswith("_") or is_internal_attribute(obj, attr))
 
-    def is_safe_callable(self, obj: t.Any) -> bool:
-        """Check if an object is safely callable. By default callables
-        are considered safe unless decorated with :func:`unsafe`.
-
-        This also recognizes the Django convention of setting
-        ``func.alters_data = True``.
+    def is_safe_callable(self, obj):
+        """Check if an object is safely callable.  Per default a function is
+        considered safe unless the `unsafe_callable` attribute exists and is
+        True.  Override this method to alter the behavior, but this won't
+        affect the `unsafe` decorator from this module.
         """
         return not (
             getattr(obj, "unsafe_callable", False) or getattr(obj, "alters_data", False)
         )
 
-    def call_binop(
-        self, context: Context, operator: str, left: t.Any, right: t.Any
-    ) -> t.Any:
+    def call_binop(self, context, operator, left, right):
         """For intercepted binary operator calls (:meth:`intercepted_binops`)
         this function is executed instead of the builtin operator.  This can
         be used to fine tune the behavior of certain operators.
@@ -276,7 +319,7 @@ class SandboxedEnvironment(Environment):
         """
         return self.binop_table[operator](left, right)
 
-    def call_unop(self, context: Context, operator: str, arg: t.Any) -> t.Any:
+    def call_unop(self, context, operator, arg):
         """For intercepted unary operator calls (:meth:`intercepted_unops`)
         this function is executed instead of the builtin operator.  This can
         be used to fine tune the behavior of certain operators.
@@ -285,7 +328,7 @@ class SandboxedEnvironment(Environment):
         """
         return self.unop_table[operator](arg)
 
-    def getitem(self, obj: t.Any, argument: str | t.Any) -> t.Any | Undefined:
+    def getitem(self, obj, argument):
         """Subscribe an object from sandboxed code."""
         try:
             return obj[argument]
@@ -301,15 +344,12 @@ class SandboxedEnvironment(Environment):
                     except AttributeError:
                         pass
                     else:
-                        fmt = self.wrap_str_format(value)
-                        if fmt is not None:
-                            return fmt
                         if self.is_safe_attribute(obj, argument, value):
                             return value
                         return self.unsafe_undefined(obj, argument)
         return self.undefined(obj=obj, name=argument)
 
-    def getattr(self, obj: t.Any, attribute: str) -> t.Any | Undefined:
+    def getattr(self, obj, attribute):
         """Subscribe an object from sandboxed code and prefer the
         attribute.  The attribute passed *must* be a bytestring.
         """
@@ -321,76 +361,49 @@ class SandboxedEnvironment(Environment):
             except (TypeError, LookupError):
                 pass
         else:
-            fmt = self.wrap_str_format(value)
-            if fmt is not None:
-                return fmt
             if self.is_safe_attribute(obj, attribute, value):
                 return value
             return self.unsafe_undefined(obj, attribute)
         return self.undefined(obj=obj, name=attribute)
 
-    def unsafe_undefined(self, obj: t.Any, attribute: str) -> Undefined:
+    def unsafe_undefined(self, obj, attribute):
         """Return an undefined object for unsafe attributes."""
         return self.undefined(
             f"access to attribute {attribute!r} of"
-            f" {type(obj).__name__!r} object is unsafe.",
+            f" {obj.__class__.__name__!r} object is unsafe.",
             name=attribute,
             obj=obj,
             exc=SecurityError,
         )
 
-    def wrap_str_format(self, value: t.Any) -> t.Callable[..., str] | None:
-        """If the given value is a ``str.format`` or ``str.format_map`` method,
-        return a new function than handles sandboxing. This is done at access
-        rather than in :meth:`call`, so that calls made without ``call`` are
-        also sandboxed.
+    def format_string(self, s, args, kwargs, format_func=None):
+        """If a format call is detected, then this is routed through this
+        method so that our safety sandbox can be used for it.
         """
-        if not isinstance(
-            value, (types.MethodType, types.BuiltinMethodType)
-        ) or value.__name__ not in ("format", "format_map"):
-            return None
-
-        f_self: t.Any = value.__self__
-
-        if not isinstance(f_self, str):
-            return None
-
-        str_type: type[str] = type(f_self)
-        is_format_map = value.__name__ == "format_map"
-        formatter: SandboxedFormatter
-
-        if isinstance(f_self, Markup):
-            formatter = SandboxedEscapeFormatter(self, escape=f_self.escape)
+        if isinstance(s, Markup):
+            formatter = SandboxedEscapeFormatter(self, s.escape)
         else:
             formatter = SandboxedFormatter(self)
 
-        vformat = formatter.vformat
+        if format_func is not None and format_func.__name__ == "format_map":
+            if len(args) != 1 or kwargs:
+                raise TypeError(
+                    "format_map() takes exactly one argument"
+                    f" {len(args) + (kwargs is not None)} given"
+                )
 
-        def wrapper(*args: t.Any, **kwargs: t.Any) -> str:
-            if is_format_map:
-                if kwargs:
-                    raise TypeError("format_map() takes no keyword arguments")
+            kwargs = args[0]
+            args = None
 
-                if len(args) != 1:
-                    raise TypeError(
-                        f"format_map() takes exactly one argument ({len(args)} given)"
-                    )
+        kwargs = _MagicFormatMapping(args, kwargs)
+        rv = formatter.vformat(s, args, kwargs)
+        return type(s)(rv)
 
-                kwargs = args[0]
-                args = ()
-
-            return str_type(vformat(f_self, args, kwargs))
-
-        return update_wrapper(wrapper, value)
-
-    def call(
-        __self,  # noqa: B902
-        __context: Context,
-        __obj: t.Any,
-        *args: t.Any,
-        **kwargs: t.Any,
-    ) -> t.Any:
+    def call(__self, __context, __obj, *args, **kwargs):  # noqa: B902
         """Call an object from sandboxed code."""
+        fmt = inspect_format_method(__obj)
+        if fmt is not None:
+            return __self.format_string(fmt, args, kwargs, __obj)
 
         # the double prefixes are to avoid double keyword argument
         # errors when proxying the call.
@@ -405,21 +418,17 @@ class ImmutableSandboxedEnvironment(SandboxedEnvironment):
     `dict` by using the :func:`modifies_known_mutable` function.
     """
 
-    def is_safe_attribute(self, obj: t.Any, attr: str, value: t.Any) -> bool:
-        if not super().is_safe_attribute(obj, attr, value):
+    def is_safe_attribute(self, obj, attr, value):
+        if not SandboxedEnvironment.is_safe_attribute(self, obj, attr, value):
             return False
-
         return not modifies_known_mutable(obj, attr)
 
 
-class SandboxedFormatter(Formatter):
-    def __init__(self, env: Environment, **kwargs: t.Any) -> None:
+class SandboxedFormatterMixin:
+    def __init__(self, env):
         self._env = env
-        super().__init__(**kwargs)
 
-    def get_field(
-        self, field_name: str, args: t.Sequence[t.Any], kwargs: t.Mapping[str, t.Any]
-    ) -> tuple[t.Any, str]:
+    def get_field(self, field_name, args, kwargs):
         first, rest = formatter_field_name_split(field_name)
         obj = self.get_value(first, args, kwargs)
         for is_attr, i in rest:
@@ -430,5 +439,13 @@ class SandboxedFormatter(Formatter):
         return obj, first
 
 
-class SandboxedEscapeFormatter(SandboxedFormatter, EscapeFormatter):
-    pass
+class SandboxedFormatter(SandboxedFormatterMixin, Formatter):
+    def __init__(self, env):
+        SandboxedFormatterMixin.__init__(self, env)
+        Formatter.__init__(self)
+
+
+class SandboxedEscapeFormatter(SandboxedFormatterMixin, EscapeFormatter):
+    def __init__(self, env, escape):
+        SandboxedFormatterMixin.__init__(self, env)
+        EscapeFormatter.__init__(self, escape)

src/jinja2/tests.py

@@ -1,33 +1,32 @@
 """Built-in template tests used with the ``is`` operator."""
-
 import operator
-import typing as t
+import re
 from collections import abc
 from numbers import Number
 
 from .runtime import Undefined
-from .utils import pass_environment
 
-if t.TYPE_CHECKING:
-    from .environment import Environment
+number_re = re.compile(r"^-?\d+(\.\d+)?$")
+regex_type = type(number_re)
+test_callable = callable
 
 
-def test_odd(value: int) -> bool:
+def test_odd(value):
     """Return true if the variable is odd."""
     return value % 2 == 1
 
 
-def test_even(value: int) -> bool:
+def test_even(value):
     """Return true if the variable is even."""
     return value % 2 == 0
 
 
-def test_divisibleby(value: int, num: int) -> bool:
+def test_divisibleby(value, num):
     """Check if a variable is divisible by a number."""
     return value % num == 0
 
 
-def test_defined(value: t.Any) -> bool:
+def test_defined(value):
     """Return true if the variable is defined:
 
     .. sourcecode:: jinja
@@ -44,57 +43,17 @@ def test_defined(value: t.Any) -> bool:
     return not isinstance(value, Undefined)
 
 
-def test_undefined(value: t.Any) -> bool:
+def test_undefined(value):
     """Like :func:`defined` but the other way round."""
     return isinstance(value, Undefined)
 
 
-@pass_environment
-def test_filter(env: "Environment", value: str) -> bool:
-    """Check if a filter exists by name. Useful if a filter may be
-    optionally available.
-
-    .. code-block:: jinja
-
-        {% if 'markdown' is filter %}
-            {{ value | markdown }}
-        {% else %}
-            {{ value }}
-        {% endif %}
-
-    .. versionadded:: 3.0
-    """
-    return value in env.filters
-
-
-@pass_environment
-def test_test(env: "Environment", value: str) -> bool:
-    """Check if a test exists by name. Useful if a test may be
-    optionally available.
-
-    .. code-block:: jinja
-
-        {% if 'loud' is test %}
-            {% if value is loud %}
-                {{ value|upper }}
-            {% else %}
-                {{ value|lower }}
-            {% endif %}
-        {% else %}
-            {{ value }}
-        {% endif %}
-
-    .. versionadded:: 3.0
-    """
-    return value in env.tests
-
-
-def test_none(value: t.Any) -> bool:
+def test_none(value):
     """Return true if the variable is none."""
     return value is None
 
 
-def test_boolean(value: t.Any) -> bool:
+def test_boolean(value):
     """Return true if the object is a boolean value.
 
     .. versionadded:: 2.11
@@ -102,7 +61,7 @@ def test_boolean(value: t.Any) -> bool:
     return value is True or value is False
 
 
-def test_false(value: t.Any) -> bool:
+def test_false(value):
     """Return true if the object is False.
 
     .. versionadded:: 2.11
@@ -110,7 +69,7 @@ def test_false(value: t.Any) -> bool:
     return value is False
 
 
-def test_true(value: t.Any) -> bool:
+def test_true(value):
     """Return true if the object is True.
 
     .. versionadded:: 2.11
@@ -119,7 +78,7 @@ def test_true(value: t.Any) -> bool:
 
 
 # NOTE: The existing 'number' test matches booleans and floats
-def test_integer(value: t.Any) -> bool:
+def test_integer(value):
     """Return true if the object is an integer.
 
     .. versionadded:: 2.11
@@ -128,7 +87,7 @@ def test_integer(value: t.Any) -> bool:
 
 
 # NOTE: The existing 'number' test matches booleans and integers
-def test_float(value: t.Any) -> bool:
+def test_float(value):
     """Return true if the object is a float.
 
     .. versionadded:: 2.11
@@ -136,22 +95,22 @@ def test_float(value: t.Any) -> bool:
     return isinstance(value, float)
 
 
-def test_lower(value: str) -> bool:
+def test_lower(value):
     """Return true if the variable is lowercased."""
     return str(value).islower()
 
 
-def test_upper(value: str) -> bool:
+def test_upper(value):
     """Return true if the variable is uppercased."""
     return str(value).isupper()
 
 
-def test_string(value: t.Any) -> bool:
+def test_string(value):
     """Return true if the object is a string."""
     return isinstance(value, str)
 
 
-def test_mapping(value: t.Any) -> bool:
+def test_mapping(value):
     """Return true if the object is a mapping (dict etc.).
 
     .. versionadded:: 2.6
@@ -159,25 +118,24 @@ def test_mapping(value: t.Any) -> bool:
     return isinstance(value, abc.Mapping)
 
 
-def test_number(value: t.Any) -> bool:
+def test_number(value):
     """Return true if the variable is a number."""
     return isinstance(value, Number)
 
 
-def test_sequence(value: t.Any) -> bool:
+def test_sequence(value):
     """Return true if the variable is a sequence. Sequences are variables
     that are iterable.
     """
     try:
         len(value)
-        value.__getitem__  # noqa B018
+        value.__getitem__
     except Exception:
         return False
-
     return True
 
 
-def test_sameas(value: t.Any, other: t.Any) -> bool:
+def test_sameas(value, other):
     """Check if an object points to the same memory address than another
     object:
 
@@ -190,22 +148,21 @@ def test_sameas(value: t.Any, other: t.Any) -> bool:
     return value is other
 
 
-def test_iterable(value: t.Any) -> bool:
+def test_iterable(value):
     """Check if it's possible to iterate over an object."""
     try:
         iter(value)
     except TypeError:
         return False
-
     return True
 
 
-def test_escaped(value: t.Any) -> bool:
+def test_escaped(value):
     """Check if the value is escaped."""
     return hasattr(value, "__html__")
 
 
-def test_in(value: t.Any, seq: t.Container[t.Any]) -> bool:
+def test_in(value, seq):
     """Check if value is in seq.
 
     .. versionadded:: 2.10
@@ -219,8 +176,6 @@ TESTS = {
     "divisibleby": test_divisibleby,
     "defined": test_defined,
     "undefined": test_undefined,
-    "filter": test_filter,
-    "test": test_test,
     "none": test_none,
     "boolean": test_boolean,
     "false": test_false,
@@ -234,7 +189,7 @@ TESTS = {
     "number": test_number,
     "sequence": test_sequence,
     "iterable": test_iterable,
-    "callable": callable,
+    "callable": test_callable,
     "sameas": test_sameas,
     "escaped": test_escaped,
     "in": test_in,

src/jinja2/utils.py

@@ -1,107 +1,85 @@
-import enum
 import json
 import os
 import re
-import typing as t
 from collections import abc
 from collections import deque
 from random import choice
 from random import randrange
 from threading import Lock
-from types import CodeType
 from urllib.parse import quote_from_bytes
 
-import markupsafe
+from markupsafe import escape
+from markupsafe import Markup
 
-if t.TYPE_CHECKING:
-    import typing_extensions as te
+_word_split_re = re.compile(r"(\s+)")
+_lead_pattern = "|".join(map(re.escape, ("(", "<", "&lt;")))
+_trail_pattern = "|".join(map(re.escape, (".", ",", ")", ">", "\n", "&gt;")))
+_punctuation_re = re.compile(
+    fr"^(?P<lead>(?:{_lead_pattern})*)(?P<middle>.*?)(?P<trail>(?:{_trail_pattern})*)$"
+)
+_simple_email_re = re.compile(r"^\S+@[a-zA-Z0-9._-]+\.[a-zA-Z0-9._-]+$")
+_striptags_re = re.compile(r"(<!--.*?-->|<[^>]*>)")
+_entity_re = re.compile(r"&([^;]+);")
+_letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+_digits = "0123456789"
 
-F = t.TypeVar("F", bound=t.Callable[..., t.Any])
+# special singleton representing missing values for the runtime
+missing = type("MissingType", (), {"__repr__": lambda x: "missing"})()
 
-
-class _MissingType:
-    def __repr__(self) -> str:
-        return "missing"
-
-    def __reduce__(self) -> str:
-        return "missing"
-
-
-missing: t.Any = _MissingType()
-"""Special singleton representing missing values for the runtime."""
-
-internal_code: t.MutableSet[CodeType] = set()
+# internal code
+internal_code = set()
 
 concat = "".join
 
+_slash_escape = "\\/" not in json.dumps("/")
 
-def pass_context(f: F) -> F:
-    """Pass the :class:`~jinja2.runtime.Context` as the first argument
-    to the decorated function when called while rendering a template.
 
-    Can be used on functions, filters, and tests.
+def contextfunction(f):
+    """This decorator can be used to mark a function or method context callable.
+    A context callable is passed the active :class:`Context` as first argument when
+    called from the template.  This is useful if a function wants to get access
+    to the context or functions provided on the context object.  For example
+    a function that returns a sorted list of template variables the current
+    template exports could look like this::
 
-    If only ``Context.eval_context`` is needed, use
-    :func:`pass_eval_context`. If only ``Context.environment`` is
-    needed, use :func:`pass_environment`.
-
-    .. versionadded:: 3.0.0
-        Replaces ``contextfunction`` and ``contextfilter``.
+        @contextfunction
+        def get_exported_names(context):
+            return sorted(context.exported_vars)
     """
-    f.jinja_pass_arg = _PassArg.context  # type: ignore
+    f.contextfunction = True
     return f
 
 
-def pass_eval_context(f: F) -> F:
-    """Pass the :class:`~jinja2.nodes.EvalContext` as the first argument
-    to the decorated function when called while rendering a template.
-    See :ref:`eval-context`.
+def evalcontextfunction(f):
+    """This decorator can be used to mark a function or method as an eval
+    context callable.  This is similar to the :func:`contextfunction`
+    but instead of passing the context, an evaluation context object is
+    passed.  For more information about the eval context, see
+    :ref:`eval-context`.
 
-    Can be used on functions, filters, and tests.
-
-    If only ``EvalContext.environment`` is needed, use
-    :func:`pass_environment`.
-
-    .. versionadded:: 3.0.0
-        Replaces ``evalcontextfunction`` and ``evalcontextfilter``.
+    .. versionadded:: 2.4
     """
-    f.jinja_pass_arg = _PassArg.eval_context  # type: ignore
+    f.evalcontextfunction = True
     return f
 
 
-def pass_environment(f: F) -> F:
-    """Pass the :class:`~jinja2.Environment` as the first argument to
-    the decorated function when called while rendering a template.
-
-    Can be used on functions, filters, and tests.
-
-    .. versionadded:: 3.0.0
-        Replaces ``environmentfunction`` and ``environmentfilter``.
+def environmentfunction(f):
+    """This decorator can be used to mark a function or method as environment
+    callable.  This decorator works exactly like the :func:`contextfunction`
+    decorator just that the first argument is the active :class:`Environment`
+    and not context.
     """
-    f.jinja_pass_arg = _PassArg.environment  # type: ignore
+    f.environmentfunction = True
     return f
 
 
-class _PassArg(enum.Enum):
-    context = enum.auto()
-    eval_context = enum.auto()
-    environment = enum.auto()
-
-    @classmethod
-    def from_obj(cls, obj: F) -> t.Optional["_PassArg"]:
-        if hasattr(obj, "jinja_pass_arg"):
-            return obj.jinja_pass_arg  # type: ignore
-
-        return None
-
-
-def internalcode(f: F) -> F:
+def internalcode(f):
     """Marks the function as internally used"""
     internal_code.add(f.__code__)
     return f
 
 
-def is_undefined(obj: t.Any) -> bool:
+def is_undefined(obj):
     """Check if the object passed is undefined.  This does nothing more than
     performing an instance check against :class:`Undefined` but looks nicer.
     This can be used for custom filters or tests that want to react to
@@ -118,26 +96,26 @@ def is_undefined(obj: t.Any) -> bool:
     return isinstance(obj, Undefined)
 
 
-def consume(iterable: t.Iterable[t.Any]) -> None:
+def consume(iterable):
     """Consumes an iterable without doing anything with it."""
     for _ in iterable:
         pass
 
 
-def clear_caches() -> None:
+def clear_caches():
     """Jinja keeps internal caches for environments and lexers.  These are
     used so that Jinja doesn't have to recreate environments and lexers all
     the time.  Normally you don't have to care about that but if you are
     measuring memory consumption you may want to clean the caches.
     """
-    from .environment import get_spontaneous_environment
+    from .environment import _spontaneous_environments
     from .lexer import _lexer_cache
 
-    get_spontaneous_environment.cache_clear()
+    _spontaneous_environments.clear()
     _lexer_cache.clear()
 
 
-def import_string(import_name: str, silent: bool = False) -> t.Any:
+def import_string(import_name, silent=False):
     """Imports an object based on a string.  This is useful if you want to
     use import paths as endpoints or something similar.  An import path can
     be specified either in dotted notation (``xml.sax.saxutils.escape``)
@@ -161,7 +139,7 @@ def import_string(import_name: str, silent: bool = False) -> t.Any:
             raise
 
 
-def open_if_exists(filename: str, mode: str = "rb") -> t.IO[t.Any] | None:
+def open_if_exists(filename, mode="rb"):
     """Returns a file descriptor for the filename if that file exists,
     otherwise ``None``.
     """
@@ -171,7 +149,7 @@ def open_if_exists(filename: str, mode: str = "rb") -> t.IO[t.Any] | None:
     return open(filename, mode)
 
 
-def object_type_repr(obj: t.Any) -> str:
+def object_type_repr(obj):
     """Returns the name of the object's type.  For some recognized
     singletons the name of the object is returned instead. (For
     example for `None` and `Ellipsis`).
@@ -189,170 +167,76 @@ def object_type_repr(obj: t.Any) -> str:
     return f"{cls.__module__}.{cls.__name__} object"
 
 
-def pformat(obj: t.Any) -> str:
-    """Format an object using :func:`pprint.pformat`."""
+def pformat(obj):
+    """Format an object using :func:`pprint.pformat`.
+    """
     from pprint import pformat
 
     return pformat(obj)
 
 
-_http_re = re.compile(
-    r"""
-    ^
-    (
-        (https?://|www\.)  # scheme or www
-        (([\w%-]+\.)+)?  # subdomain
-        (
-            [a-z]{2,63}  # basic tld
-        |
-            xn--[\w%]{2,59}  # idna tld
-        )
-    |
-        ([\w%-]{2,63}\.)+  # basic domain
-        (com|net|int|edu|gov|org|info|mil)  # basic tld
-    |
-        (https?://)  # scheme
-        (
-            (([\d]{1,3})(\.[\d]{1,3}){3})  # IPv4
-        |
-            (\[([\da-f]{0,4}:){2}([\da-f]{0,4}:?){1,6}])  # IPv6
-        )
-    )
-    (?::[\d]{1,5})?  # port
-    (?:[/?#]\S*)?  # path, query, and fragment
-    $
-    """,
-    re.IGNORECASE | re.VERBOSE,
-)
-_email_re = re.compile(r"^\S+@\w[\w.-]*\.\w+$")
+def urlize(text, trim_url_limit=None, rel=None, target=None):
+    """Converts any URLs in text into clickable links. Works on http://,
+    https:// and www. links. Links can have trailing punctuation (periods,
+    commas, close-parens) and leading punctuation (opening parens) and
+    it'll still do the right thing.
 
+    If trim_url_limit is not None, the URLs in link text will be limited
+    to trim_url_limit characters.
 
-def urlize(
-    text: str,
-    trim_url_limit: int | None = None,
-    rel: str | None = None,
-    target: str | None = None,
-    extra_schemes: t.Iterable[str] | None = None,
-) -> str:
-    """Convert URLs in text into clickable links.
+    If nofollow is True, the URLs in link text will get a rel="nofollow"
+    attribute.
 
-    This may not recognize links in some situations. Usually, a more
-    comprehensive formatter, such as a Markdown library, is a better
-    choice.
-
-    Works on ``http://``, ``https://``, ``www.``, ``mailto:``, and email
-    addresses. Links with trailing punctuation (periods, commas, closing
-    parentheses) and leading punctuation (opening parentheses) are
-    recognized excluding the punctuation. Email addresses that include
-    header fields are not recognized (for example,
-    ``mailto:address@example.com?cc=copy@example.com``).
-
-    :param text: Original text containing URLs to link.
-    :param trim_url_limit: Shorten displayed URL values to this length.
-    :param target: Add the ``target`` attribute to links.
-    :param rel: Add the ``rel`` attribute to links.
-    :param extra_schemes: Recognize URLs that start with these schemes
-        in addition to the default behavior.
-
-    .. versionchanged:: 3.0
-        The ``extra_schemes`` parameter was added.
-
-    .. versionchanged:: 3.0
-        Generate ``https://`` links for URLs without a scheme.
-
-    .. versionchanged:: 3.0
-        The parsing rules were updated. Recognize email addresses with
-        or without the ``mailto:`` scheme. Validate IP addresses. Ignore
-        parentheses and brackets in more cases.
+    If target is not None, a target attribute will be added to the link.
     """
-    if trim_url_limit is not None:
 
-        def trim_url(x: str) -> str:
-            if len(x) > trim_url_limit:
-                return f"{x[:trim_url_limit]}..."
+    def trim_url(x, limit=trim_url_limit):
+        if limit is not None:
+            return x[:limit] + ("..." if len(x) >= limit else "")
 
-            return x
+        return x
 
-    else:
-
-        def trim_url(x: str) -> str:
-            return x
-
-    words = re.split(r"(\s+)", str(markupsafe.escape(text)))
-    rel_attr = f' rel="{markupsafe.escape(rel)}"' if rel else ""
-    target_attr = f' target="{markupsafe.escape(target)}"' if target else ""
+    words = _word_split_re.split(str(escape(text)))
+    rel_attr = f' rel="{escape(rel)}"' if rel else ""
+    target_attr = f' target="{escape(target)}"' if target else ""
 
     for i, word in enumerate(words):
-        head, middle, tail = "", word, ""
-        match = re.match(r"^([(<]|&lt;)+", middle)
-
+        match = _punctuation_re.match(word)
         if match:
-            head = match.group()
-            middle = middle[match.end() :]
-
-        # Unlike lead, which is anchored to the start of the string,
-        # need to check that the string ends with any of the characters
-        # before trying to match all of them, to avoid backtracking.
-        if middle.endswith((")", ">", ".", ",", "\n", "&gt;")):
-            match = re.search(r"([)>.,\n]|&gt;)+$", middle)
-
-            if match:
-                tail = match.group()
-                middle = middle[: match.start()]
-
-        # Prefer balancing parentheses in URLs instead of ignoring a
-        # trailing character.
-        for start_char, end_char in ("(", ")"), ("<", ">"), ("&lt;", "&gt;"):
-            start_count = middle.count(start_char)
-
-            if start_count <= middle.count(end_char):
-                # Balanced, or lighter on the left
-                continue
-
-            # Move as many as possible from the tail to balance
-            for _ in range(min(start_count, tail.count(end_char))):
-                end_index = tail.index(end_char) + len(end_char)
-                # Move anything in the tail before the end char too
-                middle += tail[:end_index]
-                tail = tail[end_index:]
-
-        if _http_re.match(middle):
-            if middle.startswith("https://") or middle.startswith("http://"):
+            lead, middle, trail = match.groups()
+            if middle.startswith("www.") or (
+                "@" not in middle
+                and not middle.startswith("http://")
+                and not middle.startswith("https://")
+                and len(middle) > 0
+                and middle[0] in _letters + _digits
+                and (
+                    middle.endswith(".org")
+                    or middle.endswith(".net")
+                    or middle.endswith(".com")
+                )
+            ):
+                middle = (
+                    f'<a href="http://{middle}"{rel_attr}{target_attr}>'
+                    f"{trim_url(middle)}</a>"
+                )
+            if middle.startswith("http://") or middle.startswith("https://"):
                 middle = (
                     f'<a href="{middle}"{rel_attr}{target_attr}>{trim_url(middle)}</a>'
                 )
-            else:
-                middle = (
-                    f'<a href="https://{middle}"{rel_attr}{target_attr}>'
-                    f"{trim_url(middle)}</a>"
-                )
-
-        elif middle.startswith("mailto:") and _email_re.match(middle[7:]):
-            middle = f'<a href="{middle}">{middle[7:]}</a>'
-
-        elif (
-            "@" in middle
-            and not middle.startswith("www.")
-            # ignore values like `@a@b`
-            and not middle.startswith("@")
-            and ":" not in middle
-            and _email_re.match(middle)
-        ):
-            middle = f'<a href="mailto:{middle}">{middle}</a>'
-
-        elif extra_schemes is not None:
-            for scheme in extra_schemes:
-                if middle != scheme and middle.startswith(scheme):
-                    middle = f'<a href="{middle}"{rel_attr}{target_attr}>{middle}</a>'
-
-        words[i] = f"{head}{middle}{tail}"
-
+            if (
+                "@" in middle
+                and not middle.startswith("www.")
+                and ":" not in middle
+                and _simple_email_re.match(middle)
+            ):
+                middle = f'<a href="mailto:{middle}">{middle}</a>'
+            if lead + middle + trail != word:
+                words[i] = lead + middle + trail
     return "".join(words)
 
 
-def generate_lorem_ipsum(
-    n: int = 5, html: bool = True, min: int = 20, max: int = 100
-) -> str:
+def generate_lorem_ipsum(n=5, html=True, min=20, max=100):
     """Generate some lorem ipsum for the template."""
     from .constants import LOREM_IPSUM_WORDS
 
@@ -389,25 +273,24 @@ def generate_lorem_ipsum(
             p.append(word)
 
         # ensure that the paragraph ends with a dot.
-        p_str = " ".join(p)
-
-        if p_str.endswith(","):
-            p_str = p_str[:-1] + "."
-        elif not p_str.endswith("."):
-            p_str += "."
-
-        result.append(p_str)
+        p = " ".join(p)
+        if p.endswith(","):
+            p = p[:-1] + "."
+        elif not p.endswith("."):
+            p += "."
+        result.append(p)
 
     if not html:
         return "\n\n".join(result)
-    return markupsafe.Markup(
-        "\n".join(f"<p>{markupsafe.escape(x)}</p>" for x in result)
-    )
+    return Markup("\n".join(f"<p>{escape(x)}</p>" for x in result))
 
 
-def url_quote(obj: t.Any, charset: str = "utf-8", for_qs: bool = False) -> str:
+def url_quote(obj, charset="utf-8", for_qs=False):
     """Quote a string for use in a URL using the given charset.
 
+    This function is misnamed, it is a wrapper around
+    :func:`urllib.parse.quote`.
+
     :param obj: String or bytes to quote. Other types are converted to
         string then encoded to bytes using the given charset.
     :param charset: Encode text to bytes using this charset.
@@ -428,6 +311,18 @@ def url_quote(obj: t.Any, charset: str = "utf-8", for_qs: bool = False) -> str:
     return rv
 
 
+def unicode_urlencode(obj, charset="utf-8", for_qs=False):
+    import warnings
+
+    warnings.warn(
+        "'unicode_urlencode' has been renamed to 'url_quote'. The old"
+        " name will be removed in version 3.1.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    return url_quote(obj, charset=charset, for_qs=for_qs)
+
+
 @abc.MutableMapping.register
 class LRUCache:
     """A simple LRU Cache implementation."""
@@ -436,13 +331,13 @@ class LRUCache:
     # scale.  But as long as it's only used as storage for templates this
     # won't do any harm.
 
-    def __init__(self, capacity: int) -> None:
+    def __init__(self, capacity):
         self.capacity = capacity
-        self._mapping: dict[t.Any, t.Any] = {}
-        self._queue: deque[t.Any] = deque()
+        self._mapping = {}
+        self._queue = deque()
         self._postinit()
 
-    def _postinit(self) -> None:
+    def _postinit(self):
         # alias all queue methods for faster lookup
         self._popleft = self._queue.popleft
         self._pop = self._queue.pop
@@ -450,35 +345,35 @@ class LRUCache:
         self._wlock = Lock()
         self._append = self._queue.append
 
-    def __getstate__(self) -> t.Mapping[str, t.Any]:
+    def __getstate__(self):
         return {
             "capacity": self.capacity,
             "_mapping": self._mapping,
             "_queue": self._queue,
         }
 
-    def __setstate__(self, d: t.Mapping[str, t.Any]) -> None:
+    def __setstate__(self, d):
         self.__dict__.update(d)
         self._postinit()
 
-    def __getnewargs__(self) -> tuple[t.Any, ...]:
+    def __getnewargs__(self):
         return (self.capacity,)
 
-    def copy(self) -> "te.Self":
+    def copy(self):
         """Return a shallow copy of the instance."""
         rv = self.__class__(self.capacity)
         rv._mapping.update(self._mapping)
         rv._queue.extend(self._queue)
         return rv
 
-    def get(self, key: t.Any, default: t.Any = None) -> t.Any:
+    def get(self, key, default=None):
         """Return an item from the cache dict or `default`"""
         try:
             return self[key]
         except KeyError:
             return default
 
-    def setdefault(self, key: t.Any, default: t.Any = None) -> t.Any:
+    def setdefault(self, key, default=None):
         """Set `default` if the key is not in the cache otherwise
         leave unchanged. Return the value of this key.
         """
@@ -488,32 +383,35 @@ class LRUCache:
             self[key] = default
             return default
 
-    def clear(self) -> None:
+    def clear(self):
         """Clear the cache."""
-        with self._wlock:
+        self._wlock.acquire()
+        try:
             self._mapping.clear()
             self._queue.clear()
+        finally:
+            self._wlock.release()
 
-    def __contains__(self, key: t.Any) -> bool:
+    def __contains__(self, key):
         """Check if a key exists in this cache."""
         return key in self._mapping
 
-    def __len__(self) -> int:
+    def __len__(self):
         """Return the current size of the cache."""
         return len(self._mapping)
 
-    def __repr__(self) -> str:
-        return f"<{type(self).__name__} {self._mapping!r}>"
+    def __repr__(self):
+        return f"<{self.__class__.__name__} {self._mapping!r}>"
 
-    def __getitem__(self, key: t.Any) -> t.Any:
+    def __getitem__(self, key):
         """Get an item from the cache. Moves the item up so that it has the
         highest priority then.
 
         Raise a `KeyError` if it does not exist.
         """
-        with self._wlock:
+        self._wlock.acquire()
+        try:
             rv = self._mapping[key]
-
             if self._queue[-1] != key:
                 try:
                     self._remove(key)
@@ -522,54 +420,58 @@ class LRUCache:
                     # when we read, ignore the ValueError that we would
                     # get otherwise.
                     pass
-
                 self._append(key)
-
             return rv
+        finally:
+            self._wlock.release()
 
-    def __setitem__(self, key: t.Any, value: t.Any) -> None:
+    def __setitem__(self, key, value):
         """Sets the value for an item. Moves the item up so that it
         has the highest priority then.
         """
-        with self._wlock:
+        self._wlock.acquire()
+        try:
             if key in self._mapping:
                 self._remove(key)
             elif len(self._mapping) == self.capacity:
                 del self._mapping[self._popleft()]
-
             self._append(key)
             self._mapping[key] = value
+        finally:
+            self._wlock.release()
 
-    def __delitem__(self, key: t.Any) -> None:
+    def __delitem__(self, key):
         """Remove an item from the cache dict.
         Raise a `KeyError` if it does not exist.
         """
-        with self._wlock:
+        self._wlock.acquire()
+        try:
             del self._mapping[key]
-
             try:
                 self._remove(key)
             except ValueError:
                 pass
+        finally:
+            self._wlock.release()
 
-    def items(self) -> t.Iterable[tuple[t.Any, t.Any]]:
+    def items(self):
         """Return a list of items."""
         result = [(key, self._mapping[key]) for key in list(self._queue)]
         result.reverse()
         return result
 
-    def values(self) -> t.Iterable[t.Any]:
+    def values(self):
         """Return a list of all values."""
         return [x[1] for x in self.items()]
 
-    def keys(self) -> t.Iterable[t.Any]:
+    def keys(self):
         """Return a list of all keys ordered by most recent usage."""
         return list(self)
 
-    def __iter__(self) -> t.Iterator[t.Any]:
+    def __iter__(self):
         return reversed(tuple(self._queue))
 
-    def __reversed__(self) -> t.Iterator[t.Any]:
+    def __reversed__(self):
         """Iterate over the keys in the cache dict, oldest items
         coming first.
         """
@@ -579,11 +481,11 @@ class LRUCache:
 
 
 def select_autoescape(
-    enabled_extensions: t.Collection[str] = ("html", "htm", "xml"),
-    disabled_extensions: t.Collection[str] = (),
-    default_for_string: bool = True,
-    default: bool = False,
-) -> t.Callable[[str | None], bool]:
+    enabled_extensions=("html", "htm", "xml"),
+    disabled_extensions=(),
+    default_for_string=True,
+    default=False,
+):
     """Intelligently sets the initial value of autoescaping based on the
     filename of the template.  This is the recommended way to configure
     autoescaping if you do not want to write a custom function yourself.
@@ -621,7 +523,7 @@ def select_autoescape(
     enabled_patterns = tuple(f".{x.lstrip('.').lower()}" for x in enabled_extensions)
     disabled_patterns = tuple(f".{x.lstrip('.').lower()}" for x in disabled_extensions)
 
-    def autoescape(template_name: str | None) -> bool:
+    def autoescape(template_name):
         if template_name is None:
             return default_for_string
         template_name = template_name.lower()
@@ -634,44 +536,34 @@ def select_autoescape(
     return autoescape
 
 
-def htmlsafe_json_dumps(
-    obj: t.Any, dumps: t.Callable[..., str] | None = None, **kwargs: t.Any
-) -> markupsafe.Markup:
-    """Serialize an object to a string of JSON with :func:`json.dumps`,
-    then replace HTML-unsafe characters with Unicode escapes and mark
-    the result safe with :class:`~markupsafe.Markup`.
+def htmlsafe_json_dumps(obj, dumper=None, **kwargs):
+    """Works exactly like :func:`dumps` but is safe for use in ``<script>``
+    tags.  It accepts the same arguments and returns a JSON string.  Note that
+    this is available in templates through the ``|tojson`` filter which will
+    also mark the result as safe.  Due to how this function escapes certain
+    characters this is safe even if used outside of ``<script>`` tags.
 
-    This is available in templates as the ``|tojson`` filter.
+    The following characters are escaped in strings:
 
-    The following characters are escaped: ``<``, ``>``, ``&``, ``'``.
+    -   ``<``
+    -   ``>``
+    -   ``&``
+    -   ``'``
 
-    The returned string is safe to render in HTML documents and
-    ``<script>`` tags. The exception is in HTML attributes that are
-    double quoted; either use single quotes or the ``|forceescape``
-    filter.
-
-    :param obj: The object to serialize to JSON.
-    :param dumps: The ``dumps`` function to use. Defaults to
-        ``env.policies["json.dumps_function"]``, which defaults to
-        :func:`json.dumps`.
-    :param kwargs: Extra arguments to pass to ``dumps``. Merged onto
-        ``env.policies["json.dumps_kwargs"]``.
-
-    .. versionchanged:: 3.0
-        The ``dumper`` parameter is renamed to ``dumps``.
-
-    .. versionadded:: 2.9
+    This makes it safe to embed such strings in any place in HTML with the
+    notable exception of double quoted attributes.  In that case single
+    quote your attributes or HTML escape it in addition.
     """
-    if dumps is None:
-        dumps = json.dumps
-
-    return markupsafe.Markup(
-        dumps(obj, **kwargs)
+    if dumper is None:
+        dumper = json.dumps
+    rv = (
+        dumper(obj, **kwargs)
         .replace("<", "\\u003c")
         .replace(">", "\\u003e")
         .replace("&", "\\u0026")
         .replace("'", "\\u0027")
     )
+    return Markup(rv)
 
 
 class Cycler:
@@ -700,24 +592,24 @@ class Cycler:
     .. versionadded:: 2.1
     """
 
-    def __init__(self, *items: t.Any) -> None:
+    def __init__(self, *items):
         if not items:
             raise RuntimeError("at least one item has to be provided")
         self.items = items
         self.pos = 0
 
-    def reset(self) -> None:
+    def reset(self):
         """Resets the current item to the first item."""
         self.pos = 0
 
     @property
-    def current(self) -> t.Any:
+    def current(self):
         """Return the current item. Equivalent to the item that will be
         returned next time :meth:`next` is called.
         """
         return self.items[self.pos]
 
-    def next(self) -> t.Any:
+    def next(self):
         """Return the current item, then advance :attr:`current` to the
         next item.
         """
@@ -731,11 +623,11 @@ class Cycler:
 class Joiner:
     """A joining helper for templates."""
 
-    def __init__(self, sep: str = ", ") -> None:
+    def __init__(self, sep=", "):
         self.sep = sep
         self.used = False
 
-    def __call__(self) -> str:
+    def __call__(self):
         if not self.used:
             self.used = True
             return ""
@@ -746,21 +638,29 @@ class Namespace:
     """A namespace object that can hold arbitrary attributes.  It may be
     initialized from a dictionary or with keyword arguments."""
 
-    def __init__(*args: t.Any, **kwargs: t.Any) -> None:  # noqa: B902
+    def __init__(*args, **kwargs):  # noqa: B902
         self, args = args[0], args[1:]
         self.__attrs = dict(*args, **kwargs)
 
-    def __getattribute__(self, name: str) -> t.Any:
+    def __getattribute__(self, name):
         # __class__ is needed for the awaitable check in async mode
         if name in {"_Namespace__attrs", "__class__"}:
             return object.__getattribute__(self, name)
         try:
             return self.__attrs[name]
         except KeyError:
-            raise AttributeError(name) from None
+            raise AttributeError(name)
 
-    def __setitem__(self, name: str, value: t.Any) -> None:
+    def __setitem__(self, name, value):
         self.__attrs[name] = value
 
-    def __repr__(self) -> str:
+    def __repr__(self):
         return f"<Namespace {self.__attrs!r}>"
+
+
+# does this python version support async for in and async generators?
+try:
+    exec("async def _():\n async for _ in ():\n  yield _")
+    have_async_gen = True
+except SyntaxError:
+    have_async_gen = False

src/jinja2/visitor.py

@@ -1,17 +1,8 @@
 """API for traversing the AST nodes. Implemented by the compiler and
 meta introspection.
 """
-
-import typing as t
-
 from .nodes import Node
 
-if t.TYPE_CHECKING:
-    import typing_extensions as te
-
-    class VisitCallable(te.Protocol):
-        def __call__(self, node: Node, *args: t.Any, **kwargs: t.Any) -> t.Any: ...
-
 
 class NodeVisitor:
     """Walks the abstract syntax tree and call visitor functions for every
@@ -25,26 +16,24 @@ class NodeVisitor:
     (return value `None`) the `generic_visit` visitor is used instead.
     """
 
-    def get_visitor(self, node: Node) -> "VisitCallable | None":
+    def get_visitor(self, node):
         """Return the visitor function for this node or `None` if no visitor
         exists for this node.  In that case the generic visit function is
         used instead.
         """
-        return getattr(self, f"visit_{type(node).__name__}", None)
+        return getattr(self, f"visit_{node.__class__.__name__}", None)
 
-    def visit(self, node: Node, *args: t.Any, **kwargs: t.Any) -> t.Any:
+    def visit(self, node, *args, **kwargs):
         """Visit a node."""
         f = self.get_visitor(node)
-
         if f is not None:
             return f(node, *args, **kwargs)
-
         return self.generic_visit(node, *args, **kwargs)
 
-    def generic_visit(self, node: Node, *args: t.Any, **kwargs: t.Any) -> t.Any:
+    def generic_visit(self, node, *args, **kwargs):
         """Called if no explicit visitor function exists for a node."""
-        for child_node in node.iter_child_nodes():
-            self.visit(child_node, *args, **kwargs)
+        for node in node.iter_child_nodes():
+            self.visit(node, *args, **kwargs)
 
 
 class NodeTransformer(NodeVisitor):
@@ -58,7 +47,7 @@ class NodeTransformer(NodeVisitor):
     replacement takes place.
     """
 
-    def generic_visit(self, node: Node, *args: t.Any, **kwargs: t.Any) -> Node:
+    def generic_visit(self, node, *args, **kwargs):
         for field, old_value in node.iter_fields():
             if isinstance(old_value, list):
                 new_values = []
@@ -80,13 +69,11 @@ class NodeTransformer(NodeVisitor):
                     setattr(node, field, new_node)
         return node
 
-    def visit_list(self, node: Node, *args: t.Any, **kwargs: t.Any) -> list[Node]:
+    def visit_list(self, node, *args, **kwargs):
         """As transformers may return lists in some places this method
         can be used to enforce a list as return value.
         """
         rv = self.visit(node, *args, **kwargs)
-
         if not isinstance(rv, list):
-            return [rv]
-
+            rv = [rv]
         return rv

test_tatsu.py

@@ -0,0 +1,33 @@
+from datetime import datetime
+import pprint
+from jinja2.environment import Environment
+from jinja2.parser import Parser
+
+
+with open('grammar.ebnf', 'r') as tatsu_grammar:
+    with open('test_template.jinja', 'r') as test_template:
+        template_string = test_template.read()
+
+        env = Environment(line_statement_prefix='#', line_comment_prefix='##')
+        parser = Parser(env, template_string)
+
+        new_parse_start = datetime.now()
+
+        new_ast = parser.parse()
+
+        new_parse_end = datetime.now()
+
+        with open('tatsu_jinja.py', 'w') as new_ast_file:
+            pprint.pprint(new_ast, indent=2, stream=new_ast_file)
+
+        jinja_parse_start = datetime.now()
+
+        jinja_ast = parser.parse_old()
+
+        jinja_parse_end = datetime.now()
+
+        with open('parsed_jinja.py', 'w') as jinja_ast_file:
+            pprint.pprint(jinja_ast, indent=2, stream=jinja_ast_file)
+
+        print("New Parser", new_parse_end - new_parse_start)
+        print("Jinja Parser", jinja_parse_end - jinja_parse_start)

tests/conftest.py

@@ -1,20 +1,16 @@
-import asyncio
-from pathlib import Path
+import os
 
 import pytest
-import trio
 
+from jinja2 import Environment
 from jinja2 import loaders
-from jinja2.environment import Environment
+from jinja2.utils import have_async_gen
 
 
-def _asyncio_run(async_fn, *args):
-    return asyncio.run(async_fn(*args))
-
-
-@pytest.fixture(params=[_asyncio_run, trio.run], ids=["asyncio", "trio"])
-def run_async_fn(request):
-    return request.param
+def pytest_ignore_collect(path):
+    if "async" in path.basename and not have_async_gen:
+        return True
+    return False
 
 
 @pytest.fixture
@@ -38,8 +34,8 @@ def package_loader():
 @pytest.fixture
 def filesystem_loader():
     """returns FileSystemLoader initialized to res/templates directory"""
-    here = Path(__file__).parent.resolve()
-    return loaders.FileSystemLoader(here / "res" / "templates")
+    here = os.path.dirname(os.path.abspath(__file__))
+    return loaders.FileSystemLoader(here + "/res/templates")
 
 
 @pytest.fixture

tests/test_api.py

@@ -1,6 +1,6 @@
+import os
 import shutil
 import tempfile
-from pathlib import Path
 
 import pytest
 
@@ -18,10 +18,10 @@ from jinja2 import Undefined
 from jinja2 import UndefinedError
 from jinja2.compiler import CodeGenerator
 from jinja2.runtime import Context
+from jinja2.utils import contextfunction
 from jinja2.utils import Cycler
-from jinja2.utils import pass_context
-from jinja2.utils import pass_environment
-from jinja2.utils import pass_eval_context
+from jinja2.utils import environmentfunction
+from jinja2.utils import evalcontextfunction
 
 
 class TestExtendedAPI:
@@ -53,7 +53,7 @@ class TestExtendedAPI:
         assert t.render(value=123) == "<int>"
 
     def test_context_finalize(self):
-        @pass_context
+        @contextfunction
         def finalize(context, value):
             return value * context["scale"]
 
@@ -62,7 +62,7 @@ class TestExtendedAPI:
         assert t.render(value=5, scale=3) == "15"
 
     def test_eval_finalize(self):
-        @pass_eval_context
+        @evalcontextfunction
         def finalize(eval_ctx, value):
             return str(eval_ctx.autoescape) + value
 
@@ -71,7 +71,7 @@ class TestExtendedAPI:
         assert t.render(value="<script>") == "True&lt;script&gt;"
 
     def test_env_autoescape(self):
-        @pass_environment
+        @environmentfunction
         def finalize(env, value):
             return " ".join(
                 (env.variable_start_string, repr(value), env.variable_end_string)
@@ -150,8 +150,7 @@ class TestExtendedAPI:
         assert t.render(foo="<foo>") == "<foo>"
 
     def test_sandbox_max_range(self, env):
-        from jinja2.sandbox import MAX_RANGE
-        from jinja2.sandbox import SandboxedEnvironment
+        from jinja2.sandbox import SandboxedEnvironment, MAX_RANGE
 
         env = SandboxedEnvironment()
         t = env.from_string("{% for item in range(total) %}{{ item }}{% endfor %}")
@@ -195,22 +194,19 @@ class TestMeta:
         i = meta.find_referenced_templates(ast)
         assert list(i) == ["layout.html", "test.html", "meh.html", "muh.html"]
 
-    def test_find_included_templates(self, env):
-        ast = env.parse('{% include ["foo.html", "bar.html"] %}')
+    @pytest.mark.parametrize(
+        "include,templates",
+        (
+            ('{% include ["foo.html", "bar.html"] %}', ["foo.html", "bar.html"]),
+            ('{% include ("foo.html", "bar.html") %}', ["foo.html", "bar.html"]),
+            ('{% include ["foo.html", "bar.html", foo] %}', ["foo.html", "bar.html", None]),
+            ('{% include ("foo.html", "bar.html", foo) %}', ["foo.html", "bar.html", None])
+        )
+    )
+    def test_find_included_templates(self, env, include, templates):
+        ast = env.parse(include)
         i = meta.find_referenced_templates(ast)
-        assert list(i) == ["foo.html", "bar.html"]
-
-        ast = env.parse('{% include ("foo.html", "bar.html") %}')
-        i = meta.find_referenced_templates(ast)
-        assert list(i) == ["foo.html", "bar.html"]
-
-        ast = env.parse('{% include ["foo.html", "bar.html", foo] %}')
-        i = meta.find_referenced_templates(ast)
-        assert list(i) == ["foo.html", "bar.html", None]
-
-        ast = env.parse('{% include ("foo.html", "bar.html", foo) %}')
-        i = meta.find_referenced_templates(ast)
-        assert list(i) == ["foo.html", "bar.html", None]
+        assert list(i) == templates
 
 
 class TestStreaming:
@@ -243,12 +239,13 @@ class TestStreaming:
         assert not stream.buffered
 
     def test_dump_stream(self, env):
-        tmp = Path(tempfile.mkdtemp())
+        tmp = tempfile.mkdtemp()
         try:
             tmpl = env.from_string("\u2713")
             stream = tmpl.stream()
-            stream.dump(str(tmp / "dump.txt"), "utf-8")
-            assert (tmp / "dump.txt").read_bytes() == b"\xe2\x9c\x93"
+            stream.dump(os.path.join(tmp, "dump.txt"), "utf-8")
+            with open(os.path.join(tmp, "dump.txt"), "rb") as f:
+                assert f.read() == b"\xe2\x9c\x93"
         finally:
             shutil.rmtree(tmp)
 
@@ -265,7 +262,7 @@ class TestUndefined:
 
     def test_undefined_and_special_attributes(self):
         with pytest.raises(AttributeError):
-            Undefined("Foo").__dict__  # noqa B018
+            Undefined("Foo").__dict__
 
     def test_undefined_attribute_error(self):
         # Django's LazyObject turns the __class__ attribute into a
@@ -273,7 +270,7 @@ class TestUndefined:
         # function raises an AttributeError, printing the repr of the
         # object in the undefined message would cause a RecursionError.
         class Error:
-            @property  # type: ignore
+            @property
             def __class__(self):
                 raise AttributeError()
 
@@ -317,12 +314,13 @@ class TestUndefined:
         assert env.from_string("{{ foo.missing }}").render(foo=42) == ""
         assert env.from_string("{{ not missing }}").render() == "True"
         pytest.raises(UndefinedError, env.from_string("{{ missing - 1}}").render)
-        assert env.from_string("{{ 'foo' in missing }}").render() == "False"
         und1 = Undefined(name="x")
         und2 = Undefined(name="y")
         assert und1 == und2
         assert und1 != 42
         assert hash(und1) == hash(und2) == hash(Undefined())
+        with pytest.raises(AttributeError):
+            getattr(Undefined, "__slots__")  # noqa: B009
 
     def test_chainable_undefined(self):
         env = Environment(undefined=ChainableUndefined)
@@ -333,6 +331,8 @@ class TestUndefined:
         assert env.from_string("{{ foo.missing }}").render(foo=42) == ""
         assert env.from_string("{{ not missing }}").render() == "True"
         pytest.raises(UndefinedError, env.from_string("{{ missing - 1}}").render)
+        with pytest.raises(AttributeError):
+            getattr(ChainableUndefined, "__slots__")  # noqa: B009
 
         # The following tests ensure subclass functionality works as expected
         assert env.from_string('{{ missing.bar["baz"] }}').render() == ""
@@ -364,13 +364,14 @@ class TestUndefined:
             str(DebugUndefined(hint=undefined_hint))
             == f"{{{{ undefined value printed: {undefined_hint} }}}}"
         )
+        with pytest.raises(AttributeError):
+            getattr(DebugUndefined, "__slots__")  # noqa: B009
 
     def test_strict_undefined(self):
         env = Environment(undefined=StrictUndefined)
         pytest.raises(UndefinedError, env.from_string("{{ missing }}").render)
         pytest.raises(UndefinedError, env.from_string("{{ missing.attribute }}").render)
         pytest.raises(UndefinedError, env.from_string("{{ missing|list }}").render)
-        pytest.raises(UndefinedError, env.from_string("{{ 'foo' in missing }}").render)
         assert env.from_string("{{ missing is not defined }}").render() == "True"
         pytest.raises(
             UndefinedError, env.from_string("{{ foo.missing }}").render, foo=42
@@ -380,6 +381,8 @@ class TestUndefined:
             env.from_string('{{ missing|default("default", true) }}').render()
             == "default"
         )
+        with pytest.raises(AttributeError):
+            getattr(StrictUndefined, "__slots__")  # noqa: B009
         assert env.from_string('{{ "foo" if false }}').render() == ""
 
     def test_indexing_gives_undefined(self):
@@ -425,11 +428,3 @@ class TestLowLevel:
         env = CustomEnvironment()
         tmpl = env.from_string("{{ foo }}")
         assert tmpl.render() == "resolve-foo"
-
-
-def test_overlay_enable_async(env):
-    assert not env.is_async
-    assert not env.overlay().is_async
-    env_async = env.overlay(enable_async=True)
-    assert env_async.is_async
-    assert not env_async.overlay(enable_async=False).is_async

tests/test_async.py

@@ -1,17 +1,22 @@
+import asyncio
+
 import pytest
 
-from jinja2 import ChainableUndefined
 from jinja2 import DictLoader
 from jinja2 import Environment
 from jinja2 import Template
-from jinja2.async_utils import auto_aiter
+from jinja2.asyncsupport import auto_aiter
 from jinja2.exceptions import TemplateNotFound
 from jinja2.exceptions import TemplatesNotFound
 from jinja2.exceptions import UndefinedError
-from jinja2.nativetypes import NativeEnvironment
 
 
-def test_basic_async(run_async_fn):
+def run(coro):
+    loop = asyncio.get_event_loop()
+    return loop.run_until_complete(coro)
+
+
+def test_basic_async():
     t = Template(
         "{% for item in [1, 2, 3] %}[{{ item }}]{% endfor %}", enable_async=True
     )
@@ -19,11 +24,11 @@ def test_basic_async(run_async_fn):
     async def func():
         return await t.render_async()
 
-    rv = run_async_fn(func)
+    rv = run(func())
     assert rv == "[1][2][3]"
 
 
-def test_await_on_calls(run_async_fn):
+def test_await_on_calls():
     t = Template("{{ async_func() + normal_func() }}", enable_async=True)
 
     async def async_func():
@@ -35,7 +40,7 @@ def test_await_on_calls(run_async_fn):
     async def func():
         return await t.render_async(async_func=async_func, normal_func=normal_func)
 
-    rv = run_async_fn(func)
+    rv = run(func())
     assert rv == "65"
 
 
@@ -49,10 +54,11 @@ def test_await_on_calls_normal_render():
         return 23
 
     rv = t.render(async_func=async_func, normal_func=normal_func)
+
     assert rv == "65"
 
 
-def test_await_and_macros(run_async_fn):
+def test_await_and_macros():
     t = Template(
         "{% macro foo(x) %}[{{ x }}][{{ async_func() }}]{% endmacro %}{{ foo(42) }}",
         enable_async=True,
@@ -64,11 +70,11 @@ def test_await_and_macros(run_async_fn):
     async def func():
         return await t.render_async(async_func=async_func)
 
-    rv = run_async_fn(func)
+    rv = run(func())
     assert rv == "[42][42]"
 
 
-def test_async_blocks(run_async_fn):
+def test_async_blocks():
     t = Template(
         "{% block foo %}<Test>{% endblock %}{{ self.foo() }}",
         enable_async=True,
@@ -78,7 +84,7 @@ def test_async_blocks(run_async_fn):
     async def func():
         return await t.render_async()
 
-    rv = run_async_fn(func)
+    rv = run(func())
     assert rv == "<Test><Test>"
 
 
@@ -154,46 +160,24 @@ class TestAsyncImports:
         test_env_async.from_string('{% from "foo" import bar, with, context %}')
         test_env_async.from_string('{% from "foo" import bar, with with context %}')
 
-    def test_exports(self, test_env_async, run_async_fn):
-        coro_fn = test_env_async.from_string(
-            """
+    def test_exports(self, test_env_async):
+        m = run(
+            test_env_async.from_string(
+                """
             {% macro toplevel() %}...{% endmacro %}
             {% macro __private() %}...{% endmacro %}
             {% set variable = 42 %}
             {% for item in [1] %}
                 {% macro notthere() %}{% endmacro %}
             {% endfor %}
-            """
-        )._get_default_module_async
-        m = run_async_fn(coro_fn)
-        assert run_async_fn(m.toplevel) == "..."
+        """
+            )._get_default_module_async()
+        )
+        assert run(m.toplevel()) == "..."
         assert not hasattr(m, "__missing")
         assert m.variable == 42
         assert not hasattr(m, "notthere")
 
-    def test_import_with_globals(self, test_env_async):
-        t = test_env_async.from_string(
-            '{% import "module" as m %}{{ m.test() }}', globals={"foo": 42}
-        )
-        assert t.render() == "[42|23]"
-
-        t = test_env_async.from_string('{% import "module" as m %}{{ m.test() }}')
-        assert t.render() == "[|23]"
-
-    def test_import_with_globals_override(self, test_env_async):
-        t = test_env_async.from_string(
-            '{% set foo = 41 %}{% import "module" as m %}{{ m.test() }}',
-            globals={"foo": 42},
-        )
-        assert t.render() == "[42|23]"
-
-    def test_from_import_with_globals(self, test_env_async):
-        t = test_env_async.from_string(
-            '{% from "module" import test %}{{ test() }}',
-            globals={"foo": 42},
-        )
-        assert t.render() == "[42|23]"
-
 
 class TestAsyncIncludes:
     def test_context_include(self, test_env_async):
@@ -274,7 +258,7 @@ class TestAsyncIncludes:
 
     def test_unoptimized_scopes_autoescape(self):
         env = Environment(
-            loader=DictLoader({"o_printer": "({{ o }})"}),
+            loader=DictLoader(dict(o_printer="({{ o }})",)),
             autoescape=True,
             enable_async=True,
         )
@@ -449,23 +433,23 @@ class TestAsyncForLoop:
 
     def test_reversed_bug(self, test_env_async):
         tmpl = test_env_async.from_string(
-            "{% for i in items %}{{ i }}{% if not loop.last %},{% endif %}{% endfor %}"
+            "{% for i in items %}{{ i }}"
+            "{% if not loop.last %}"
+            ",{% endif %}{% endfor %}"
         )
         assert tmpl.render(items=reversed([3, 2, 1])) == "1,2,3"
 
-    def test_loop_errors(self, test_env_async, run_async_fn):
+    def test_loop_errors(self, test_env_async):
         tmpl = test_env_async.from_string(
             """{% for item in [1] if loop.index
                                       == 0 %}...{% endfor %}"""
         )
-        with pytest.raises(UndefinedError):
-            run_async_fn(tmpl.render_async)
-
+        pytest.raises(UndefinedError, tmpl.render)
         tmpl = test_env_async.from_string(
             """{% for item in [] %}...{% else
             %}{{ loop }}{% endfor %}"""
         )
-        assert run_async_fn(tmpl.render_async) == ""
+        assert tmpl.render() == ""
 
     def test_loop_filter(self, test_env_async):
         tmpl = test_env_async.from_string(
@@ -595,7 +579,7 @@ class TestAsyncForLoop:
         assert t.render(a=dict(b=[1, 2, 3])) == "1"
 
 
-def test_namespace_awaitable(test_env_async, run_async_fn):
+def test_namespace_awaitable(test_env_async):
     async def _test():
         t = test_env_async.from_string(
             '{% set ns = namespace(foo="Bar") %}{{ ns.foo }}'
@@ -603,118 +587,4 @@ def test_namespace_awaitable(test_env_async, run_async_fn):
         actual = await t.render_async()
         assert actual == "Bar"
 
-    run_async_fn(_test)
-
-
-def test_chainable_undefined_aiter(run_async_fn):
-    async def _test():
-        t = Template(
-            "{% for x in a['b']['c'] %}{{ x }}{% endfor %}",
-            enable_async=True,
-            undefined=ChainableUndefined,
-        )
-        rv = await t.render_async(a={})
-        assert rv == ""
-
-    run_async_fn(_test)
-
-
-@pytest.fixture
-def async_native_env():
-    return NativeEnvironment(enable_async=True)
-
-
-def test_native_async(async_native_env, run_async_fn):
-    async def _test():
-        t = async_native_env.from_string("{{ x }}")
-        rv = await t.render_async(x=23)
-        assert rv == 23
-
-    run_async_fn(_test)
-
-
-def test_native_list_async(async_native_env, run_async_fn):
-    async def _test():
-        t = async_native_env.from_string("{{ x }}")
-        rv = await t.render_async(x=list(range(3)))
-        assert rv == [0, 1, 2]
-
-    run_async_fn(_test)
-
-
-def test_getitem_after_filter():
-    env = Environment(enable_async=True)
-    env.filters["add_each"] = lambda v, x: [i + x for i in v]
-    t = env.from_string("{{ (a|add_each(2))[1:] }}")
-    out = t.render(a=range(3))
-    assert out == "[3, 4]"
-
-
-def test_getitem_after_call():
-    env = Environment(enable_async=True)
-    env.globals["add_each"] = lambda v, x: [i + x for i in v]
-    t = env.from_string("{{ add_each(a, 2)[1:] }}")
-    out = t.render(a=range(3))
-    assert out == "[3, 4]"
-
-
-def test_basic_generate_async(run_async_fn):
-    t = Template(
-        "{% for item in [1, 2, 3] %}[{{ item }}]{% endfor %}", enable_async=True
-    )
-
-    async def func():
-        agen = t.generate_async()
-        try:
-            return await agen.__anext__()
-        finally:
-            await agen.aclose()
-
-    rv = run_async_fn(func)
-    assert rv == "["
-
-
-def test_include_generate_async(run_async_fn, test_env_async):
-    t = test_env_async.from_string('{% include "header" %}')
-
-    async def func():
-        agen = t.generate_async()
-        try:
-            return await agen.__anext__()
-        finally:
-            await agen.aclose()
-
-    rv = run_async_fn(func)
-    assert rv == "["
-
-
-def test_blocks_generate_async(run_async_fn):
-    t = Template(
-        "{% block foo %}<Test>{% endblock %}{{ self.foo() }}",
-        enable_async=True,
-        autoescape=True,
-    )
-
-    async def func():
-        agen = t.generate_async()
-        try:
-            return await agen.__anext__()
-        finally:
-            await agen.aclose()
-
-    rv = run_async_fn(func)
-    assert rv == "<Test>"
-
-
-def test_async_extend(run_async_fn, test_env_async):
-    t = test_env_async.from_string('{% extends "header" %}')
-
-    async def func():
-        agen = t.generate_async()
-        try:
-            return await agen.__anext__()
-        finally:
-            await agen.aclose()
-
-    rv = run_async_fn(func)
-    assert rv == "["
+    run(_test())

tests/test_asyncfilters.py

@@ -1,11 +1,9 @@
-import contextlib
 from collections import namedtuple
 
 import pytest
-from markupsafe import Markup
 
 from jinja2 import Environment
-from jinja2.async_utils import auto_aiter
+from jinja2.utils import Markup
 
 
 async def make_aiter(iter):
@@ -27,30 +25,10 @@ def env_async():
     return Environment(enable_async=True)
 
 
-@contextlib.asynccontextmanager
-async def closing_factory():
-    async with contextlib.AsyncExitStack() as stack:
-
-        def closing(maybe_agen):
-            try:
-                aclose = maybe_agen.aclose
-            except AttributeError:
-                pass
-            else:
-                stack.push_async_callback(aclose)
-            return maybe_agen
-
-        yield closing
-
-
 @mark_dualiter("foo", lambda: range(10))
-def test_first(env_async, foo, run_async_fn):
-    async def test():
-        async with closing_factory() as closing:
-            tmpl = env_async.from_string("{{ closing(foo())|first }}")
-            return await tmpl.render_async(foo=foo, closing=closing)
-
-    out = run_async_fn(test)
+def test_first(env_async, foo):
+    tmpl = env_async.from_string("{{ foo()|first }}")
+    out = tmpl.render(foo=foo)
     assert out == "0"
 
 
@@ -78,26 +56,6 @@ def test_groupby(env_async, items):
     ]
 
 
-@pytest.mark.parametrize(
-    ("case_sensitive", "expect"),
-    [
-        (False, "a: 1, 3\nb: 2\n"),
-        (True, "A: 3\na: 1\nb: 2\n"),
-    ],
-)
-def test_groupby_case(env_async, case_sensitive, expect):
-    tmpl = env_async.from_string(
-        "{% for k, vs in data|groupby('k', case_sensitive=cs) %}"
-        "{{ k }}: {{ vs|join(', ', attribute='v') }}\n"
-        "{% endfor %}"
-    )
-    out = tmpl.render(
-        data=[{"k": "a", "v": 1}, {"k": "b", "v": 2}, {"k": "A", "v": 3}],
-        cs=case_sensitive,
-    )
-    assert out == expect
-
-
 @mark_dualiter("items", lambda: [("a", 1), ("a", 2), ("b", 1)])
 def test_groupby_tuple_index(env_async, items):
     tmpl = env_async.from_string(
@@ -184,7 +142,7 @@ def test_bool_select(env_async, items):
     assert tmpl.render(items=items) == "1|2|3|4|5"
 
 
-def make_users():  # type: ignore
+def make_users():
     User = namedtuple("User", "name,is_active")
     return [
         User("john", True),
@@ -264,47 +222,3 @@ def test_slice(env_async, items):
         "[[0, 1, 2, 3], [4, 5, 6], [7, 8, 9]]|"
         "[[0, 1, 2, 3], [4, 5, 6, 'X'], [7, 8, 9, 'X']]"
     )
-
-
-def test_unique_with_async_gen(env_async):
-    items = ["a", "b", "c", "c", "a", "d", "z"]
-    tmpl = env_async.from_string("{{ items|reject('==', 'z')|unique|list }}")
-    out = tmpl.render(items=items)
-    assert out == "['a', 'b', 'c', 'd']"
-
-
-def test_custom_async_filter(env_async, run_async_fn):
-    async def customfilter(val):
-        return str(val)
-
-    async def test():
-        env_async.filters["customfilter"] = customfilter
-        tmpl = env_async.from_string(
-            "{{ 'static'|customfilter }} {{ arg|customfilter }}"
-        )
-        return await tmpl.render_async(arg="dynamic")
-
-    out = run_async_fn(test)
-    assert out == "static dynamic"
-
-
-@mark_dualiter("items", lambda: range(10))
-def test_custom_async_iteratable_filter(env_async, items, run_async_fn):
-    async def customfilter(iterable):
-        items = []
-        async for item in auto_aiter(iterable):
-            items.append(str(item))
-            if len(items) == 3:
-                break
-        return ",".join(items)
-
-    async def test():
-        async with closing_factory() as closing:
-            env_async.filters["customfilter"] = customfilter
-            tmpl = env_async.from_string(
-                "{{ closing(items())|customfilter }} .. {{ [3, 4, 5, 6]|customfilter }}"
-            )
-            return await tmpl.render_async(items=items, closing=closing)
-
-    out = run_async_fn(test)
-    assert out == "0,1,2 .. 3,4,5"

tests/test_compile.py

@@ -1,108 +0,0 @@
-import os
-import re
-
-import pytest
-
-from jinja2 import UndefinedError
-from jinja2.environment import Environment
-from jinja2.loaders import DictLoader
-
-
-def test_filters_deterministic(tmp_path):
-    src = "".join(f"{{{{ {i}|filter{i} }}}}" for i in range(10))
-    env = Environment(loader=DictLoader({"foo": src}))
-    env.filters.update(dict.fromkeys((f"filter{i}" for i in range(10)), lambda: None))
-    env.compile_templates(tmp_path, zip=None)
-    name = os.listdir(tmp_path)[0]
-    content = (tmp_path / name).read_text("utf8")
-    expect = [f"filters['filter{i}']" for i in range(10)]
-    found = re.findall(r"filters\['filter\d']", content)
-    assert found == expect
-
-
-def test_import_as_with_context_deterministic(tmp_path):
-    src = "\n".join(f'{{% import "bar" as bar{i} with context %}}' for i in range(10))
-    env = Environment(loader=DictLoader({"foo": src}))
-    env.compile_templates(tmp_path, zip=None)
-    name = os.listdir(tmp_path)[0]
-    content = (tmp_path / name).read_text("utf8")
-    expect = [f"'bar{i}': " for i in range(10)]
-    found = re.findall(r"'bar\d': ", content)[:10]
-    assert found == expect
-
-
-def test_top_level_set_vars_unpacking_deterministic(tmp_path):
-    src = "\n".join(f"{{% set a{i}, b{i}, c{i} = tuple_var{i} %}}" for i in range(10))
-    env = Environment(loader=DictLoader({"foo": src}))
-    env.compile_templates(tmp_path, zip=None)
-    name = os.listdir(tmp_path)[0]
-    content = (tmp_path / name).read_text("utf8")
-    expect = [
-        f"context.vars.update({{'a{i}': l_0_a{i}, 'b{i}': l_0_b{i}, 'c{i}': l_0_c{i}}})"
-        for i in range(10)
-    ]
-    found = re.findall(
-        r"context\.vars\.update\(\{'a\d': l_0_a\d, 'b\d': l_0_b\d, 'c\d': l_0_c\d\}\)",
-        content,
-    )[:10]
-    assert found == expect
-    expect = [
-        f"context.exported_vars.update(('a{i}', 'b{i}', 'c{i}'))" for i in range(10)
-    ]
-    found = re.findall(
-        r"context\.exported_vars\.update\(\('a\d', 'b\d', 'c\d'\)\)",
-        content,
-    )[:10]
-    assert found == expect
-
-
-def test_loop_set_vars_unpacking_deterministic(tmp_path):
-    src = "\n".join(f"  {{% set a{i}, b{i}, c{i} = tuple_var{i} %}}" for i in range(10))
-    src = f"{{% for i in seq %}}\n{src}\n{{% endfor %}}"
-    env = Environment(loader=DictLoader({"foo": src}))
-    env.compile_templates(tmp_path, zip=None)
-    name = os.listdir(tmp_path)[0]
-    content = (tmp_path / name).read_text("utf8")
-    expect = [
-        f"_loop_vars.update({{'a{i}': l_1_a{i}, 'b{i}': l_1_b{i}, 'c{i}': l_1_c{i}}})"
-        for i in range(10)
-    ]
-    found = re.findall(
-        r"_loop_vars\.update\(\{'a\d': l_1_a\d, 'b\d': l_1_b\d, 'c\d': l_1_c\d\}\)",
-        content,
-    )[:10]
-    assert found == expect
-
-
-def test_block_set_vars_unpacking_deterministic(tmp_path):
-    src = "\n".join(f"  {{% set a{i}, b{i}, c{i} = tuple_var{i} %}}" for i in range(10))
-    src = f"{{% block test %}}\n{src}\n{{% endblock test %}}"
-    env = Environment(loader=DictLoader({"foo": src}))
-    env.compile_templates(tmp_path, zip=None)
-    name = os.listdir(tmp_path)[0]
-    content = (tmp_path / name).read_text("utf8")
-    expect = [
-        f"_block_vars.update({{'a{i}': l_0_a{i}, 'b{i}': l_0_b{i}, 'c{i}': l_0_c{i}}})"
-        for i in range(10)
-    ]
-    found = re.findall(
-        r"_block_vars\.update\(\{'a\d': l_0_a\d, 'b\d': l_0_b\d, 'c\d': l_0_c\d\}\)",
-        content,
-    )[:10]
-    assert found == expect
-
-
-def test_undefined_import_curly_name():
-    env = Environment(
-        loader=DictLoader(
-            {
-                "{bad}": "{% from 'macro' import m %}{{ m() }}",
-                "macro": "",
-            }
-        )
-    )
-
-    # Must not raise `NameError: 'bad' is not defined`, as that would indicate
-    # that `{bad}` is being interpreted as an f-string. It must be escaped.
-    with pytest.raises(UndefinedError):
-        env.get_template("{bad}").render()

tests/test_core_tags.py

@@ -191,7 +191,9 @@ class TestForLoop:
 
     def test_reversed_bug(self, env):
         tmpl = env.from_string(
-            "{% for i in items %}{{ i }}{% if not loop.last %},{% endif %}{% endfor %}"
+            "{% for i in items %}{{ i }}"
+            "{% if not loop.last %}"
+            ",{% endif %}{% endfor %}"
         )
         assert tmpl.render(items=reversed([3, 2, 1])) == "1,2,3"
 
@@ -536,14 +538,6 @@ class TestSet:
         )
         assert tmpl.render() == "13|37"
 
-    def test_namespace_set_tuple(self, env_trim):
-        tmpl = env_trim.from_string(
-            "{% set ns = namespace(a=12, b=36) %}"
-            "{% set ns.a, ns.b = ns.a + 1, ns.b + 1 %}"
-            "{{ ns.a }}|{{ ns.b }}"
-        )
-        assert tmpl.render() == "13|37"
-
     def test_block_escaping_filtered(self):
         env = Environment(autoescape=True)
         tmpl = env.from_string(

tests/test_debug.py

@@ -23,9 +23,9 @@ class TestDebug:
 
         tb = format_exception(exc_info.type, exc_info.value, exc_info.tb)
         m = re.search(expected_tb.strip(), "".join(tb))
-        assert m is not None, (
-            f"Traceback did not match:\n\n{''.join(tb)}\nexpected:\n{expected_tb}"
-        )
+        assert (
+            m is not None
+        ), "Traceback did not match:\n\n{''.join(tb)}\nexpected:\n{expected_tb}"
 
     def test_runtime_error(self, fs_env):
         def test():
@@ -36,11 +36,9 @@ class TestDebug:
             test,
             r"""
   File ".*?broken.html", line 2, in (top-level template code|<module>)
-    \{\{ fail\(\) \}\}(
-    \^{12})?
+    \{\{ fail\(\) \}\}
   File ".*debug?.pyc?", line \d+, in <lambda>
-    tmpl\.render\(fail=lambda: 1 / 0\)(
-                             ~~\^~~)?
+    tmpl\.render\(fail=lambda: 1 / 0\)
 ZeroDivisionError: (int(eger)? )?division (or modulo )?by zero
 """,
         )
@@ -68,8 +66,7 @@ to be closed is 'for'.
             test,
             r"""
   File ".*debug.pyc?", line \d+, in test
-    raise TemplateSyntaxError\("wtf", 42\)(
-    \^{36})?
+    raise TemplateSyntaxError\("wtf", 42\)
 (jinja2\.exceptions\.)?TemplateSyntaxError: wtf
   line 42""",
         )

tests/test_ext.py

@@ -3,11 +3,10 @@ from io import BytesIO
 
 import pytest
 
+from jinja2 import contextfunction
 from jinja2 import DictLoader
 from jinja2 import Environment
 from jinja2 import nodes
-from jinja2 import pass_context
-from jinja2 import TemplateSyntaxError
 from jinja2.exceptions import TemplateAssertionError
 from jinja2.ext import Extension
 from jinja2.lexer import count_newlines
@@ -19,9 +18,9 @@ _gettext_re = re.compile(r"_\((.*?)\)", re.DOTALL)
 
 
 i18n_templates = {
-    "default.html": '<title>{{ page_title|default(_("missing")) }}</title>'
+    "master.html": '<title>{{ page_title|default(_("missing")) }}</title>'
     "{% block body %}{% endblock %}",
-    "child.html": '{% extends "default.html" %}{% block body %}'
+    "child.html": '{% extends "master.html" %}{% block body %}'
     "{% trans %}watch out{% endtrans %}{% endblock %}",
     "plural.html": "{% trans user_count %}One user online{% pluralize %}"
     "{{ user_count }} users online{% endtrans %}",
@@ -31,9 +30,9 @@ i18n_templates = {
 }
 
 newstyle_i18n_templates = {
-    "default.html": '<title>{{ page_title|default(_("missing")) }}</title>'
+    "master.html": '<title>{{ page_title|default(_("missing")) }}</title>'
     "{% block body %}{% endblock %}",
-    "child.html": '{% extends "default.html" %}{% block body %}'
+    "child.html": '{% extends "master.html" %}{% block body %}'
     "{% trans %}watch out{% endtrans %}{% endblock %}",
     "plural.html": "{% trans user_count %}One user online{% pluralize %}"
     "{{ user_count }} users online{% endtrans %}",
@@ -41,12 +40,6 @@ newstyle_i18n_templates = {
     "ngettext.html": '{{ ngettext("%(num)s apple", "%(num)s apples", apples) }}',
     "ngettext_long.html": "{% trans num=apples %}{{ num }} apple{% pluralize %}"
     "{{ num }} apples{% endtrans %}",
-    "pgettext.html": '{{ pgettext("fruit", "Apple") }}',
-    "npgettext.html": '{{ npgettext("fruit", "%(num)s apple", "%(num)s apples",'
-    " apples) }}",
-    "pgettext_block": "{% trans 'fruit' num=apples %}Apple{% endtrans %}",
-    "npgettext_block": "{% trans 'fruit' num=apples %}{{ num }} apple"
-    "{% pluralize %}{{ num }} apples{% endtrans %}",
     "transvars1.html": "{% trans %}User: {{ num }}{% endtrans %}",
     "transvars2.html": "{% trans num=count %}User: {{ num }}{% endtrans %}",
     "transvars3.html": "{% trans count=num %}User: {{ count }}{% endtrans %}",
@@ -64,89 +57,40 @@ languages = {
         "%(user_count)s users online": "%(user_count)s Benutzer online",
         "User: %(num)s": "Benutzer: %(num)s",
         "User: %(count)s": "Benutzer: %(count)s",
-        "Apple": {None: "Apfel", "fruit": "Apple"},
-        "%(num)s apple": {None: "%(num)s Apfel", "fruit": "%(num)s Apple"},
-        "%(num)s apples": {None: "%(num)s Äpfel", "fruit": "%(num)s Apples"},
+        "%(num)s apple": "%(num)s Apfel",
+        "%(num)s apples": "%(num)s Äpfel",
     }
 }
 
 
-def _get_with_context(value, ctx=None):
-    if isinstance(value, dict):
-        return value.get(ctx, value)
-
-    return value
-
-
-@pass_context
+@contextfunction
 def gettext(context, string):
     language = context.get("LANGUAGE", "en")
-    value = languages.get(language, {}).get(string, string)
-    return _get_with_context(value)
+    return languages.get(language, {}).get(string, string)
 
 
-@pass_context
+@contextfunction
 def ngettext(context, s, p, n):
     language = context.get("LANGUAGE", "en")
-
     if n != 1:
-        value = languages.get(language, {}).get(p, p)
-        return _get_with_context(value)
-
-    value = languages.get(language, {}).get(s, s)
-    return _get_with_context(value)
-
-
-@pass_context
-def pgettext(context, c, s):
-    language = context.get("LANGUAGE", "en")
-    value = languages.get(language, {}).get(s, s)
-    return _get_with_context(value, c)
-
-
-@pass_context
-def npgettext(context, c, s, p, n):
-    language = context.get("LANGUAGE", "en")
-
-    if n != 1:
-        value = languages.get(language, {}).get(p, p)
-        return _get_with_context(value, c)
-
-    value = languages.get(language, {}).get(s, s)
-    return _get_with_context(value, c)
+        return languages.get(language, {}).get(p, p)
+    return languages.get(language, {}).get(s, s)
 
 
 i18n_env = Environment(
     loader=DictLoader(i18n_templates), extensions=["jinja2.ext.i18n"]
 )
-i18n_env.globals.update(
-    {
-        "_": gettext,
-        "gettext": gettext,
-        "ngettext": ngettext,
-        "pgettext": pgettext,
-        "npgettext": npgettext,
-    }
-)
+i18n_env.globals.update({"_": gettext, "gettext": gettext, "ngettext": ngettext})
 i18n_env_trimmed = Environment(extensions=["jinja2.ext.i18n"])
-
 i18n_env_trimmed.policies["ext.i18n.trimmed"] = True
 i18n_env_trimmed.globals.update(
-    {
-        "_": gettext,
-        "gettext": gettext,
-        "ngettext": ngettext,
-        "pgettext": pgettext,
-        "npgettext": npgettext,
-    }
+    {"_": gettext, "gettext": gettext, "ngettext": ngettext}
 )
 
 newstyle_i18n_env = Environment(
     loader=DictLoader(newstyle_i18n_templates), extensions=["jinja2.ext.i18n"]
 )
-newstyle_i18n_env.install_gettext_callables(  # type: ignore
-    gettext, ngettext, newstyle=True, pgettext=pgettext, npgettext=npgettext
-)
+newstyle_i18n_env.install_gettext_callables(gettext, ngettext, newstyle=True)
 
 
 class ExampleExtension(Extension):
@@ -177,7 +121,7 @@ class ExampleExtension(Extension):
 
 
 class DerivedExampleExtension(ExampleExtension):
-    context_reference_node_cls = nodes.DerivedContextReference  # type: ignore
+    context_reference_node_cls = nodes.DerivedContextReference
 
 
 class PreprocessorExtension(Extension):
@@ -197,7 +141,7 @@ class StreamFilterExtension(Extension):
         pos = 0
         end = len(token.value)
         lineno = token.lineno
-        while True:
+        while 1:
             match = _gettext_re.search(token.value, pos)
             if match is None:
                 break
@@ -219,6 +163,7 @@ class StreamFilterExtension(Extension):
 class TestExtensions:
     def test_extend_late(self):
         env = Environment()
+        env.add_extension("jinja2.ext.autoescape")
         t = env.from_string('{% autoescape true %}{{ "<test>" }}{% endautoescape %}')
         assert t.render() == "&lt;test&gt;"
 
@@ -455,32 +400,6 @@ class TestInternationalization:
             (6, "ngettext", ("%(users)s user", "%(users)s users", None), ["third"]),
         ]
 
-    def test_extract_context(self):
-        from jinja2.ext import babel_extract
-
-        source = BytesIO(
-            b"""
-             {{ pgettext("babel", "Hello World") }}
-             {{ npgettext("babel", "%(users)s user", "%(users)s users", users) }}
-             """
-        )
-        assert list(babel_extract(source, ("pgettext", "npgettext", "_"), [], {})) == [
-            (2, "pgettext", ("babel", "Hello World"), []),
-            (3, "npgettext", ("babel", "%(users)s user", "%(users)s users", None), []),
-        ]
-
-    def test_nested_trans_error(self):
-        s = "{% trans %}foo{% trans %}{% endtrans %}"
-        with pytest.raises(TemplateSyntaxError) as excinfo:
-            i18n_env.from_string(s)
-        assert "trans blocks can't be nested" in str(excinfo.value)
-
-    def test_trans_block_error(self):
-        s = "{% trans %}foo{% wibble bar %}{% endwibble %}{% endtrans %}"
-        with pytest.raises(TemplateSyntaxError) as excinfo:
-            i18n_env.from_string(s)
-        assert "saw `wibble`" in str(excinfo.value)
-
 
 class TestScope:
     def test_basic_scope_behavior(self):
@@ -547,20 +466,21 @@ class TestNewstyleInternationalization:
         assert tmpl.render(LANGUAGE="de", apples=5) == "5 Äpfel"
 
     def test_autoescape_support(self):
-        env = Environment(extensions=["jinja2.ext.i18n"])
+        env = Environment(extensions=["jinja2.ext.autoescape", "jinja2.ext.i18n"])
         env.install_gettext_callables(
             lambda x: "<strong>Wert: %(name)s</strong>",
             lambda s, p, n: s,
             newstyle=True,
         )
         t = env.from_string(
-            '{% autoescape ae %}{{ gettext("foo", name="<test>") }}{% endautoescape %}'
+            '{% autoescape ae %}{{ gettext("foo", name='
+            '"<test>") }}{% endautoescape %}'
         )
         assert t.render(ae=True) == "<strong>Wert: &lt;test&gt;</strong>"
         assert t.render(ae=False) == "<strong>Wert: <test></strong>"
 
     def test_autoescape_macros(self):
-        env = Environment(autoescape=False)
+        env = Environment(autoescape=False, extensions=["jinja2.ext.autoescape"])
         template = (
             "{% macro m() %}<html>{% endmacro %}"
             "{% autoescape true %}{{ m() }}{% endautoescape %}"
@@ -604,28 +524,10 @@ class TestNewstyleInternationalization:
         t = newstyle_i18n_env.get_template("explicitvars.html")
         assert t.render() == "%(foo)s"
 
-    def test_context(self):
-        tmpl = newstyle_i18n_env.get_template("pgettext.html")
-        assert tmpl.render(LANGUAGE="de") == "Apple"
-
-    def test_context_plural(self):
-        tmpl = newstyle_i18n_env.get_template("npgettext.html")
-        assert tmpl.render(LANGUAGE="de", apples=1) == "1 Apple"
-        assert tmpl.render(LANGUAGE="de", apples=5) == "5 Apples"
-
-    def test_context_block(self):
-        tmpl = newstyle_i18n_env.get_template("pgettext_block")
-        assert tmpl.render(LANGUAGE="de") == "Apple"
-
-    def test_context_plural_block(self):
-        tmpl = newstyle_i18n_env.get_template("npgettext_block")
-        assert tmpl.render(LANGUAGE="de", apples=1) == "1 Apple"
-        assert tmpl.render(LANGUAGE="de", apples=5) == "5 Apples"
-
 
 class TestAutoEscape:
     def test_scoped_setting(self):
-        env = Environment(autoescape=True)
+        env = Environment(extensions=["jinja2.ext.autoescape"], autoescape=True)
         tmpl = env.from_string(
             """
             {{ "<HelloWorld>" }}
@@ -641,7 +543,7 @@ class TestAutoEscape:
             "&lt;HelloWorld&gt;",
         ]
 
-        env = Environment(autoescape=False)
+        env = Environment(extensions=["jinja2.ext.autoescape"], autoescape=False)
         tmpl = env.from_string(
             """
             {{ "<HelloWorld>" }}
@@ -658,7 +560,7 @@ class TestAutoEscape:
         ]
 
     def test_nonvolatile(self):
-        env = Environment(autoescape=True)
+        env = Environment(extensions=["jinja2.ext.autoescape"], autoescape=True)
         tmpl = env.from_string('{{ {"foo": "<test>"}|xmlattr|escape }}')
         assert tmpl.render() == ' foo="&lt;test&gt;"'
         tmpl = env.from_string(
@@ -668,7 +570,7 @@ class TestAutoEscape:
         assert tmpl.render() == " foo=&#34;&amp;lt;test&amp;gt;&#34;"
 
     def test_volatile(self):
-        env = Environment(autoescape=True)
+        env = Environment(extensions=["jinja2.ext.autoescape"], autoescape=True)
         tmpl = env.from_string(
             '{% autoescape foo %}{{ {"foo": "<test>"}'
             "|xmlattr|escape }}{% endautoescape %}"
@@ -677,7 +579,7 @@ class TestAutoEscape:
         assert tmpl.render(foo=True) == ' foo="&lt;test&gt;"'
 
     def test_scoping(self):
-        env = Environment()
+        env = Environment(extensions=["jinja2.ext.autoescape"])
         tmpl = env.from_string(
             '{% autoescape true %}{% set x = "<x>" %}{{ x }}'
             '{% endautoescape %}{{ x }}{{ "<y>" }}'
@@ -685,7 +587,7 @@ class TestAutoEscape:
         assert tmpl.render(x=1) == "&lt;x&gt;1<y>"
 
     def test_volatile_scoping(self):
-        env = Environment()
+        env = Environment(extensions=["jinja2.ext.autoescape"])
         tmplsource = """
         {% autoescape val %}
             {% macro foo(x) %}
@@ -701,11 +603,11 @@ class TestAutoEscape:
 
         # looking at the source we should see <testing> there in raw
         # (and then escaped as well)
-        env = Environment()
+        env = Environment(extensions=["jinja2.ext.autoescape"])
         pysource = env.compile(tmplsource, raw=True)
         assert "<testing>\\n" in pysource
 
-        env = Environment(autoescape=True)
+        env = Environment(extensions=["jinja2.ext.autoescape"], autoescape=True)
         pysource = env.compile(tmplsource, raw=True)
         assert "&lt;testing&gt;\\n" in pysource
 

tests/test_features.py

@@ -0,0 +1,14 @@
+import pytest
+
+from jinja2 import Template
+
+
+# Python < 3.7
+def test_generator_stop():
+    class X:
+        def __getattr__(self, name):
+            raise StopIteration()
+
+    t = Template("a{{ bad.bar() }}b")
+    with pytest.raises(RuntimeError):
+        t.render(bad=X())

tests/test_filters.py

@@ -2,13 +2,11 @@ import random
 from collections import namedtuple
 
 import pytest
-from markupsafe import Markup
 
 from jinja2 import Environment
+from jinja2 import Markup
 from jinja2 import StrictUndefined
-from jinja2 import TemplateRuntimeError
 from jinja2 import UndefinedError
-from jinja2.exceptions import TemplateAssertionError
 
 
 class Magic:
@@ -185,10 +183,6 @@ class TestFilter:
         """
         self._test_indent_multiline_template(env, markup=True)
 
-    def test_indent_width_string(self, env):
-        t = env.from_string("{{ 'jinja\nflask'|indent(width='>>> ', first=True) }}")
-        assert t.render() == ">>> jinja\n>>> flask"
-
     @pytest.mark.parametrize(
         ("value", "expect"),
         (
@@ -196,7 +190,6 @@ class TestFilter:
             ("abc", "0"),
             ("32.32", "32"),
             ("12345678901234567890", "12345678901234567890"),
-            ("1e10000", "0"),
         ),
     )
     def test_int(self, env, value, expect):
@@ -205,7 +198,7 @@ class TestFilter:
 
     @pytest.mark.parametrize(
         ("value", "base", "expect"),
-        (("0x4d32", 16, "19762"), ("011", 8, "9"), ("0x33Z", 16, "0")),
+        (("0x4d32", 16, "19762"), ("011", 8, "9"), ("0x33Z", 16, "0"),),
     )
     def test_int_base(self, env, value, base, expect):
         t = env.from_string("{{ value|int(base=base) }}")
@@ -252,17 +245,6 @@ class TestFilter:
         out = tmpl.render()
         assert out == "foo"
 
-    def test_items(self, env):
-        d = {i: c for i, c in enumerate("abc")}
-        tmpl = env.from_string("""{{ d|items|list }}""")
-        out = tmpl.render(d=d)
-        assert out == "[(0, 'a'), (1, 'b'), (2, 'c')]"
-
-    def test_items_undefined(self, env):
-        tmpl = env.from_string("""{{ d|items|list }}""")
-        out = tmpl.render()
-        assert out == "[]"
-
     def test_pprint(self, env):
         from pprint import pformat
 
@@ -355,23 +337,11 @@ class TestFilter:
         assert tmpl.render() == "FOO"
 
     def test_urlize(self, env):
-        tmpl = env.from_string('{{ "foo example.org bar"|urlize }}')
-        assert tmpl.render() == (
-            'foo <a href="https://example.org" rel="noopener">example.org</a> bar'
-        )
         tmpl = env.from_string('{{ "foo http://www.example.com/ bar"|urlize }}')
         assert tmpl.render() == (
             'foo <a href="http://www.example.com/" rel="noopener">'
             "http://www.example.com/</a> bar"
         )
-        tmpl = env.from_string('{{ "foo mailto:email@example.com bar"|urlize }}')
-        assert tmpl.render() == (
-            'foo <a href="mailto:email@example.com">email@example.com</a> bar'
-        )
-        tmpl = env.from_string('{{ "foo email@example.com bar"|urlize }}')
-        assert tmpl.render() == (
-            'foo <a href="mailto:email@example.com">email@example.com</a> bar'
-        )
 
     def test_urlize_rel_policy(self):
         env = Environment()
@@ -391,17 +361,6 @@ class TestFilter:
             "http://www.example.com/</a> bar"
         )
 
-    def test_urlize_extra_schemes_parameter(self, env):
-        tmpl = env.from_string(
-            '{{ "foo tel:+1-514-555-1234 ftp://localhost bar"|'
-            'urlize(extra_schemes=["tel:", "ftp:"]) }}'
-        )
-        assert tmpl.render() == (
-            'foo <a href="tel:+1-514-555-1234" rel="noopener">'
-            'tel:+1-514-555-1234</a> <a href="ftp://localhost" rel="noopener">'
-            "ftp://localhost</a> bar"
-        )
-
     def test_wordcount(self, env):
         tmpl = env.from_string('{{ "foo bar baz"|wordcount }}')
         assert tmpl.render() == "3"
@@ -475,13 +434,6 @@ class TestFilter:
         assert 'bar="23"' in out
         assert 'blub:blub="&lt;?&gt;"' in out
 
-    @pytest.mark.parametrize("sep", ("\t", "\n", "\f", " ", "/", ">", "="))
-    def test_xmlattr_key_invalid(self, env: Environment, sep: str) -> None:
-        with pytest.raises(ValueError, match="Invalid character"):
-            env.from_string("{{ {key: 'my_class'}|xmlattr }}").render(
-                key=f"class{sep}onclick=alert(1)"
-            )
-
     def test_sort1(self, env):
         tmpl = env.from_string("{{ [2, 3, 1]|sort }}|{{ [2, 3, 1]|sort(true) }}")
         assert tmpl.render() == "[1, 2, 3]|[3, 2, 1]"
@@ -565,7 +517,7 @@ class TestFilter:
         t = env.from_string(source)
         assert t.render() == expect
 
-    @pytest.mark.parametrize(("name", "expect"), [("min", "1"), ("max", "9")])
+    @pytest.mark.parametrize("name,expect", (("min", "1"), ("max", "9"),))
     def test_min_max_attribute(self, env, name, expect):
         t = env.from_string("{{ items|" + name + '(attribute="value") }}')
         assert t.render(items=map(Magic, [5, 1, 9])) == expect
@@ -612,40 +564,6 @@ class TestFilter:
             "",
         ]
 
-    def test_groupby_default(self, env):
-        tmpl = env.from_string(
-            "{% for city, items in users|groupby('city', default='NY') %}"
-            "{{ city }}: {{ items|map(attribute='name')|join(', ') }}\n"
-            "{% endfor %}"
-        )
-        out = tmpl.render(
-            users=[
-                {"name": "emma", "city": "NY"},
-                {"name": "smith", "city": "WA"},
-                {"name": "john"},
-            ]
-        )
-        assert out == "NY: emma, john\nWA: smith\n"
-
-    @pytest.mark.parametrize(
-        ("case_sensitive", "expect"),
-        [
-            (False, "a: 1, 3\nb: 2\n"),
-            (True, "A: 3\na: 1\nb: 2\n"),
-        ],
-    )
-    def test_groupby_case(self, env, case_sensitive, expect):
-        tmpl = env.from_string(
-            "{% for k, vs in data|groupby('k', case_sensitive=cs) %}"
-            "{{ k }}: {{ vs|join(', ', attribute='v') }}\n"
-            "{% endfor %}"
-        )
-        out = tmpl.render(
-            data=[{"k": "a", "v": 1}, {"k": "b", "v": 2}, {"k": "A", "v": 3}],
-            cs=case_sensitive,
-        )
-        assert out == expect
-
     def test_filtertag(self, env):
         tmpl = env.from_string(
             "{% filter upper|replace('FOO', 'foo') %}foobar{% endfilter %}"
@@ -725,12 +643,6 @@ class TestFilter:
         tmpl = env.from_string(
             '{{ users|map(attribute="lastname", default="smith")|join(", ") }}'
         )
-        test_list = env.from_string(
-            '{{ users|map(attribute="lastname", default=["smith","x"])|join(", ") }}'
-        )
-        test_str = env.from_string(
-            '{{ users|map(attribute="lastname", default="")|join(", ") }}'
-        )
         users = [
             Fullname("john", "lennon"),
             Fullname("jane", "edwards"),
@@ -738,8 +650,6 @@ class TestFilter:
             Firstname("mike"),
         ]
         assert tmpl.render(users=users) == "lennon, edwards, None, smith"
-        assert test_list.render(users=users) == "lennon, edwards, None, ['smith', 'x']"
-        assert test_str.render(users=users) == "lennon, edwards, None, "
 
     def test_simple_select(self, env):
         env = Environment()
@@ -833,51 +743,3 @@ class TestFilter:
         t = env.from_string("{{ s|wordwrap(20) }}")
         result = t.render(s="Hello!\nThis is Jinja saying something.")
         assert result == "Hello!\nThis is Jinja saying\nsomething."
-
-    def test_filter_undefined(self, env):
-        with pytest.raises(TemplateAssertionError, match="No filter named 'f'"):
-            env.from_string("{{ var|f }}")
-
-    def test_filter_undefined_in_if(self, env):
-        t = env.from_string("{%- if x is defined -%}{{ x|f }}{%- else -%}x{% endif %}")
-        assert t.render() == "x"
-        with pytest.raises(TemplateRuntimeError, match="No filter named 'f'"):
-            t.render(x=42)
-
-    def test_filter_undefined_in_elif(self, env):
-        t = env.from_string(
-            "{%- if x is defined -%}{{ x }}{%- elif y is defined -%}"
-            "{{ y|f }}{%- else -%}foo{%- endif -%}"
-        )
-        assert t.render() == "foo"
-        with pytest.raises(TemplateRuntimeError, match="No filter named 'f'"):
-            t.render(y=42)
-
-    def test_filter_undefined_in_else(self, env):
-        t = env.from_string(
-            "{%- if x is not defined -%}foo{%- else -%}{{ x|f }}{%- endif -%}"
-        )
-        assert t.render() == "foo"
-        with pytest.raises(TemplateRuntimeError, match="No filter named 'f'"):
-            t.render(x=42)
-
-    def test_filter_undefined_in_nested_if(self, env):
-        t = env.from_string(
-            "{%- if x is not defined -%}foo{%- else -%}{%- if y "
-            "is defined -%}{{ y|f }}{%- endif -%}{{ x }}{%- endif -%}"
-        )
-        assert t.render() == "foo"
-        assert t.render(x=42) == "42"
-        with pytest.raises(TemplateRuntimeError, match="No filter named 'f'"):
-            t.render(x=24, y=42)
-
-    def test_filter_undefined_in_condexpr(self, env):
-        t1 = env.from_string("{{ x|f if x is defined else 'foo' }}")
-        t2 = env.from_string("{{ 'foo' if x is not defined else x|f }}")
-        assert t1.render() == t2.render() == "foo"
-
-        with pytest.raises(TemplateRuntimeError, match="No filter named 'f'"):
-            t1.render(x=42)
-
-        with pytest.raises(TemplateRuntimeError, match="No filter named 'f'"):
-            t2.render(x=42)

tests/test_idtracking.py

@@ -38,7 +38,7 @@ def test_basics():
 
 def test_complex():
     title_block = nodes.Block(
-        "title", [nodes.Output([nodes.TemplateData("Page Title")])], False, False
+        "title", [nodes.Output([nodes.TemplateData("Page Title")])], False
     )
 
     render_title_macro = nodes.Macro(
@@ -137,7 +137,6 @@ def test_complex():
             nodes.Output([nodes.TemplateData("\n  </ul>\n")]),
         ],
         False,
-        False,
     )
 
     tmpl = nodes.Template(

tests/test_imports.py

@@ -98,29 +98,6 @@ class TestImports:
         with pytest.raises(UndefinedError, match="does not export the requested name"):
             t.render()
 
-    def test_import_with_globals(self, test_env):
-        t = test_env.from_string(
-            '{% import "module" as m %}{{ m.test() }}', globals={"foo": 42}
-        )
-        assert t.render() == "[42|23]"
-
-        t = test_env.from_string('{% import "module" as m %}{{ m.test() }}')
-        assert t.render() == "[|23]"
-
-    def test_import_with_globals_override(self, test_env):
-        t = test_env.from_string(
-            '{% set foo = 41 %}{% import "module" as m %}{{ m.test() }}',
-            globals={"foo": 42},
-        )
-        assert t.render() == "[42|23]"
-
-    def test_from_import_with_globals(self, test_env):
-        t = test_env.from_string(
-            '{% from "module" import test %}{{ test() }}',
-            globals={"foo": 42},
-        )
-        assert t.render() == "[42|23]"
-
 
 class TestIncludes:
     def test_context_include(self, test_env):

tests/test_inheritance.py

@@ -3,7 +3,6 @@ import pytest
 from jinja2 import DictLoader
 from jinja2 import Environment
 from jinja2 import TemplateRuntimeError
-from jinja2 import TemplateSyntaxError
 
 LAYOUTTEMPLATE = """\
 |{% block block1 %}block 1 from layout{% endblock %}
@@ -37,7 +36,7 @@ WORKINGTEMPLATE = """\
 {% block block1 %}
   {% if false %}
     {% block block2 %}
-      this should work
+      this should workd
     {% endblock %}
   {% endif %}
 {% endblock %}
@@ -49,7 +48,7 @@ DOUBLEEXTENDS = """\
 {% block block1 %}
   {% if false %}
     {% block block2 %}
-      this should work
+      this should workd
     {% endblock %}
   {% endif %}
 {% endblock %}
@@ -149,46 +148,43 @@ class TestInheritance:
         env = Environment(
             loader=DictLoader(
                 {
-                    "default1": "DEFAULT1{% block x %}{% endblock %}",
-                    "default2": "DEFAULT2{% block x %}{% endblock %}",
-                    "child": "{% extends default %}{% block x %}CHILD{% endblock %}",
+                    "master1": "MASTER1{% block x %}{% endblock %}",
+                    "master2": "MASTER2{% block x %}{% endblock %}",
+                    "child": "{% extends master %}{% block x %}CHILD{% endblock %}",
                 }
             )
         )
         tmpl = env.get_template("child")
         for m in range(1, 3):
-            assert tmpl.render(default=f"default{m}") == f"DEFAULT{m}CHILD"
+            assert tmpl.render(master=f"master{m}") == f"MASTER{m}CHILD"
 
     def test_multi_inheritance(self, env):
         env = Environment(
             loader=DictLoader(
                 {
-                    "default1": "DEFAULT1{% block x %}{% endblock %}",
-                    "default2": "DEFAULT2{% block x %}{% endblock %}",
-                    "child": (
-                        "{% if default %}{% extends default %}{% else %}"
-                        "{% extends 'default1' %}{% endif %}"
-                        "{% block x %}CHILD{% endblock %}"
-                    ),
+                    "master1": "MASTER1{% block x %}{% endblock %}",
+                    "master2": "MASTER2{% block x %}{% endblock %}",
+                    "child": """{% if master %}{% extends master %}{% else %}{% extends
+                'master1' %}{% endif %}{% block x %}CHILD{% endblock %}""",
                 }
             )
         )
         tmpl = env.get_template("child")
-        assert tmpl.render(default="default2") == "DEFAULT2CHILD"
-        assert tmpl.render(default="default1") == "DEFAULT1CHILD"
-        assert tmpl.render() == "DEFAULT1CHILD"
+        assert tmpl.render(master="master2") == "MASTER2CHILD"
+        assert tmpl.render(master="master1") == "MASTER1CHILD"
+        assert tmpl.render() == "MASTER1CHILD"
 
     def test_scoped_block(self, env):
         env = Environment(
             loader=DictLoader(
                 {
-                    "default.html": "{% for item in seq %}[{% block item scoped %}"
+                    "master.html": "{% for item in seq %}[{% block item scoped %}"
                     "{% endblock %}]{% endfor %}"
                 }
             )
         )
         t = env.from_string(
-            "{% extends 'default.html' %}{% block item %}{{ item }}{% endblock %}"
+            "{% extends 'master.html' %}{% block item %}{{ item }}{% endblock %}"
         )
         assert t.render(seq=list(range(5))) == "[0][1][2][3][4]"
 
@@ -196,13 +192,13 @@ class TestInheritance:
         env = Environment(
             loader=DictLoader(
                 {
-                    "default.html": "{% for item in seq %}[{% block item scoped %}"
+                    "master.html": "{% for item in seq %}[{% block item scoped %}"
                     "{{ item }}{% endblock %}]{% endfor %}"
                 }
             )
         )
         t = env.from_string(
-            '{% extends "default.html" %}{% block item %}'
+            '{% extends "master.html" %}{% block item %}'
             "{{ super() }}|{{ item * 2 }}{% endblock %}"
         )
         assert t.render(seq=list(range(5))) == "[0|0][1|2][2|4][3|6][4|8]"
@@ -234,141 +230,14 @@ class TestInheritance:
         rv = env.get_template("index.html").render(the_foo=42).split()
         assert rv == ["43", "44", "45"]
 
-    def test_level1_required(self, env):
-        env = Environment(
-            loader=DictLoader(
-                {
-                    "default": "{% block x required %}{# comment #}\n {% endblock %}",
-                    "level1": "{% extends 'default' %}{% block x %}[1]{% endblock %}",
-                }
-            )
-        )
-        rv = env.get_template("level1").render()
-        assert rv == "[1]"
-
-    def test_level2_required(self, env):
-        env = Environment(
-            loader=DictLoader(
-                {
-                    "default": "{% block x required %}{% endblock %}",
-                    "level1": "{% extends 'default' %}{% block x %}[1]{% endblock %}",
-                    "level2": "{% extends 'default' %}{% block x %}[2]{% endblock %}",
-                }
-            )
-        )
-        rv1 = env.get_template("level1").render()
-        rv2 = env.get_template("level2").render()
-
-        assert rv1 == "[1]"
-        assert rv2 == "[2]"
-
-    def test_level3_required(self, env):
-        env = Environment(
-            loader=DictLoader(
-                {
-                    "default": "{% block x required %}{% endblock %}",
-                    "level1": "{% extends 'default' %}",
-                    "level2": "{% extends 'level1' %}{% block x %}[2]{% endblock %}",
-                    "level3": "{% extends 'level2' %}",
-                }
-            )
-        )
-        t1 = env.get_template("level1")
-        t2 = env.get_template("level2")
-        t3 = env.get_template("level3")
-
-        with pytest.raises(TemplateRuntimeError, match="Required block 'x' not found"):
-            assert t1.render()
-
-        assert t2.render() == "[2]"
-        assert t3.render() == "[2]"
-
-    def test_invalid_required(self, env):
-        env = Environment(
-            loader=DictLoader(
-                {
-                    "empty": "{% block x required %}{% endblock %}",
-                    "blank": "{% block x required %} {# c #}{% endblock %}",
-                    "text": "{% block x required %}data {# c #}{% endblock %}",
-                    "block": "{% block x required %}{% block y %}"
-                    "{% endblock %}{% endblock %}",
-                    "if": "{% block x required %}{% if true %}"
-                    "{% endif %}{% endblock %}",
-                    "top": "{% extends t %}{% block x %}CHILD{% endblock %}",
-                }
-            )
-        )
-        t = env.get_template("top")
-        assert t.render(t="empty") == "CHILD"
-        assert t.render(t="blank") == "CHILD"
-
-        required_block_check = pytest.raises(
-            TemplateSyntaxError,
-            match="Required blocks can only contain comments or whitespace",
-        )
-
-        with required_block_check:
-            t.render(t="text")
-
-        with required_block_check:
-            t.render(t="block")
-
-        with required_block_check:
-            t.render(t="if")
-
-    def test_required_with_scope(self, env):
-        env = Environment(
-            loader=DictLoader(
-                {
-                    "default1": "{% for item in seq %}[{% block item scoped required %}"
-                    "{% endblock %}]{% endfor %}",
-                    "child1": "{% extends 'default1' %}{% block item %}"
-                    "{{ item }}{% endblock %}",
-                    "default2": "{% for item in seq %}[{% block item required scoped %}"
-                    "{% endblock %}]{% endfor %}",
-                    "child2": "{% extends 'default2' %}{% block item %}"
-                    "{{ item }}{% endblock %}",
-                }
-            )
-        )
-        t1 = env.get_template("child1")
-        t2 = env.get_template("child2")
-
-        assert t1.render(seq=list(range(3))) == "[0][1][2]"
-
-        # scoped must come before required
-        with pytest.raises(TemplateSyntaxError):
-            t2.render(seq=list(range(3)))
-
-    def test_duplicate_required_or_scoped(self, env):
-        env = Environment(
-            loader=DictLoader(
-                {
-                    "default1": "{% for item in seq %}[{% block item "
-                    "scoped scoped %}}{{% endblock %}}]{{% endfor %}}",
-                    "default2": "{% for item in seq %}[{% block item "
-                    "required required %}}{{% endblock %}}]{{% endfor %}}",
-                    "child": "{% if default %}{% extends default %}{% else %}"
-                    "{% extends 'default1' %}{% endif %}{%- block x %}"
-                    "CHILD{% endblock %}",
-                }
-            )
-        )
-        tmpl = env.get_template("child")
-
-        with pytest.raises(TemplateSyntaxError):
-            tmpl.render(default="default1", seq=list(range(3)))
-
-        with pytest.raises(TemplateSyntaxError):
-            tmpl.render(default="default2", seq=list(range(3)))
-
 
 class TestBugFix:
     def test_fixed_macro_scoping_bug(self, env):
-        assert Environment(
-            loader=DictLoader(
-                {
-                    "test.html": """\
+        assert (
+            Environment(
+                loader=DictLoader(
+                    {
+                        "test.html": """\
         {% extends 'details.html' %}
 
         {% macro my_macro() %}
@@ -379,7 +248,7 @@ class TestBugFix:
             {{ my_macro() }}
         {% endblock %}
             """,
-                    "details.html": """\
+                        "details.html": """\
         {% extends 'standard.html' %}
 
         {% macro my_macro() %}
@@ -395,12 +264,17 @@ class TestBugFix:
             {% endblock %}
         {% endblock %}
         """,
-                    "standard.html": """
+                        "standard.html": """
         {% block content %} {% endblock %}
         """,
-                }
+                    }
+                )
             )
-        ).get_template("test.html").render().split() == ["outer_box", "my_macro"]
+            .get_template("test.html")
+            .render()
+            .split()
+            == ["outer_box", "my_macro"]
+        )
 
     def test_double_extends(self, env):
         """Ensures that a template with more than 1 {% extends ... %} usage

tests/test_lexnparse.py

@@ -43,7 +43,8 @@ class TestTokenStream:
 class TestLexer:
     def test_raw1(self, env):
         tmpl = env.from_string(
-            "{% raw %}foo{% endraw %}|{%raw%}{{ bar }}|{% baz %}{%       endraw    %}"
+            "{% raw %}foo{% endraw %}|"
+            "{%raw%}{{ bar }}|{% baz %}{%       endraw    %}"
         )
         assert tmpl.render() == "foo|{{ bar }}|{% baz %}"
 
@@ -411,13 +412,6 @@ class TestSyntax:
             ("2.5e+100", "2.5e+100"),
             ("25.6e-10", "2.56e-09"),
             ("1_2.3_4e5_6", "1.234e+57"),
-            ("0", "0"),
-            ("0_00", "0"),
-            ("0b1001_1111", "159"),
-            ("0o123", "83"),
-            ("0o1_23", "83"),
-            ("0x123abc", "1194684"),
-            ("0x12_3abc", "1194684"),
         ),
     )
     def test_numeric_literal(self, env, value, expect):
@@ -455,8 +449,9 @@ class TestSyntax:
         tmpl = env.from_string('{{ "foo"|upper + "bar"|upper }}')
         assert tmpl.render() == "FOOBAR"
 
-    def test_function_calls(self, env):
-        tests = [
+    @pytest.mark.parametrize(
+        "should_fail,sig",
+        (
             (True, "*foo, bar"),
             (True, "*foo, *bar"),
             (True, "**foo, *bar"),
@@ -472,16 +467,18 @@ class TestSyntax:
             (False, "*foo, **bar"),
             (False, "*foo, bar=42, **baz"),
             (False, "foo, *args, bar=23, **baz"),
-        ]
-        for should_fail, sig in tests:
-            if should_fail:
-                with pytest.raises(TemplateSyntaxError):
-                    env.from_string(f"{{{{ foo({sig}) }}}}")
-            else:
-                env.from_string(f"foo({sig})")
+        )
+    )
+    def test_function_calls(self, env, should_fail, sig):
+        if should_fail:
+            with pytest.raises(TemplateSyntaxError):
+                env.from_string(f"{{{{ foo({sig}) }}}}")
+        else:
+            env.from_string(f"foo({sig})")
 
-    def test_tuple_expr(self, env):
-        for tmpl in [
+    @pytest.mark.parametrize(
+        "tmpl",
+        (
             "{{ () }}",
             "{{ (1, 2) }}",
             "{{ (1, 2,) }}",
@@ -490,8 +487,10 @@ class TestSyntax:
             "{% for foo, bar in seq %}...{% endfor %}",
             "{% for x in foo, bar %}...{% endfor %}",
             "{% for x in foo, %}...{% endfor %}",
-        ]:
-            assert env.from_string(tmpl)
+        )
+    )
+    def test_tuple_expr(self, env, tmpl):
+        assert env.from_string(tmpl)
 
     def test_trailing_comma(self, env):
         tmpl = env.from_string("{{ (1, 2,) }}|{{ [1, 2,] }}|{{ {1: 2,} }}")
@@ -909,121 +908,3 @@ ${item} ## the rest of the stuff
 <!--- endfor -->"""
         )
         assert tmpl.render(seq=range(5)) == "01234"
-
-
-class TestTrimBlocks:
-    def test_trim(self, env):
-        env = Environment(trim_blocks=True, lstrip_blocks=False)
-        tmpl = env.from_string("    {% if True %}\n    {% endif %}")
-        assert tmpl.render() == "        "
-
-    def test_no_trim(self, env):
-        env = Environment(trim_blocks=True, lstrip_blocks=False)
-        tmpl = env.from_string("    {% if True +%}\n    {% endif %}")
-        assert tmpl.render() == "    \n    "
-
-    def test_no_trim_outer(self, env):
-        env = Environment(trim_blocks=True, lstrip_blocks=False)
-        tmpl = env.from_string("{% if True %}X{% endif +%}\nmore things")
-        assert tmpl.render() == "X\nmore things"
-
-    def test_lstrip_no_trim(self, env):
-        env = Environment(trim_blocks=True, lstrip_blocks=True)
-        tmpl = env.from_string("    {% if True +%}\n    {% endif %}")
-        assert tmpl.render() == "\n"
-
-    def test_trim_blocks_false_with_no_trim(self, env):
-        # Test that + is a NOP (but does not cause an error) if trim_blocks=False
-        env = Environment(trim_blocks=False, lstrip_blocks=False)
-        tmpl = env.from_string("    {% if True %}\n    {% endif %}")
-        assert tmpl.render() == "    \n    "
-        tmpl = env.from_string("    {% if True +%}\n    {% endif %}")
-        assert tmpl.render() == "    \n    "
-
-        tmpl = env.from_string("    {# comment #}\n    ")
-        assert tmpl.render() == "    \n    "
-        tmpl = env.from_string("    {# comment +#}\n    ")
-        assert tmpl.render() == "    \n    "
-
-        tmpl = env.from_string("    {% raw %}{% endraw %}\n    ")
-        assert tmpl.render() == "    \n    "
-        tmpl = env.from_string("    {% raw %}{% endraw +%}\n    ")
-        assert tmpl.render() == "    \n    "
-
-    def test_trim_nested(self, env):
-        env = Environment(trim_blocks=True, lstrip_blocks=True)
-        tmpl = env.from_string(
-            "    {% if True %}\na {% if True %}\nb {% endif %}\nc {% endif %}"
-        )
-        assert tmpl.render() == "a b c "
-
-    def test_no_trim_nested(self, env):
-        env = Environment(trim_blocks=True, lstrip_blocks=True)
-        tmpl = env.from_string(
-            "    {% if True +%}\na {% if True +%}\nb {% endif +%}\nc {% endif %}"
-        )
-        assert tmpl.render() == "\na \nb \nc "
-
-    def test_comment_trim(self, env):
-        env = Environment(trim_blocks=True, lstrip_blocks=True)
-        tmpl = env.from_string("""    {# comment #}\n\n  """)
-        assert tmpl.render() == "\n  "
-
-    def test_comment_no_trim(self, env):
-        env = Environment(trim_blocks=True, lstrip_blocks=True)
-        tmpl = env.from_string("""    {# comment +#}\n\n  """)
-        assert tmpl.render() == "\n\n  "
-
-    def test_multiple_comment_trim_lstrip(self, env):
-        env = Environment(trim_blocks=True, lstrip_blocks=True)
-        tmpl = env.from_string(
-            "   {# comment #}\n\n{# comment2 #}\n   \n{# comment3 #}\n\n "
-        )
-        assert tmpl.render() == "\n   \n\n "
-
-    def test_multiple_comment_no_trim_lstrip(self, env):
-        env = Environment(trim_blocks=True, lstrip_blocks=True)
-        tmpl = env.from_string(
-            "   {# comment +#}\n\n{# comment2 +#}\n   \n{# comment3 +#}\n\n "
-        )
-        assert tmpl.render() == "\n\n\n   \n\n\n "
-
-    def test_raw_trim_lstrip(self, env):
-        env = Environment(trim_blocks=True, lstrip_blocks=True)
-        tmpl = env.from_string("{{x}}{% raw %}\n\n    {% endraw %}\n\n{{ y }}")
-        assert tmpl.render(x=1, y=2) == "1\n\n\n2"
-
-    def test_raw_no_trim_lstrip(self, env):
-        env = Environment(trim_blocks=False, lstrip_blocks=True)
-        tmpl = env.from_string("{{x}}{% raw %}\n\n      {% endraw +%}\n\n{{ y }}")
-        assert tmpl.render(x=1, y=2) == "1\n\n\n\n2"
-
-        # raw blocks do not process inner text, so start tag cannot ignore trim
-        with pytest.raises(TemplateSyntaxError):
-            tmpl = env.from_string("{{x}}{% raw +%}\n\n  {% endraw +%}\n\n{{ y }}")
-
-    def test_no_trim_angle_bracket(self, env):
-        env = Environment(
-            "<%", "%>", "${", "}", "<%#", "%>", lstrip_blocks=True, trim_blocks=True
-        )
-        tmpl = env.from_string("    <% if True +%>\n\n    <% endif %>")
-        assert tmpl.render() == "\n\n"
-
-        tmpl = env.from_string("    <%# comment +%>\n\n   ")
-        assert tmpl.render() == "\n\n   "
-
-    def test_no_trim_php_syntax(self, env):
-        env = Environment(
-            "<?",
-            "?>",
-            "<?=",
-            "?>",
-            "<!--",
-            "-->",
-            lstrip_blocks=False,
-            trim_blocks=True,
-        )
-        tmpl = env.from_string("    <? if True +?>\n\n    <? endif ?>")
-        assert tmpl.render() == "    \n\n    "
-        tmpl = env.from_string("    <!-- comment +-->\n\n    ")
-        assert tmpl.render() == "    \n\n    "

tests/test_loader.py

@@ -2,12 +2,12 @@ import importlib.abc
 import importlib.machinery
 import importlib.util
 import os
+import platform
 import shutil
 import sys
 import tempfile
 import time
 import weakref
-from pathlib import Path
 
 import pytest
 
@@ -32,7 +32,8 @@ class TestLoaders:
         pytest.raises(TemplateNotFound, env.get_template, "missing.html")
 
     def test_filesystem_loader_overlapping_names(self, filesystem_loader):
-        t2_dir = Path(filesystem_loader.searchpath[0]) / ".." / "templates2"
+        res = os.path.dirname(filesystem_loader.searchpath[0])
+        t2_dir = os.path.join(res, "templates2")
         # Make "foo" show up before "foo/test.html".
         filesystem_loader.searchpath.insert(0, t2_dir)
         e = Environment(loader=filesystem_loader)
@@ -117,7 +118,9 @@ class TestLoaders:
 
 
 class TestFileSystemLoader:
-    searchpath = (Path(__file__) / ".." / "res" / "templates").resolve()
+    searchpath = os.path.join(
+        os.path.dirname(os.path.abspath(__file__)), "res", "templates"
+    )
 
     @staticmethod
     def _test_common(env):
@@ -128,20 +131,24 @@ class TestFileSystemLoader:
         pytest.raises(TemplateNotFound, env.get_template, "missing.html")
 
     def test_searchpath_as_str(self):
-        filesystem_loader = loaders.FileSystemLoader(str(self.searchpath))
+        filesystem_loader = loaders.FileSystemLoader(self.searchpath)
 
         env = Environment(loader=filesystem_loader)
         self._test_common(env)
 
     def test_searchpath_as_pathlib(self):
-        filesystem_loader = loaders.FileSystemLoader(self.searchpath)
+        import pathlib
+
+        searchpath = pathlib.Path(self.searchpath)
+        filesystem_loader = loaders.FileSystemLoader(searchpath)
         env = Environment(loader=filesystem_loader)
         self._test_common(env)
 
     def test_searchpath_as_list_including_pathlib(self):
-        filesystem_loader = loaders.FileSystemLoader(
-            ["/tmp/templates", self.searchpath]
-        )
+        import pathlib
+
+        searchpath = pathlib.Path(self.searchpath)
+        filesystem_loader = loaders.FileSystemLoader(["/tmp/templates", searchpath])
         env = Environment(loader=filesystem_loader)
         self._test_common(env)
 
@@ -153,7 +160,7 @@ class TestFileSystemLoader:
         tmpl2 = env.get_template("test.html")
         assert tmpl1 is tmpl2
 
-        os.utime(self.searchpath / "test.html", (time.time(), time.time()))
+        os.utime(os.path.join(self.searchpath, "test.html"), (time.time(), time.time()))
         tmpl3 = env.get_template("test.html")
         assert tmpl1 is not tmpl3
 
@@ -170,37 +177,9 @@ class TestFileSystemLoader:
         t = e.get_template("mojibake.txt")
         assert t.render() == expect
 
-    def test_filename_normpath(self):
-        """Nested template names should only contain ``os.sep`` in the
-        loaded filename.
-        """
-        loader = loaders.FileSystemLoader(self.searchpath)
-        e = Environment(loader=loader)
-        t = e.get_template("foo/test.html")
-        assert t.filename == str(self.searchpath / "foo" / "test.html")
-
-    def test_error_includes_paths(self, env, filesystem_loader):
-        env.loader = filesystem_loader
-
-        with pytest.raises(TemplateNotFound) as info:
-            env.get_template("missing")
-
-        e_str = str(info.value)
-        assert e_str.startswith("'missing' not found in search path: ")
-
-        filesystem_loader.searchpath.append("other")
-
-        with pytest.raises(TemplateNotFound) as info:
-            env.get_template("missing")
-
-        e_str = str(info.value)
-        assert e_str.startswith("'missing' not found in search paths: ")
-        assert ", 'other'" in e_str
-
 
 class TestModuleLoader:
     archive = None
-    mod_env = None
 
     def compile_down(self, prefix_loader, zip="deflated"):
         log = []
@@ -214,14 +193,13 @@ class TestModuleLoader:
         self.mod_env = Environment(loader=loaders.ModuleLoader(self.archive))
         return "".join(log)
 
-    def teardown_method(self):
-        if self.archive is not None:
+    def teardown(self):
+        if hasattr(self, "mod_env"):
             if os.path.isfile(self.archive):
                 os.remove(self.archive)
             else:
                 shutil.rmtree(self.archive)
             self.archive = None
-            self.mod_env = None
 
     def test_log(self, prefix_loader):
         log = self.compile_down(prefix_loader)
@@ -304,7 +282,10 @@ class TestModuleLoader:
         self.compile_down(prefix_loader)
 
         mod_path = self.mod_env.loader.module.__path__[0]
-        mod_loader = loaders.ModuleLoader(Path(mod_path))
+
+        import pathlib
+
+        mod_loader = loaders.ModuleLoader(pathlib.Path(mod_path))
         self.mod_env = Environment(loader=mod_loader)
 
         self._test_common()
@@ -313,7 +294,10 @@ class TestModuleLoader:
         self.compile_down(prefix_loader)
 
         mod_path = self.mod_env.loader.module.__path__[0]
-        mod_loader = loaders.ModuleLoader([Path(mod_path), "/tmp/templates"])
+
+        import pathlib
+
+        mod_loader = loaders.ModuleLoader([pathlib.Path(mod_path), "/tmp/templates"])
         self.mod_env = Environment(loader=mod_loader)
 
         self._test_common()
@@ -321,7 +305,7 @@ class TestModuleLoader:
 
 @pytest.fixture()
 def package_dir_loader(monkeypatch):
-    monkeypatch.syspath_prepend(Path(__file__).parent)
+    monkeypatch.syspath_prepend(os.path.dirname(__file__))
     return PackageLoader("res")
 
 
@@ -341,32 +325,11 @@ def test_package_dir_list(package_dir_loader):
     assert "test.html" in templates
 
 
-@pytest.fixture()
-def package_file_loader(monkeypatch):
-    monkeypatch.syspath_prepend(Path(__file__).parent / "res")
-    return PackageLoader("__init__")
-
-
-@pytest.mark.parametrize(
-    ("template", "expect"), [("foo/test.html", "FOO"), ("test.html", "BAR")]
-)
-def test_package_file_source(package_file_loader, template, expect):
-    source, name, up_to_date = package_file_loader.get_source(None, template)
-    assert source.rstrip() == expect
-    assert name.endswith(os.path.join(*split_template_path(template)))
-    assert up_to_date()
-
-
-def test_package_file_list(package_file_loader):
-    templates = package_file_loader.list_templates()
-    assert "foo/test.html" in templates
-    assert "test.html" in templates
-
-
 @pytest.fixture()
 def package_zip_loader(monkeypatch):
-    package_zip = (Path(__file__) / ".." / "res" / "package.zip").resolve()
-    monkeypatch.syspath_prepend(package_zip)
+    monkeypatch.syspath_prepend(
+        os.path.join(os.path.dirname(__file__), "res", "package.zip")
+    )
     return PackageLoader("t_pack")
 
 
@@ -381,25 +344,14 @@ def test_package_zip_source(package_zip_loader, template, expect):
 
 
 @pytest.mark.xfail(
-    sys.implementation.name == "pypy",
-    reason="zipimporter doesn't have a '_files' attribute",
+    platform.python_implementation() == "PyPy",
+    reason="PyPy's zipimporter doesn't have a '_files' attribute.",
     raises=TypeError,
 )
 def test_package_zip_list(package_zip_loader):
     assert package_zip_loader.list_templates() == ["foo/test.html", "test.html"]
 
 
-@pytest.mark.parametrize("package_path", ["", ".", "./"])
-def test_package_zip_omit_curdir(package_zip_loader, package_path):
-    """PackageLoader should not add or include "." or "./" in the root
-    path, it is invalid in zip paths.
-    """
-    loader = PackageLoader("t_pack", package_path)
-    assert loader.package_path == ""
-    source, _, _ = loader.get_source(None, "templates/foo/test.html")
-    assert source.rstrip() == "FOO"
-
-
 def test_pep_451_import_hook():
     class ImportHook(importlib.abc.MetaPathFinder, importlib.abc.Loader):
         def find_spec(self, name, path=None, target=None):
@@ -429,8 +381,3 @@ def test_pep_451_import_hook():
         assert "test.html" in package_loader.list_templates()
     finally:
         sys.meta_path[:] = before
-
-
-def test_package_loader_no_dir() -> None:
-    with pytest.raises(ValueError, match="could not find a 'templates' directory"):
-        PackageLoader("jinja2")

tests/test_nativetypes.py

@@ -13,11 +13,6 @@ def env():
     return NativeEnvironment()
 
 
-@pytest.fixture
-def async_native_env():
-    return NativeEnvironment(enable_async=True)
-
-
 def test_is_defined_native_return(env):
     t = env.from_string("{{ missing is defined }}")
     assert not t.render()
@@ -127,18 +122,6 @@ def test_string_top_level(env):
     assert result == "Jinja"
 
 
-def test_string_concatenation(async_native_env, run_async_fn):
-    async def async_render():
-        t = async_native_env.from_string(
-            "{%- macro x(y) -%}{{ y }}{%- endmacro -%}{{- x('not') }} {{ x('bad') -}}"
-        )
-        result = await t.render_async()
-        assert isinstance(result, str)
-        assert result == "not bad"
-
-    run_async_fn(async_render)
-
-
 def test_tuple_of_variable_strings(env):
     t = env.from_string("'{{ a }}', 'data', '{{ b }}', b'{{ c }}'")
     result = t.render(a=1, b=2, c="bytes")
@@ -164,26 +147,3 @@ def test_no_intermediate_eval(env):
 def test_spontaneous_env():
     t = NativeTemplate("{{ true }}")
     assert isinstance(t.environment, NativeEnvironment)
-
-
-def test_leading_spaces(env):
-    t = env.from_string(" {{ True }}")
-    result = t.render()
-    assert result == " True"
-
-
-def test_macro(env):
-    t = env.from_string("{%- macro x() -%}{{- [1,2] -}}{%- endmacro -%}{{- x()[1] -}}")
-    result = t.render()
-    assert result == 2
-    assert isinstance(result, int)
-
-
-def test_block(env):
-    t = env.from_string(
-        "{% block b %}{% for i in range(1) %}{{ loop.index }}{% endfor %}"
-        "{% endblock %}{{ self.b() }}"
-    )
-    result = t.render()
-    assert result == 11
-    assert isinstance(result, int)

tests/test_nodes.py

@@ -1,3 +0,0 @@
-def test_template_hash(env):
-    template = env.parse("hash test")
-    hash(template)

tests/test_pickle.py

@@ -1,6 +0,0 @@
-import pickle
-
-
-def test_environment(env):
-    env = pickle.loads(pickle.dumps(env))
-    assert env.from_string("x={{ x }}").render(x=42) == "x=42"

tests/test_regression.py

@@ -7,7 +7,6 @@ from jinja2 import Template
 from jinja2 import TemplateAssertionError
 from jinja2 import TemplateNotFound
 from jinja2 import TemplateSyntaxError
-from jinja2.utils import pass_context
 
 
 class TestCorner:
@@ -110,15 +109,6 @@ class TestBug:
             "http://www.example.org/&lt;foo</a>"
         )
 
-    def test_urlize_filter_closing_punctuation(self, env):
-        tmpl = env.from_string(
-            '{{ "(see http://www.example.org/?page=subj_<desc.h>)"|urlize }}'
-        )
-        assert tmpl.render() == (
-            '(see <a href="http://www.example.org/?page=subj_&lt;desc.h&gt;" '
-            'rel="noopener">http://www.example.org/?page=subj_&lt;desc.h&gt;</a>)'
-        )
-
     def test_loop_call_loop(self, env):
         tmpl = env.from_string(
             """
@@ -298,9 +288,11 @@ class TestBug:
 
         assert e.value.name == "foo/bar.html"
 
-    def test_pass_context_callable_class(self, env):
+    def test_contextfunction_callable_classes(self, env):
+        from jinja2.utils import contextfunction
+
         class CallableClass:
-            @pass_context
+            @contextfunction
             def __call__(self, ctx):
                 return ctx.resolve("hello")
 
@@ -363,7 +355,9 @@ class TestBug:
         assert t.render().strip() == "45|6"
 
     def test_macro_escaping(self):
-        env = Environment(autoescape=lambda x: False)
+        env = Environment(
+            autoescape=lambda x: False, extensions=["jinja2.ext.autoescape"]
+        )
         template = "{% macro m() %}<html>{% endmacro %}"
         template += "{% autoescape true %}{{ m() }}{% endautoescape %}"
         assert env.from_string(template).render()
@@ -591,6 +585,21 @@ class TestBug:
         env = MyEnvironment(loader=loader)
         assert env.get_template("test").render(foobar="test") == "test"
 
+    def test_legacy_custom_context(self, env):
+        from jinja2.runtime import Context, missing
+
+        class MyContext(Context):
+            def resolve(self, name):
+                if name == "foo":
+                    return 42
+                return super().resolve(name)
+
+        x = MyContext(env, parent={"bar": 23}, name="foo", blocks={})
+        assert x._legacy_resolve_mode
+        assert x.resolve_or_missing("foo") == 42
+        assert x.resolve_or_missing("bar") == 23
+        assert x.resolve_or_missing("baz") is missing
+
     def test_recursive_loop_bug(self, env):
         tmpl = env.from_string(
             "{%- for value in values recursive %}1{% else %}0{% endfor -%}"
@@ -598,170 +607,7 @@ class TestBug:
         assert tmpl.render(values=[]) == "0"
 
     def test_markup_and_chainable_undefined(self):
-        from markupsafe import Markup
-
+        from jinja2 import Markup
         from jinja2.runtime import ChainableUndefined
 
         assert str(Markup(ChainableUndefined())) == ""
-
-    def test_scoped_block_loop_vars(self, env):
-        tmpl = env.from_string(
-            """\
-Start
-{% for i in ["foo", "bar"] -%}
-{% block body scoped -%}
-{{ loop.index }}) {{ i }}{% if loop.last %} last{% endif -%}
-{%- endblock %}
-{% endfor -%}
-End"""
-        )
-        assert tmpl.render() == "Start\n1) foo\n2) bar last\nEnd"
-
-    def test_pass_context_loop_vars(self, env):
-        @pass_context
-        def test(ctx):
-            return f"{ctx['i']}{ctx['j']}"
-
-        tmpl = env.from_string(
-            """\
-{% set i = 42 %}
-{%- for idx in range(2) -%}
-{{ i }}{{ j }}
-{% set i = idx -%}
-{%- set j = loop.index -%}
-{{ test() }}
-{{ i }}{{ j }}
-{% endfor -%}
-{{ i }}{{ j }}"""
-        )
-        tmpl.globals["test"] = test
-        assert tmpl.render() == "42\n01\n01\n42\n12\n12\n42"
-
-    def test_pass_context_scoped_loop_vars(self, env):
-        @pass_context
-        def test(ctx):
-            return f"{ctx['i']}"
-
-        tmpl = env.from_string(
-            """\
-{% set i = 42 %}
-{%- for idx in range(2) -%}
-{{ i }}
-{%- set i = loop.index0 -%}
-{% block body scoped %}
-{{ test() }}
-{% endblock -%}
-{% endfor -%}
-{{ i }}"""
-        )
-        tmpl.globals["test"] = test
-        assert tmpl.render() == "42\n0\n42\n1\n42"
-
-    def test_pass_context_in_blocks(self, env):
-        @pass_context
-        def test(ctx):
-            return f"{ctx['i']}"
-
-        tmpl = env.from_string(
-            """\
-{%- set i = 42 -%}
-{{ i }}
-{% block body -%}
-{% set i = 24 -%}
-{{ test() }}
-{% endblock -%}
-{{ i }}"""
-        )
-        tmpl.globals["test"] = test
-        assert tmpl.render() == "42\n24\n42"
-
-    def test_pass_context_block_and_loop(self, env):
-        @pass_context
-        def test(ctx):
-            return f"{ctx['i']}"
-
-        tmpl = env.from_string(
-            """\
-{%- set i = 42 -%}
-{% for idx in range(2) -%}
-{{ test() }}
-{%- set i = idx -%}
-{% block body scoped %}
-{{ test() }}
-{% set i = 24 -%}
-{{ test() }}
-{% endblock -%}
-{{ test() }}
-{% endfor -%}
-{{ test() }}"""
-        )
-        tmpl.globals["test"] = test
-
-        # values set within a block or loop should not
-        # show up outside of it
-        assert tmpl.render() == "42\n0\n24\n0\n42\n1\n24\n1\n42"
-
-    @pytest.mark.parametrize("op", ["extends", "include"])
-    def test_cached_extends(self, op):
-        env = Environment(
-            loader=DictLoader(
-                {"base": "{{ x }} {{ y }}", "main": f"{{% {op} 'base' %}}"}
-            )
-        )
-        env.globals["x"] = "x"
-        env.globals["y"] = "y"
-
-        # template globals overlay env globals
-        tmpl = env.get_template("main", globals={"x": "bar"})
-        assert tmpl.render() == "bar y"
-
-        # base was loaded indirectly, it just has env globals
-        tmpl = env.get_template("base")
-        assert tmpl.render() == "x y"
-
-        # set template globals for base, no longer uses env globals
-        tmpl = env.get_template("base", globals={"x": 42})
-        assert tmpl.render() == "42 y"
-
-        # templates are cached, they keep template globals set earlier
-        tmpl = env.get_template("main")
-        assert tmpl.render() == "bar y"
-
-        tmpl = env.get_template("base")
-        assert tmpl.render() == "42 y"
-
-    def test_nested_loop_scoping(self, env):
-        tmpl = env.from_string(
-            "{% set output %}{% for x in [1,2,3] %}hello{% endfor %}"
-            "{% endset %}{{ output }}"
-        )
-        assert tmpl.render() == "hellohellohello"
-
-    def test_pass_context_with_select(self, env):
-        @pass_context
-        def is_foo(ctx, s):
-            assert ctx is not None
-            return s == "foo"
-
-        env.tests["foo"] = is_foo
-        tmpl = env.from_string(
-            "{% for x in ['one', 'foo'] | select('foo') %}{{ x }}{% endfor %}"
-        )
-        assert tmpl.render() == "foo"
-
-
-def test_load_parameter_when_set_in_all_if_branches(env):
-    tmpl = env.from_string(
-        "{% if True %}{{ a.b }}{% set a = 1 %}"
-        "{% elif False %}{% set a = 2 %}"
-        "{% else %}{% set a = 3 %}{% endif %}"
-        "{{ a }}"
-    )
-    assert tmpl.render(a={"b": 0}) == "01"
-
-
-@pytest.mark.parametrize("unicode_char", ["\N{FORM FEED}", "\x85"])
-def test_unicode_whitespace(env, unicode_char):
-    content = "Lorem ipsum\n" + unicode_char + "\nMore text"
-    tmpl = env.from_string(content)
-    assert tmpl.render() == content

tests/test_runtime.py

@@ -1,15 +1,6 @@
-import copy
 import itertools
-import pickle
 
-import pytest
-
-from jinja2 import ChainableUndefined
-from jinja2 import DebugUndefined
-from jinja2 import StrictUndefined
 from jinja2 import Template
-from jinja2 import TemplateRuntimeError
-from jinja2 import Undefined
 from jinja2.runtime import LoopContext
 
 TEST_IDX_TEMPLATE_STR_1 = (
@@ -65,10 +56,10 @@ def test_iterator_not_advanced_early():
     assert out == "1 [(1, 'a'), (1, 'b')]\n2 [(2, 'c')]\n3 [(3, 'd')]\n"
 
 
-def test_mock_not_pass_arg_marker():
+def test_mock_not_contextfunction():
     """If a callable class has a ``__getattr__`` that returns True-like
     values for arbitrary attrs, it should not be incorrectly identified
-    as a ``pass_context`` function.
+    as a ``contextfunction``.
     """
 
     class Calc:
@@ -82,44 +73,3 @@ def test_mock_not_pass_arg_marker():
     out = t.render(calc=Calc())
     # Would be "1" if context argument was passed.
     assert out == "0"
-
-
-_undefined_types = (Undefined, ChainableUndefined, DebugUndefined, StrictUndefined)
-
-
-@pytest.mark.parametrize("undefined_type", _undefined_types)
-def test_undefined_copy(undefined_type):
-    undef = undefined_type("a hint", ["foo"], "a name", TemplateRuntimeError)
-    copied = copy.copy(undef)
-
-    assert copied is not undef
-    assert copied._undefined_hint is undef._undefined_hint
-    assert copied._undefined_obj is undef._undefined_obj
-    assert copied._undefined_name is undef._undefined_name
-    assert copied._undefined_exception is undef._undefined_exception
-
-
-@pytest.mark.parametrize("undefined_type", _undefined_types)
-def test_undefined_deepcopy(undefined_type):
-    undef = undefined_type("a hint", ["foo"], "a name", TemplateRuntimeError)
-    copied = copy.deepcopy(undef)
-
-    assert copied._undefined_hint is undef._undefined_hint
-    assert copied._undefined_obj is not undef._undefined_obj
-    assert copied._undefined_obj == undef._undefined_obj
-    assert copied._undefined_name is undef._undefined_name
-    assert copied._undefined_exception is undef._undefined_exception
-
-
-@pytest.mark.parametrize("undefined_type", _undefined_types)
-def test_undefined_pickle(undefined_type):
-    undef = undefined_type("a hint", ["foo"], "a name", TemplateRuntimeError)
-    copied = pickle.loads(pickle.dumps(undef))
-
-    assert copied._undefined_hint is not undef._undefined_hint
-    assert copied._undefined_hint == undef._undefined_hint
-    assert copied._undefined_obj is not undef._undefined_obj
-    assert copied._undefined_obj == undef._undefined_obj
-    assert copied._undefined_name is not undef._undefined_name
-    assert copied._undefined_name == undef._undefined_name
-    assert copied._undefined_exception is undef._undefined_exception

tests/test_security.py

@@ -1,7 +1,7 @@
 import pytest
-from markupsafe import escape
 
 from jinja2 import Environment
+from jinja2 import escape
 from jinja2.exceptions import SecurityError
 from jinja2.exceptions import TemplateRuntimeError
 from jinja2.exceptions import TemplateSyntaxError
@@ -58,8 +58,6 @@ class TestSandbox:
     def test_immutable_environment(self, env):
         env = ImmutableSandboxedEnvironment()
         pytest.raises(SecurityError, env.from_string("{{ [].append(23) }}").render)
-        pytest.raises(SecurityError, env.from_string("{{ [].clear() }}").render)
-        pytest.raises(SecurityError, env.from_string("{{ [1].pop() }}").render)
         pytest.raises(SecurityError, env.from_string("{{ {1:2}.clear() }}").render)
 
     def test_restricted(self, env):
@@ -112,19 +110,25 @@ class TestSandbox:
             with pytest.raises(TemplateRuntimeError):
                 t.render(ctx)
 
-    def test_unary_operator_intercepting(self, env):
+    @pytest.mark.parametrize(
+        "expr,ctx,rv",
+        (
+            ("-1", {}, "-1"),
+            ("-a", {"a": 2}, "-2")
+        )
+    )
+    def test_unary_operator_intercepting(self, env, expr, ctx, rv):
         def disable_op(arg):
             raise TemplateRuntimeError("that operator so does not work")
 
-        for expr, ctx, rv in ("-1", {}, "-1"), ("-a", {"a": 2}, "-2"):
-            env = SandboxedEnvironment()
-            env.unop_table["-"] = disable_op
-            t = env.from_string(f"{{{{ {expr} }}}}")
-            assert t.render(ctx) == rv
-            env.intercepted_unops = frozenset(["-"])
-            t = env.from_string(f"{{{{ {expr} }}}}")
-            with pytest.raises(TemplateRuntimeError):
-                t.render(ctx)
+        env = SandboxedEnvironment()
+        env.unop_table["-"] = disable_op
+        t = env.from_string(f"{{{{ {expr} }}}}")
+        assert t.render(ctx) == rv
+        env.intercepted_unops = frozenset(["-"])
+        t = env.from_string(f"{{{{ {expr} }}}}")
+        with pytest.raises(TemplateRuntimeError):
+            t.render(ctx)
 
 
 class TestStringFormat:
@@ -148,13 +152,6 @@ class TestStringFormat:
         t = env.from_string('{{ ("a{0.foo}b{1}"|safe).format({"foo": 42}, "<foo>") }}')
         assert t.render() == "a42b&lt;foo&gt;"
 
-    def test_empty_braces_format(self):
-        env = SandboxedEnvironment()
-        t1 = env.from_string('{{ ("a{}b{}").format("foo", "42")}}')
-        t2 = env.from_string('{{ ("a{}b{}"|safe).format(42, "<foo>") }}')
-        assert t1.render() == "afoob42"
-        assert t2.render() == "a42b&lt;foo&gt;"
-
 
 class TestStringFormatMap:
     def test_basic_format_safety(self):
@@ -173,30 +170,3 @@ class TestStringFormatMap:
             '{{ ("a{x.foo}b{y}"|safe).format_map({"x":{"foo": 42}, "y":"<foo>"}) }}'
         )
         assert t.render() == "a42b&lt;foo&gt;"
-
-    def test_indirect_call(self):
-        def run(value, arg):
-            return value.run(arg)
-
-        env = SandboxedEnvironment()
-        env.filters["run"] = run
-        t = env.from_string(
-            """{% set
-                ns = namespace(run="{0.__call__.__builtins__[__import__]}".format)
-            %}
-            {{ ns | run(not_here) }}
-            """
-        )
-
-        with pytest.raises(SecurityError):
-            t.render()
-
-    def test_attr_filter(self) -> None:
-        env = SandboxedEnvironment()
-        t = env.from_string(
-            """{{ "{0.__call__.__builtins__[__import__]}"
-                  | attr("format")(not_here) }}"""
-        )
-
-        with pytest.raises(SecurityError):
-            t.render()