diff --git a/pep_sphinx_extensions/__init__.py b/pep_sphinx_extensions/__init__.py index 1c3383dfd81..936cd8efef0 100644 --- a/pep_sphinx_extensions/__init__.py +++ b/pep_sphinx_extensions/__init__.py @@ -13,6 +13,7 @@ create_rss_feed, get_from_doctree, ) +from pep_sphinx_extensions.lexers import get_custom_lexers from pep_sphinx_extensions.pep_processor.html import ( pep_html_builder, pep_html_translator, @@ -109,6 +110,10 @@ def setup(app: Sphinx) -> dict[str, bool]: app.add_directive("superseded", pep_banner_directive.SupersededBanner) app.add_directive("withdrawn", pep_banner_directive.WithdrawnBanner) + # Register custom lexers + for lexer in get_custom_lexers(): + app.add_lexer(lexer.name, lexer) + # Register event callbacks app.connect( "builder-inited", _update_config_for_builder diff --git a/pep_sphinx_extensions/lexers/__init__.py b/pep_sphinx_extensions/lexers/__init__.py new file mode 100644 index 00000000000..e76f0d26339 --- /dev/null +++ b/pep_sphinx_extensions/lexers/__init__.py @@ -0,0 +1,21 @@ +# This file is placed in the public domain or under the +# CC0-1.0-Universal license, whichever is more permissive. + +from __future__ import annotations + +import importlib +import pkgutil +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pygments.lexer import Lexer + + +def get_custom_lexers() -> list[type[Lexer]]: + lexers: list[type[Lexer]] = [] + for module_info in pkgutil.walk_packages(__path__, prefix=f"{__name__}."): + module = importlib.import_module(module_info.name) + if (register_func := getattr(module, "register", None)) is None: + continue + lexers.extend(register_func()) + return lexers diff --git a/pep_sphinx_extensions/lexers/pep823_lexer.py b/pep_sphinx_extensions/lexers/pep823_lexer.py new file mode 100644 index 00000000000..3be76b8ede7 --- /dev/null +++ b/pep_sphinx_extensions/lexers/pep823_lexer.py @@ -0,0 +1,44 @@ +# This file is placed in the public domain or under the +# CC0-1.0-Universal license, whichever is more permissive. + +"""Custom lexer for PEP 823.""" + +from pygments.lexer import DelegatingLexer, inherit +from pygments.lexers.python import ( + PythonLexer, + PythonTracebackLexer, + _PythonConsoleLexerBase, +) +from pygments.token import Operator, Other + + +class Py823Lexer(PythonLexer): + name = "py823" + + tokens = { + "expr": [ + (r"maybe\b", Operator.Word), + (r"\?", Operator), + inherit, + ], + } + + +class Py823ConsoleLexer(DelegatingLexer): + name = "py823-console" + + def __init__(self, **options): + pylexer = Py823Lexer + tblexer = PythonTracebackLexer + + class _ReplaceInnerCode(DelegatingLexer): + def __init__(self, **options): + super().__init__( + pylexer, _PythonConsoleLexerBase, Other.Code, **options + ) + + super().__init__(tblexer, _ReplaceInnerCode, Other.Traceback, **options) + + +def register(): + return [Py823Lexer, Py823ConsoleLexer] diff --git a/pep_sphinx_extensions/lexers/pep824_lexer.py b/pep_sphinx_extensions/lexers/pep824_lexer.py new file mode 100644 index 00000000000..afef3caaecf --- /dev/null +++ b/pep_sphinx_extensions/lexers/pep824_lexer.py @@ -0,0 +1,25 @@ +# This file is placed in the public domain or under the +# CC0-1.0-Universal license, whichever is more permissive. + +"""Custom lexer for PEP 824.""" + +from pygments.lexer import inherit +from pygments.lexers.python import PythonLexer +from pygments.token import Operator + + +class Py824Lexer(PythonLexer): + name = "py824" + + tokens = { + "expr": [ + (r"\?\?(?=\s)", Operator.Word), + (r"otherwise\b", Operator.Word), + (r"\?", Operator), + inherit, + ], + } + + +def register(): + return [Py824Lexer] diff --git a/peps/pep-0823.rst b/peps/pep-0823.rst index 4f84bbb92e6..ccb74052c6c 100644 --- a/peps/pep-0823.rst +++ b/peps/pep-0823.rst @@ -149,7 +149,7 @@ kinds of expressions much simpler while being predictable and doing the correct things intuitively. Using these operators, the function could instead be written as: -:: +.. code-block:: py823 def get_customer_name(data: Data) -> str | None: return data.customer?.user?.name.lower() @@ -198,7 +198,7 @@ least for dictionaries a useful helper method is ``dict.get(key)``. Writing it using ``?.`` and ``?[ ]`` would look like this: -:: +.. code-block:: py823 def get_customer_name(data: Data) -> str | None: return data.get("customer")?["user"]?["name"].lower() @@ -221,14 +221,14 @@ hide in plain sight. Attribute and function names have been shortened. If code relied on this property, the expression cannot necessarily be replaced with ``?.`` or ``?[ ]``. -:: +.. code-block:: py823 # In assignments x = a.b if (a is not None) else None x = a?.b -:: +.. code-block:: py823 # In if statements often used as guard clause with early # return or raising of an exception @@ -242,7 +242,7 @@ hide in plain sight. Attribute and function names have been shortened. if a is None or a.b is None: ... if a?.b is None: ... -:: +.. code-block:: py823 # Misc expressions @@ -336,7 +336,7 @@ for trying to get a subscript of ``None`` are omitted. It is therefore not necessary to change subsequent ``.`` or ``[ ]`` on the right-hand side just because a ``?.`` or ``?[ ]`` is used prior. -:: +.. code-block:: py823-console >>> a = None >>> print(a?.b.c[0].some_function()) @@ -348,7 +348,7 @@ their ``None``-aware counterparts, and call expressions). As a rule of thumb, short-circuiting is broken once an operator other than ``.``, ``[ ]``, ``?.``, ``?[ ]`` is reached. -:: +.. code-block:: py823-console >>> a = None >>> print(a?.b.c) @@ -370,7 +370,7 @@ be broken. For example function arguments or subscripts are evaluated on their own and would not short-circuit the remaining ``tail`` of the outer expression. -:: +.. code-block:: py823 # func(a?.b).c[d?.e] @@ -389,7 +389,7 @@ if ``a is None``. This is conceptually identical to extracting the group contents and storing the result in a temporary variable before substituting it back into the original expression. -:: +.. code-block:: py823 # (a?.b).c @@ -400,7 +400,7 @@ Common use cases for ``None``-aware access operators in groups are boolean or conditional expressions which can provide a fallback value in case the first part evaluates to ``None``. -:: +.. code-block:: py823 (a.b?.c or d).e?.func() @@ -419,7 +419,7 @@ Assignments ``None``-aware expressions may only be used in a ``Load`` context. Assignments are not permitted and will raise a ``SyntaxError``. -:: +.. code-block:: py823-console >>> a?.b = 1 File "", line 1 @@ -437,7 +437,7 @@ This does not apply if the ``None``-aware expressions is only part of a larger expression and evaluated on its own, for example as a function argument. -:: +.. code-block:: py823-console >>> a = None >>> def f(a): @@ -512,7 +512,7 @@ their needs, especially code formatters might prefer a style which conforms better to their existing preferences. An example of what is possible: -:: +.. code-block:: py823 def get_customer_name(data: Data) -> str | None: return ( @@ -675,7 +675,7 @@ because it might be too difficult to understand. Developers should instead change any subsequent attribute access or subscript to their ``None``-aware variants. -:: +.. code-block:: py823 # before a.b.optional?.c.d.e @@ -706,7 +706,7 @@ instead of two new operators, it may also be **too general**, in a sense that it can be combine with any other operator. For example it is not clear what the following expressions would mean: -:: +.. code-block:: py823-console >>> x? + 1 >>> x? -= 1 @@ -717,7 +717,7 @@ clear what the following expressions would mean: Even if a default meaning of ``is not None else None`` is assumed, the expressions are likely to raise errors at some point. -:: +.. code-block:: py823-console >>> x? + 1 >>> (_t1 if ((_t1 := x) is not None) else None) + 1 @@ -896,7 +896,7 @@ the substitution principle. An expression ``(a?.b).c`` should behave the same whether or not ``a?.b`` is written inline inside a group or defined as a separate variable. -:: +.. code-block:: py823 (a?.b).c @@ -1053,7 +1053,7 @@ for an ``optional`` value evaluates to ``None``, the result will be will be skipped. In the example below, if ``a.b`` is ``None``, so will be ``a.b?.c``: -:: +.. code-block:: py823 a.b?.c ^^^ diff --git a/peps/pep-0824.rst b/peps/pep-0824.rst index 45f94657d3d..d0943b0bf61 100644 --- a/peps/pep-0824.rst +++ b/peps/pep-0824.rst @@ -111,7 +111,7 @@ Using the "``None``-coalescing" operator ``??`` instead, helps to keep the expression short and predictable while still clearly communicating the intent. -:: +.. code-block:: py824 def show_user_age(user: User): age = user.age ?? "unknown" @@ -134,7 +134,7 @@ Using the "``None``-coalesce assignment" operator ``??=`` helps to avoid repeating the expression. Especially for more complex once, this will make it easier to read and write. -:: +.. code-block:: py824 def fix_user_name(user: User): user.name ??= "unknown" @@ -157,7 +157,7 @@ and assign the fallback value inside the function itself. This could be rewritten as: -:: +.. code-block:: py824 def show_user_name(user: User | None): user ??= create_default_user() @@ -188,7 +188,7 @@ conditional expressions. Parentheses can be added as necessary to modify the precedence of individual expressions. A few examples of how implicit parentheses would be placed: -:: +.. code-block:: py824 # x or y ?? 2 (x or y) ?? 2 @@ -335,7 +335,7 @@ The following is therefore merely meant as a suggestion. +---------------------------+--------------------------+----------------------------+ | Code | Pattern | Example | +===========================+==========================+============================+ -| :: | "... or ... if None" | "user dot age ``or`` | +| .. code-block:: py824 | "... or ... if None" | "user dot age ``or`` | | | | unknown ``if None``" | + user.age ?? "unknown" +--------------------------+----------------------------+ | | "... coalesce with ..." | "user dot age | @@ -348,7 +348,7 @@ The following is therefore merely meant as a suggestion. +-----------------------------+------------------------------+------------------------------------+ | Code | Pattern | Example | +=============================+==============================+====================================+ -| :: | "if ... is None, assign ..." | "``if`` user dot name ``is None``, | +| .. code-block:: py824 | "if ... is None, assign ..." | "``if`` user dot name ``is None``, | | | | ``assign`` unknown" | + user.name ??= "unknown" +------------------------------+------------------------------------+ | | "assign ... to ... if None" | "``assign`` unknown ``to`` user | @@ -416,7 +416,7 @@ programming languages. Lastly, using a (soft-) keyword for the "``None``-coalescing assignment" operator poses additional questions and readability concerns. -:: +.. code-block:: py824 a = otherwise b