[config.git] / djavu-asus / elpy / rpc-venv / lib / python3.11 / site-packages / black / comments.py

import re
from dataclasses import dataclass
from functools import lru_cache
from typing import Final, Iterator, List, Optional, Union

from black.nodes import (
    CLOSING_BRACKETS,
    STANDALONE_COMMENT,
    WHITESPACE,
    container_of,
    first_leaf_of,
    preceding_leaf,
    syms,
)
from blib2to3.pgen2 import token
from blib2to3.pytree import Leaf, Node

# types
LN = Union[Leaf, Node]

FMT_OFF: Final = {"# fmt: off", "# fmt:off", "# yapf: disable"}
FMT_SKIP: Final = {"# fmt: skip", "# fmt:skip"}
FMT_PASS: Final = {*FMT_OFF, *FMT_SKIP}
FMT_ON: Final = {"# fmt: on", "# fmt:on", "# yapf: enable"}

COMMENT_EXCEPTIONS = " !:#'"


@dataclass
class ProtoComment:
    """Describes a piece of syntax that is a comment.

    It's not a :class:`blib2to3.pytree.Leaf` so that:

    * it can be cached (`Leaf` objects should not be reused more than once as
      they store their lineno, column, prefix, and parent information);
    * `newlines` and `consumed` fields are kept separate from the `value`. This
      simplifies handling of special marker comments like ``# fmt: off/on``.
    """

    type: int  # token.COMMENT or STANDALONE_COMMENT
    value: str  # content of the comment
    newlines: int  # how many newlines before the comment
    consumed: int  # how many characters of the original leaf's prefix did we consume


def generate_comments(leaf: LN) -> Iterator[Leaf]:
    """Clean the prefix of the `leaf` and generate comments from it, if any.

    Comments in lib2to3 are shoved into the whitespace prefix.  This happens
    in `pgen2/driver.py:Driver.parse_tokens()`.  This was a brilliant implementation
    move because it does away with modifying the grammar to include all the
    possible places in which comments can be placed.

    The sad consequence for us though is that comments don't "belong" anywhere.
    This is why this function generates simple parentless Leaf objects for
    comments.  We simply don't know what the correct parent should be.

    No matter though, we can live without this.  We really only need to
    differentiate between inline and standalone comments.  The latter don't
    share the line with any code.

    Inline comments are emitted as regular token.COMMENT leaves.  Standalone
    are emitted with a fake STANDALONE_COMMENT token identifier.
    """
    for pc in list_comments(leaf.prefix, is_endmarker=leaf.type == token.ENDMARKER):
        yield Leaf(pc.type, pc.value, prefix="\n" * pc.newlines)


@lru_cache(maxsize=4096)
def list_comments(prefix: str, *, is_endmarker: bool) -> List[ProtoComment]:
    """Return a list of :class:`ProtoComment` objects parsed from the given `prefix`."""
    result: List[ProtoComment] = []
    if not prefix or "#" not in prefix:
        return result

    consumed = 0
    nlines = 0
    ignored_lines = 0
    for index, line in enumerate(re.split("\r?\n", prefix)):
        consumed += len(line) + 1  # adding the length of the split '\n'
        line = line.lstrip()
        if not line:
            nlines += 1
        if not line.startswith("#"):
            # Escaped newlines outside of a comment are not really newlines at
            # all. We treat a single-line comment following an escaped newline
            # as a simple trailing comment.
            if line.endswith("\\"):
                ignored_lines += 1
            continue

        if index == ignored_lines and not is_endmarker:
            comment_type = token.COMMENT  # simple trailing comment
        else:
            comment_type = STANDALONE_COMMENT
        comment = make_comment(line)
        result.append(
            ProtoComment(
                type=comment_type, value=comment, newlines=nlines, consumed=consumed
            )
        )
        nlines = 0
    return result


def make_comment(content: str) -> str:
    """Return a consistently formatted comment from the given `content` string.

    All comments (except for "##", "#!", "#:", '#'") should have a single
    space between the hash sign and the content.

    If `content` didn't start with a hash sign, one is provided.
    """
    content = content.rstrip()
    if not content:
        return "#"

    if content[0] == "#":
        content = content[1:]
    NON_BREAKING_SPACE = " "
    if (
        content
        and content[0] == NON_BREAKING_SPACE
        and not content.lstrip().startswith("type:")
    ):
        content = " " + content[1:]  # Replace NBSP by a simple space
    if content and content[0] not in COMMENT_EXCEPTIONS:
        content = " " + content
    return "#" + content


def normalize_fmt_off(node: Node) -> None:
    """Convert content between `# fmt: off`/`# fmt: on` into standalone comments."""
    try_again = True
    while try_again:
        try_again = convert_one_fmt_off_pair(node)


def convert_one_fmt_off_pair(node: Node) -> bool:
    """Convert content of a single `# fmt: off`/`# fmt: on` into a standalone comment.

    Returns True if a pair was converted.
    """
    for leaf in node.leaves():
        previous_consumed = 0
        for comment in list_comments(leaf.prefix, is_endmarker=False):
            if comment.value not in FMT_PASS:
                previous_consumed = comment.consumed
                continue
            # We only want standalone comments. If there's no previous leaf or
            # the previous leaf is indentation, it's a standalone comment in
            # disguise.
            if comment.value in FMT_PASS and comment.type != STANDALONE_COMMENT:
                prev = preceding_leaf(leaf)
                if prev:
                    if comment.value in FMT_OFF and prev.type not in WHITESPACE:
                        continue
                    if comment.value in FMT_SKIP and prev.type in WHITESPACE:
                        continue

            ignored_nodes = list(generate_ignored_nodes(leaf, comment))
            if not ignored_nodes:
                continue

            first = ignored_nodes[0]  # Can be a container node with the `leaf`.
            parent = first.parent
            prefix = first.prefix
            if comment.value in FMT_OFF:
                first.prefix = prefix[comment.consumed :]
            if comment.value in FMT_SKIP:
                first.prefix = ""
                standalone_comment_prefix = prefix
            else:
                standalone_comment_prefix = (
                    prefix[:previous_consumed] + "\n" * comment.newlines
                )
            hidden_value = "".join(str(n) for n in ignored_nodes)
            if comment.value in FMT_OFF:
                hidden_value = comment.value + "\n" + hidden_value
            if comment.value in FMT_SKIP:
                hidden_value += "  " + comment.value
            if hidden_value.endswith("\n"):
                # That happens when one of the `ignored_nodes` ended with a NEWLINE
                # leaf (possibly followed by a DEDENT).
                hidden_value = hidden_value[:-1]
            first_idx: Optional[int] = None
            for ignored in ignored_nodes:
                index = ignored.remove()
                if first_idx is None:
                    first_idx = index
            assert parent is not None, "INTERNAL ERROR: fmt: on/off handling (1)"
            assert first_idx is not None, "INTERNAL ERROR: fmt: on/off handling (2)"
            parent.insert_child(
                first_idx,
                Leaf(
                    STANDALONE_COMMENT,
                    hidden_value,
                    prefix=standalone_comment_prefix,
                    fmt_pass_converted_first_leaf=first_leaf_of(first),
                ),
            )
            return True

    return False


def generate_ignored_nodes(leaf: Leaf, comment: ProtoComment) -> Iterator[LN]:
    """Starting from the container of `leaf`, generate all leaves until `# fmt: on`.

    If comment is skip, returns leaf only.
    Stops at the end of the block.
    """
    if comment.value in FMT_SKIP:
        yield from _generate_ignored_nodes_from_fmt_skip(leaf, comment)
        return
    container: Optional[LN] = container_of(leaf)
    while container is not None and container.type != token.ENDMARKER:
        if is_fmt_on(container):
            return

        # fix for fmt: on in children
        if children_contains_fmt_on(container):
            for index, child in enumerate(container.children):
                if isinstance(child, Leaf) and is_fmt_on(child):
                    if child.type in CLOSING_BRACKETS:
                        # This means `# fmt: on` is placed at a different bracket level
                        # than `# fmt: off`. This is an invalid use, but as a courtesy,
                        # we include this closing bracket in the ignored nodes.
                        # The alternative is to fail the formatting.
                        yield child
                    return
                if (
                    child.type == token.INDENT
                    and index < len(container.children) - 1
                    and children_contains_fmt_on(container.children[index + 1])
                ):
                    # This means `# fmt: on` is placed right after an indentation
                    # level, and we shouldn't swallow the previous INDENT token.
                    return
                if children_contains_fmt_on(child):
                    return
                yield child
        else:
            if container.type == token.DEDENT and container.next_sibling is None:
                # This can happen when there is no matching `# fmt: on` comment at the
                # same level as `# fmt: on`. We need to keep this DEDENT.
                return
            yield container
            container = container.next_sibling


def _generate_ignored_nodes_from_fmt_skip(
    leaf: Leaf, comment: ProtoComment
) -> Iterator[LN]:
    """Generate all leaves that should be ignored by the `# fmt: skip` from `leaf`."""
    prev_sibling = leaf.prev_sibling
    parent = leaf.parent
    # Need to properly format the leaf prefix to compare it to comment.value,
    # which is also formatted
    comments = list_comments(leaf.prefix, is_endmarker=False)
    if not comments or comment.value != comments[0].value:
        return
    if prev_sibling is not None:
        leaf.prefix = ""
        siblings = [prev_sibling]
        while "\n" not in prev_sibling.prefix and prev_sibling.prev_sibling is not None:
            prev_sibling = prev_sibling.prev_sibling
            siblings.insert(0, prev_sibling)
        yield from siblings
    elif (
        parent is not None and parent.type == syms.suite and leaf.type == token.NEWLINE
    ):
        # The `# fmt: skip` is on the colon line of the if/while/def/class/...
        # statements. The ignored nodes should be previous siblings of the
        # parent suite node.
        leaf.prefix = ""
        ignored_nodes: List[LN] = []
        parent_sibling = parent.prev_sibling
        while parent_sibling is not None and parent_sibling.type != syms.suite:
            ignored_nodes.insert(0, parent_sibling)
            parent_sibling = parent_sibling.prev_sibling
        # Special case for `async_stmt` where the ASYNC token is on the
        # grandparent node.
        grandparent = parent.parent
        if (
            grandparent is not None
            and grandparent.prev_sibling is not None
            and grandparent.prev_sibling.type == token.ASYNC
        ):
            ignored_nodes.insert(0, grandparent.prev_sibling)
        yield from iter(ignored_nodes)


def is_fmt_on(container: LN) -> bool:
    """Determine whether formatting is switched on within a container.
    Determined by whether the last `# fmt:` comment is `on` or `off`.
    """
    fmt_on = False
    for comment in list_comments(container.prefix, is_endmarker=False):
        if comment.value in FMT_ON:
            fmt_on = True
        elif comment.value in FMT_OFF:
            fmt_on = False
    return fmt_on


def children_contains_fmt_on(container: LN) -> bool:
    """Determine if children have formatting switched on."""
    for child in container.children:
        leaf = first_leaf_of(child)
        if leaf is not None and is_fmt_on(leaf):
            return True

    return False


def contains_pragma_comment(comment_list: List[Leaf]) -> bool:
    """
    Returns:
        True iff one of the comments in @comment_list is a pragma used by one
        of the more common static analysis tools for python (e.g. mypy, flake8,
        pylint).
    """
    for comment in comment_list:
        if comment.value.startswith(("# type:", "# noqa", "# pylint:")):
            return True

    return False
Commit	Line	Data
53e6db90 DC	1	import re
	2	from dataclasses import dataclass
	3	from functools import lru_cache
	4	from typing import Final, Iterator, List, Optional, Union
	5
	6	from black.nodes import (
	7	CLOSING_BRACKETS,
	8	STANDALONE_COMMENT,
	9	WHITESPACE,
	10	container_of,
	11	first_leaf_of,
	12	preceding_leaf,
	13	syms,
	14	)
	15	from blib2to3.pgen2 import token
	16	from blib2to3.pytree import Leaf, Node
	17
	18	# types
	19	LN = Union[Leaf, Node]
	20
	21	FMT_OFF: Final = {"# fmt: off", "# fmt:off", "# yapf: disable"}
	22	FMT_SKIP: Final = {"# fmt: skip", "# fmt:skip"}
	23	FMT_PASS: Final = {FMT_OFF, FMT_SKIP}
	24	FMT_ON: Final = {"# fmt: on", "# fmt:on", "# yapf: enable"}
	25
	26	COMMENT_EXCEPTIONS = " !:#'"
	27
	28
	29	@dataclass
	30	class ProtoComment:
	31	"""Describes a piece of syntax that is a comment.
	32
	33	It's not a :class:`blib2to3.pytree.Leaf` so that:
	34
	35	* it can be cached (`Leaf` objects should not be reused more than once as
	36	they store their lineno, column, prefix, and parent information);
	37	* `newlines` and `consumed` fields are kept separate from the `value`. This
	38	simplifies handling of special marker comments like ``# fmt: off/on``.
	39	"""
	40
	41	type: int # token.COMMENT or STANDALONE_COMMENT
	42	value: str # content of the comment
	43	newlines: int # how many newlines before the comment
	44	consumed: int # how many characters of the original leaf's prefix did we consume
	45
	46
	47	def generate_comments(leaf: LN) -> Iterator[Leaf]:
	48	"""Clean the prefix of the `leaf` and generate comments from it, if any.
	49
	50	Comments in lib2to3 are shoved into the whitespace prefix. This happens
	51	in `pgen2/driver.py:Driver.parse_tokens()`. This was a brilliant implementation
	52	move because it does away with modifying the grammar to include all the
	53	possible places in which comments can be placed.
	54
	55	The sad consequence for us though is that comments don't "belong" anywhere.
	56	This is why this function generates simple parentless Leaf objects for
	57	comments. We simply don't know what the correct parent should be.
	58
	59	No matter though, we can live without this. We really only need to
	60	differentiate between inline and standalone comments. The latter don't
	61	share the line with any code.
	62
	63	Inline comments are emitted as regular token.COMMENT leaves. Standalone
	64	are emitted with a fake STANDALONE_COMMENT token identifier.
65	"""
66	for pc in list_comments(leaf.prefix, is_endmarker=leaf.type == token.ENDMARKER):
67	yield Leaf(pc.type, pc.value, prefix="\n" * pc.newlines)
68
69
70	@lru_cache(maxsize=4096)
71	def list_comments(prefix: str, *, is_endmarker: bool) -> List[ProtoComment]:
72	"""Return a list of :class:`ProtoComment` objects parsed from the given `prefix`."""
73	result: List[ProtoComment] = []
74	if not prefix or "#" not in prefix:
75	return result
76
77	consumed = 0
78	nlines = 0
79	ignored_lines = 0
80	for index, line in enumerate(re.split("\r?\n", prefix)):
81	consumed += len(line) + 1 # adding the length of the split '\n'
82	line = line.lstrip()
83	if not line:
84	nlines += 1
85	if not line.startswith("#"):
86	# Escaped newlines outside of a comment are not really newlines at
87	# all. We treat a single-line comment following an escaped newline
88	# as a simple trailing comment.
89	if line.endswith("\\"):
90	ignored_lines += 1
91	continue
92
93	if index == ignored_lines and not is_endmarker:
94	comment_type = token.COMMENT # simple trailing comment
95	else:
96	comment_type = STANDALONE_COMMENT
97	comment = make_comment(line)
98	result.append(
99	ProtoComment(
100	type=comment_type, value=comment, newlines=nlines, consumed=consumed
101	)
102	)
103	nlines = 0
104	return result
105
106
107	def make_comment(content: str) -> str:
108	"""Return a consistently formatted comment from the given `content` string.
109
110	All comments (except for "##", "#!", "#:", '#'") should have a single
111	space between the hash sign and the content.
112
113	If `content` didn't start with a hash sign, one is provided.
114	"""
115	content = content.rstrip()
116	if not content:
117	return "#"
118
119	if content[0] == "#":
120	content = content[1:]
121	NON_BREAKING_SPACE = " "
122	if (
123	content
124	and content[0] == NON_BREAKING_SPACE
125	and not content.lstrip().startswith("type:")
126	):
127	content = " " + content[1:] # Replace NBSP by a simple space
128	if content and content[0] not in COMMENT_EXCEPTIONS:
129	content = " " + content
130	return "#" + content
131
132
133	def normalize_fmt_off(node: Node) -> None:
134	"""Convert content between `# fmt: off`/`# fmt: on` into standalone comments."""
135	try_again = True
136	while try_again:
137	try_again = convert_one_fmt_off_pair(node)
138
139
140	def convert_one_fmt_off_pair(node: Node) -> bool:
141	"""Convert content of a single `# fmt: off`/`# fmt: on` into a standalone comment.
142
143	Returns True if a pair was converted.
144	"""
145	for leaf in node.leaves():
146	previous_consumed = 0
147	for comment in list_comments(leaf.prefix, is_endmarker=False):
148	if comment.value not in FMT_PASS:
149	previous_consumed = comment.consumed
150	continue
151	# We only want standalone comments. If there's no previous leaf or
152	# the previous leaf is indentation, it's a standalone comment in
153	# disguise.
154	if comment.value in FMT_PASS and comment.type != STANDALONE_COMMENT:
155	prev = preceding_leaf(leaf)
156	if prev:
157	if comment.value in FMT_OFF and prev.type not in WHITESPACE:
158	continue
159	if comment.value in FMT_SKIP and prev.type in WHITESPACE:
160	continue
161
162	ignored_nodes = list(generate_ignored_nodes(leaf, comment))
163	if not ignored_nodes:
164	continue
165
166	first = ignored_nodes[0] # Can be a container node with the `leaf`.
167	parent = first.parent
168	prefix = first.prefix
169	if comment.value in FMT_OFF:
170	first.prefix = prefix[comment.consumed :]
171	if comment.value in FMT_SKIP:
172	first.prefix = ""
173	standalone_comment_prefix = prefix
174	else:
175	standalone_comment_prefix = (
176	prefix[:previous_consumed] + "\n" * comment.newlines
177	)
178	hidden_value = "".join(str(n) for n in ignored_nodes)
179	if comment.value in FMT_OFF:
180	hidden_value = comment.value + "\n" + hidden_value
181	if comment.value in FMT_SKIP:
182	hidden_value += " " + comment.value
183	if hidden_value.endswith("\n"):
184	# That happens when one of the `ignored_nodes` ended with a NEWLINE
185	# leaf (possibly followed by a DEDENT).
186	hidden_value = hidden_value[:-1]
187	first_idx: Optional[int] = None
188	for ignored in ignored_nodes:
189	index = ignored.remove()
190	if first_idx is None:
191	first_idx = index
192	assert parent is not None, "INTERNAL ERROR: fmt: on/off handling (1)"
193	assert first_idx is not None, "INTERNAL ERROR: fmt: on/off handling (2)"
194	parent.insert_child(
195	first_idx,
196	Leaf(
197	STANDALONE_COMMENT,
198	hidden_value,
199	prefix=standalone_comment_prefix,
200	fmt_pass_converted_first_leaf=first_leaf_of(first),
201	),
202	)
203	return True
204
205	return False
206
207
208	def generate_ignored_nodes(leaf: Leaf, comment: ProtoComment) -> Iterator[LN]:
209	"""Starting from the container of `leaf`, generate all leaves until `# fmt: on`.
210
211	If comment is skip, returns leaf only.
212	Stops at the end of the block.
213	"""
214	if comment.value in FMT_SKIP:
215	yield from _generate_ignored_nodes_from_fmt_skip(leaf, comment)
216	return
217	container: Optional[LN] = container_of(leaf)
218	while container is not None and container.type != token.ENDMARKER:
219	if is_fmt_on(container):
220	return
221
222	# fix for fmt: on in children
223	if children_contains_fmt_on(container):
224	for index, child in enumerate(container.children):
225	if isinstance(child, Leaf) and is_fmt_on(child):
226	if child.type in CLOSING_BRACKETS:
227	# This means `# fmt: on` is placed at a different bracket level
228	# than `# fmt: off`. This is an invalid use, but as a courtesy,
229	# we include this closing bracket in the ignored nodes.
230	# The alternative is to fail the formatting.
231	yield child
232	return
233	if (
234	child.type == token.INDENT
235	and index < len(container.children) - 1
236	and children_contains_fmt_on(container.children[index + 1])
237	):
238	# This means `# fmt: on` is placed right after an indentation
239	# level, and we shouldn't swallow the previous INDENT token.
240	return
241	if children_contains_fmt_on(child):
242	return
243	yield child
244	else:
245	if container.type == token.DEDENT and container.next_sibling is None:
246	# This can happen when there is no matching `# fmt: on` comment at the
247	# same level as `# fmt: on`. We need to keep this DEDENT.
248	return
249	yield container
250	container = container.next_sibling
251
252
253	def _generate_ignored_nodes_from_fmt_skip(
254	leaf: Leaf, comment: ProtoComment
255	) -> Iterator[LN]:
256	"""Generate all leaves that should be ignored by the `# fmt: skip` from `leaf`."""
257	prev_sibling = leaf.prev_sibling
258	parent = leaf.parent
259	# Need to properly format the leaf prefix to compare it to comment.value,
260	# which is also formatted
261	comments = list_comments(leaf.prefix, is_endmarker=False)
262	if not comments or comment.value != comments[0].value:
263	return
264	if prev_sibling is not None:
265	leaf.prefix = ""
266	siblings = [prev_sibling]
267	while "\n" not in prev_sibling.prefix and prev_sibling.prev_sibling is not None:
268	prev_sibling = prev_sibling.prev_sibling
269	siblings.insert(0, prev_sibling)
270	yield from siblings
271	elif (
272	parent is not None and parent.type == syms.suite and leaf.type == token.NEWLINE
273	):
274	# The `# fmt: skip` is on the colon line of the if/while/def/class/...
275	# statements. The ignored nodes should be previous siblings of the
276	# parent suite node.
277	leaf.prefix = ""
278	ignored_nodes: List[LN] = []
279	parent_sibling = parent.prev_sibling
280	while parent_sibling is not None and parent_sibling.type != syms.suite:
281	ignored_nodes.insert(0, parent_sibling)
282	parent_sibling = parent_sibling.prev_sibling
283	# Special case for `async_stmt` where the ASYNC token is on the
284	# grandparent node.
285	grandparent = parent.parent
286	if (
287	grandparent is not None
288	and grandparent.prev_sibling is not None
289	and grandparent.prev_sibling.type == token.ASYNC
290	):
291	ignored_nodes.insert(0, grandparent.prev_sibling)
292	yield from iter(ignored_nodes)
293
294
295	def is_fmt_on(container: LN) -> bool:
296	"""Determine whether formatting is switched on within a container.
297	Determined by whether the last `# fmt:` comment is `on` or `off`.
298	"""
299	fmt_on = False
300	for comment in list_comments(container.prefix, is_endmarker=False):
301	if comment.value in FMT_ON:
302	fmt_on = True
303	elif comment.value in FMT_OFF:
304	fmt_on = False
305	return fmt_on
306
307
308	def children_contains_fmt_on(container: LN) -> bool:
309	"""Determine if children have formatting switched on."""
310	for child in container.children:
311	leaf = first_leaf_of(child)
312	if leaf is not None and is_fmt_on(leaf):
313	return True
314
315	return False
316
317
318	def contains_pragma_comment(comment_list: List[Leaf]) -> bool:
319	"""
320	Returns:
321	True iff one of the comments in @comment_list is a pragma used by one
322	of the more common static analysis tools for python (e.g. mypy, flake8,
323	pylint).
324	"""
325	for comment in comment_list:
326	if comment.value.startswith(("# type:", "# noqa", "# pylint:")):
327	return True
328
329	return False