formatting

src/blib2to3/pgen2/driver.py

@@ -190,9 +190,7 @@ def parse_file(
 
     def parse_string(self, text: str, debug: bool = False) -> NL:
         """Parse a string and return the syntax tree."""
-        tokens = tokenize.tokenize(
-            text, grammar=self.grammar
-        )
+        tokens = tokenize.tokenize(text, grammar=self.grammar)
         return self.parse_tokens(tokens, debug)
 
     def _partially_consume_prefix(self, prefix: str, column: int) -> tuple[str, str]:

src/blib2to3/pgen2/tokenize.py

@@ -97,12 +97,13 @@
     TokenType.fstring_start: FSTRING_START,
     TokenType.fstring_middle: FSTRING_MIDDLE,
     TokenType.fstring_end: FSTRING_END,
-    TokenType.endmarker: ENDMARKER
-
+    TokenType.endmarker: ENDMARKER,
 }
 
+
 class TokenError(Exception): ...
 
+
 def token_type(token: pytokens.Token, source: str) -> int:
     tok_type = TOKEN_TYPE_MAP[token.type]
     if tok_type == NAME:
@@ -114,6 +115,7 @@ def token_type(token: pytokens.Token, source: str) -> int:
 
     return tok_type
 
+
 def tokenize(source: str, grammar: Grammar | None = None) -> Iterator[TokenInfo]:
     lines = source.split("\n")
     lines += [""]  # For newline tokens in files that don't end in a newline
@@ -124,9 +126,9 @@ def tokenize(source: str, grammar: Grammar | None = None) -> Iterator[TokenInfo]
             if token.type == TokenType.whitespace:
                 continue
 
-            token_string = source[token.start_index:token.end_index]
+            token_string = source[token.start_index : token.end_index]
 
-            if token.type == TokenType.newline and token_string == '':
+            if token.type == TokenType.newline and token_string == "":
                 # Black doesn't yield empty newline tokens at the end of a file
                 # if there's no newline at the end of a file.
                 continue
@@ -141,12 +143,25 @@ def tokenize(source: str, grammar: Grammar | None = None) -> Iterator[TokenInfo]
                 token_string = "."
                 for start_col in range(token.start_col, token.start_col + 3):
                     end_col = start_col + 1
-                    yield (token_type(token, token_string), token_string, (token.start_line, start_col), (token.end_line, end_col), source_line)
+                    yield (
+                        token_type(token, token_string),
+                        token_string,
+                        (token.start_line, start_col),
+                        (token.end_line, end_col),
+                        source_line,
+                    )
             else:
-                yield (token_type(token, token_string), token_string, (token.start_line, token.start_col), (token.end_line, token.end_col), source_line)
+                yield (
+                    token_type(token, token_string),
+                    token_string,
+                    (token.start_line, token.start_col),
+                    (token.end_line, token.end_col),
+                    source_line,
+                )
     except Exception as exc:  # TODO:
         raise TokenError(repr(exc), (line, column))
 
+
 def printtoken(
     type: int, token: str, srow_col: Coord, erow_col: Coord, line: str
 ) -> None:  # for testing
@@ -156,6 +171,7 @@ def printtoken(
         "%d,%d-%d,%d:\t%s\t%s" % (srow, scol, erow, ecol, tok_name[type], repr(token))
     )
 
+
 if __name__ == "__main__":  # testing
     if len(sys.argv) > 1:
         token_iterator = tokenize(open(sys.argv[1]).read())

tests/test_tokenize.py

@@ -8,7 +8,6 @@
 import black
 from blib2to3.pgen2 import token, tokenize
 
-token._name
 
 @dataclass
 class Token:
@@ -20,7 +19,10 @@ class Token:
 
 def get_tokens(text: str) -> list[Token]:
     """Return the tokens produced by the tokenizer."""
-    return [Token(token.tok_name[tok_type], string, start, end) for tok_type, string, start, end, _ in tokenize.tokenize(text)]
+    return [
+        Token(token.tok_name[tok_type], string, start, end)
+        for tok_type, string, start, end, _ in tokenize.tokenize(text)
+    ]
 
 
 def assert_tokenizes(text: str, tokens: list[Token]) -> None: