Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reject invalid ParamSpec locations #18278

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
+158 −127
Open

Reject invalid ParamSpec locations #18278

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
ab6f17e
Add compliance test, stop ignoring some weird paramspec positions
sterliakov Dec 11, 2024
6ef6d89
Move ParamSpec validity checks to typeanal entirely
sterliakov Dec 11, 2024
5d9d703
Remove "special" treatment for ParamSpec parts encountered during ana…
sterliakov Dec 11, 2024
d461ba2
Fix broken `Callable[[Unpack[SomeTypedDict]], None]`
sterliakov Dec 11, 2024
dfa0887
Recover Unpack sanity checks
sterliakov Dec 11, 2024
643a0db
Fix runtime crash in prefect, clean up naming
sterliakov Dec 11, 2024
caeb13e
Merge branch 'master' into bugfix/st-paramspec-locations
sterliakov Dec 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 0 additions & 60 deletions mypy/semanal.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,6 @@
from mypy.nodes import (
ARG_NAMED,
ARG_POS,
ARG_STAR,
ARG_STAR2,
CONTRAVARIANT,
COVARIANT,
Expand Down Expand Up @@ -980,7 +979,6 @@ def analyze_func_def(self, defn: FuncDef) -> None:
defn.type = result
self.add_type_alias_deps(analyzer.aliases_used)
self.check_function_signature(defn)
self.check_paramspec_definition(defn)
if isinstance(defn, FuncDef):
assert isinstance(defn.type, CallableType)
defn.type = set_callable_name(defn.type, defn)
Expand Down Expand Up @@ -1609,64 +1607,6 @@ def check_function_signature(self, fdef: FuncItem) -> None:
elif len(sig.arg_types) > len(fdef.arguments):
self.fail("Type signature has too many arguments", fdef, blocker=True)

def check_paramspec_definition(self, defn: FuncDef) -> None:
func = defn.type
assert isinstance(func, CallableType)

if not any(isinstance(var, ParamSpecType) for var in func.variables):
return # Function does not have param spec variables

args = func.var_arg()
kwargs = func.kw_arg()
if args is None and kwargs is None:
return # Looks like this function does not have starred args

args_defn_type = None
kwargs_defn_type = None
for arg_def, arg_kind in zip(defn.arguments, defn.arg_kinds):
if arg_kind == ARG_STAR:
args_defn_type = arg_def.type_annotation
elif arg_kind == ARG_STAR2:
kwargs_defn_type = arg_def.type_annotation

# This may happen on invalid `ParamSpec` args / kwargs definition,
# type analyzer sets types of arguments to `Any`, but keeps
# definition types as `UnboundType` for now.
if not (
(isinstance(args_defn_type, UnboundType) and args_defn_type.name.endswith(".args"))
or (
isinstance(kwargs_defn_type, UnboundType)
and kwargs_defn_type.name.endswith(".kwargs")
)
):
# Looks like both `*args` and `**kwargs` are not `ParamSpec`
# It might be something else, skipping.
return

args_type = args.typ if args is not None else None
kwargs_type = kwargs.typ if kwargs is not None else None

if (
not isinstance(args_type, ParamSpecType)
or not isinstance(kwargs_type, ParamSpecType)
or args_type.name != kwargs_type.name
):
if isinstance(args_defn_type, UnboundType) and args_defn_type.name.endswith(".args"):
param_name = args_defn_type.name.split(".")[0]
elif isinstance(kwargs_defn_type, UnboundType) and kwargs_defn_type.name.endswith(
".kwargs"
):
param_name = kwargs_defn_type.name.split(".")[0]
else:
# Fallback for cases that probably should not ever happen:
param_name = "P"

self.fail(
f'ParamSpec must have "*args" typed as "{param_name}.args" and "**kwargs" typed as "{param_name}.kwargs"',
func,
code=codes.VALID_TYPE,
)

def visit_decorator(self, dec: Decorator) -> None:
self.statement = dec
# TODO: better don't modify them at all.
Expand Down
119 changes: 67 additions & 52 deletions mypy/typeanal.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,15 @@ def not_declared_in_type_params(self, tvar_name: str) -> bool:

def visit_unbound_type_nonoptional(self, t: UnboundType, defining_literal: bool) -> Type:
sym = self.lookup_qualified(t.name, t)
param_spec_name = None
if t.name.endswith((".args", ".kwargs")):
param_spec_name = t.name.rsplit(".", 1)[0]
maybe_param_spec = self.lookup_qualified(param_spec_name, t)
if maybe_param_spec and isinstance(maybe_param_spec.node, ParamSpecExpr):
sym = maybe_param_spec
else:
param_spec_name = None

if sym is not None:
node = sym.node
if isinstance(node, PlaceholderNode):
Expand Down Expand Up @@ -359,17 +368,23 @@ def visit_unbound_type_nonoptional(self, t: UnboundType, defining_literal: bool)
if tvar_def is None:
if self.allow_unbound_tvars:
return t
name = param_spec_name or t.name
if self.defining_alias and self.not_declared_in_type_params(t.name):
msg = f'ParamSpec "{t.name}" is not included in type_params'
msg = f'ParamSpec "{name}" is not included in type_params'
else:
msg = f'ParamSpec "{t.name}" is unbound'
msg = f'ParamSpec "{name}" is unbound'
self.fail(msg, t, code=codes.VALID_TYPE)
return AnyType(TypeOfAny.from_error)
assert isinstance(tvar_def, ParamSpecType)
if len(t.args) > 0:
self.fail(
f'ParamSpec "{t.name}" used with arguments', t, code=codes.VALID_TYPE
)
if param_spec_name is not None and not self.allow_param_spec_literals:
self.fail(
"ParamSpec components are not allowed here", t, code=codes.VALID_TYPE
)
return AnyType(TypeOfAny.from_error)
# Change the line number
return ParamSpecType(
tvar_def.name,
Expand Down Expand Up @@ -1109,46 +1124,57 @@ def visit_callable_type(
variables, _ = self.bind_function_type_variables(t, t)
type_guard = self.anal_type_guard(t.ret_type)
type_is = self.anal_type_is(t.ret_type)

arg_kinds = t.arg_kinds
if len(arg_kinds) >= 2 and arg_kinds[-2] == ARG_STAR and arg_kinds[-1] == ARG_STAR2:
arg_types = self.anal_array(t.arg_types[:-2], nested=nested) + [
self.anal_star_arg_type(t.arg_types[-2], ARG_STAR, nested=nested),
self.anal_star_arg_type(t.arg_types[-1], ARG_STAR2, nested=nested),
]
# If nested is True, it means we are analyzing a Callable[...] type, rather
# than a function definition type. We need to "unpack" ** TypedDict annotation
# here (for function definitions it is done in semanal).
if nested and isinstance(arg_types[-1], UnpackType):
arg_types = []
param_spec_with_args = param_spec_with_kwargs = None
param_spec_invalid = False
for kind, ut in zip(arg_kinds, t.arg_types):
if kind == ARG_STAR:
param_spec_with_args, at = self.anal_star_arg_type(ut, kind, nested=nested)
elif kind == ARG_STAR2:
param_spec_with_kwargs, at = self.anal_star_arg_type(ut, kind, nested=nested)
else:
if param_spec_with_args:
param_spec_invalid = True
self.fail(
"Arguments not allowed after ParamSpec.args", t, code=codes.VALID_TYPE
)
at = self.anal_type(ut, nested=nested, allow_unpack=False)
arg_types.append(at)

if nested and arg_types:
# If we've got a Callable[[Unpack[SomeTypedDict]], None], make sure
# Unpack is interpreted as `**` and not as `*`.
last = arg_types[-1]
if isinstance(last, UnpackType):
# TODO: it would be better to avoid this get_proper_type() call.
unpacked = get_proper_type(arg_types[-1].type)
if isinstance(unpacked, TypedDictType):
arg_types[-1] = unpacked
p_at = get_proper_type(last.type)
if isinstance(p_at, TypedDictType) and not last.from_star_syntax:
# Automatically detect Unpack[Foo] in Callable as backwards
# compatible syntax for **Foo, if Foo is a TypedDict.
arg_kinds[-1] = ARG_STAR2
arg_types[-1] = p_at
unpacked_kwargs = True
arg_types = self.check_unpacks_in_list(arg_types)
else:
star_index = None
arg_types = self.check_unpacks_in_list(arg_types)

if not param_spec_invalid and param_spec_with_args != param_spec_with_kwargs:
# If already invalid, do not report more errors - definition has
# to be fixed anyway
name = param_spec_with_args or param_spec_with_kwargs
self.fail(
f'ParamSpec must have "*args" typed as "{name}.args" and "**kwargs" typed as "{name}.kwargs"',
t,
code=codes.VALID_TYPE,
)
param_spec_invalid = True

if param_spec_invalid:
if ARG_STAR in arg_kinds:
star_index = arg_kinds.index(ARG_STAR)
star2_index = None
arg_types[arg_kinds.index(ARG_STAR)] = AnyType(TypeOfAny.from_error)
if ARG_STAR2 in arg_kinds:
star2_index = arg_kinds.index(ARG_STAR2)
arg_types = []
for i, ut in enumerate(t.arg_types):
at = self.anal_type(
ut, nested=nested, allow_unpack=i in (star_index, star2_index)
)
if nested and isinstance(at, UnpackType) and i == star_index:
# TODO: it would be better to avoid this get_proper_type() call.
p_at = get_proper_type(at.type)
if isinstance(p_at, TypedDictType) and not at.from_star_syntax:
# Automatically detect Unpack[Foo] in Callable as backwards
# compatible syntax for **Foo, if Foo is a TypedDict.
at = p_at
arg_kinds[i] = ARG_STAR2
unpacked_kwargs = True
arg_types.append(at)
if nested:
arg_types = self.check_unpacks_in_list(arg_types)
arg_types[arg_kinds.index(ARG_STAR2)] = AnyType(TypeOfAny.from_error)

# If there were multiple (invalid) unpacks, the arg types list will become shorter,
# we need to trim the kinds/names as well to avoid crashes.
arg_kinds = t.arg_kinds[: len(arg_types)]
Expand Down Expand Up @@ -1203,7 +1229,7 @@ def anal_type_is_arg(self, t: UnboundType, fullname: str) -> Type | None:
return self.anal_type(t.args[0])
return None

def anal_star_arg_type(self, t: Type, kind: ArgKind, nested: bool) -> Type:
def anal_star_arg_type(self, t: Type, kind: ArgKind, nested: bool) -> tuple[str | None, Type]:
"""Analyze signature argument type for *args and **kwargs argument."""
if isinstance(t, UnboundType) and t.name and "." in t.name and not t.args:
components = t.name.split(".")
Expand All @@ -1230,15 +1256,15 @@ def anal_star_arg_type(self, t: Type, kind: ArgKind, nested: bool) -> Type:
)
else:
assert False, kind
return make_paramspec(
return tvar_name, make_paramspec(
tvar_def.name,
tvar_def.fullname,
tvar_def.id,
named_type_func=self.named_type,
line=t.line,
column=t.column,
)
return self.anal_type(t, nested=nested, allow_unpack=True)
return None, self.anal_type(t, nested=nested, allow_unpack=True)

def visit_overloaded(self, t: Overloaded) -> Type:
# Overloaded types are manually constructed in semanal.py by analyzing the
Expand Down Expand Up @@ -2584,18 +2610,7 @@ def _seems_like_callable(self, type: UnboundType) -> bool:

def visit_unbound_type(self, t: UnboundType) -> None:
name = t.name
node = None

# Special case P.args and P.kwargs for ParamSpecs only.
if name.endswith("args"):
if name.endswith((".args", ".kwargs")):
base = ".".join(name.split(".")[:-1])
n = self.api.lookup_qualified(base, t)
if n is not None and isinstance(n.node, ParamSpecExpr):
node = n
name = base
if node is None:
node = self.api.lookup_qualified(name, t)
node = self.api.lookup_qualified(name, t)
if node and node.fullname in SELF_TYPE_NAMES:
self.has_self_type = True
if (
Expand Down
Loading
Loading