diff --git a/crates/ty/docs/configuration.md b/crates/ty/docs/configuration.md index 11cd490c2b..f7198e395a 100644 --- a/crates/ty/docs/configuration.md +++ b/crates/ty/docs/configuration.md @@ -449,6 +449,97 @@ Defaults to `true`, and to `false` under the `ty-compatible` type checking prese --- +### `lexical-return-callable-typevars` + +Whether a type variable that only appears in a function's returned callable stays bound to +the function, as it is written. + +```python +def counter[T]() -> Callable[[T], T]: + state: list[T] = [] + + def remember(t: T) -> T: + if state: + return state[0] + state.append(t) + return t + + return remember +``` + +`T` belongs to `counter`: one call of `counter` makes one `remember`, and every call of that +`remember` shares the one `state`. By default, `T` is instead moved onto the returned +callable, which then reads as the generic `[T](T) -> T`. That is the reading mypy and ty +give it, and it is unsound here: it accepts `remember(1)` followed by +`remember("a") + "b"`, where the second call returns the `int` the first one stored. + +When set to `true`, `T` stays on `counter` as it is written. This applies to `.py` and +`.by` files alike. + +Defaults to `false`. + +**Default value**: `false` + +**Type**: `bool` + +**Example usage**: + +=== "pyproject.toml" + + ```toml + [tool.ty.analysis] + # Keep a type variable that only a returned callable mentions on the function + lexical-return-callable-typevars = true + ``` + +=== "ty.toml" + + ```toml + [analysis] + # Keep a type variable that only a returned callable mentions on the function + lexical-return-callable-typevars = true + ``` + +--- + +### `lexical-return-callable-typevars-in-typeshed` + +Whether [`lexical-return-callable-typevars`](#lexical-return-callable-typevars) also applies +to the standard library's stubs. + +typeshed is written for the reading that moves the type variable onto the returned +callable. `unittest.skip(reason)` is declared to return a `Callable[[F], F]` that relies on +it, so under the lexical reading a function decorated with `@unittest.skip("...")` would +lose its type. A stub resolved from the standard library search path — the bundled +typeshed or a custom one set with `environment.typeshed` — therefore keeps that reading +unless this is set to `true`. + +Defaults to `false`. + +**Default value**: `false` + +**Type**: `bool` + +**Example usage**: + +=== "pyproject.toml" + + ```toml + [tool.ty.analysis] + # Read typeshed's returned callables lexically too + lexical-return-callable-typevars-in-typeshed = true + ``` + +=== "ty.toml" + + ```toml + [analysis] + # Read typeshed's returned callables lexically too + lexical-return-callable-typevars-in-typeshed = true + ``` + +--- + ### `overlapping-condition-assume-truthy-instances` Whether an instance with no `__bool__` and no `__len__` counts as always truthy when @@ -2191,6 +2282,109 @@ Defaults to `true`, and to `false` under the `ty-compatible` type checking prese --- +#### `lexical-return-callable-typevars` + +Whether a type variable that only appears in a function's returned callable stays bound to +the function, as it is written. + +```python +def counter[T]() -> Callable[[T], T]: + state: list[T] = [] + + def remember(t: T) -> T: + if state: + return state[0] + state.append(t) + return t + + return remember +``` + +`T` belongs to `counter`: one call of `counter` makes one `remember`, and every call of that +`remember` shares the one `state`. By default, `T` is instead moved onto the returned +callable, which then reads as the generic `[T](T) -> T`. That is the reading mypy and ty +give it, and it is unsound here: it accepts `remember(1)` followed by +`remember("a") + "b"`, where the second call returns the `int` the first one stored. + +When set to `true`, `T` stays on `counter` as it is written. This applies to `.py` and +`.by` files alike. + +Defaults to `false`. + +**Default value**: `false` + +**Type**: `bool` + +**Example usage**: + +=== "pyproject.toml" + + ```toml + [[tool.ty.overrides]] + include = ["src"] + + [tool.ty.overrides.analysis] + # Keep a type variable that only a returned callable mentions on the function + lexical-return-callable-typevars = true + ``` + +=== "ty.toml" + + ```toml + [[overrides]] + include = ["src"] + + [overrides.analysis] + # Keep a type variable that only a returned callable mentions on the function + lexical-return-callable-typevars = true + ``` + +--- + +#### `lexical-return-callable-typevars-in-typeshed` + +Whether [`lexical-return-callable-typevars`](#lexical-return-callable-typevars) also applies +to the standard library's stubs. + +typeshed is written for the reading that moves the type variable onto the returned +callable. `unittest.skip(reason)` is declared to return a `Callable[[F], F]` that relies on +it, so under the lexical reading a function decorated with `@unittest.skip("...")` would +lose its type. A stub resolved from the standard library search path — the bundled +typeshed or a custom one set with `environment.typeshed` — therefore keeps that reading +unless this is set to `true`. + +Defaults to `false`. + +**Default value**: `false` + +**Type**: `bool` + +**Example usage**: + +=== "pyproject.toml" + + ```toml + [[tool.ty.overrides]] + include = ["src"] + + [tool.ty.overrides.analysis] + # Read typeshed's returned callables lexically too + lexical-return-callable-typevars-in-typeshed = true + ``` + +=== "ty.toml" + + ```toml + [[overrides]] + include = ["src"] + + [overrides.analysis] + # Read typeshed's returned callables lexically too + lexical-return-callable-typevars-in-typeshed = true + ``` + +--- + #### `overlapping-condition-assume-truthy-instances` Whether an instance with no `__bool__` and no `__len__` counts as always truthy when diff --git a/crates/ty/tests/cli/analysis_options.rs b/crates/ty/tests/cli/analysis_options.rs index 4f1c835832..ae667b1df3 100644 --- a/crates/ty/tests/cli/analysis_options.rs +++ b/crates/ty/tests/cli/analysis_options.rs @@ -363,6 +363,89 @@ fn precise_unsolved_typevars_is_per_module() -> anyhow::Result<()> { Ok(()) } +/// `lexical-return-callable-typevars` is resolved per module: the module that *declares* a +/// function decides whether a type variable only its returned callable mentions stays on the +/// function, and callers see the result regardless of their own setting. +#[test] +fn lexical_return_callable_typevars_is_per_module() -> anyhow::Result<()> { + let case = CliTest::with_files([ + ( + "pyproject.toml", + r#" + [tool.ty.environment] + python-version = "3.13" + + [[tool.ty.overrides]] + include = ["lexical/**"] + + [tool.ty.overrides.analysis] + lexical-return-callable-typevars = true + "#, + ), + ( + "lexical/lib.py", + r#" + from typing import Callable + + def f[T]() -> Callable[[T], T]: + raise NotImplementedError + "#, + ), + ( + "moved/lib.py", + r#" + from typing import Callable + + def g[T]() -> Callable[[T], T]: + raise NotImplementedError + "#, + ), + ( + "moved/main.py", + r#" + from lexical.lib import f + from typing_extensions import reveal_type + + # `f` is declared in a lexical module, so `T` stays on `f` here too + reveal_type(f) + "#, + ), + ( + "lexical/main.py", + r#" + from moved.lib import g + from typing_extensions import reveal_type + + # `g` is declared in a module that moves `T`, so its returned callable is generic + reveal_type(g) + "#, + ), + ])?; + + assert_cmd_snapshot!(case.command(), @" + success: true + exit_code: 0 + ----- stdout ----- + info[revealed-type]: Revealed type + --> lexical/main.py:6:13 + | + 6 | reveal_type(g) + | ^ `def g() -> ((T'return, /) -> T'return)` + + info[revealed-type]: Revealed type + --> moved/main.py:6:13 + | + 6 | reveal_type(f) + | ^ `def f[T]() -> ((T, /) -> T)` + + Found 2 diagnostics + + ----- stderr ----- + "); + + Ok(()) +} + /// `bivariant-private-attributes` is resolved per module: the module that *declares* a class /// governs how its variance is inferred, and consumers see the result regardless of their own /// setting. diff --git a/crates/ty_project/src/metadata/options.rs b/crates/ty_project/src/metadata/options.rs index c11711c086..efe648e8d0 100644 --- a/crates/ty_project/src/metadata/options.rs +++ b/crates/ty_project/src/metadata/options.rs @@ -2586,6 +2586,65 @@ pub struct AnalysisOptions { )] pub precise_unsolved_typevars: Option, + /// Whether a type variable that only appears in a function's returned callable stays bound to + /// the function, as it is written. + /// + /// ```python + /// def counter[T]() -> Callable[[T], T]: + /// state: list[T] = [] + /// + /// def remember(t: T) -> T: + /// if state: + /// return state[0] + /// state.append(t) + /// return t + /// + /// return remember + /// ``` + /// + /// `T` belongs to `counter`: one call of `counter` makes one `remember`, and every call of that + /// `remember` shares the one `state`. By default, `T` is instead moved onto the returned + /// callable, which then reads as the generic `[T](T) -> T`. That is the reading mypy and ty + /// give it, and it is unsound here: it accepts `remember(1)` followed by + /// `remember("a") + "b"`, where the second call returns the `int` the first one stored. + /// + /// When set to `true`, `T` stays on `counter` as it is written. This applies to `.py` and + /// `.by` files alike. + /// + /// Defaults to `false`. + #[serde(skip_serializing_if = "Option::is_none")] + #[option( + default = r#"false"#, + value_type = "bool", + example = r#" + # Keep a type variable that only a returned callable mentions on the function + lexical-return-callable-typevars = true + "# + )] + pub lexical_return_callable_typevars: Option, + + /// Whether [`lexical-return-callable-typevars`](#lexical-return-callable-typevars) also applies + /// to the standard library's stubs. + /// + /// typeshed is written for the reading that moves the type variable onto the returned + /// callable. `unittest.skip(reason)` is declared to return a `Callable[[F], F]` that relies on + /// it, so under the lexical reading a function decorated with `@unittest.skip("...")` would + /// lose its type. A stub resolved from the standard library search path — the bundled + /// typeshed or a custom one set with `environment.typeshed` — therefore keeps that reading + /// unless this is set to `true`. + /// + /// Defaults to `false`. + #[serde(skip_serializing_if = "Option::is_none")] + #[option( + default = r#"false"#, + value_type = "bool", + example = r#" + # Read typeshed's returned callables lexically too + lexical-return-callable-typevars-in-typeshed = true + "# + )] + pub lexical_return_callable_typevars_in_typeshed: Option, + /// A list of classes whose values do not count as a distinct member of an /// [`overlapping-condition`](rules.md#overlapping-condition). /// @@ -2803,6 +2862,8 @@ impl AnalysisOptions { infer_unannotated_signatures, bivariant_private_attributes, precise_unsolved_typevars, + lexical_return_callable_typevars, + lexical_return_callable_typevars_in_typeshed, overlapping_condition_exempt_types, overlapping_condition_assume_truthy_instances, implicit_object_repr_exempt_types, @@ -2826,6 +2887,9 @@ impl AnalysisOptions { infer_unannotated_signatures: infer_unannotated_signatures_default, bivariant_private_attributes: bivariant_private_attributes_default, precise_unsolved_typevars: precise_unsolved_typevars_default, + lexical_return_callable_typevars: lexical_return_callable_typevars_default, + lexical_return_callable_typevars_in_typeshed: + lexical_return_callable_typevars_in_typeshed_default, overlapping_condition_exempt_types: overlapping_condition_exempt_types_default, overlapping_condition_assume_truthy_instances: overlapping_condition_assume_truthy_instances_default, @@ -2880,6 +2944,11 @@ impl AnalysisOptions { .unwrap_or(bivariant_private_attributes_default), precise_unsolved_typevars: precise_unsolved_typevars .unwrap_or(precise_unsolved_typevars_default), + lexical_return_callable_typevars: lexical_return_callable_typevars + .unwrap_or(lexical_return_callable_typevars_default), + lexical_return_callable_typevars_in_typeshed: + lexical_return_callable_typevars_in_typeshed + .unwrap_or(lexical_return_callable_typevars_in_typeshed_default), overlapping_condition_exempt_types: overlapping_condition_exempt_types .as_ref() .map(|types| { diff --git a/crates/ty_python_semantic/resources/mdtest/basedpython_lexical_return_callable_typevars.md b/crates/ty_python_semantic/resources/mdtest/basedpython_lexical_return_callable_typevars.md new file mode 100644 index 0000000000..6802e49215 --- /dev/null +++ b/crates/ty_python_semantic/resources/mdtest/basedpython_lexical_return_callable_typevars.md @@ -0,0 +1,198 @@ +# basedpython: lexical return callable type variables + +mypy, and ty, move a type variable that a function's signature only mentions in the callable it +returns onto that callable, reading `def f[T]() -> Callable[[T], T]` as returning the generic +`[T](T) -> T`. that reading is unsound as soon as the body ties the returned callable to a single +solution. + +`analysis.lexical-return-callable-typevars` keeps the type variable on the function, as it is +written. it is off by default, and applies to `.py` and `.by` files alike. + +## off by default + +```toml +[environment] +python-version = "3.13" +``` + +by default, the type variable moves onto the returned callable, and the unsound call is accepted: + +```py +from typing import Callable + +def remembers_first[T]() -> Callable[[T], T]: + raise NotImplementedError + +reveal_type(remembers_first) # revealed: def remembers_first() -> ((T'return, /) -> T'return) + +remember = remembers_first() +reveal_type(remember) # revealed: [T'return](T'return, /) -> T'return +reveal_type(remember("a")) # revealed: Literal["a"] +``` + +## the type variable stays on the function + +```toml +[environment] +python-version = "3.13" + +[analysis] +lexical-return-callable-typevars = true +``` + +```py +from typing import Callable + +def remembers_first[T]() -> Callable[[T], T]: + state: list[T] = [] + + def remember(t: T) -> T: + if state: + return state[0] + state.append(t) + return t + + return remember + +reveal_type(remembers_first) # revealed: def remembers_first[T]() -> ((T, /) -> T) +``` + +every call of one `remember` shares one `state`, so the second call below returns the `int` the +first one stored. once the call solves `T`, the mismatch is reported: + +```py +remember: Callable[[int], int] = remembers_first() +remember(1) + 1 + +# error: [invalid-argument-type] +remember("a") +``` + +a call that leaves `T` unsolved gets the gradual answer, rather than a generic callable it has no +grounds for. `T` is both a parameter and a return type there, so it is not solved to `Never` either: + +```py +unsolved = remembers_first() +reveal_type(unsolved) # revealed: (Unknown, /) -> Unknown +``` + +the legacy spelling says the same thing: + +```py +from typing import TypeVar + +U = TypeVar("U") + +def legacy_remembers_first() -> Callable[[U], U]: + raise NotImplementedError + +reveal_type(legacy_remembers_first) # revealed: def legacy_remembers_first[U]() -> ((U, /) -> U) +``` + +## basedpython syntax + +```toml +[environment] +python-version = "3.13" + +[analysis] +lexical-return-callable-typevars = true +``` + +```by +def remembers_first[T]() -> (T) -> T: + state: list[T] = [] + + def remember(t: T) -> T: + if state: + return state[0] + state.append(t) + return t + + return remember + +reveal_type(remembers_first) # revealed: def remembers_first[T]() -> ((T, /) -> T) + +remember: (int) -> int = remembers_first() +# error: [invalid-argument-type] +remember("a") +``` + +## returning a generic callable + +a function that really does return a generic callable says so with a protocol whose `__call__` is +generic: + +```toml +[environment] +python-version = "3.13" + +[analysis] +lexical-return-callable-typevars = true +``` + +```py +from typing import Protocol + +class Identity(Protocol): + def __call__[T](self, t: T, /) -> T: ... + +def identity_factory() -> Identity: + def identity[T](t: T) -> T: + return t + + return identity + +identity = identity_factory() +reveal_type(identity(1)) # revealed: Literal[1] +reveal_type(identity("a")) # revealed: Literal["a"] +``` + +## typeshed keeps the moved reading + +typeshed is written for the reading that moves the type variable. `unittest.skip(reason)` is +declared to return a `Callable[[F], F]` it relies on being generic, so a stub from the standard +library keeps that reading even with the option on: + +```toml +[environment] +python-version = "3.13" + +[analysis] +lexical-return-callable-typevars = true +``` + +```py +import unittest + +@unittest.skip("not yet") +def skipped(x: int) -> str: + return "" + +reveal_type(skipped) # revealed: def skipped(x: int) -> str +``` + +## typeshed opted in + +`analysis.lexical-return-callable-typevars-in-typeshed` applies the lexical reading to the standard +library's stubs too: + +```toml +[environment] +python-version = "3.13" + +[analysis] +lexical-return-callable-typevars = true +lexical-return-callable-typevars-in-typeshed = true +``` + +```py +import unittest + +# error: [dynamic-function-decorator-return] +@unittest.skip("not yet") +def skipped(x: int) -> str: + return "" + +reveal_type(skipped) # revealed: Unknown +``` diff --git a/crates/ty_python_semantic/src/lib.rs b/crates/ty_python_semantic/src/lib.rs index 82181e8d4e..29d46c2794 100644 --- a/crates/ty_python_semantic/src/lib.rs +++ b/crates/ty_python_semantic/src/lib.rs @@ -278,6 +278,23 @@ pub struct AnalysisSettings { /// occurrence stays gradual, since `Never` there would say nothing can be written or passed. pub precise_unsolved_typevars: bool, + /// Whether a type variable that only appears in a function's returned callable stays bound to + /// the function. + /// + /// When this is disabled, `def f[T]() -> Callable[[T], T]` is read as returning a generic + /// callable `[T](T) -> T`: the type variable is moved off the function and onto the callable it + /// returns, so every call of the result solves it afresh. That reading is unsound whenever the + /// function's body ties the returned callable to one solution, such as through state it + /// closes over. + pub lexical_return_callable_typevars: bool, + + /// [`Self::lexical_return_callable_typevars`] for the standard library's stubs. + /// + /// typeshed is written for the other reading — `unittest.skip(reason)` returns a + /// `Callable[[F], F]` it relies on being generic — so it keeps that reading unless this is + /// enabled. + pub lexical_return_callable_typevars_in_typeshed: bool, + /// Classes whose values do not count as a distinct member of an overlapping condition. /// /// Entries are qualified class names (`decimal.Decimal`); a class in `builtins` may also be @@ -384,6 +401,8 @@ impl AnalysisSettings { infer_unannotated_signatures: basedpython, bivariant_private_attributes: basedpython, precise_unsolved_typevars: basedpython, + lexical_return_callable_typevars: false, + lexical_return_callable_typevars_in_typeshed: false, overlapping_condition_exempt_types: Box::default(), overlapping_condition_assume_truthy_instances: false, implicit_object_repr_exempt_types: Box::default(), diff --git a/crates/ty_python_semantic/src/types/signatures.rs b/crates/ty_python_semantic/src/types/signatures.rs index 25d3331a52..e52024587b 100644 --- a/crates/ty_python_semantic/src/types/signatures.rs +++ b/crates/ty_python_semantic/src/types/signatures.rs @@ -66,6 +66,7 @@ use ruff_python_ast::helpers::ReturnGuardForm; use ruff_python_ast::{self as ast, ParameterBorrow, name::Name}; use ruff_text_size::Ranged; use std::borrow::Cow; +use ty_module_resolver::{SearchPath, file_to_module}; use ty_python_core::definition::{Definition, DefinitionKind, ParameterDefinitionNodeKind}; /// Selects which binding context to use for type variables that only appear in a return-position @@ -78,6 +79,23 @@ pub(super) enum ReturnCallableTypeVarScope { Public, } +/// basedpython: whether a type variable that only appears in the callable `definition` returns +/// stays bound to `definition`, rather than moving onto that callable. +/// +/// The module that declares the function governs it. The standard library's stubs have their own +/// setting, because typeshed is written for the reading that moves the type variable. +fn lexical_return_callable_typevars<'db>(db: &'db dyn Db, definition: Definition<'db>) -> bool { + let settings = db.analysis_settings(definition.file(db)); + let is_typeshed = file_to_module(db, definition.program_file(db).resolver_file(db)) + .and_then(|module| module.search_path(db)) + .is_some_and(SearchPath::is_standard_library); + if is_typeshed { + settings.lexical_return_callable_typevars_in_typeshed + } else { + settings.lexical_return_callable_typevars + } +} + /// Infer the type of a parameter or return annotation in a function signature. /// /// This is very similar to `definition_expression_type`, but knows that `TypeInferenceBuilder` @@ -1394,6 +1412,11 @@ impl<'db> Signature<'db> { legacy_generic_context, ); + let return_callable_typevar_scope = if lexical_return_callable_typevars(db, definition) { + ReturnCallableTypeVarScope::Lexical + } else { + return_callable_typevar_scope + }; let (generic_context, return_ty) = match return_callable_typevar_scope { ReturnCallableTypeVarScope::Lexical => (full_generic_context, return_ty), ReturnCallableTypeVarScope::Public => { diff --git a/crates/ty_server/tests/e2e/snapshots/e2e__commands__debug_command.snap b/crates/ty_server/tests/e2e/snapshots/e2e__commands__debug_command.snap index 654a32a60b..0f32bf9d71 100644 --- a/crates/ty_server/tests/e2e/snapshots/e2e__commands__debug_command.snap +++ b/crates/ty_server/tests/e2e/snapshots/e2e__commands__debug_command.snap @@ -228,6 +228,8 @@ Settings: Settings { infer_unannotated_signatures: true, bivariant_private_attributes: true, precise_unsolved_typevars: true, + lexical_return_callable_typevars: false, + lexical_return_callable_typevars_in_typeshed: false, overlapping_condition_exempt_types: [], overlapping_condition_assume_truthy_instances: false, implicit_object_repr_exempt_types: [], diff --git a/crates/ty_test/src/config.rs b/crates/ty_test/src/config.rs index 4b6018eb15..afe9209e3f 100644 --- a/crates/ty_test/src/config.rs +++ b/crates/ty_test/src/config.rs @@ -280,6 +280,10 @@ pub(crate) struct Analysis { pub(crate) precise_unsolved_typevars: Option, + pub(crate) lexical_return_callable_typevars: Option, + + pub(crate) lexical_return_callable_typevars_in_typeshed: Option, + pub(crate) overlapping_condition_exempt_types: Option>, pub(crate) overlapping_condition_assume_truthy_instances: Option, diff --git a/crates/ty_test/src/db.rs b/crates/ty_test/src/db.rs index 66803e65ad..345542cdd5 100644 --- a/crates/ty_test/src/db.rs +++ b/crates/ty_test/src/db.rs @@ -406,6 +406,9 @@ fn mdtest_analysis_settings( infer_unannotated_signatures: infer_unannotated_signatures_default, bivariant_private_attributes: bivariant_private_attributes_default, precise_unsolved_typevars: precise_unsolved_typevars_default, + lexical_return_callable_typevars: lexical_return_callable_typevars_default, + lexical_return_callable_typevars_in_typeshed: + lexical_return_callable_typevars_in_typeshed_default, overlapping_condition_exempt_types: overlapping_condition_exempt_types_default, overlapping_condition_assume_truthy_instances: overlapping_condition_assume_truthy_instances_default, @@ -474,6 +477,12 @@ fn mdtest_analysis_settings( precise_unsolved_typevars: options .precise_unsolved_typevars .unwrap_or(precise_unsolved_typevars_default), + lexical_return_callable_typevars: options + .lexical_return_callable_typevars + .unwrap_or(lexical_return_callable_typevars_default), + lexical_return_callable_typevars_in_typeshed: options + .lexical_return_callable_typevars_in_typeshed + .unwrap_or(lexical_return_callable_typevars_in_typeshed_default), overlapping_condition_exempt_types: options .overlapping_condition_exempt_types .as_deref() diff --git a/docs/basedpython/features/index.md b/docs/basedpython/features/index.md index 7dab9b8871..69ec16fd3d 100644 --- a/docs/basedpython/features/index.md +++ b/docs/basedpython/features/index.md @@ -73,6 +73,8 @@ type-checking improvements with no new syntax — they work in `.by` and `.py` f - [sound types](sound-types.md) — infer precise types instead of gradual ones - [precise unsolved type variables](precise-unsolved-typevars.md) — an unsolved type variable is `Never`, not `Unknown` +- [lexical return callable type variables](lexical-return-callable-typevars.md) — a type variable + only a returned callable mentions still belongs to the function - [regex group types](regex-groups.md) — type a match from the pattern it came from - [boolean conditions](conditions.md) — catch a test that conflates two members, or asks nothing - [refutable unpacking](refutable-unpacking.md) — catch `a, b = ...` over a value of unknown length diff --git a/docs/basedpython/features/lexical-return-callable-typevars.md b/docs/basedpython/features/lexical-return-callable-typevars.md new file mode 100644 index 0000000000..5273b19706 --- /dev/null +++ b/docs/basedpython/features/lexical-return-callable-typevars.md @@ -0,0 +1,76 @@ +# lexical return callable type variables + +with `analysis.lexical-return-callable-typevars` on, a type variable written on a function belongs +to that function, even when the only place the signature mentions it is the callable the function +returns + +```python +from typing import Callable + +def remembers_first[T]() -> Callable[[T], T]: + state: list[T] = [] + + def remember(t: T) -> T: + if state: + return state[0] + state.append(t) + return t + + return remember + +remember: Callable[[int], int] = remembers_first() +remember(1) + 1 +remember("a") # error: [invalid-argument-type] +``` + +mypy, and ty, read that signature differently: they move `T` off `remembers_first` and onto the +callable it returns, so `remembers_first()` is the generic `[T](T) -> T` and every call of it +solves `T` afresh. that accepts `remember(1)` followed by `remember("a") + "b"` — and the second call +returns the `int` the first one stored, so the program fails at runtime + +with this option on, basedpython keeps `T` where it is written. one call of `remembers_first` makes one `remember`, with +one solution for `T`. a call that does not solve it gets the gradual `(Unknown) -> Unknown` rather +than a generic callable it has no grounds for + +this is off by default, and applies to `.py` files as well as `.by` when enabled + +```toml +[analysis] +lexical-return-callable-typevars = true +``` + +## returning a generic callable + +a function that really does return a generic callable — the usual decorator factory — says so with +a protocol whose `__call__` is generic + +```python +from typing import Protocol + +class Identity(Protocol): + def __call__[T](self, t: T, /) -> T: ... + +def identity_factory() -> Identity: + def identity[T](t: T) -> T: + return t + + return identity + +identity = identity_factory() +identity(1) # int +identity("a") # str +``` + +## typeshed + +typeshed is written for the other reading. `unittest.skip(reason)` is declared to return a +`Callable[[F], F]` that relies on `F` moving onto the returned callable, so a function decorated with +`@unittest.skip("...")` would lose its type under this one + +a stub from the standard library — the bundled typeshed, or one set with `environment.typeshed` — +therefore keeps the moved reading. to apply this one there too + +```toml +[analysis] +lexical-return-callable-typevars-in-typeshed = true +``` diff --git a/ty.schema.json b/ty.schema.json index dede19cad0..fa3328f711 100644 --- a/ty.schema.json +++ b/ty.schema.json @@ -226,6 +226,20 @@ "null" ] }, + "lexical-return-callable-typevars": { + "description": "Whether a type variable that only appears in a function's returned callable stays bound to\nthe function, as it is written.\n\n```python\ndef counter[T]() -> Callable[[T], T]:\n state: list[T] = []\n\n def remember(t: T) -> T:\n if state:\n return state[0]\n state.append(t)\n return t\n\n return remember\n```\n\n`T` belongs to `counter`: one call of `counter` makes one `remember`, and every call of that\n`remember` shares the one `state`. By default, `T` is instead moved onto the returned\ncallable, which then reads as the generic `[T](T) -> T`. That is the reading mypy and ty\ngive it, and it is unsound here: it accepts `remember(1)` followed by\n`remember(\"a\") + \"b\"`, where the second call returns the `int` the first one stored.\n\nWhen set to `true`, `T` stays on `counter` as it is written. This applies to `.py` and\n`.by` files alike.\n\nDefaults to `false`.", + "type": [ + "boolean", + "null" + ] + }, + "lexical-return-callable-typevars-in-typeshed": { + "description": "Whether [`lexical-return-callable-typevars`](#lexical-return-callable-typevars) also applies\nto the standard library's stubs.\n\ntypeshed is written for the reading that moves the type variable onto the returned\ncallable. `unittest.skip(reason)` is declared to return a `Callable[[F], F]` that relies on\nit, so under the lexical reading a function decorated with `@unittest.skip(\"...\")` would\nlose its type. A stub resolved from the standard library search path — the bundled\ntypeshed or a custom one set with `environment.typeshed` — therefore keeps that reading\nunless this is set to `true`.\n\nDefaults to `false`.", + "type": [ + "boolean", + "null" + ] + }, "overlapping-condition-assume-truthy-instances": { "description": "Whether an instance with no `__bool__` and no `__len__` counts as always truthy when\nlooking for an [`overlapping-condition`](rules.md#overlapping-condition).\n\nSuch an instance is only *ambiguously* truthy — a subclass may define `__bool__` — so by\ndefault it is a falsy member of `if not x` just as `False` is. Enabling this assumes the\nclass means what it looks like it means, which drops the reports for `if not x` over an\ninstance or a `False` sentinel. (`truthiness-test-of-none-union`, which reports a `None`\nsentinel, always makes this assumption.)\n\nDefaults to `false`.", "type": [ diff --git a/zensical.toml b/zensical.toml index d852eaee25..73634f3693 100644 --- a/zensical.toml +++ b/zensical.toml @@ -103,6 +103,7 @@ features = [ { "fluid specializations" = "features/fluid-specializations.md" }, { "sound types" = "features/sound-types.md" }, { "precise unsolved type variables" = "features/precise-unsolved-typevars.md" }, + { "lexical return callable type variables" = "features/lexical-return-callable-typevars.md" }, { "regex group types" = "features/regex-groups.md" }, { "boolean conditions" = "features/conditions.md" }, { "refutable unpacking" = "features/refutable-unpacking.md" },