Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions docs/core_concepts/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,15 @@ example `{"feature_0": "rbf"}`. Check `numerical_features_` / `categorical_featu
overrides, rather than guessing the column order.
```

At `transform`, array columns are matched by position, so the training and serving
containers may differ, as in scikit-learn. A `Preprocessor` fitted on a `DataFrame` accepts
the same data as an array of the same width (for example `df.to_numpy()` in NumPy-based
serving or an explainer such as `shap.KernelExplainer`). It warns that `X` does not have
valid feature names and re-infers the numeric columns of an object array. One fitted on an
array accepts a `DataFrame` of the same width the same way, with a warning that `X` has
feature names. Input matched by position with a different number of columns raises the usual
`X has 4 features, but Preprocessor is expecting 3 features as input.` error.

## Per-feature overrides

Columns rarely want identical treatment. The `feature_preprocessing` dict assigns a strategy
Expand Down
56 changes: 54 additions & 2 deletions pretab/preprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -613,7 +613,12 @@ def transform(self, X, embeddings=None, return_array=None):
Parameters
----------
X : pandas.DataFrame, numpy.ndarray, or dict
Input features to transform.
Input features to transform. A DataFrame or dict is matched to the fitted
columns by name. A NumPy array is matched by position: after a fit on named
columns it takes those names (numeric columns of an object array are
re-inferred), with scikit-learn's usual warning that ``X`` does not have
valid feature names. Likewise, a DataFrame passed after a fit on an array
is matched by position, with a warning that ``X`` has feature names.
embeddings : np.ndarray or list of np.ndarray, optional
External embeddings to attach to dictionary output. Required when
embeddings were supplied during ``fit`` and unsupported when the
Expand All @@ -632,11 +637,17 @@ def transform(self, X, embeddings=None, return_array=None):
(or ``return_array=False``); a SciPy CSR matrix (or CSR blocks) when
``output_format`` resolves to ``"sparse"``; or a pandas / polars DataFrame
when configured via :meth:`set_output`.

Raises
------
PretabDataError
If input matched by position has a different number of columns than
``n_features_in_``: it could never be routed correctly.
"""

check_is_fitted(self)

X = to_dataframe(X, copy=True)
X = self._align_input(X)

if self.missing_policy == "error":
self._reject_missing(X)
Expand Down Expand Up @@ -822,6 +833,47 @@ def output_dims_(self) -> dict:
dims[columns[0]] = width
return dims

def _align_input(self, X):
"""Return ``X`` as a DataFrame whose columns line up with the fitted columns.

A NumPy array is matched by position. Once the preprocessor was fitted on
named columns, an array of the fitted width takes those column labels
(numeric columns of an object array are re-inferred), and a DataFrame
passed to a preprocessor fitted on an array is matched by position too --
both with scikit-learn's usual warning about the missing or unexpected
feature names. Any other DataFrame is matched by column label.

Raises
------
PretabDataError
If array-like input matched by position has a different number of
columns than seen during ``fit``: it could never be routed correctly.
"""
# The ColumnTransformer records the labels it was fitted on whenever they are
# all strings; an array fit shows up as the synthetic positional labels.
fitted_labels = getattr(self.column_transformer_, "feature_names_in_", None)
positional_labels = [f"feature_{i}" for i in range(self.n_features_in_)]
fitted_on_array = fitted_labels is not None and list(fitted_labels) == positional_labels

is_array = isinstance(X, np.ndarray)
X = to_dataframe(X, copy=True)
if not (is_array or (fitted_on_array and list(X.columns) != positional_labels)):
return X
if X.shape[1] != self.n_features_in_:
raise PretabDataError(
f"X has {X.shape[1]} features, but {type(self).__name__} is expecting "
f"{self.n_features_in_} features as input."
)
if fitted_labels is None or list(X.columns) == list(fitted_labels):
return X
if fitted_on_array:
message = f"X has feature names, but {type(self).__name__} was fitted without feature names"
else:
message = f"X does not have valid feature names, but {type(self).__name__} was fitted with feature names"
warnings.warn(message, UserWarning, stacklevel=3)
X = X.set_axis(list(fitted_labels), axis=1)
return X.infer_objects() if is_array else X

def _output_itemsize(self) -> int:
"""Bytes per element of the dense transformed array.

Expand Down
49 changes: 49 additions & 0 deletions tests/integration/test_preprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -377,3 +377,52 @@ def test_unknown_feature_preprocessing_key_raises(sample_data):
X, y = sample_data
with pytest.raises(InvalidParamError, match="feature_preprocessing"):
Preprocessor(feature_preprocessing={"nnum1": "minmax"}).fit(X, y)


# --- positional input after a named fit (and vice versa) (issue #78) ---------------


@pytest.fixture
def mixed_named_frame():
rng = np.random.default_rng(0)
frame = pd.DataFrame(
{
"age": rng.normal(40, 10, 100),
"income": rng.normal(50, 10, 100),
"city": rng.choice(["a", "b", "c"], 100),
}
)
return frame, rng.normal(size=100)


def test_frame_fit_accepts_an_array_of_the_fitted_width(mixed_named_frame):
"""Regression guard for issue #78: a DataFrame-fitted preprocessor rejected the same
data as an array ('columns are missing'), breaking NumPy-based serving and explainers."""
from sklearn.linear_model import Ridge
from sklearn.pipeline import make_pipeline

X, y = mixed_named_frame
pipe = make_pipeline(Preprocessor(random_state=0), Ridge()).fit(X, y)
with pytest.warns(UserWarning, match="does not have valid feature names"):
from_array = pipe.predict(X.to_numpy())
np.testing.assert_allclose(from_array, pipe.predict(X))


def test_array_fit_accepts_a_frame_of_the_fitted_width(mixed_named_frame):
X, y = mixed_named_frame
numeric = X[["age", "income"]]
pre = Preprocessor(numerical_method="minmax").fit(numeric.to_numpy(), y)
with pytest.warns(UserWarning, match="fitted without feature names"):
from_frame = pre.transform(numeric.rename(columns={"age": "u", "income": "v"}))
np.testing.assert_array_equal(from_frame, pre.transform(numeric.to_numpy()))


def test_positional_input_of_the_wrong_width_is_rejected(mixed_named_frame):
X, y = mixed_named_frame
pre = Preprocessor(random_state=0).fit(X, y)
with pytest.raises(PretabDataError, match="X has 2 features, but Preprocessor is expecting 3"):
pre.transform(X.to_numpy()[:, :2])
numeric = X[["age", "income"]]
array_fit = Preprocessor(numerical_method="minmax").fit(numeric.to_numpy(), y)
with pytest.raises(PretabDataError, match="X has 1 features"):
array_fit.transform(X[["age"]])
Loading