diff --git a/ngboost/api.py b/ngboost/api.py index 085bd942..8ca65f9c 100644 --- a/ngboost/api.py +++ b/ngboost/api.py @@ -1,4 +1,5 @@ "The NGBoost library API" + # pylint: disable=too-many-arguments from sklearn.base import BaseEstimator from sklearn.utils import check_array @@ -31,8 +32,15 @@ class NGBRegressor(NGBoost, BaseEstimator): A distribution from ngboost.distns, e.g. Normal Score : rule to compare probabilistic predictions P̂ to the observed data y. A score from ngboost.scores, e.g. LogScore - Base : base learner to use in the boosting algorithm. - Any instantiated sklearn regressor, e.g. DecisionTreeRegressor() + Base : base learner(s) to use in the boosting algorithm. + Pass a single instantiated sklearn regressor, e.g. + DecisionTreeRegressor(), to use the same learner for + every distribution parameter. To use different learners + per distribution parameter, pass a list/tuple of + instantiated sklearn regressors with length equal to + Dist.n_params. The sequence order matches the distribution + parameter order; for example, Normal uses + [loc_learner, scale_learner]. natural_gradient : logical flag indicating whether the natural gradient should be used n_estimators : the number of boosting iterations to fit learning_rate : the learning rate @@ -127,8 +135,15 @@ class NGBClassifier(NGBoost, BaseEstimator): A distribution from ngboost.distns, e.g. Bernoulli Score : rule to compare probabilistic predictions P̂ to the observed data y. A score from ngboost.scores, e.g. LogScore - Base : base learner to use in the boosting algorithm. - Any instantiated sklearn regressor, e.g. DecisionTreeRegressor() + Base : base learner(s) to use in the boosting algorithm. + Pass a single instantiated sklearn regressor, e.g. + DecisionTreeRegressor(), to use the same learner for + every distribution parameter. To use different learners + per distribution parameter, pass a list/tuple of + instantiated sklearn regressors with length equal to + Dist.n_params. The sequence order matches the distribution + parameter order; for example, Normal uses + [loc_learner, scale_learner]. natural_gradient : logical flag indicating whether the natural gradient should be used n_estimators : the number of boosting iterations to fit learning_rate : the learning rate @@ -221,8 +236,15 @@ class NGBSurvival(NGBoost, BaseEstimator): A distribution from ngboost.distns, e.g. LogNormal Score : rule to compare probabilistic predictions P̂ to the observed data y. A score from ngboost.scores, e.g. LogScore - Base : base learner to use in the boosting algorithm. - Any instantiated sklearn regressor, e.g. DecisionTreeRegressor() + Base : base learner(s) to use in the boosting algorithm. + Pass a single instantiated sklearn regressor, e.g. + DecisionTreeRegressor(), to use the same learner for + every distribution parameter. To use different learners + per distribution parameter, pass a list/tuple of + instantiated sklearn regressors with length equal to + Dist.n_params. The sequence order matches the distribution + parameter order; for example, Normal uses + [loc_learner, scale_learner]. natural_gradient : logical flag indicating whether the natural gradient should be used n_estimators : the number of boosting iterations to fit learning_rate : the learning rate diff --git a/ngboost/ngboost.py b/ngboost/ngboost.py index 2406e574..2bae7a2b 100644 --- a/ngboost/ngboost.py +++ b/ngboost/ngboost.py @@ -29,8 +29,15 @@ class NGBoost: A distribution from ngboost.distns, e.g. Normal Score : rule to compare probabilistic predictions P̂ to the observed data y. A score from ngboost.scores, e.g. LogScore - Base : base learner to use in the boosting algorithm. - Any instantiated sklearn regressor, e.g. DecisionTreeRegressor() + Base : base learner(s) to use in the boosting algorithm. + Pass a single instantiated sklearn regressor, e.g. + DecisionTreeRegressor(), to use the same learner for + every distribution parameter. To use different learners + per distribution parameter, pass a list/tuple of + instantiated sklearn regressors with length equal to + Dist.n_params. The sequence order matches the distribution + parameter order; for example, Normal uses + [loc_learner, scale_learner]. natural_gradient : logical flag indicating whether the natural gradient should be used n_estimators : the number of boosting iterations to fit learning_rate : the learning rate @@ -167,12 +174,27 @@ def sample(self, X, Y, sample_weight, params): params[idxs, :], ) + def _base_learners(self): + if not isinstance(self.Base, (list, tuple)): + return [self.Base] * self.Manifold.n_params + + if len(self.Base) != self.Manifold.n_params: + raise ValueError( + "Base must be a single estimator or a sequence with one estimator " + "per distribution parameter " + f"({self.Manifold.n_params} for {self.Dist.__name__}); " + f"got {len(self.Base)}." + ) + return self.Base + def fit_base(self, X, grads, sample_weight=None): + base_learners = self._base_learners() if sample_weight is None: - models = [clone(self.Base).fit(X, g) for g in grads.T] + models = [clone(base).fit(X, g) for base, g in zip(base_learners, grads.T)] else: models = [ - clone(self.Base).fit(X, g, sample_weight=sample_weight) for g in grads.T + clone(base).fit(X, g, sample_weight=sample_weight) + for base, g in zip(base_learners, grads.T) ] fitted = np.array([m.predict(X) for m in models]).T self.base_models.append(models) @@ -320,6 +342,7 @@ def partial_fit( raise RuntimeError( "Base models, scalings, and col_idxs are not the same length" ) + self._base_learners() # if early stopping is specified, split X,Y and sample weights (if given) into training and validation sets # This will overwrite any X_val and Y_val values passed by the user directly. @@ -491,16 +514,65 @@ def partial_fit( return self + def _set_base_params(self, parameters): + if not isinstance(self.Base, (list, tuple)): + self.Base.set_params(**parameters) + return + + base_learners = list(self.Base) + for key, value in parameters.items(): + index, delim, sub_key = key.partition("__") + try: + index = int(index) + except ValueError as exc: + raise ValueError( + "Base sequence parameters must use integer positions, " + f"for example Base__0__max_depth. Got Base__{key}." + ) from exc + if index < 0 or index >= len(base_learners): + raise ValueError( + f"Base sequence index {index} is out of range for " + f"{len(base_learners)} estimators." + ) + if delim: + base_learners[index].set_params(**{sub_key: value}) + else: + base_learners[index] = value + self.Base = type(self.Base)(base_learners) + def set_params(self, **parameters): - for parameter, value in parameters.items(): - setattr(self, parameter, value) + if not parameters: + return self + + valid_params = self.get_params(deep=True) + nested_params = {} + for key, value in parameters.items(): + key, delim, sub_key = key.partition("__") + if key not in valid_params: + local_valid_params = self.get_params(deep=False).keys() + raise ValueError( + f"Invalid parameter {key!r} for estimator {self}. " + f"Valid parameters are: {sorted(local_valid_params)!r}." + ) + if delim: + nested_params.setdefault(key, {})[sub_key] = value + else: + setattr(self, key, value) + valid_params[key] = value + + for key, sub_params in nested_params.items(): + if key == "Base": + self._set_base_params(sub_params) + else: + valid_params[key].set_params(**sub_params) return self def get_params(self, deep=True): """ Parameters ---------- - deep : Ignored. (for compatibility with sklearn) + deep : bool, default=True + If True, return nested parameters for the base learner or learners. Returns ---------- params : returns an dictionary of parameters. @@ -515,11 +587,26 @@ def get_params(self, deep=True): "minibatch_frac": self.minibatch_frac, "col_sample": self.col_sample, "verbose": self.verbose, + "verbose_eval": self.verbose_eval, + "tol": self.tol, "random_state": self.random_state, "validation_fraction": self.validation_fraction, "early_stopping_rounds": self.early_stopping_rounds, } + if not deep: + return params + + if isinstance(self.Base, (list, tuple)): + for i, base in enumerate(self.Base): + params[f"Base__{i}"] = base + if hasattr(base, "get_params"): + for key, value in base.get_params(deep=True).items(): + params[f"Base__{i}__{key}"] = value + elif hasattr(self.Base, "get_params"): + for key, value in self.Base.get_params(deep=True).items(): + params[f"Base__{key}"] = value + return params def score(self, X, Y): # for sklearn @@ -616,8 +703,12 @@ def feature_importances_(self): # Check whether the model is fitted if not self.base_models: return None - # Check whether the base model is DecisionTreeRegressor - if not isinstance(self.base_models[0][0], DecisionTreeRegressor): + # Check whether all base models are DecisionTreeRegressor + if any( + not isinstance(model, DecisionTreeRegressor) + for models in self.base_models + for model in models + ): return None # Reshape the base_models params_trees = zip(*self.base_models) diff --git a/tests/test_basic.py b/tests/test_basic.py index 59c9ea73..ddf4bced 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -1,5 +1,23 @@ +import numpy as np +import pytest +from sklearn.base import BaseEstimator, RegressorMixin, clone +from sklearn.ensemble import HistGradientBoostingRegressor +from sklearn.linear_model import Ridge +from sklearn.tree import DecisionTreeRegressor + from ngboost import NGBClassifier, NGBRegressor -from ngboost.distns import Bernoulli, Normal +from ngboost.distns import Bernoulli, Normal, k_categorical + + +class RecordingRegressor(BaseEstimator, RegressorMixin): + # pylint: disable=attribute-defined-outside-init,unused-argument + def fit(self, X, y, sample_weight=None): + self.sample_weight_ = None if sample_weight is None else sample_weight.copy() + self.prediction_ = np.average(y, weights=sample_weight) + return self + + def predict(self, X): + return np.full(X.shape[0], self.prediction_) # TODO: This is non-deterministic in the model fitting @@ -54,3 +72,243 @@ def test_regression(california_housing_data): score = mean_squared_error(y_test, preds) assert score <= 15 + + +def test_regression_accepts_base_learner_per_distribution_parameter(): + rng = np.random.default_rng(0) + X = rng.normal(size=(80, 4)) + Y = X[:, 0] - 0.5 * X[:, 1] + rng.normal(scale=0.1, size=80) + + ngb = NGBRegressor( + Dist=Normal, + Base=[DecisionTreeRegressor(max_depth=1), Ridge(alpha=0.0)], + n_estimators=2, + natural_gradient=False, + verbose=False, + ) + + ngb.fit(X, Y) + + assert len(ngb.base_models) == 2 + for models in ngb.base_models: + assert len(models) == Normal.n_params + assert isinstance(models[0], DecisionTreeRegressor) + assert isinstance(models[1], Ridge) + + preds = ngb.predict(X[:5]) + dist = ngb.pred_dist(X[:5]) + + assert preds.shape == (5,) + assert isinstance(dist, Normal) + + +def test_classification_accepts_base_learner_per_distribution_parameter(): + rng = np.random.default_rng(4) + X = rng.normal(size=(90, 3)) + Y = np.argmax(np.column_stack([X[:, 0], X[:, 1], -X[:, 0] - X[:, 1]]), axis=1) + + ngb = NGBClassifier( + Dist=k_categorical(3), + Base=[DecisionTreeRegressor(max_depth=1), DecisionTreeRegressor(max_depth=2)], + n_estimators=2, + natural_gradient=False, + verbose=False, + ) + + ngb.fit(X, Y) + + for models in ngb.base_models: + assert len(models) == 2 + assert models[0].max_depth == 1 + assert models[1].max_depth == 2 + + assert ngb.predict_proba(X[:5]).shape == (5, 3) + + +def test_base_learner_sequence_must_match_distribution_parameter_count(): + rng = np.random.default_rng(1) + X = rng.normal(size=(20, 2)) + Y = rng.normal(size=20) + + ngb = NGBRegressor( + Dist=Normal, + Base=[DecisionTreeRegressor(max_depth=1)], + n_estimators=1, + verbose=False, + ) + + with pytest.raises(ValueError, match="one estimator per distribution parameter"): + ngb.fit(X, Y) + + assert not ngb.base_models + assert not ngb.scalings + assert not ngb.col_idxs + + +def test_base_learner_sequence_accepts_tuple(): + rng = np.random.default_rng(5) + X = rng.normal(size=(40, 2)) + Y = X[:, 0] + rng.normal(scale=0.1, size=40) + + ngb = NGBRegressor( + Dist=Normal, + Base=(DecisionTreeRegressor(max_depth=1), DecisionTreeRegressor(max_depth=2)), + n_estimators=1, + natural_gradient=False, + verbose=False, + ).fit(X, Y) + + assert isinstance(ngb.Base, tuple) + assert len(ngb.base_models[0]) == Normal.n_params + assert ngb.base_models[0][0].max_depth == 1 + assert ngb.base_models[0][1].max_depth == 2 + + +def test_parameter_base_learners_support_monotonic_constraints(): + rng = np.random.default_rng(6) + X = rng.normal(size=(80, 3)) + Y = 2 * X[:, 0] - X[:, 1] + rng.normal(scale=0.1, size=80) + loc_learner = HistGradientBoostingRegressor( + max_iter=4, + max_leaf_nodes=3, + monotonic_cst=[1, 0, 0], + random_state=0, + ) + + ngb = NGBRegressor( + Dist=Normal, + Base=[loc_learner, DecisionTreeRegressor(max_depth=1)], + n_estimators=1, + natural_gradient=False, + verbose=False, + ).fit(X, Y) + + assert isinstance(ngb.base_models[0][0], HistGradientBoostingRegressor) + assert ngb.base_models[0][0].monotonic_cst == [1, 0, 0] + assert isinstance(ngb.base_models[0][1], DecisionTreeRegressor) + + +def test_parameter_base_learners_receive_sample_weight(): + rng = np.random.default_rng(7) + X = rng.normal(size=(30, 2)) + Y = X[:, 0] + rng.normal(scale=0.1, size=30) + sample_weight = np.linspace(1.0, 2.0, num=30) + + ngb = NGBRegressor( + Dist=Normal, + Base=[RecordingRegressor(), RecordingRegressor()], + n_estimators=1, + natural_gradient=False, + verbose=False, + ).fit(X, Y, sample_weight=sample_weight) + + for model in ngb.base_models[0]: + np.testing.assert_allclose(model.sample_weight_, sample_weight) + + +def test_sklearn_clone_accepts_base_learner_sequence(): + ngb = NGBRegressor( + Dist=Normal, + Base=[DecisionTreeRegressor(max_depth=1), DecisionTreeRegressor(max_depth=2)], + n_estimators=1, + verbose=False, + ) + + cloned = clone(ngb) + + assert cloned is not ngb + assert isinstance(cloned.Base, list) + assert cloned.Base is not ngb.Base + assert cloned.Base[0] is not ngb.Base[0] + assert cloned.Base[1] is not ngb.Base[1] + assert cloned.Base[0].max_depth == 1 + assert cloned.Base[1].max_depth == 2 + + +def test_single_base_nested_set_params_updates_base(): + ngb = NGBRegressor( + Dist=Normal, + Base=DecisionTreeRegressor(max_depth=1), + n_estimators=1, + verbose=False, + ) + + ngb.set_params(Base__max_depth=4) + + assert ngb.Base.max_depth == 4 + assert not hasattr(ngb, "Base__max_depth") + + +def test_sequence_base_nested_set_params_updates_element(): + ngb = NGBRegressor( + Dist=Normal, + Base=[DecisionTreeRegressor(max_depth=1), DecisionTreeRegressor(max_depth=2)], + n_estimators=1, + verbose=False, + ) + + ngb.set_params(Base__0__max_depth=5) + + assert ngb.Base[0].max_depth == 5 + assert ngb.Base[1].max_depth == 2 + assert not hasattr(ngb, "Base__0__max_depth") + + +def test_single_base_learner_matches_repeated_base_learner_sequence(): + rng = np.random.default_rng(2) + X = rng.normal(size=(60, 3)) + Y = X[:, 0] + rng.normal(scale=0.1, size=60) + base = DecisionTreeRegressor(max_depth=2, random_state=0) + + single_base = NGBRegressor( + Dist=Normal, + Base=base, + n_estimators=3, + natural_gradient=False, + verbose=False, + ).fit(X, Y) + repeated_base = NGBRegressor( + Dist=Normal, + Base=[base, base], + n_estimators=3, + natural_gradient=False, + verbose=False, + ).fit(X, Y) + + np.testing.assert_allclose(single_base.pred_param(X), repeated_base.pred_param(X)) + + +def test_feature_importances_have_parameter_rows_for_tree_sequence(): + rng = np.random.default_rng(8) + X = rng.normal(size=(60, 3)) + Y = X[:, 0] + 0.5 * X[:, 1] + rng.normal(scale=0.1, size=60) + + ngb = NGBRegressor( + Dist=Normal, + Base=[DecisionTreeRegressor(max_depth=1), DecisionTreeRegressor(max_depth=2)], + n_estimators=2, + natural_gradient=False, + verbose=False, + ).fit(X, Y) + + importances = ngb.feature_importances_ + + assert importances.shape == (Normal.n_params, X.shape[1]) + assert np.isfinite(importances).all() + np.testing.assert_allclose(importances.sum(axis=1), np.ones(Normal.n_params)) + + +def test_feature_importances_are_none_for_mixed_base_learners(): + rng = np.random.default_rng(3) + X = rng.normal(size=(40, 3)) + Y = X[:, 0] + rng.normal(scale=0.1, size=40) + + ngb = NGBRegressor( + Dist=Normal, + Base=[DecisionTreeRegressor(max_depth=1), Ridge(alpha=0.0)], + n_estimators=1, + natural_gradient=False, + verbose=False, + ).fit(X, Y) + + assert ngb.feature_importances_ is None