""" Testing for the gradient boosting module (sklearn.ensemble.gradient_boosting). """ import re import warnings import numpy as np import pytest from numpy.testing import assert_allclose from sklearn import datasets from sklearn.base import clone from sklearn.datasets import make_classification, make_regression from sklearn.dummy import DummyClassifier, DummyRegressor from sklearn.ensemble import GradientBoostingClassifier, GradientBoostingRegressor from sklearn.ensemble._gb import _safe_divide from sklearn.ensemble._gradient_boosting import predict_stages from sklearn.exceptions import DataConversionWarning, NotFittedError from sklearn.linear_model import LinearRegression from sklearn.metrics import mean_squared_error from sklearn.model_selection import train_test_split from sklearn.pipeline import make_pipeline from sklearn.preprocessing import scale from sklearn.svm import NuSVR from sklearn.utils import check_random_state, tosequence from sklearn.utils._mocking import NoSampleWeightWrapper from sklearn.utils._param_validation import InvalidParameterError from sklearn.utils._testing import ( assert_array_almost_equal, assert_array_equal, skip_if_32bit, ) from sklearn.utils.fixes import COO_CONTAINERS, CSC_CONTAINERS, CSR_CONTAINERS GRADIENT_BOOSTING_ESTIMATORS = [GradientBoostingClassifier, GradientBoostingRegressor] # toy sample X = [[-2, -1], [-1, -1], [-1, -2], [1, 1], [1, 2], [2, 1]] y = [-1, -1, -1, 1, 1, 1] T = [[-1, -1], [2, 2], [3, 2]] true_result = [-1, 1, 1] # also make regression dataset X_reg, y_reg = make_regression( n_samples=100, n_features=4, n_informative=8, noise=10, random_state=7 ) y_reg = scale(y_reg) rng = np.random.RandomState(0) # also load the iris dataset # and randomly permute it iris = datasets.load_iris() perm = rng.permutation(iris.target.size) iris.data = iris.data[perm] iris.target = iris.target[perm] def test_exponential_n_classes_gt_2(): """Test exponential loss raises for n_classes > 2.""" clf = GradientBoostingClassifier(loss="exponential") msg = "loss='exponential' is only suitable for a binary classification" with pytest.raises(ValueError, match=msg): clf.fit(iris.data, iris.target) def test_raise_if_init_has_no_predict_proba(): """Test raise if init_ has no predict_proba method.""" clf = GradientBoostingClassifier(init=GradientBoostingRegressor) msg = ( "The 'init' parameter of GradientBoostingClassifier must be a str among " "{'zero'}, None or an object implementing 'fit' and 'predict_proba'." ) with pytest.raises(ValueError, match=msg): clf.fit(X, y) @pytest.mark.parametrize("loss", ("log_loss", "exponential")) def test_classification_toy(loss, global_random_seed): # Check classification on a toy dataset. clf = GradientBoostingClassifier( loss=loss, n_estimators=10, random_state=global_random_seed ) with pytest.raises(ValueError): clf.predict(T) clf.fit(X, y) assert_array_equal(clf.predict(T), true_result) assert 10 == len(clf.estimators_) log_loss_decrease = clf.train_score_[:-1] - clf.train_score_[1:] assert np.any(log_loss_decrease >= 0.0) leaves = clf.apply(X) assert leaves.shape == (6, 10, 1) @pytest.mark.parametrize("loss", ("log_loss", "exponential")) def test_classification_synthetic(loss, global_random_seed): # Test GradientBoostingClassifier on synthetic dataset used by # Hastie et al. in ESLII - Figure 10.9 # Note that Figure 10.9 reuses the dataset generated for figure 10.2 # and should have 2_000 train data points and 10_000 test data points. # Here we intentionally use a smaller variant to make the test run faster, # but the conclusions are still the same, despite the smaller datasets. X, y = datasets.make_hastie_10_2(n_samples=2000, random_state=global_random_seed) split_idx = 500 X_train, X_test = X[:split_idx], X[split_idx:] y_train, y_test = y[:split_idx], y[split_idx:] # Increasing the number of trees should decrease the test error common_params = { "max_depth": 1, "learning_rate": 1.0, "loss": loss, "random_state": global_random_seed, } gbrt_10_stumps = GradientBoostingClassifier(n_estimators=10, **common_params) gbrt_10_stumps.fit(X_train, y_train) gbrt_50_stumps = GradientBoostingClassifier(n_estimators=50, **common_params) gbrt_50_stumps.fit(X_train, y_train) assert gbrt_10_stumps.score(X_test, y_test) < gbrt_50_stumps.score(X_test, y_test) # Decision stumps are better suited for this dataset with a large number of # estimators. common_params = { "n_estimators": 200, "learning_rate": 1.0, "loss": loss, "random_state": global_random_seed, } gbrt_stumps = GradientBoostingClassifier(max_depth=1, **common_params) gbrt_stumps.fit(X_train, y_train) gbrt_10_nodes = GradientBoostingClassifier(max_leaf_nodes=10, **common_params) gbrt_10_nodes.fit(X_train, y_train) assert gbrt_stumps.score(X_test, y_test) > gbrt_10_nodes.score(X_test, y_test) @pytest.mark.parametrize("loss", ("squared_error", "absolute_error", "huber")) @pytest.mark.parametrize("subsample", (1.0, 0.5)) def test_regression_dataset(loss, subsample, global_random_seed): # Check consistency on regression dataset with least squares # and least absolute deviation. ones = np.ones(len(y_reg)) last_y_pred = None for sample_weight in [None, ones, 2 * ones]: # learning_rate, max_depth and n_estimators were adjusted to get a mode # that is accurate enough to reach a low MSE on the training set while # keeping the resource used to execute this test low enough. reg = GradientBoostingRegressor( n_estimators=30, loss=loss, max_depth=4, subsample=subsample, min_samples_split=2, random_state=global_random_seed, learning_rate=0.5, ) reg.fit(X_reg, y_reg, sample_weight=sample_weight) leaves = reg.apply(X_reg) assert leaves.shape == (100, 30) y_pred = reg.predict(X_reg) mse = mean_squared_error(y_reg, y_pred) assert mse < 0.05 if last_y_pred is not None: # FIXME: We temporarily bypass this test. This is due to the fact # that GBRT with and without `sample_weight` do not use the same # implementation of the median during the initialization with the # `DummyRegressor`. In the future, we should make sure that both # implementations should be the same. See PR #17377 for more. # assert_allclose(last_y_pred, y_pred) pass last_y_pred = y_pred @pytest.mark.parametrize("subsample", (1.0, 0.5)) @pytest.mark.parametrize("sample_weight", (None, 1)) def test_iris(subsample, sample_weight, global_random_seed): if sample_weight == 1: sample_weight = np.ones(len(iris.target)) # Check consistency on dataset iris. clf = GradientBoostingClassifier( n_estimators=100, loss="log_loss", random_state=global_random_seed, subsample=subsample, ) clf.fit(iris.data, iris.target, sample_weight=sample_weight) score = clf.score(iris.data, iris.target) assert score > 0.9 leaves = clf.apply(iris.data) assert leaves.shape == (150, 100, 3) def test_regression_synthetic(global_random_seed): # Test on synthetic regression datasets used in Leo Breiman, # `Bagging Predictors?. Machine Learning 24(2): 123-140 (1996). random_state = check_random_state(global_random_seed) regression_params = { "n_estimators": 100, "max_depth": 4, "min_samples_split": 2, "learning_rate": 0.1, "loss": "squared_error", "random_state": global_random_seed, } # Friedman1 X, y = datasets.make_friedman1(n_samples=1200, random_state=random_state, noise=1.0) X_train, y_train = X[:200], y[:200] X_test, y_test = X[200:], y[200:] clf = GradientBoostingRegressor(**regression_params) clf.fit(X_train, y_train) mse = mean_squared_error(y_test, clf.predict(X_test)) assert mse < 6.5 # Friedman2 X, y = datasets.make_friedman2(n_samples=1200, random_state=random_state) X_train, y_train = X[:200], y[:200] X_test, y_test = X[200:], y[200:] clf = GradientBoostingRegressor(**regression_params) clf.fit(X_train, y_train) mse = mean_squared_error(y_test, clf.predict(X_test)) assert mse < 2500.0 # Friedman3 X, y = datasets.make_friedman3(n_samples=1200, random_state=random_state) X_train, y_train = X[:200], y[:200] X_test, y_test = X[200:], y[200:] clf = GradientBoostingRegressor(**regression_params) clf.fit(X_train, y_train) mse = mean_squared_error(y_test, clf.predict(X_test)) assert mse < 0.025 @pytest.mark.parametrize( "GradientBoosting, X, y", [ (GradientBoostingRegressor, X_reg, y_reg), (GradientBoostingClassifier, iris.data, iris.target), ], ) def test_feature_importances(GradientBoosting, X, y): # smoke test to check that the gradient boosting expose an attribute # feature_importances_ gbdt = GradientBoosting() assert not hasattr(gbdt, "feature_importances_") gbdt.fit(X, y) assert hasattr(gbdt, "feature_importances_") def test_probability_log(global_random_seed): # Predict probabilities. clf = GradientBoostingClassifier(n_estimators=100, random_state=global_random_seed) with pytest.raises(ValueError): clf.predict_proba(T) clf.fit(X, y) assert_array_equal(clf.predict(T), true_result) # check if probabilities are in [0, 1]. y_proba = clf.predict_proba(T) assert np.all(y_proba >= 0.0) assert np.all(y_proba <= 1.0) # derive predictions from probabilities y_pred = clf.classes_.take(y_proba.argmax(axis=1), axis=0) assert_array_equal(y_pred, true_result) def test_single_class_with_sample_weight(): sample_weight = [0, 0, 0, 1, 1, 1] clf = GradientBoostingClassifier(n_estimators=100, random_state=1) msg = ( "y contains 1 class after sample_weight trimmed classes with " "zero weights, while a minimum of 2 classes are required." ) with pytest.raises(ValueError, match=msg): clf.fit(X, y, sample_weight=sample_weight) @pytest.mark.parametrize("csc_container", CSC_CONTAINERS) def test_check_inputs_predict_stages(csc_container): # check that predict_stages through an error if the type of X is not # supported x, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) x_sparse_csc = csc_container(x) clf = GradientBoostingClassifier(n_estimators=100, random_state=1) clf.fit(x, y) score = np.zeros((y.shape)).reshape(-1, 1) err_msg = "When X is a sparse matrix, a CSR format is expected" with pytest.raises(ValueError, match=err_msg): predict_stages(clf.estimators_, x_sparse_csc, clf.learning_rate, score) x_fortran = np.asfortranarray(x) with pytest.raises(ValueError, match="X should be C-ordered np.ndarray"): predict_stages(clf.estimators_, x_fortran, clf.learning_rate, score) def test_max_feature_regression(global_random_seed): # Test to make sure random state is set properly. X, y = datasets.make_hastie_10_2(n_samples=12000, random_state=global_random_seed) X_train, X_test = X[:2000], X[2000:] y_train, y_test = y[:2000], y[2000:] gbrt = GradientBoostingClassifier( n_estimators=100, min_samples_split=5, max_depth=2, learning_rate=0.1, max_features=2, random_state=global_random_seed, ) gbrt.fit(X_train, y_train) log_loss = gbrt._loss(y_test, gbrt.decision_function(X_test)) assert log_loss < 0.5, "GB failed with deviance %.4f" % log_loss def test_feature_importance_regression( fetch_california_housing_fxt, global_random_seed ): """Test that Gini importance is calculated correctly. This test follows the example from [1]_ (pg. 373). .. [1] Friedman, J., Hastie, T., & Tibshirani, R. (2001). The elements of statistical learning. New York: Springer series in statistics. """ california = fetch_california_housing_fxt() X, y = california.data, california.target X_train, X_test, y_train, y_test = train_test_split( X, y, random_state=global_random_seed ) reg = GradientBoostingRegressor( loss="huber", learning_rate=0.1, max_leaf_nodes=6, n_estimators=100, random_state=global_random_seed, ) reg.fit(X_train, y_train) sorted_idx = np.argsort(reg.feature_importances_)[::-1] sorted_features = [california.feature_names[s] for s in sorted_idx] # The most important feature is the median income by far. assert sorted_features[0] == "MedInc" # The three subsequent features are the following. Their relative ordering # might change a bit depending on the randomness of the trees and the # train / test split. assert set(sorted_features[1:4]) == {"Longitude", "AveOccup", "Latitude"} def test_max_features(): # Test if max features is set properly for floats and str. X, y = datasets.make_hastie_10_2(n_samples=12000, random_state=1) _, n_features = X.shape X_train = X[:2000] y_train = y[:2000] gbrt = GradientBoostingClassifier(n_estimators=1, max_features=None) gbrt.fit(X_train, y_train) assert gbrt.max_features_ == n_features gbrt = GradientBoostingRegressor(n_estimators=1, max_features=None) gbrt.fit(X_train, y_train) assert gbrt.max_features_ == n_features gbrt = GradientBoostingRegressor(n_estimators=1, max_features=0.3) gbrt.fit(X_train, y_train) assert gbrt.max_features_ == int(n_features * 0.3) gbrt = GradientBoostingRegressor(n_estimators=1, max_features="sqrt") gbrt.fit(X_train, y_train) assert gbrt.max_features_ == int(np.sqrt(n_features)) gbrt = GradientBoostingRegressor(n_estimators=1, max_features="log2") gbrt.fit(X_train, y_train) assert gbrt.max_features_ == int(np.log2(n_features)) gbrt = GradientBoostingRegressor(n_estimators=1, max_features=0.01 / X.shape[1]) gbrt.fit(X_train, y_train) assert gbrt.max_features_ == 1 def test_staged_predict(): # Test whether staged decision function eventually gives # the same prediction. X, y = datasets.make_friedman1(n_samples=1200, random_state=1, noise=1.0) X_train, y_train = X[:200], y[:200] X_test = X[200:] clf = GradientBoostingRegressor() # test raise ValueError if not fitted with pytest.raises(ValueError): np.fromiter(clf.staged_predict(X_test), dtype=np.float64) clf.fit(X_train, y_train) y_pred = clf.predict(X_test) # test if prediction for last stage equals ``predict`` for y in clf.staged_predict(X_test): assert y.shape == y_pred.shape assert_array_almost_equal(y_pred, y) def test_staged_predict_proba(): # Test whether staged predict proba eventually gives # the same prediction. X, y = datasets.make_hastie_10_2(n_samples=1200, random_state=1) X_train, y_train = X[:200], y[:200] X_test, y_test = X[200:], y[200:] clf = GradientBoostingClassifier(n_estimators=20) # test raise NotFittedError if not with pytest.raises(NotFittedError): np.fromiter(clf.staged_predict_proba(X_test), dtype=np.float64) clf.fit(X_train, y_train) # test if prediction for last stage equals ``predict`` for y_pred in clf.staged_predict(X_test): assert y_test.shape == y_pred.shape assert_array_equal(clf.predict(X_test), y_pred) # test if prediction for last stage equals ``predict_proba`` for staged_proba in clf.staged_predict_proba(X_test): assert y_test.shape[0] == staged_proba.shape[0] assert 2 == staged_proba.shape[1] assert_array_almost_equal(clf.predict_proba(X_test), staged_proba) @pytest.mark.parametrize("Estimator", GRADIENT_BOOSTING_ESTIMATORS) def test_staged_functions_defensive(Estimator, global_random_seed): # test that staged_functions make defensive copies rng = np.random.RandomState(global_random_seed) X = rng.uniform(size=(10, 3)) y = (4 * X[:, 0]).astype(int) + 1 # don't predict zeros estimator = Estimator() estimator.fit(X, y) for func in ["predict", "decision_function", "predict_proba"]: staged_func = getattr(estimator, "staged_" + func, None) if staged_func is None: # regressor has no staged_predict_proba continue with warnings.catch_warnings(record=True): staged_result = list(staged_func(X)) staged_result[1][:] = 0 assert np.all(staged_result[0] != 0) def test_serialization(): # Check model serialization. clf = GradientBoostingClassifier(n_estimators=100, random_state=1) clf.fit(X, y) assert_array_equal(clf.predict(T), true_result) assert 100 == len(clf.estimators_) try: import cPickle as pickle except ImportError: import pickle serialized_clf = pickle.dumps(clf, protocol=pickle.HIGHEST_PROTOCOL) clf = None clf = pickle.loads(serialized_clf) assert_array_equal(clf.predict(T), true_result) assert 100 == len(clf.estimators_) def test_degenerate_targets(): # Check if we can fit even though all targets are equal. clf = GradientBoostingClassifier(n_estimators=100, random_state=1) # classifier should raise exception with pytest.raises(ValueError): clf.fit(X, np.ones(len(X))) clf = GradientBoostingRegressor(n_estimators=100, random_state=1) clf.fit(X, np.ones(len(X))) clf.predict([rng.rand(2)]) assert_array_equal(np.ones((1,), dtype=np.float64), clf.predict([rng.rand(2)])) def test_quantile_loss(global_random_seed): # Check if quantile loss with alpha=0.5 equals absolute_error. clf_quantile = GradientBoostingRegressor( n_estimators=100, loss="quantile", max_depth=4, alpha=0.5, random_state=global_random_seed, ) clf_quantile.fit(X_reg, y_reg) y_quantile = clf_quantile.predict(X_reg) clf_ae = GradientBoostingRegressor( n_estimators=100, loss="absolute_error", max_depth=4, random_state=global_random_seed, ) clf_ae.fit(X_reg, y_reg) y_ae = clf_ae.predict(X_reg) assert_allclose(y_quantile, y_ae) def test_symbol_labels(): # Test with non-integer class labels. clf = GradientBoostingClassifier(n_estimators=100, random_state=1) symbol_y = tosequence(map(str, y)) clf.fit(X, symbol_y) assert_array_equal(clf.predict(T), tosequence(map(str, true_result))) assert 100 == len(clf.estimators_) def test_float_class_labels(): # Test with float class labels. clf = GradientBoostingClassifier(n_estimators=100, random_state=1) float_y = np.asarray(y, dtype=np.float32) clf.fit(X, float_y) assert_array_equal(clf.predict(T), np.asarray(true_result, dtype=np.float32)) assert 100 == len(clf.estimators_) def test_shape_y(): # Test with float class labels. clf = GradientBoostingClassifier(n_estimators=100, random_state=1) y_ = np.asarray(y, dtype=np.int32) y_ = y_[:, np.newaxis] # This will raise a DataConversionWarning that we want to # "always" raise, elsewhere the warnings gets ignored in the # later tests, and the tests that check for this warning fail warn_msg = ( "A column-vector y was passed when a 1d array was expected. " "Please change the shape of y to \\(n_samples, \\), for " "example using ravel()." ) with pytest.warns(DataConversionWarning, match=warn_msg): clf.fit(X, y_) assert_array_equal(clf.predict(T), true_result) assert 100 == len(clf.estimators_) def test_mem_layout(): # Test with different memory layouts of X and y X_ = np.asfortranarray(X) clf = GradientBoostingClassifier(n_estimators=100, random_state=1) clf.fit(X_, y) assert_array_equal(clf.predict(T), true_result) assert 100 == len(clf.estimators_) X_ = np.ascontiguousarray(X) clf = GradientBoostingClassifier(n_estimators=100, random_state=1) clf.fit(X_, y) assert_array_equal(clf.predict(T), true_result) assert 100 == len(clf.estimators_) y_ = np.asarray(y, dtype=np.int32) y_ = np.ascontiguousarray(y_) clf = GradientBoostingClassifier(n_estimators=100, random_state=1) clf.fit(X, y_) assert_array_equal(clf.predict(T), true_result) assert 100 == len(clf.estimators_) y_ = np.asarray(y, dtype=np.int32) y_ = np.asfortranarray(y_) clf = GradientBoostingClassifier(n_estimators=100, random_state=1) clf.fit(X, y_) assert_array_equal(clf.predict(T), true_result) assert 100 == len(clf.estimators_) @pytest.mark.parametrize("GradientBoostingEstimator", GRADIENT_BOOSTING_ESTIMATORS) def test_oob_improvement(GradientBoostingEstimator): # Test if oob improvement has correct shape and regression test. estimator = GradientBoostingEstimator( n_estimators=100, random_state=1, subsample=0.5 ) estimator.fit(X, y) assert estimator.oob_improvement_.shape[0] == 100 # hard-coded regression test - change if modification in OOB computation assert_array_almost_equal( estimator.oob_improvement_[:5], np.array([0.19, 0.15, 0.12, -0.11, 0.11]), decimal=2, ) @pytest.mark.parametrize("GradientBoostingEstimator", GRADIENT_BOOSTING_ESTIMATORS) def test_oob_scores(GradientBoostingEstimator): # Test if oob scores has correct shape and regression test. X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) estimator = GradientBoostingEstimator( n_estimators=100, random_state=1, subsample=0.5 ) estimator.fit(X, y) assert estimator.oob_scores_.shape[0] == 100 assert estimator.oob_scores_[-1] == pytest.approx(estimator.oob_score_) estimator = GradientBoostingEstimator( n_estimators=100, random_state=1, subsample=0.5, n_iter_no_change=5, ) estimator.fit(X, y) assert estimator.oob_scores_.shape[0] < 100 assert estimator.oob_scores_[-1] == pytest.approx(estimator.oob_score_) @pytest.mark.parametrize( "GradientBoostingEstimator, oob_attribute", [ (GradientBoostingClassifier, "oob_improvement_"), (GradientBoostingClassifier, "oob_scores_"), (GradientBoostingClassifier, "oob_score_"), (GradientBoostingRegressor, "oob_improvement_"), (GradientBoostingRegressor, "oob_scores_"), (GradientBoostingRegressor, "oob_score_"), ], ) def test_oob_attributes_error(GradientBoostingEstimator, oob_attribute): """ Check that we raise an AttributeError when the OOB statistics were not computed. """ X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) estimator = GradientBoostingEstimator( n_estimators=100, random_state=1, subsample=1.0, ) estimator.fit(X, y) with pytest.raises(AttributeError): estimator.oob_attribute def test_oob_multilcass_iris(): # Check OOB improvement on multi-class dataset. estimator = GradientBoostingClassifier( n_estimators=100, loss="log_loss", random_state=1, subsample=0.5 ) estimator.fit(iris.data, iris.target) score = estimator.score(iris.data, iris.target) assert score > 0.9 assert estimator.oob_improvement_.shape[0] == estimator.n_estimators assert estimator.oob_scores_.shape[0] == estimator.n_estimators assert estimator.oob_scores_[-1] == pytest.approx(estimator.oob_score_) estimator = GradientBoostingClassifier( n_estimators=100, loss="log_loss", random_state=1, subsample=0.5, n_iter_no_change=5, ) estimator.fit(iris.data, iris.target) score = estimator.score(iris.data, iris.target) assert estimator.oob_improvement_.shape[0] < estimator.n_estimators assert estimator.oob_scores_.shape[0] < estimator.n_estimators assert estimator.oob_scores_[-1] == pytest.approx(estimator.oob_score_) # hard-coded regression test - change if modification in OOB computation # FIXME: the following snippet does not yield the same results on 32 bits # assert_array_almost_equal(estimator.oob_improvement_[:5], # np.array([12.68, 10.45, 8.18, 6.43, 5.13]), # decimal=2) def test_verbose_output(): # Check verbose=1 does not cause error. import sys from io import StringIO old_stdout = sys.stdout sys.stdout = StringIO() clf = GradientBoostingClassifier( n_estimators=100, random_state=1, verbose=1, subsample=0.8 ) clf.fit(X, y) verbose_output = sys.stdout sys.stdout = old_stdout # check output verbose_output.seek(0) header = verbose_output.readline().rstrip() # with OOB true_header = " ".join(["%10s"] + ["%16s"] * 3) % ( "Iter", "Train Loss", "OOB Improve", "Remaining Time", ) assert true_header == header n_lines = sum(1 for l in verbose_output.readlines()) # one for 1-10 and then 9 for 20-100 assert 10 + 9 == n_lines def test_more_verbose_output(): # Check verbose=2 does not cause error. import sys from io import StringIO old_stdout = sys.stdout sys.stdout = StringIO() clf = GradientBoostingClassifier(n_estimators=100, random_state=1, verbose=2) clf.fit(X, y) verbose_output = sys.stdout sys.stdout = old_stdout # check output verbose_output.seek(0) header = verbose_output.readline().rstrip() # no OOB true_header = " ".join(["%10s"] + ["%16s"] * 2) % ( "Iter", "Train Loss", "Remaining Time", ) assert true_header == header n_lines = sum(1 for l in verbose_output.readlines()) # 100 lines for n_estimators==100 assert 100 == n_lines @pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) def test_warm_start(Cls, global_random_seed): # Test if warm start equals fit. X, y = datasets.make_hastie_10_2(n_samples=100, random_state=global_random_seed) est = Cls(n_estimators=200, max_depth=1, random_state=global_random_seed) est.fit(X, y) est_ws = Cls( n_estimators=100, max_depth=1, warm_start=True, random_state=global_random_seed ) est_ws.fit(X, y) est_ws.set_params(n_estimators=200) est_ws.fit(X, y) if Cls is GradientBoostingRegressor: assert_allclose(est_ws.predict(X), est.predict(X)) else: # Random state is preserved and hence predict_proba must also be # same assert_array_equal(est_ws.predict(X), est.predict(X)) assert_allclose(est_ws.predict_proba(X), est.predict_proba(X)) @pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) def test_warm_start_n_estimators(Cls, global_random_seed): # Test if warm start equals fit - set n_estimators. X, y = datasets.make_hastie_10_2(n_samples=100, random_state=global_random_seed) est = Cls(n_estimators=300, max_depth=1, random_state=global_random_seed) est.fit(X, y) est_ws = Cls( n_estimators=100, max_depth=1, warm_start=True, random_state=global_random_seed ) est_ws.fit(X, y) est_ws.set_params(n_estimators=300) est_ws.fit(X, y) assert_allclose(est_ws.predict(X), est.predict(X)) @pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) def test_warm_start_max_depth(Cls): # Test if possible to fit trees of different depth in ensemble. X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) est = Cls(n_estimators=100, max_depth=1, warm_start=True) est.fit(X, y) est.set_params(n_estimators=110, max_depth=2) est.fit(X, y) # last 10 trees have different depth assert est.estimators_[0, 0].max_depth == 1 for i in range(1, 11): assert est.estimators_[-i, 0].max_depth == 2 @pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) def test_warm_start_clear(Cls): # Test if fit clears state. X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) est = Cls(n_estimators=100, max_depth=1) est.fit(X, y) est_2 = Cls(n_estimators=100, max_depth=1, warm_start=True) est_2.fit(X, y) # inits state est_2.set_params(warm_start=False) est_2.fit(X, y) # clears old state and equals est assert_array_almost_equal(est_2.predict(X), est.predict(X)) @pytest.mark.parametrize("GradientBoosting", GRADIENT_BOOSTING_ESTIMATORS) def test_warm_start_state_oob_scores(GradientBoosting): """ Check that the states of the OOB scores are cleared when used with `warm_start`. """ X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) n_estimators = 100 estimator = GradientBoosting( n_estimators=n_estimators, max_depth=1, subsample=0.5, warm_start=True, random_state=1, ) estimator.fit(X, y) oob_scores, oob_score = estimator.oob_scores_, estimator.oob_score_ assert len(oob_scores) == n_estimators assert oob_scores[-1] == pytest.approx(oob_score) n_more_estimators = 200 estimator.set_params(n_estimators=n_more_estimators).fit(X, y) assert len(estimator.oob_scores_) == n_more_estimators assert_allclose(estimator.oob_scores_[:n_estimators], oob_scores) estimator.set_params(n_estimators=n_estimators, warm_start=False).fit(X, y) assert estimator.oob_scores_ is not oob_scores assert estimator.oob_score_ is not oob_score assert_allclose(estimator.oob_scores_, oob_scores) assert estimator.oob_score_ == pytest.approx(oob_score) assert oob_scores[-1] == pytest.approx(oob_score) @pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) def test_warm_start_smaller_n_estimators(Cls): # Test if warm start with smaller n_estimators raises error X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) est = Cls(n_estimators=100, max_depth=1, warm_start=True) est.fit(X, y) est.set_params(n_estimators=99) with pytest.raises(ValueError): est.fit(X, y) @pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) def test_warm_start_equal_n_estimators(Cls): # Test if warm start with equal n_estimators does nothing X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) est = Cls(n_estimators=100, max_depth=1) est.fit(X, y) est2 = clone(est) est2.set_params(n_estimators=est.n_estimators, warm_start=True) est2.fit(X, y) assert_array_almost_equal(est2.predict(X), est.predict(X)) @pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) def test_warm_start_oob_switch(Cls): # Test if oob can be turned on during warm start. X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) est = Cls(n_estimators=100, max_depth=1, warm_start=True) est.fit(X, y) est.set_params(n_estimators=110, subsample=0.5) est.fit(X, y) assert_array_equal(est.oob_improvement_[:100], np.zeros(100)) assert_array_equal(est.oob_scores_[:100], np.zeros(100)) # the last 10 are not zeros assert (est.oob_improvement_[-10:] != 0.0).all() assert (est.oob_scores_[-10:] != 0.0).all() assert est.oob_scores_[-1] == pytest.approx(est.oob_score_) @pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) def test_warm_start_oob(Cls): # Test if warm start OOB equals fit. X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) est = Cls(n_estimators=200, max_depth=1, subsample=0.5, random_state=1) est.fit(X, y) est_ws = Cls( n_estimators=100, max_depth=1, subsample=0.5, random_state=1, warm_start=True ) est_ws.fit(X, y) est_ws.set_params(n_estimators=200) est_ws.fit(X, y) assert_array_almost_equal(est_ws.oob_improvement_[:100], est.oob_improvement_[:100]) assert_array_almost_equal(est_ws.oob_scores_[:100], est.oob_scores_[:100]) assert est.oob_scores_[-1] == pytest.approx(est.oob_score_) assert est_ws.oob_scores_[-1] == pytest.approx(est_ws.oob_score_) @pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) @pytest.mark.parametrize( "sparse_container", COO_CONTAINERS + CSC_CONTAINERS + CSR_CONTAINERS ) def test_warm_start_sparse(Cls, sparse_container): # Test that all sparse matrix types are supported X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) est_dense = Cls( n_estimators=100, max_depth=1, subsample=0.5, random_state=1, warm_start=True ) est_dense.fit(X, y) est_dense.predict(X) est_dense.set_params(n_estimators=200) est_dense.fit(X, y) y_pred_dense = est_dense.predict(X) X_sparse = sparse_container(X) est_sparse = Cls( n_estimators=100, max_depth=1, subsample=0.5, random_state=1, warm_start=True, ) est_sparse.fit(X_sparse, y) est_sparse.predict(X) est_sparse.set_params(n_estimators=200) est_sparse.fit(X_sparse, y) y_pred_sparse = est_sparse.predict(X) assert_array_almost_equal( est_dense.oob_improvement_[:100], est_sparse.oob_improvement_[:100] ) assert est_dense.oob_scores_[-1] == pytest.approx(est_dense.oob_score_) assert_array_almost_equal(est_dense.oob_scores_[:100], est_sparse.oob_scores_[:100]) assert est_sparse.oob_scores_[-1] == pytest.approx(est_sparse.oob_score_) assert_array_almost_equal(y_pred_dense, y_pred_sparse) @pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) def test_warm_start_fortran(Cls, global_random_seed): # Test that feeding a X in Fortran-ordered is giving the same results as # in C-ordered X, y = datasets.make_hastie_10_2(n_samples=100, random_state=global_random_seed) est_c = Cls(n_estimators=1, random_state=global_random_seed, warm_start=True) est_fortran = Cls(n_estimators=1, random_state=global_random_seed, warm_start=True) est_c.fit(X, y) est_c.set_params(n_estimators=11) est_c.fit(X, y) X_fortran = np.asfortranarray(X) est_fortran.fit(X_fortran, y) est_fortran.set_params(n_estimators=11) est_fortran.fit(X_fortran, y) assert_allclose(est_c.predict(X), est_fortran.predict(X)) def early_stopping_monitor(i, est, locals): """Returns True on the 10th iteration.""" if i == 9: return True else: return False @pytest.mark.parametrize("Cls", GRADIENT_BOOSTING_ESTIMATORS) def test_monitor_early_stopping(Cls): # Test if monitor return value works. X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) est = Cls(n_estimators=20, max_depth=1, random_state=1, subsample=0.5) est.fit(X, y, monitor=early_stopping_monitor) assert est.n_estimators == 20 # this is not altered assert est.estimators_.shape[0] == 10 assert est.train_score_.shape[0] == 10 assert est.oob_improvement_.shape[0] == 10 assert est.oob_scores_.shape[0] == 10 assert est.oob_scores_[-1] == pytest.approx(est.oob_score_) # try refit est.set_params(n_estimators=30) est.fit(X, y) assert est.n_estimators == 30 assert est.estimators_.shape[0] == 30 assert est.train_score_.shape[0] == 30 assert est.oob_improvement_.shape[0] == 30 assert est.oob_scores_.shape[0] == 30 assert est.oob_scores_[-1] == pytest.approx(est.oob_score_) est = Cls( n_estimators=20, max_depth=1, random_state=1, subsample=0.5, warm_start=True ) est.fit(X, y, monitor=early_stopping_monitor) assert est.n_estimators == 20 assert est.estimators_.shape[0] == 10 assert est.train_score_.shape[0] == 10 assert est.oob_improvement_.shape[0] == 10 assert est.oob_scores_.shape[0] == 10 assert est.oob_scores_[-1] == pytest.approx(est.oob_score_) # try refit est.set_params(n_estimators=30, warm_start=False) est.fit(X, y) assert est.n_estimators == 30 assert est.train_score_.shape[0] == 30 assert est.estimators_.shape[0] == 30 assert est.oob_improvement_.shape[0] == 30 assert est.oob_scores_.shape[0] == 30 assert est.oob_scores_[-1] == pytest.approx(est.oob_score_) def test_complete_classification(): # Test greedy trees with max_depth + 1 leafs. from sklearn.tree._tree import TREE_LEAF X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) k = 4 est = GradientBoostingClassifier( n_estimators=20, max_depth=None, random_state=1, max_leaf_nodes=k + 1 ) est.fit(X, y) tree = est.estimators_[0, 0].tree_ assert tree.max_depth == k assert tree.children_left[tree.children_left == TREE_LEAF].shape[0] == k + 1 def test_complete_regression(): # Test greedy trees with max_depth + 1 leafs. from sklearn.tree._tree import TREE_LEAF k = 4 est = GradientBoostingRegressor( n_estimators=20, max_depth=None, random_state=1, max_leaf_nodes=k + 1 ) est.fit(X_reg, y_reg) tree = est.estimators_[-1, 0].tree_ assert tree.children_left[tree.children_left == TREE_LEAF].shape[0] == k + 1 def test_zero_estimator_reg(global_random_seed): # Test if init='zero' works for regression by checking that it is better # than a simple baseline. baseline = DummyRegressor(strategy="mean").fit(X_reg, y_reg) mse_baseline = mean_squared_error(baseline.predict(X_reg), y_reg) est = GradientBoostingRegressor( n_estimators=5, max_depth=1, random_state=global_random_seed, init="zero", learning_rate=0.5, ) est.fit(X_reg, y_reg) y_pred = est.predict(X_reg) mse_gbdt = mean_squared_error(y_reg, y_pred) assert mse_gbdt < mse_baseline def test_zero_estimator_clf(global_random_seed): # Test if init='zero' works for classification. X = iris.data y = np.array(iris.target) est = GradientBoostingClassifier( n_estimators=20, max_depth=1, random_state=global_random_seed, init="zero" ) est.fit(X, y) assert est.score(X, y) > 0.96 # binary clf mask = y != 0 y[mask] = 1 y[~mask] = 0 est = GradientBoostingClassifier( n_estimators=20, max_depth=1, random_state=global_random_seed, init="zero" ) est.fit(X, y) assert est.score(X, y) > 0.96 @pytest.mark.parametrize("GBEstimator", GRADIENT_BOOSTING_ESTIMATORS) def test_max_leaf_nodes_max_depth(GBEstimator): # Test precedence of max_leaf_nodes over max_depth. X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) k = 4 est = GBEstimator(max_depth=1, max_leaf_nodes=k).fit(X, y) tree = est.estimators_[0, 0].tree_ assert tree.max_depth == 1 est = GBEstimator(max_depth=1).fit(X, y) tree = est.estimators_[0, 0].tree_ assert tree.max_depth == 1 @pytest.mark.parametrize("GBEstimator", GRADIENT_BOOSTING_ESTIMATORS) def test_min_impurity_decrease(GBEstimator): X, y = datasets.make_hastie_10_2(n_samples=100, random_state=1) est = GBEstimator(min_impurity_decrease=0.1) est.fit(X, y) for tree in est.estimators_.flat: # Simply check if the parameter is passed on correctly. Tree tests # will suffice for the actual working of this param assert tree.min_impurity_decrease == 0.1 def test_warm_start_wo_nestimators_change(): # Test if warm_start does nothing if n_estimators is not changed. # Regression test for #3513. clf = GradientBoostingClassifier(n_estimators=10, warm_start=True) clf.fit([[0, 1], [2, 3]], [0, 1]) assert clf.estimators_.shape[0] == 10 clf.fit([[0, 1], [2, 3]], [0, 1]) assert clf.estimators_.shape[0] == 10 @pytest.mark.parametrize( ("loss", "value"), [ ("squared_error", 0.5), ("absolute_error", 0.0), ("huber", 0.5), ("quantile", 0.5), ], ) def test_non_uniform_weights_toy_edge_case_reg(loss, value): X = [[1, 0], [1, 0], [1, 0], [0, 1]] y = [0, 0, 1, 0] # ignore the first 2 training samples by setting their weight to 0 sample_weight = [0, 0, 1, 1] gb = GradientBoostingRegressor(learning_rate=1.0, n_estimators=2, loss=loss) gb.fit(X, y, sample_weight=sample_weight) assert gb.predict([[1, 0]])[0] >= value def test_non_uniform_weights_toy_edge_case_clf(): X = [[1, 0], [1, 0], [1, 0], [0, 1]] y = [0, 0, 1, 0] # ignore the first 2 training samples by setting their weight to 0 sample_weight = [0, 0, 1, 1] for loss in ("log_loss", "exponential"): gb = GradientBoostingClassifier(n_estimators=5, loss=loss) gb.fit(X, y, sample_weight=sample_weight) assert_array_equal(gb.predict([[1, 0]]), [1]) @skip_if_32bit @pytest.mark.parametrize( "EstimatorClass", (GradientBoostingClassifier, GradientBoostingRegressor) ) @pytest.mark.parametrize( "sparse_container", COO_CONTAINERS + CSC_CONTAINERS + CSR_CONTAINERS ) def test_sparse_input(EstimatorClass, sparse_container): y, X = datasets.make_multilabel_classification( random_state=0, n_samples=50, n_features=1, n_classes=20 ) y = y[:, 0] X_sparse = sparse_container(X) dense = EstimatorClass( n_estimators=10, random_state=0, max_depth=2, min_impurity_decrease=1e-7 ).fit(X, y) sparse = EstimatorClass( n_estimators=10, random_state=0, max_depth=2, min_impurity_decrease=1e-7 ).fit(X_sparse, y) assert_array_almost_equal(sparse.apply(X), dense.apply(X)) assert_array_almost_equal(sparse.predict(X), dense.predict(X)) assert_array_almost_equal(sparse.feature_importances_, dense.feature_importances_) assert_array_almost_equal(sparse.predict(X_sparse), dense.predict(X)) assert_array_almost_equal(dense.predict(X_sparse), sparse.predict(X)) if issubclass(EstimatorClass, GradientBoostingClassifier): assert_array_almost_equal(sparse.predict_proba(X), dense.predict_proba(X)) assert_array_almost_equal( sparse.predict_log_proba(X), dense.predict_log_proba(X) ) assert_array_almost_equal( sparse.decision_function(X_sparse), sparse.decision_function(X) ) assert_array_almost_equal( dense.decision_function(X_sparse), sparse.decision_function(X) ) for res_sparse, res in zip( sparse.staged_decision_function(X_sparse), sparse.staged_decision_function(X), ): assert_array_almost_equal(res_sparse, res) @pytest.mark.parametrize( "GradientBoostingEstimator", [GradientBoostingClassifier, GradientBoostingRegressor] ) def test_gradient_boosting_early_stopping(GradientBoostingEstimator): # Check if early stopping works as expected, that is empirically check that the # number of trained estimators is increasing when the tolerance decreases. X, y = make_classification(n_samples=1000, random_state=0) n_estimators = 1000 gb_large_tol = GradientBoostingEstimator( n_estimators=n_estimators, n_iter_no_change=10, learning_rate=0.1, max_depth=3, random_state=42, tol=1e-1, ) gb_small_tol = GradientBoostingEstimator( n_estimators=n_estimators, n_iter_no_change=10, learning_rate=0.1, max_depth=3, random_state=42, tol=1e-3, ) X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42) gb_large_tol.fit(X_train, y_train) gb_small_tol.fit(X_train, y_train) assert gb_large_tol.n_estimators_ < gb_small_tol.n_estimators_ < n_estimators assert gb_large_tol.score(X_test, y_test) > 0.7 assert gb_small_tol.score(X_test, y_test) > 0.7 def test_gradient_boosting_without_early_stopping(): # When early stopping is not used, the number of trained estimators # must be the one specified. X, y = make_classification(n_samples=1000, random_state=0) gbc = GradientBoostingClassifier( n_estimators=50, learning_rate=0.1, max_depth=3, random_state=42 ) gbc.fit(X, y) gbr = GradientBoostingRegressor( n_estimators=30, learning_rate=0.1, max_depth=3, random_state=42 ) gbr.fit(X, y) # The number of trained estimators must be the one specified. assert gbc.n_estimators_ == 50 assert gbr.n_estimators_ == 30 def test_gradient_boosting_validation_fraction(): X, y = make_classification(n_samples=1000, random_state=0) gbc = GradientBoostingClassifier( n_estimators=100, n_iter_no_change=10, validation_fraction=0.1, learning_rate=0.1, max_depth=3, random_state=42, ) gbc2 = clone(gbc).set_params(validation_fraction=0.3) gbc3 = clone(gbc).set_params(n_iter_no_change=20) gbr = GradientBoostingRegressor( n_estimators=100, n_iter_no_change=10, learning_rate=0.1, max_depth=3, validation_fraction=0.1, random_state=42, ) gbr2 = clone(gbr).set_params(validation_fraction=0.3) gbr3 = clone(gbr).set_params(n_iter_no_change=20) X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42) # Check if validation_fraction has an effect gbc.fit(X_train, y_train) gbc2.fit(X_train, y_train) assert gbc.n_estimators_ != gbc2.n_estimators_ gbr.fit(X_train, y_train) gbr2.fit(X_train, y_train) assert gbr.n_estimators_ != gbr2.n_estimators_ # Check if n_estimators_ increase monotonically with n_iter_no_change # Set validation gbc3.fit(X_train, y_train) gbr3.fit(X_train, y_train) assert gbr.n_estimators_ < gbr3.n_estimators_ assert gbc.n_estimators_ < gbc3.n_estimators_ def test_early_stopping_stratified(): # Make sure data splitting for early stopping is stratified X = [[1, 2], [2, 3], [3, 4], [4, 5]] y = [0, 0, 0, 1] gbc = GradientBoostingClassifier(n_iter_no_change=5) with pytest.raises( ValueError, match="The least populated class in y has only 1 member" ): gbc.fit(X, y) def _make_multiclass(): return make_classification(n_classes=3, n_clusters_per_class=1) @pytest.mark.parametrize( "gb, dataset_maker, init_estimator", [ (GradientBoostingClassifier, make_classification, DummyClassifier), (GradientBoostingClassifier, _make_multiclass, DummyClassifier), (GradientBoostingRegressor, make_regression, DummyRegressor), ], ids=["binary classification", "multiclass classification", "regression"], ) def test_gradient_boosting_with_init( gb, dataset_maker, init_estimator, global_random_seed ): # Check that GradientBoostingRegressor works when init is a sklearn # estimator. # Check that an error is raised if trying to fit with sample weight but # initial estimator does not support sample weight X, y = dataset_maker() sample_weight = np.random.RandomState(global_random_seed).rand(100) # init supports sample weights init_est = init_estimator() gb(init=init_est).fit(X, y, sample_weight=sample_weight) # init does not support sample weights init_est = NoSampleWeightWrapper(init_estimator()) gb(init=init_est).fit(X, y) # ok no sample weights with pytest.raises(ValueError, match="estimator.*does not support sample weights"): gb(init=init_est).fit(X, y, sample_weight=sample_weight) def test_gradient_boosting_with_init_pipeline(): # Check that the init estimator can be a pipeline (see issue #13466) X, y = make_regression(random_state=0) init = make_pipeline(LinearRegression()) gb = GradientBoostingRegressor(init=init) gb.fit(X, y) # pipeline without sample_weight works fine with pytest.raises( ValueError, match="The initial estimator Pipeline does not support sample weights", ): gb.fit(X, y, sample_weight=np.ones(X.shape[0])) # Passing sample_weight to a pipeline raises a ValueError. This test makes # sure we make the distinction between ValueError raised by a pipeline that # was passed sample_weight, and a InvalidParameterError raised by a regular # estimator whose input checking failed. invalid_nu = 1.5 err_msg = ( "The 'nu' parameter of NuSVR must be a float in the" f" range (0.0, 1.0]. Got {invalid_nu} instead." ) with pytest.raises(InvalidParameterError, match=re.escape(err_msg)): # Note that NuSVR properly supports sample_weight init = NuSVR(gamma="auto", nu=invalid_nu) gb = GradientBoostingRegressor(init=init) gb.fit(X, y, sample_weight=np.ones(X.shape[0])) def test_early_stopping_n_classes(): # when doing early stopping (_, , y_train, _ = train_test_split(X, y)) # there might be classes in y that are missing in y_train. As the init # estimator will be trained on y_train, we need to raise an error if this # happens. X = [[1]] * 10 y = [0, 0] + [1] * 8 # only 2 negative class over 10 samples gb = GradientBoostingClassifier( n_iter_no_change=5, random_state=0, validation_fraction=0.8 ) with pytest.raises( ValueError, match="The training data after the early stopping split" ): gb.fit(X, y) # No error if we let training data be big enough gb = GradientBoostingClassifier( n_iter_no_change=5, random_state=0, validation_fraction=0.4 ) def test_gbr_degenerate_feature_importances(): # growing an ensemble of single node trees. See #13620 X = np.zeros((10, 10)) y = np.ones((10,)) gbr = GradientBoostingRegressor().fit(X, y) assert_array_equal(gbr.feature_importances_, np.zeros(10, dtype=np.float64)) def test_huber_vs_mean_and_median(): """Check that huber lies between absolute and squared error.""" n_rep = 100 n_samples = 10 y = np.tile(np.arange(n_samples), n_rep) x1 = np.minimum(y, n_samples / 2) x2 = np.minimum(-y, -n_samples / 2) X = np.c_[x1, x2] rng = np.random.RandomState(42) # We want an asymmetric distribution. y = y + rng.exponential(scale=1, size=y.shape) gbt_absolute_error = GradientBoostingRegressor(loss="absolute_error").fit(X, y) gbt_huber = GradientBoostingRegressor(loss="huber").fit(X, y) gbt_squared_error = GradientBoostingRegressor().fit(X, y) gbt_huber_predictions = gbt_huber.predict(X) assert np.all(gbt_absolute_error.predict(X) <= gbt_huber_predictions) assert np.all(gbt_huber_predictions <= gbt_squared_error.predict(X)) def test_safe_divide(): """Test that _safe_divide handles division by zero.""" with warnings.catch_warnings(): warnings.simplefilter("error") assert _safe_divide(np.float64(1e300), 0) == 0 assert _safe_divide(np.float64(0.0), np.float64(0.0)) == 0 with pytest.warns(RuntimeWarning, match="overflow"): # np.finfo(float).max = 1.7976931348623157e+308 _safe_divide(np.float64(1e300), 1e-10) def test_squared_error_exact_backward_compat(): """Test squared error GBT backward compat on a simple dataset. The results to compare against are taken from scikit-learn v1.2.0. """ n_samples = 10 y = np.arange(n_samples) x1 = np.minimum(y, n_samples / 2) x2 = np.minimum(-y, -n_samples / 2) X = np.c_[x1, x2] gbt = GradientBoostingRegressor(loss="squared_error", n_estimators=100).fit(X, y) pred_result = np.array( [ 1.39245726e-04, 1.00010468e00, 2.00007043e00, 3.00004051e00, 4.00000802e00, 4.99998972e00, 5.99996312e00, 6.99993395e00, 7.99989372e00, 8.99985660e00, ] ) assert_allclose(gbt.predict(X), pred_result, rtol=1e-8) train_score = np.array( [ 4.87246390e-08, 3.95590036e-08, 3.21267865e-08, 2.60970300e-08, 2.11820178e-08, 1.71995782e-08, 1.39695549e-08, 1.13391770e-08, 9.19931587e-09, 7.47000575e-09, ] ) assert_allclose(gbt.train_score_[-10:], train_score, rtol=1e-8) # Same but with sample_weights sample_weights = np.tile([1, 10], n_samples // 2) gbt = GradientBoostingRegressor(loss="squared_error", n_estimators=100).fit( X, y, sample_weight=sample_weights ) pred_result = np.array( [ 1.52391462e-04, 1.00011168e00, 2.00007724e00, 3.00004638e00, 4.00001302e00, 4.99999873e00, 5.99997093e00, 6.99994329e00, 7.99991290e00, 8.99988727e00, ] ) assert_allclose(gbt.predict(X), pred_result, rtol=1e-6, atol=1e-5) train_score = np.array( [ 4.12445296e-08, 3.34418322e-08, 2.71151383e-08, 2.19782469e-08, 1.78173649e-08, 1.44461976e-08, 1.17120123e-08, 9.49485678e-09, 7.69772505e-09, 6.24155316e-09, ] ) assert_allclose(gbt.train_score_[-10:], train_score, rtol=1e-3, atol=1e-11) @skip_if_32bit def test_huber_exact_backward_compat(): """Test huber GBT backward compat on a simple dataset. The results to compare against are taken from scikit-learn v1.2.0. """ n_samples = 10 y = np.arange(n_samples) x1 = np.minimum(y, n_samples / 2) x2 = np.minimum(-y, -n_samples / 2) X = np.c_[x1, x2] gbt = GradientBoostingRegressor(loss="huber", n_estimators=100, alpha=0.8).fit(X, y) assert_allclose(gbt._loss.closs.delta, 0.0001655688041282133) pred_result = np.array( [ 1.48120765e-04, 9.99949174e-01, 2.00116957e00, 2.99986716e00, 4.00012064e00, 5.00002462e00, 5.99998898e00, 6.99692549e00, 8.00006356e00, 8.99985099e00, ] ) assert_allclose(gbt.predict(X), pred_result, rtol=1e-8) train_score = np.array( [ 2.59484709e-07, 2.19165900e-07, 1.89644782e-07, 1.64556454e-07, 1.38705110e-07, 1.20373736e-07, 1.04746082e-07, 9.13835687e-08, 8.20245756e-08, 7.17122188e-08, ] ) assert_allclose(gbt.train_score_[-10:], train_score, rtol=1e-8) def test_binomial_error_exact_backward_compat(): """Test binary log_loss GBT backward compat on a simple dataset. The results to compare against are taken from scikit-learn v1.2.0. """ n_samples = 10 y = np.arange(n_samples) % 2 x1 = np.minimum(y, n_samples / 2) x2 = np.minimum(-y, -n_samples / 2) X = np.c_[x1, x2] gbt = GradientBoostingClassifier(loss="log_loss", n_estimators=100).fit(X, y) pred_result = np.array( [ [9.99978098e-01, 2.19017313e-05], [2.19017313e-05, 9.99978098e-01], [9.99978098e-01, 2.19017313e-05], [2.19017313e-05, 9.99978098e-01], [9.99978098e-01, 2.19017313e-05], [2.19017313e-05, 9.99978098e-01], [9.99978098e-01, 2.19017313e-05], [2.19017313e-05, 9.99978098e-01], [9.99978098e-01, 2.19017313e-05], [2.19017313e-05, 9.99978098e-01], ] ) assert_allclose(gbt.predict_proba(X), pred_result, rtol=1e-8) train_score = np.array( [ 1.07742210e-04, 9.74889078e-05, 8.82113863e-05, 7.98167784e-05, 7.22210566e-05, 6.53481907e-05, 5.91293869e-05, 5.35023988e-05, 4.84109045e-05, 4.38039423e-05, ] ) assert_allclose(gbt.train_score_[-10:], train_score, rtol=1e-8) def test_multinomial_error_exact_backward_compat(): """Test multiclass log_loss GBT backward compat on a simple dataset. The results to compare against are taken from scikit-learn v1.2.0. """ n_samples = 10 y = np.arange(n_samples) % 4 x1 = np.minimum(y, n_samples / 2) x2 = np.minimum(-y, -n_samples / 2) X = np.c_[x1, x2] gbt = GradientBoostingClassifier(loss="log_loss", n_estimators=100).fit(X, y) pred_result = np.array( [ [9.99999727e-01, 1.11956255e-07, 8.04921671e-08, 8.04921668e-08], [1.11956254e-07, 9.99999727e-01, 8.04921671e-08, 8.04921668e-08], [1.19417637e-07, 1.19417637e-07, 9.99999675e-01, 8.60526098e-08], [1.19417637e-07, 1.19417637e-07, 8.60526088e-08, 9.99999675e-01], [9.99999727e-01, 1.11956255e-07, 8.04921671e-08, 8.04921668e-08], [1.11956254e-07, 9.99999727e-01, 8.04921671e-08, 8.04921668e-08], [1.19417637e-07, 1.19417637e-07, 9.99999675e-01, 8.60526098e-08], [1.19417637e-07, 1.19417637e-07, 8.60526088e-08, 9.99999675e-01], [9.99999727e-01, 1.11956255e-07, 8.04921671e-08, 8.04921668e-08], [1.11956254e-07, 9.99999727e-01, 8.04921671e-08, 8.04921668e-08], ] ) assert_allclose(gbt.predict_proba(X), pred_result, rtol=1e-8) train_score = np.array( [ 1.13300150e-06, 9.75183397e-07, 8.39348103e-07, 7.22433588e-07, 6.21804338e-07, 5.35191943e-07, 4.60643966e-07, 3.96479930e-07, 3.41253434e-07, 2.93719550e-07, ] ) assert_allclose(gbt.train_score_[-10:], train_score, rtol=1e-8) def test_gb_denominator_zero(global_random_seed): """Test _update_terminal_regions denominator is not zero. For instance for log loss based binary classification, the line search step might become nan/inf as denominator = hessian = prob * (1 - prob) and prob = 0 or 1 can happen. Here, we create a situation were this happens (at least with roughly 80%) based on the random seed. """ X, y = datasets.make_hastie_10_2(n_samples=100, random_state=20) params = { "learning_rate": 1.0, "subsample": 0.5, "n_estimators": 100, "max_leaf_nodes": 4, "max_depth": None, "random_state": global_random_seed, "min_samples_leaf": 2, } clf = GradientBoostingClassifier(**params) # _safe_devide would raise a RuntimeWarning with warnings.catch_warnings(): warnings.simplefilter("error") clf.fit(X, y)