import re import warnings import numpy as np import pytest from scipy import stats from sklearn import datasets, svm from sklearn.datasets import make_multilabel_classification from sklearn.exceptions import UndefinedMetricWarning from sklearn.linear_model import LogisticRegression from sklearn.metrics import ( accuracy_score, auc, average_precision_score, coverage_error, dcg_score, det_curve, label_ranking_average_precision_score, label_ranking_loss, ndcg_score, precision_recall_curve, roc_auc_score, roc_curve, top_k_accuracy_score, ) from sklearn.metrics._ranking import _dcg_sample_scores, _ndcg_sample_scores from sklearn.model_selection import train_test_split from sklearn.preprocessing import label_binarize from sklearn.random_projection import _sparse_random_matrix from sklearn.utils._testing import ( assert_allclose, assert_almost_equal, assert_array_almost_equal, assert_array_equal, ) from sklearn.utils.extmath import softmax from sklearn.utils.fixes import CSR_CONTAINERS from sklearn.utils.validation import ( check_array, check_consistent_length, check_random_state, ) ############################################################################### # Utilities for testing CURVE_FUNCS = [ det_curve, precision_recall_curve, roc_curve, ] def make_prediction(dataset=None, binary=False): """Make some classification predictions on a toy dataset using a SVC If binary is True restrict to a binary classification problem instead of a multiclass classification problem """ if dataset is None: # import some data to play with dataset = datasets.load_iris() X = dataset.data y = dataset.target if binary: # restrict to a binary classification task X, y = X[y < 2], y[y < 2] n_samples, n_features = X.shape p = np.arange(n_samples) rng = check_random_state(37) rng.shuffle(p) X, y = X[p], y[p] half = int(n_samples / 2) # add noisy features to make the problem harder and avoid perfect results rng = np.random.RandomState(0) X = np.c_[X, rng.randn(n_samples, 200 * n_features)] # run classifier, get class probabilities and label predictions clf = svm.SVC(kernel="linear", probability=True, random_state=0) y_score = clf.fit(X[:half], y[:half]).predict_proba(X[half:]) if binary: # only interested in probabilities of the positive case # XXX: do we really want a special API for the binary case? y_score = y_score[:, 1] y_pred = clf.predict(X[half:]) y_true = y[half:] return y_true, y_pred, y_score ############################################################################### # Tests def _auc(y_true, y_score): """Alternative implementation to check for correctness of `roc_auc_score`.""" pos_label = np.unique(y_true)[1] # Count the number of times positive samples are correctly ranked above # negative samples. pos = y_score[y_true == pos_label] neg = y_score[y_true != pos_label] diff_matrix = pos.reshape(1, -1) - neg.reshape(-1, 1) n_correct = np.sum(diff_matrix > 0) return n_correct / float(len(pos) * len(neg)) def _average_precision(y_true, y_score): """Alternative implementation to check for correctness of `average_precision_score`. Note that this implementation fails on some edge cases. For example, for constant predictions e.g. [0.5, 0.5, 0.5], y_true = [1, 0, 0] returns an average precision of 0.33... but y_true = [0, 0, 1] returns 1.0. """ pos_label = np.unique(y_true)[1] n_pos = np.sum(y_true == pos_label) order = np.argsort(y_score)[::-1] y_score = y_score[order] y_true = y_true[order] score = 0 for i in range(len(y_score)): if y_true[i] == pos_label: # Compute precision up to document i # i.e, percentage of relevant documents up to document i. prec = 0 for j in range(0, i + 1): if y_true[j] == pos_label: prec += 1.0 prec /= i + 1.0 score += prec return score / n_pos def _average_precision_slow(y_true, y_score): """A second alternative implementation of average precision that closely follows the Wikipedia article's definition (see References). This should give identical results as `average_precision_score` for all inputs. References ---------- .. [1] `Wikipedia entry for the Average precision `_ """ precision, recall, threshold = precision_recall_curve(y_true, y_score) precision = list(reversed(precision)) recall = list(reversed(recall)) average_precision = 0 for i in range(1, len(precision)): average_precision += precision[i] * (recall[i] - recall[i - 1]) return average_precision def _partial_roc_auc_score(y_true, y_predict, max_fpr): """Alternative implementation to check for correctness of `roc_auc_score` with `max_fpr` set. """ def _partial_roc(y_true, y_predict, max_fpr): fpr, tpr, _ = roc_curve(y_true, y_predict) new_fpr = fpr[fpr <= max_fpr] new_fpr = np.append(new_fpr, max_fpr) new_tpr = tpr[fpr <= max_fpr] idx_out = np.argmax(fpr > max_fpr) idx_in = idx_out - 1 x_interp = [fpr[idx_in], fpr[idx_out]] y_interp = [tpr[idx_in], tpr[idx_out]] new_tpr = np.append(new_tpr, np.interp(max_fpr, x_interp, y_interp)) return (new_fpr, new_tpr) new_fpr, new_tpr = _partial_roc(y_true, y_predict, max_fpr) partial_auc = auc(new_fpr, new_tpr) # Formula (5) from McClish 1989 fpr1 = 0 fpr2 = max_fpr min_area = 0.5 * (fpr2 - fpr1) * (fpr2 + fpr1) max_area = fpr2 - fpr1 return 0.5 * (1 + (partial_auc - min_area) / (max_area - min_area)) @pytest.mark.parametrize("drop", [True, False]) def test_roc_curve(drop): # Test Area under Receiver Operating Characteristic (ROC) curve y_true, _, y_score = make_prediction(binary=True) expected_auc = _auc(y_true, y_score) fpr, tpr, thresholds = roc_curve(y_true, y_score, drop_intermediate=drop) roc_auc = auc(fpr, tpr) assert_array_almost_equal(roc_auc, expected_auc, decimal=2) assert_almost_equal(roc_auc, roc_auc_score(y_true, y_score)) assert fpr.shape == tpr.shape assert fpr.shape == thresholds.shape def test_roc_curve_end_points(): # Make sure that roc_curve returns a curve start at 0 and ending and # 1 even in corner cases rng = np.random.RandomState(0) y_true = np.array([0] * 50 + [1] * 50) y_pred = rng.randint(3, size=100) fpr, tpr, thr = roc_curve(y_true, y_pred, drop_intermediate=True) assert fpr[0] == 0 assert fpr[-1] == 1 assert fpr.shape == tpr.shape assert fpr.shape == thr.shape def test_roc_returns_consistency(): # Test whether the returned threshold matches up with tpr # make small toy dataset y_true, _, y_score = make_prediction(binary=True) fpr, tpr, thresholds = roc_curve(y_true, y_score) # use the given thresholds to determine the tpr tpr_correct = [] for t in thresholds: tp = np.sum((y_score >= t) & y_true) p = np.sum(y_true) tpr_correct.append(1.0 * tp / p) # compare tpr and tpr_correct to see if the thresholds' order was correct assert_array_almost_equal(tpr, tpr_correct, decimal=2) assert fpr.shape == tpr.shape assert fpr.shape == thresholds.shape def test_roc_curve_multi(): # roc_curve not applicable for multi-class problems y_true, _, y_score = make_prediction(binary=False) with pytest.raises(ValueError): roc_curve(y_true, y_score) def test_roc_curve_confidence(): # roc_curve for confidence scores y_true, _, y_score = make_prediction(binary=True) fpr, tpr, thresholds = roc_curve(y_true, y_score - 0.5) roc_auc = auc(fpr, tpr) assert_array_almost_equal(roc_auc, 0.90, decimal=2) assert fpr.shape == tpr.shape assert fpr.shape == thresholds.shape def test_roc_curve_hard(): # roc_curve for hard decisions y_true, pred, y_score = make_prediction(binary=True) # always predict one trivial_pred = np.ones(y_true.shape) fpr, tpr, thresholds = roc_curve(y_true, trivial_pred) roc_auc = auc(fpr, tpr) assert_array_almost_equal(roc_auc, 0.50, decimal=2) assert fpr.shape == tpr.shape assert fpr.shape == thresholds.shape # always predict zero trivial_pred = np.zeros(y_true.shape) fpr, tpr, thresholds = roc_curve(y_true, trivial_pred) roc_auc = auc(fpr, tpr) assert_array_almost_equal(roc_auc, 0.50, decimal=2) assert fpr.shape == tpr.shape assert fpr.shape == thresholds.shape # hard decisions fpr, tpr, thresholds = roc_curve(y_true, pred) roc_auc = auc(fpr, tpr) assert_array_almost_equal(roc_auc, 0.78, decimal=2) assert fpr.shape == tpr.shape assert fpr.shape == thresholds.shape def test_roc_curve_one_label(): y_true = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1] y_pred = [0, 1, 0, 1, 0, 1, 0, 1, 0, 1] # assert there are warnings expected_message = ( "No negative samples in y_true, false positive value should be meaningless" ) with pytest.warns(UndefinedMetricWarning, match=expected_message): fpr, tpr, thresholds = roc_curve(y_true, y_pred) # all true labels, all fpr should be nan assert_array_equal(fpr, np.full(len(thresholds), np.nan)) assert fpr.shape == tpr.shape assert fpr.shape == thresholds.shape # assert there are warnings expected_message = ( "No positive samples in y_true, true positive value should be meaningless" ) with pytest.warns(UndefinedMetricWarning, match=expected_message): fpr, tpr, thresholds = roc_curve([1 - x for x in y_true], y_pred) # all negative labels, all tpr should be nan assert_array_equal(tpr, np.full(len(thresholds), np.nan)) assert fpr.shape == tpr.shape assert fpr.shape == thresholds.shape def test_roc_curve_toydata(): # Binary classification y_true = [0, 1] y_score = [0, 1] tpr, fpr, _ = roc_curve(y_true, y_score) roc_auc = roc_auc_score(y_true, y_score) assert_array_almost_equal(tpr, [0, 0, 1]) assert_array_almost_equal(fpr, [0, 1, 1]) assert_almost_equal(roc_auc, 1.0) y_true = [0, 1] y_score = [1, 0] tpr, fpr, _ = roc_curve(y_true, y_score) roc_auc = roc_auc_score(y_true, y_score) assert_array_almost_equal(tpr, [0, 1, 1]) assert_array_almost_equal(fpr, [0, 0, 1]) assert_almost_equal(roc_auc, 0.0) y_true = [1, 0] y_score = [1, 1] tpr, fpr, _ = roc_curve(y_true, y_score) roc_auc = roc_auc_score(y_true, y_score) assert_array_almost_equal(tpr, [0, 1]) assert_array_almost_equal(fpr, [0, 1]) assert_almost_equal(roc_auc, 0.5) y_true = [1, 0] y_score = [1, 0] tpr, fpr, _ = roc_curve(y_true, y_score) roc_auc = roc_auc_score(y_true, y_score) assert_array_almost_equal(tpr, [0, 0, 1]) assert_array_almost_equal(fpr, [0, 1, 1]) assert_almost_equal(roc_auc, 1.0) y_true = [1, 0] y_score = [0.5, 0.5] tpr, fpr, _ = roc_curve(y_true, y_score) roc_auc = roc_auc_score(y_true, y_score) assert_array_almost_equal(tpr, [0, 1]) assert_array_almost_equal(fpr, [0, 1]) assert_almost_equal(roc_auc, 0.5) y_true = [0, 0] y_score = [0.25, 0.75] # assert UndefinedMetricWarning because of no positive sample in y_true expected_message = ( "No positive samples in y_true, true positive value should be meaningless" ) with pytest.warns(UndefinedMetricWarning, match=expected_message): tpr, fpr, _ = roc_curve(y_true, y_score) with pytest.raises(ValueError): roc_auc_score(y_true, y_score) assert_array_almost_equal(tpr, [0.0, 0.5, 1.0]) assert_array_almost_equal(fpr, [np.nan, np.nan, np.nan]) y_true = [1, 1] y_score = [0.25, 0.75] # assert UndefinedMetricWarning because of no negative sample in y_true expected_message = ( "No negative samples in y_true, false positive value should be meaningless" ) with pytest.warns(UndefinedMetricWarning, match=expected_message): tpr, fpr, _ = roc_curve(y_true, y_score) with pytest.raises(ValueError): roc_auc_score(y_true, y_score) assert_array_almost_equal(tpr, [np.nan, np.nan, np.nan]) assert_array_almost_equal(fpr, [0.0, 0.5, 1.0]) # Multi-label classification task y_true = np.array([[0, 1], [0, 1]]) y_score = np.array([[0, 1], [0, 1]]) with pytest.raises(ValueError): roc_auc_score(y_true, y_score, average="macro") with pytest.raises(ValueError): roc_auc_score(y_true, y_score, average="weighted") assert_almost_equal(roc_auc_score(y_true, y_score, average="samples"), 1.0) assert_almost_equal(roc_auc_score(y_true, y_score, average="micro"), 1.0) y_true = np.array([[0, 1], [0, 1]]) y_score = np.array([[0, 1], [1, 0]]) with pytest.raises(ValueError): roc_auc_score(y_true, y_score, average="macro") with pytest.raises(ValueError): roc_auc_score(y_true, y_score, average="weighted") assert_almost_equal(roc_auc_score(y_true, y_score, average="samples"), 0.5) assert_almost_equal(roc_auc_score(y_true, y_score, average="micro"), 0.5) y_true = np.array([[1, 0], [0, 1]]) y_score = np.array([[0, 1], [1, 0]]) assert_almost_equal(roc_auc_score(y_true, y_score, average="macro"), 0) assert_almost_equal(roc_auc_score(y_true, y_score, average="weighted"), 0) assert_almost_equal(roc_auc_score(y_true, y_score, average="samples"), 0) assert_almost_equal(roc_auc_score(y_true, y_score, average="micro"), 0) y_true = np.array([[1, 0], [0, 1]]) y_score = np.array([[0.5, 0.5], [0.5, 0.5]]) assert_almost_equal(roc_auc_score(y_true, y_score, average="macro"), 0.5) assert_almost_equal(roc_auc_score(y_true, y_score, average="weighted"), 0.5) assert_almost_equal(roc_auc_score(y_true, y_score, average="samples"), 0.5) assert_almost_equal(roc_auc_score(y_true, y_score, average="micro"), 0.5) def test_roc_curve_drop_intermediate(): # Test that drop_intermediate drops the correct thresholds y_true = [0, 0, 0, 0, 1, 1] y_score = [0.0, 0.2, 0.5, 0.6, 0.7, 1.0] tpr, fpr, thresholds = roc_curve(y_true, y_score, drop_intermediate=True) assert_array_almost_equal(thresholds, [np.inf, 1.0, 0.7, 0.0]) # Test dropping thresholds with repeating scores y_true = [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1] y_score = [0.0, 0.1, 0.6, 0.6, 0.7, 0.8, 0.9, 0.6, 0.7, 0.8, 0.9, 0.9, 1.0] tpr, fpr, thresholds = roc_curve(y_true, y_score, drop_intermediate=True) assert_array_almost_equal(thresholds, [np.inf, 1.0, 0.9, 0.7, 0.6, 0.0]) def test_roc_curve_fpr_tpr_increasing(): # Ensure that fpr and tpr returned by roc_curve are increasing. # Construct an edge case with float y_score and sample_weight # when some adjacent values of fpr and tpr are actually the same. y_true = [0, 0, 1, 1, 1] y_score = [0.1, 0.7, 0.3, 0.4, 0.5] sample_weight = np.repeat(0.2, 5) fpr, tpr, _ = roc_curve(y_true, y_score, sample_weight=sample_weight) assert (np.diff(fpr) < 0).sum() == 0 assert (np.diff(tpr) < 0).sum() == 0 def test_auc(): # Test Area Under Curve (AUC) computation x = [0, 1] y = [0, 1] assert_array_almost_equal(auc(x, y), 0.5) x = [1, 0] y = [0, 1] assert_array_almost_equal(auc(x, y), 0.5) x = [1, 0, 0] y = [0, 1, 1] assert_array_almost_equal(auc(x, y), 0.5) x = [0, 1] y = [1, 1] assert_array_almost_equal(auc(x, y), 1) x = [0, 0.5, 1] y = [0, 0.5, 1] assert_array_almost_equal(auc(x, y), 0.5) def test_auc_errors(): # Incompatible shapes with pytest.raises(ValueError): auc([0.0, 0.5, 1.0], [0.1, 0.2]) # Too few x values with pytest.raises(ValueError): auc([0.0], [0.1]) # x is not in order x = [2, 1, 3, 4] y = [5, 6, 7, 8] error_message = "x is neither increasing nor decreasing : {}".format(np.array(x)) with pytest.raises(ValueError, match=re.escape(error_message)): auc(x, y) @pytest.mark.parametrize( "y_true, labels", [ (np.array([0, 1, 0, 2]), [0, 1, 2]), (np.array([0, 1, 0, 2]), None), (["a", "b", "a", "c"], ["a", "b", "c"]), (["a", "b", "a", "c"], None), ], ) def test_multiclass_ovo_roc_auc_toydata(y_true, labels): # Tests the one-vs-one multiclass ROC AUC algorithm # on a small example, representative of an expected use case. y_scores = np.array( [[0.1, 0.8, 0.1], [0.3, 0.4, 0.3], [0.35, 0.5, 0.15], [0, 0.2, 0.8]] ) # Used to compute the expected output. # Consider labels 0 and 1: # positive label is 0, negative label is 1 score_01 = roc_auc_score([1, 0, 1], [0.1, 0.3, 0.35]) # positive label is 1, negative label is 0 score_10 = roc_auc_score([0, 1, 0], [0.8, 0.4, 0.5]) average_score_01 = (score_01 + score_10) / 2 # Consider labels 0 and 2: score_02 = roc_auc_score([1, 1, 0], [0.1, 0.35, 0]) score_20 = roc_auc_score([0, 0, 1], [0.1, 0.15, 0.8]) average_score_02 = (score_02 + score_20) / 2 # Consider labels 1 and 2: score_12 = roc_auc_score([1, 0], [0.4, 0.2]) score_21 = roc_auc_score([0, 1], [0.3, 0.8]) average_score_12 = (score_12 + score_21) / 2 # Unweighted, one-vs-one multiclass ROC AUC algorithm ovo_unweighted_score = (average_score_01 + average_score_02 + average_score_12) / 3 assert_almost_equal( roc_auc_score(y_true, y_scores, labels=labels, multi_class="ovo"), ovo_unweighted_score, ) # Weighted, one-vs-one multiclass ROC AUC algorithm # Each term is weighted by the prevalence for the positive label. pair_scores = [average_score_01, average_score_02, average_score_12] prevalence = [0.75, 0.75, 0.50] ovo_weighted_score = np.average(pair_scores, weights=prevalence) assert_almost_equal( roc_auc_score( y_true, y_scores, labels=labels, multi_class="ovo", average="weighted" ), ovo_weighted_score, ) # Check that average=None raises NotImplemented error error_message = "average=None is not implemented for multi_class='ovo'." with pytest.raises(NotImplementedError, match=error_message): roc_auc_score(y_true, y_scores, labels=labels, multi_class="ovo", average=None) @pytest.mark.parametrize( "y_true, labels", [ (np.array([0, 2, 0, 2]), [0, 1, 2]), (np.array(["a", "d", "a", "d"]), ["a", "b", "d"]), ], ) def test_multiclass_ovo_roc_auc_toydata_binary(y_true, labels): # Tests the one-vs-one multiclass ROC AUC algorithm for binary y_true # # on a small example, representative of an expected use case. y_scores = np.array( [[0.2, 0.0, 0.8], [0.6, 0.0, 0.4], [0.55, 0.0, 0.45], [0.4, 0.0, 0.6]] ) # Used to compute the expected output. # Consider labels 0 and 1: # positive label is 0, negative label is 1 score_01 = roc_auc_score([1, 0, 1, 0], [0.2, 0.6, 0.55, 0.4]) # positive label is 1, negative label is 0 score_10 = roc_auc_score([0, 1, 0, 1], [0.8, 0.4, 0.45, 0.6]) ovo_score = (score_01 + score_10) / 2 assert_almost_equal( roc_auc_score(y_true, y_scores, labels=labels, multi_class="ovo"), ovo_score ) # Weighted, one-vs-one multiclass ROC AUC algorithm assert_almost_equal( roc_auc_score( y_true, y_scores, labels=labels, multi_class="ovo", average="weighted" ), ovo_score, ) @pytest.mark.parametrize( "y_true, labels", [ (np.array([0, 1, 2, 2]), None), (["a", "b", "c", "c"], None), ([0, 1, 2, 2], [0, 1, 2]), (["a", "b", "c", "c"], ["a", "b", "c"]), ], ) def test_multiclass_ovr_roc_auc_toydata(y_true, labels): # Tests the unweighted, one-vs-rest multiclass ROC AUC algorithm # on a small example, representative of an expected use case. y_scores = np.array( [[1.0, 0.0, 0.0], [0.1, 0.5, 0.4], [0.1, 0.1, 0.8], [0.3, 0.3, 0.4]] ) # Compute the expected result by individually computing the 'one-vs-rest' # ROC AUC scores for classes 0, 1, and 2. out_0 = roc_auc_score([1, 0, 0, 0], y_scores[:, 0]) out_1 = roc_auc_score([0, 1, 0, 0], y_scores[:, 1]) out_2 = roc_auc_score([0, 0, 1, 1], y_scores[:, 2]) assert_almost_equal( roc_auc_score(y_true, y_scores, multi_class="ovr", labels=labels, average=None), [out_0, out_1, out_2], ) # Compute unweighted results (default behaviour is average="macro") result_unweighted = (out_0 + out_1 + out_2) / 3.0 assert_almost_equal( roc_auc_score(y_true, y_scores, multi_class="ovr", labels=labels), result_unweighted, ) # Tests the weighted, one-vs-rest multiclass ROC AUC algorithm # on the same input (Provost & Domingos, 2000) result_weighted = out_0 * 0.25 + out_1 * 0.25 + out_2 * 0.5 assert_almost_equal( roc_auc_score( y_true, y_scores, multi_class="ovr", labels=labels, average="weighted" ), result_weighted, ) @pytest.mark.parametrize( "multi_class, average", [ ("ovr", "macro"), ("ovr", "micro"), ("ovo", "macro"), ], ) def test_perfect_imperfect_chance_multiclass_roc_auc(multi_class, average): y_true = np.array([3, 1, 2, 0]) # Perfect classifier (from a ranking point of view) has roc_auc_score = 1.0 y_perfect = [ [0.0, 0.0, 0.0, 1.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.75, 0.05, 0.05, 0.15], ] assert_almost_equal( roc_auc_score(y_true, y_perfect, multi_class=multi_class, average=average), 1.0, ) # Imperfect classifier has roc_auc_score < 1.0 y_imperfect = [ [0.0, 0.0, 0.0, 1.0], [0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0], ] assert ( roc_auc_score(y_true, y_imperfect, multi_class=multi_class, average=average) < 1.0 ) # Chance level classifier has roc_auc_score = 5.0 y_chance = 0.25 * np.ones((4, 4)) assert roc_auc_score( y_true, y_chance, multi_class=multi_class, average=average ) == pytest.approx(0.5) def test_micro_averaged_ovr_roc_auc(global_random_seed): seed = global_random_seed # Let's generate a set of random predictions and matching true labels such # that the predictions are not perfect. To make the problem more interesting, # we use an imbalanced class distribution (by using different parameters # in the Dirichlet prior (conjugate prior of the multinomial distribution). y_pred = stats.dirichlet.rvs([2.0, 1.0, 0.5], size=1000, random_state=seed) y_true = np.asarray( [ stats.multinomial.rvs(n=1, p=y_pred_i, random_state=seed).argmax() for y_pred_i in y_pred ] ) y_onehot = label_binarize(y_true, classes=[0, 1, 2]) fpr, tpr, _ = roc_curve(y_onehot.ravel(), y_pred.ravel()) roc_auc_by_hand = auc(fpr, tpr) roc_auc_auto = roc_auc_score(y_true, y_pred, multi_class="ovr", average="micro") assert roc_auc_by_hand == pytest.approx(roc_auc_auto) @pytest.mark.parametrize( "msg, y_true, labels", [ ("Parameter 'labels' must be unique", np.array([0, 1, 2, 2]), [0, 2, 0]), ( "Parameter 'labels' must be unique", np.array(["a", "b", "c", "c"]), ["a", "a", "b"], ), ( ( "Number of classes in y_true not equal to the number of columns " "in 'y_score'" ), np.array([0, 2, 0, 2]), None, ), ( "Parameter 'labels' must be ordered", np.array(["a", "b", "c", "c"]), ["a", "c", "b"], ), ( ( "Number of given labels, 2, not equal to the number of columns in " "'y_score', 3" ), np.array([0, 1, 2, 2]), [0, 1], ), ( ( "Number of given labels, 2, not equal to the number of columns in " "'y_score', 3" ), np.array(["a", "b", "c", "c"]), ["a", "b"], ), ( ( "Number of given labels, 4, not equal to the number of columns in " "'y_score', 3" ), np.array([0, 1, 2, 2]), [0, 1, 2, 3], ), ( ( "Number of given labels, 4, not equal to the number of columns in " "'y_score', 3" ), np.array(["a", "b", "c", "c"]), ["a", "b", "c", "d"], ), ( "'y_true' contains labels not in parameter 'labels'", np.array(["a", "b", "c", "e"]), ["a", "b", "c"], ), ( "'y_true' contains labels not in parameter 'labels'", np.array(["a", "b", "c", "d"]), ["a", "b", "c"], ), ( "'y_true' contains labels not in parameter 'labels'", np.array([0, 1, 2, 3]), [0, 1, 2], ), ], ) @pytest.mark.parametrize("multi_class", ["ovo", "ovr"]) def test_roc_auc_score_multiclass_labels_error(msg, y_true, labels, multi_class): y_scores = np.array( [[0.1, 0.8, 0.1], [0.3, 0.4, 0.3], [0.35, 0.5, 0.15], [0, 0.2, 0.8]] ) with pytest.raises(ValueError, match=msg): roc_auc_score(y_true, y_scores, labels=labels, multi_class=multi_class) @pytest.mark.parametrize( "msg, kwargs", [ ( ( r"average must be one of \('macro', 'weighted', None\) for " r"multiclass problems" ), {"average": "samples", "multi_class": "ovo"}, ), ( ( r"average must be one of \('micro', 'macro', 'weighted', None\) for " r"multiclass problems" ), {"average": "samples", "multi_class": "ovr"}, ), ( ( r"sample_weight is not supported for multiclass one-vs-one " r"ROC AUC, 'sample_weight' must be None in this case" ), {"multi_class": "ovo", "sample_weight": []}, ), ( ( r"Partial AUC computation not available in multiclass setting, " r"'max_fpr' must be set to `None`, received `max_fpr=0.5` " r"instead" ), {"multi_class": "ovo", "max_fpr": 0.5}, ), (r"multi_class must be in \('ovo', 'ovr'\)", {}), ], ) def test_roc_auc_score_multiclass_error(msg, kwargs): # Test that roc_auc_score function returns an error when trying # to compute multiclass AUC for parameters where an output # is not defined. rng = check_random_state(404) y_score = rng.rand(20, 3) y_prob = softmax(y_score) y_true = rng.randint(0, 3, size=20) with pytest.raises(ValueError, match=msg): roc_auc_score(y_true, y_prob, **kwargs) def test_auc_score_non_binary_class(): # Test that roc_auc_score function returns an error when trying # to compute AUC for non-binary class values. rng = check_random_state(404) y_pred = rng.rand(10) # y_true contains only one class value y_true = np.zeros(10, dtype="int") err_msg = "ROC AUC score is not defined" with pytest.raises(ValueError, match=err_msg): roc_auc_score(y_true, y_pred) y_true = np.ones(10, dtype="int") with pytest.raises(ValueError, match=err_msg): roc_auc_score(y_true, y_pred) y_true = np.full(10, -1, dtype="int") with pytest.raises(ValueError, match=err_msg): roc_auc_score(y_true, y_pred) with warnings.catch_warnings(record=True): rng = check_random_state(404) y_pred = rng.rand(10) # y_true contains only one class value y_true = np.zeros(10, dtype="int") with pytest.raises(ValueError, match=err_msg): roc_auc_score(y_true, y_pred) y_true = np.ones(10, dtype="int") with pytest.raises(ValueError, match=err_msg): roc_auc_score(y_true, y_pred) y_true = np.full(10, -1, dtype="int") with pytest.raises(ValueError, match=err_msg): roc_auc_score(y_true, y_pred) @pytest.mark.parametrize("curve_func", CURVE_FUNCS) def test_binary_clf_curve_multiclass_error(curve_func): rng = check_random_state(404) y_true = rng.randint(0, 3, size=10) y_pred = rng.rand(10) msg = "multiclass format is not supported" with pytest.raises(ValueError, match=msg): curve_func(y_true, y_pred) @pytest.mark.parametrize("curve_func", CURVE_FUNCS) def test_binary_clf_curve_implicit_pos_label(curve_func): # Check that using string class labels raises an informative # error for any supported string dtype: msg = ( "y_true takes value in {'a', 'b'} and pos_label is " "not specified: either make y_true take " "value in {0, 1} or {-1, 1} or pass pos_label " "explicitly." ) with pytest.raises(ValueError, match=msg): curve_func(np.array(["a", "b"], dtype="= 0 and y_score.max() <= 1 else 0 y_pred = (y_score > threshold).astype(np.int64) if k == 1 else y_true score = top_k_accuracy_score(y_true, y_score, k=k) score_acc = accuracy_score(y_true, y_pred) assert score == score_acc == pytest.approx(true_score) @pytest.mark.parametrize( "y_true, true_score, labels", [ (np.array([0, 1, 1, 2]), 0.75, [0, 1, 2, 3]), (np.array([0, 1, 1, 1]), 0.5, [0, 1, 2, 3]), (np.array([1, 1, 1, 1]), 0.5, [0, 1, 2, 3]), (np.array(["a", "e", "e", "a"]), 0.75, ["a", "b", "d", "e"]), ], ) @pytest.mark.parametrize("labels_as_ndarray", [True, False]) def test_top_k_accuracy_score_multiclass_with_labels( y_true, true_score, labels, labels_as_ndarray ): """Test when labels and y_score are multiclass.""" if labels_as_ndarray: labels = np.asarray(labels) y_score = np.array( [ [0.4, 0.3, 0.2, 0.1], [0.1, 0.3, 0.4, 0.2], [0.4, 0.1, 0.2, 0.3], [0.3, 0.2, 0.4, 0.1], ] ) score = top_k_accuracy_score(y_true, y_score, k=2, labels=labels) assert score == pytest.approx(true_score) def test_top_k_accuracy_score_increasing(): # Make sure increasing k leads to a higher score X, y = datasets.make_classification( n_classes=10, n_samples=1000, n_informative=10, random_state=0 ) X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0) clf = LogisticRegression(random_state=0) clf.fit(X_train, y_train) for X, y in zip((X_train, X_test), (y_train, y_test)): scores = [ top_k_accuracy_score(y, clf.predict_proba(X), k=k) for k in range(2, 10) ] assert np.all(np.diff(scores) > 0) @pytest.mark.parametrize( "y_true, k, true_score", [ ([0, 1, 2, 3], 1, 0.25), ([0, 1, 2, 3], 2, 0.5), ([0, 1, 2, 3], 3, 1), ], ) def test_top_k_accuracy_score_ties(y_true, k, true_score): # Make sure highest indices labels are chosen first in case of ties y_score = np.array( [ [5, 5, 7, 0], [1, 5, 5, 5], [0, 0, 3, 3], [1, 1, 1, 1], ] ) assert top_k_accuracy_score(y_true, y_score, k=k) == pytest.approx(true_score) @pytest.mark.parametrize( "y_true, k", [ ([0, 1, 2, 3], 4), ([0, 1, 2, 3], 5), ], ) def test_top_k_accuracy_score_warning(y_true, k): y_score = np.array( [ [0.4, 0.3, 0.2, 0.1], [0.1, 0.4, 0.3, 0.2], [0.2, 0.1, 0.4, 0.3], [0.3, 0.2, 0.1, 0.4], ] ) expected_message = ( r"'k' \(\d+\) greater than or equal to 'n_classes' \(\d+\) will result in a " "perfect score and is therefore meaningless." ) with pytest.warns(UndefinedMetricWarning, match=expected_message): score = top_k_accuracy_score(y_true, y_score, k=k) assert score == 1 @pytest.mark.parametrize( "y_true, y_score, labels, msg", [ ( [0, 0.57, 1, 2], [ [0.2, 0.1, 0.7], [0.4, 0.3, 0.3], [0.3, 0.4, 0.3], [0.4, 0.5, 0.1], ], None, "y type must be 'binary' or 'multiclass', got 'continuous'", ), ( [0, 1, 2, 3], [ [0.2, 0.1, 0.7], [0.4, 0.3, 0.3], [0.3, 0.4, 0.3], [0.4, 0.5, 0.1], ], None, r"Number of classes in 'y_true' \(4\) not equal to the number of " r"classes in 'y_score' \(3\).", ), ( ["c", "c", "a", "b"], [ [0.2, 0.1, 0.7], [0.4, 0.3, 0.3], [0.3, 0.4, 0.3], [0.4, 0.5, 0.1], ], ["a", "b", "c", "c"], "Parameter 'labels' must be unique.", ), ( ["c", "c", "a", "b"], [ [0.2, 0.1, 0.7], [0.4, 0.3, 0.3], [0.3, 0.4, 0.3], [0.4, 0.5, 0.1], ], ["a", "c", "b"], "Parameter 'labels' must be ordered.", ), ( [0, 0, 1, 2], [ [0.2, 0.1, 0.7], [0.4, 0.3, 0.3], [0.3, 0.4, 0.3], [0.4, 0.5, 0.1], ], [0, 1, 2, 3], r"Number of given labels \(4\) not equal to the number of classes in " r"'y_score' \(3\).", ), ( [0, 0, 1, 2], [ [0.2, 0.1, 0.7], [0.4, 0.3, 0.3], [0.3, 0.4, 0.3], [0.4, 0.5, 0.1], ], [0, 1, 3], "'y_true' contains labels not in parameter 'labels'.", ), ( [0, 1], [[0.5, 0.2, 0.2], [0.3, 0.4, 0.2]], None, ( "`y_true` is binary while y_score is 2d with 3 classes. If" " `y_true` does not contain all the labels, `labels` must be provided" ), ), ], ) def test_top_k_accuracy_score_error(y_true, y_score, labels, msg): with pytest.raises(ValueError, match=msg): top_k_accuracy_score(y_true, y_score, k=2, labels=labels) @pytest.mark.parametrize("csr_container", CSR_CONTAINERS) def test_label_ranking_avg_precision_score_should_allow_csr_matrix_for_y_true_input( csr_container, ): # Test that label_ranking_avg_precision_score accept sparse y_true. # Non-regression test for #22575 y_true = csr_container([[1, 0, 0], [0, 0, 1]]) y_score = np.array([[0.5, 0.9, 0.6], [0, 0, 1]]) result = label_ranking_average_precision_score(y_true, y_score) assert result == pytest.approx(2 / 3) @pytest.mark.parametrize( "metric", [average_precision_score, det_curve, precision_recall_curve, roc_curve] ) @pytest.mark.parametrize( "classes", [(False, True), (0, 1), (0.0, 1.0), ("zero", "one")] ) def test_ranking_metric_pos_label_types(metric, classes): """Check that the metric works with different types of `pos_label`. We can expect `pos_label` to be a bool, an integer, a float, a string. No error should be raised for those types. """ rng = np.random.RandomState(42) n_samples, pos_label = 10, classes[-1] y_true = rng.choice(classes, size=n_samples, replace=True) y_proba = rng.rand(n_samples) result = metric(y_true, y_proba, pos_label=pos_label) if isinstance(result, float): assert not np.isnan(result) else: metric_1, metric_2, thresholds = result assert not np.isnan(metric_1).any() assert not np.isnan(metric_2).any() assert not np.isnan(thresholds).any() def test_roc_curve_with_probablity_estimates(global_random_seed): """Check that thresholds do not exceed 1.0 when `y_score` is a probability estimate. Non-regression test for: https://github.com/scikit-learn/scikit-learn/issues/26193 """ rng = np.random.RandomState(global_random_seed) y_true = rng.randint(0, 2, size=10) y_score = rng.rand(10) _, _, thresholds = roc_curve(y_true, y_score) assert np.isinf(thresholds[0])