576 lines
19 KiB
Python
576 lines
19 KiB
Python
# Authors: Rob Zinkov, Mathieu Blondel
|
|
# License: BSD 3 clause
|
|
from numbers import Real
|
|
|
|
from ..base import _fit_context
|
|
from ..utils._param_validation import Interval, StrOptions
|
|
from ._stochastic_gradient import DEFAULT_EPSILON, BaseSGDClassifier, BaseSGDRegressor
|
|
|
|
|
|
class PassiveAggressiveClassifier(BaseSGDClassifier):
|
|
"""Passive Aggressive Classifier.
|
|
|
|
Read more in the :ref:`User Guide <passive_aggressive>`.
|
|
|
|
Parameters
|
|
----------
|
|
C : float, default=1.0
|
|
Maximum step size (regularization). Defaults to 1.0.
|
|
|
|
fit_intercept : bool, default=True
|
|
Whether the intercept should be estimated or not. If False, the
|
|
data is assumed to be already centered.
|
|
|
|
max_iter : int, default=1000
|
|
The maximum number of passes over the training data (aka epochs).
|
|
It only impacts the behavior in the ``fit`` method, and not the
|
|
:meth:`~sklearn.linear_model.PassiveAggressiveClassifier.partial_fit` method.
|
|
|
|
.. versionadded:: 0.19
|
|
|
|
tol : float or None, default=1e-3
|
|
The stopping criterion. If it is not None, the iterations will stop
|
|
when (loss > previous_loss - tol).
|
|
|
|
.. versionadded:: 0.19
|
|
|
|
early_stopping : bool, default=False
|
|
Whether to use early stopping to terminate training when validation
|
|
score is not improving. If set to True, it will automatically set aside
|
|
a stratified fraction of training data as validation and terminate
|
|
training when validation score is not improving by at least `tol` for
|
|
`n_iter_no_change` consecutive epochs.
|
|
|
|
.. versionadded:: 0.20
|
|
|
|
validation_fraction : float, default=0.1
|
|
The proportion of training data to set aside as validation set for
|
|
early stopping. Must be between 0 and 1.
|
|
Only used if early_stopping is True.
|
|
|
|
.. versionadded:: 0.20
|
|
|
|
n_iter_no_change : int, default=5
|
|
Number of iterations with no improvement to wait before early stopping.
|
|
|
|
.. versionadded:: 0.20
|
|
|
|
shuffle : bool, default=True
|
|
Whether or not the training data should be shuffled after each epoch.
|
|
|
|
verbose : int, default=0
|
|
The verbosity level.
|
|
|
|
loss : str, default="hinge"
|
|
The loss function to be used:
|
|
hinge: equivalent to PA-I in the reference paper.
|
|
squared_hinge: equivalent to PA-II in the reference paper.
|
|
|
|
n_jobs : int or None, default=None
|
|
The number of CPUs to use to do the OVA (One Versus All, for
|
|
multi-class problems) computation.
|
|
``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
|
|
``-1`` means using all processors. See :term:`Glossary <n_jobs>`
|
|
for more details.
|
|
|
|
random_state : int, RandomState instance, default=None
|
|
Used to shuffle the training data, when ``shuffle`` is set to
|
|
``True``. Pass an int for reproducible output across multiple
|
|
function calls.
|
|
See :term:`Glossary <random_state>`.
|
|
|
|
warm_start : bool, default=False
|
|
When set to True, reuse the solution of the previous call to fit as
|
|
initialization, otherwise, just erase the previous solution.
|
|
See :term:`the Glossary <warm_start>`.
|
|
|
|
Repeatedly calling fit or partial_fit when warm_start is True can
|
|
result in a different solution than when calling fit a single time
|
|
because of the way the data is shuffled.
|
|
|
|
class_weight : dict, {class_label: weight} or "balanced" or None, \
|
|
default=None
|
|
Preset for the class_weight fit parameter.
|
|
|
|
Weights associated with classes. If not given, all classes
|
|
are supposed to have weight one.
|
|
|
|
The "balanced" mode uses the values of y to automatically adjust
|
|
weights inversely proportional to class frequencies in the input data
|
|
as ``n_samples / (n_classes * np.bincount(y))``.
|
|
|
|
.. versionadded:: 0.17
|
|
parameter *class_weight* to automatically weight samples.
|
|
|
|
average : bool or int, default=False
|
|
When set to True, computes the averaged SGD weights and stores the
|
|
result in the ``coef_`` attribute. If set to an int greater than 1,
|
|
averaging will begin once the total number of samples seen reaches
|
|
average. So average=10 will begin averaging after seeing 10 samples.
|
|
|
|
.. versionadded:: 0.19
|
|
parameter *average* to use weights averaging in SGD.
|
|
|
|
Attributes
|
|
----------
|
|
coef_ : ndarray of shape (1, n_features) if n_classes == 2 else \
|
|
(n_classes, n_features)
|
|
Weights assigned to the features.
|
|
|
|
intercept_ : ndarray of shape (1,) if n_classes == 2 else (n_classes,)
|
|
Constants in decision function.
|
|
|
|
n_features_in_ : int
|
|
Number of features seen during :term:`fit`.
|
|
|
|
.. versionadded:: 0.24
|
|
|
|
feature_names_in_ : ndarray of shape (`n_features_in_`,)
|
|
Names of features seen during :term:`fit`. Defined only when `X`
|
|
has feature names that are all strings.
|
|
|
|
.. versionadded:: 1.0
|
|
|
|
n_iter_ : int
|
|
The actual number of iterations to reach the stopping criterion.
|
|
For multiclass fits, it is the maximum over every binary fit.
|
|
|
|
classes_ : ndarray of shape (n_classes,)
|
|
The unique classes labels.
|
|
|
|
t_ : int
|
|
Number of weight updates performed during training.
|
|
Same as ``(n_iter_ * n_samples + 1)``.
|
|
|
|
loss_function_ : callable
|
|
Loss function used by the algorithm.
|
|
|
|
See Also
|
|
--------
|
|
SGDClassifier : Incrementally trained logistic regression.
|
|
Perceptron : Linear perceptron classifier.
|
|
|
|
References
|
|
----------
|
|
Online Passive-Aggressive Algorithms
|
|
<http://jmlr.csail.mit.edu/papers/volume7/crammer06a/crammer06a.pdf>
|
|
K. Crammer, O. Dekel, J. Keshat, S. Shalev-Shwartz, Y. Singer - JMLR (2006)
|
|
|
|
Examples
|
|
--------
|
|
>>> from sklearn.linear_model import PassiveAggressiveClassifier
|
|
>>> from sklearn.datasets import make_classification
|
|
>>> X, y = make_classification(n_features=4, random_state=0)
|
|
>>> clf = PassiveAggressiveClassifier(max_iter=1000, random_state=0,
|
|
... tol=1e-3)
|
|
>>> clf.fit(X, y)
|
|
PassiveAggressiveClassifier(random_state=0)
|
|
>>> print(clf.coef_)
|
|
[[0.26642044 0.45070924 0.67251877 0.64185414]]
|
|
>>> print(clf.intercept_)
|
|
[1.84127814]
|
|
>>> print(clf.predict([[0, 0, 0, 0]]))
|
|
[1]
|
|
"""
|
|
|
|
_parameter_constraints: dict = {
|
|
**BaseSGDClassifier._parameter_constraints,
|
|
"loss": [StrOptions({"hinge", "squared_hinge"})],
|
|
"C": [Interval(Real, 0, None, closed="right")],
|
|
}
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
C=1.0,
|
|
fit_intercept=True,
|
|
max_iter=1000,
|
|
tol=1e-3,
|
|
early_stopping=False,
|
|
validation_fraction=0.1,
|
|
n_iter_no_change=5,
|
|
shuffle=True,
|
|
verbose=0,
|
|
loss="hinge",
|
|
n_jobs=None,
|
|
random_state=None,
|
|
warm_start=False,
|
|
class_weight=None,
|
|
average=False,
|
|
):
|
|
super().__init__(
|
|
penalty=None,
|
|
fit_intercept=fit_intercept,
|
|
max_iter=max_iter,
|
|
tol=tol,
|
|
early_stopping=early_stopping,
|
|
validation_fraction=validation_fraction,
|
|
n_iter_no_change=n_iter_no_change,
|
|
shuffle=shuffle,
|
|
verbose=verbose,
|
|
random_state=random_state,
|
|
eta0=1.0,
|
|
warm_start=warm_start,
|
|
class_weight=class_weight,
|
|
average=average,
|
|
n_jobs=n_jobs,
|
|
)
|
|
|
|
self.C = C
|
|
self.loss = loss
|
|
|
|
@_fit_context(prefer_skip_nested_validation=True)
|
|
def partial_fit(self, X, y, classes=None):
|
|
"""Fit linear model with Passive Aggressive algorithm.
|
|
|
|
Parameters
|
|
----------
|
|
X : {array-like, sparse matrix} of shape (n_samples, n_features)
|
|
Subset of the training data.
|
|
|
|
y : array-like of shape (n_samples,)
|
|
Subset of the target values.
|
|
|
|
classes : ndarray of shape (n_classes,)
|
|
Classes across all calls to partial_fit.
|
|
Can be obtained by via `np.unique(y_all)`, where y_all is the
|
|
target vector of the entire dataset.
|
|
This argument is required for the first call to partial_fit
|
|
and can be omitted in the subsequent calls.
|
|
Note that y doesn't need to contain all labels in `classes`.
|
|
|
|
Returns
|
|
-------
|
|
self : object
|
|
Fitted estimator.
|
|
"""
|
|
if not hasattr(self, "classes_"):
|
|
self._more_validate_params(for_partial_fit=True)
|
|
|
|
if self.class_weight == "balanced":
|
|
raise ValueError(
|
|
"class_weight 'balanced' is not supported for "
|
|
"partial_fit. For 'balanced' weights, use "
|
|
"`sklearn.utils.compute_class_weight` with "
|
|
"`class_weight='balanced'`. In place of y you "
|
|
"can use a large enough subset of the full "
|
|
"training set target to properly estimate the "
|
|
"class frequency distributions. Pass the "
|
|
"resulting weights as the class_weight "
|
|
"parameter."
|
|
)
|
|
|
|
lr = "pa1" if self.loss == "hinge" else "pa2"
|
|
return self._partial_fit(
|
|
X,
|
|
y,
|
|
alpha=1.0,
|
|
C=self.C,
|
|
loss="hinge",
|
|
learning_rate=lr,
|
|
max_iter=1,
|
|
classes=classes,
|
|
sample_weight=None,
|
|
coef_init=None,
|
|
intercept_init=None,
|
|
)
|
|
|
|
@_fit_context(prefer_skip_nested_validation=True)
|
|
def fit(self, X, y, coef_init=None, intercept_init=None):
|
|
"""Fit linear model with Passive Aggressive algorithm.
|
|
|
|
Parameters
|
|
----------
|
|
X : {array-like, sparse matrix} of shape (n_samples, n_features)
|
|
Training data.
|
|
|
|
y : array-like of shape (n_samples,)
|
|
Target values.
|
|
|
|
coef_init : ndarray of shape (n_classes, n_features)
|
|
The initial coefficients to warm-start the optimization.
|
|
|
|
intercept_init : ndarray of shape (n_classes,)
|
|
The initial intercept to warm-start the optimization.
|
|
|
|
Returns
|
|
-------
|
|
self : object
|
|
Fitted estimator.
|
|
"""
|
|
self._more_validate_params()
|
|
|
|
lr = "pa1" if self.loss == "hinge" else "pa2"
|
|
return self._fit(
|
|
X,
|
|
y,
|
|
alpha=1.0,
|
|
C=self.C,
|
|
loss="hinge",
|
|
learning_rate=lr,
|
|
coef_init=coef_init,
|
|
intercept_init=intercept_init,
|
|
)
|
|
|
|
|
|
class PassiveAggressiveRegressor(BaseSGDRegressor):
|
|
"""Passive Aggressive Regressor.
|
|
|
|
Read more in the :ref:`User Guide <passive_aggressive>`.
|
|
|
|
Parameters
|
|
----------
|
|
|
|
C : float, default=1.0
|
|
Maximum step size (regularization). Defaults to 1.0.
|
|
|
|
fit_intercept : bool, default=True
|
|
Whether the intercept should be estimated or not. If False, the
|
|
data is assumed to be already centered. Defaults to True.
|
|
|
|
max_iter : int, default=1000
|
|
The maximum number of passes over the training data (aka epochs).
|
|
It only impacts the behavior in the ``fit`` method, and not the
|
|
:meth:`~sklearn.linear_model.PassiveAggressiveRegressor.partial_fit` method.
|
|
|
|
.. versionadded:: 0.19
|
|
|
|
tol : float or None, default=1e-3
|
|
The stopping criterion. If it is not None, the iterations will stop
|
|
when (loss > previous_loss - tol).
|
|
|
|
.. versionadded:: 0.19
|
|
|
|
early_stopping : bool, default=False
|
|
Whether to use early stopping to terminate training when validation.
|
|
score is not improving. If set to True, it will automatically set aside
|
|
a fraction of training data as validation and terminate
|
|
training when validation score is not improving by at least tol for
|
|
n_iter_no_change consecutive epochs.
|
|
|
|
.. versionadded:: 0.20
|
|
|
|
validation_fraction : float, default=0.1
|
|
The proportion of training data to set aside as validation set for
|
|
early stopping. Must be between 0 and 1.
|
|
Only used if early_stopping is True.
|
|
|
|
.. versionadded:: 0.20
|
|
|
|
n_iter_no_change : int, default=5
|
|
Number of iterations with no improvement to wait before early stopping.
|
|
|
|
.. versionadded:: 0.20
|
|
|
|
shuffle : bool, default=True
|
|
Whether or not the training data should be shuffled after each epoch.
|
|
|
|
verbose : int, default=0
|
|
The verbosity level.
|
|
|
|
loss : str, default="epsilon_insensitive"
|
|
The loss function to be used:
|
|
epsilon_insensitive: equivalent to PA-I in the reference paper.
|
|
squared_epsilon_insensitive: equivalent to PA-II in the reference
|
|
paper.
|
|
|
|
epsilon : float, default=0.1
|
|
If the difference between the current prediction and the correct label
|
|
is below this threshold, the model is not updated.
|
|
|
|
random_state : int, RandomState instance, default=None
|
|
Used to shuffle the training data, when ``shuffle`` is set to
|
|
``True``. Pass an int for reproducible output across multiple
|
|
function calls.
|
|
See :term:`Glossary <random_state>`.
|
|
|
|
warm_start : bool, default=False
|
|
When set to True, reuse the solution of the previous call to fit as
|
|
initialization, otherwise, just erase the previous solution.
|
|
See :term:`the Glossary <warm_start>`.
|
|
|
|
Repeatedly calling fit or partial_fit when warm_start is True can
|
|
result in a different solution than when calling fit a single time
|
|
because of the way the data is shuffled.
|
|
|
|
average : bool or int, default=False
|
|
When set to True, computes the averaged SGD weights and stores the
|
|
result in the ``coef_`` attribute. If set to an int greater than 1,
|
|
averaging will begin once the total number of samples seen reaches
|
|
average. So average=10 will begin averaging after seeing 10 samples.
|
|
|
|
.. versionadded:: 0.19
|
|
parameter *average* to use weights averaging in SGD.
|
|
|
|
Attributes
|
|
----------
|
|
coef_ : array, shape = [1, n_features] if n_classes == 2 else [n_classes,\
|
|
n_features]
|
|
Weights assigned to the features.
|
|
|
|
intercept_ : array, shape = [1] if n_classes == 2 else [n_classes]
|
|
Constants in decision function.
|
|
|
|
n_features_in_ : int
|
|
Number of features seen during :term:`fit`.
|
|
|
|
.. versionadded:: 0.24
|
|
|
|
feature_names_in_ : ndarray of shape (`n_features_in_`,)
|
|
Names of features seen during :term:`fit`. Defined only when `X`
|
|
has feature names that are all strings.
|
|
|
|
.. versionadded:: 1.0
|
|
|
|
n_iter_ : int
|
|
The actual number of iterations to reach the stopping criterion.
|
|
|
|
t_ : int
|
|
Number of weight updates performed during training.
|
|
Same as ``(n_iter_ * n_samples + 1)``.
|
|
|
|
See Also
|
|
--------
|
|
SGDRegressor : Linear model fitted by minimizing a regularized
|
|
empirical loss with SGD.
|
|
|
|
References
|
|
----------
|
|
Online Passive-Aggressive Algorithms
|
|
<http://jmlr.csail.mit.edu/papers/volume7/crammer06a/crammer06a.pdf>
|
|
K. Crammer, O. Dekel, J. Keshat, S. Shalev-Shwartz, Y. Singer - JMLR (2006).
|
|
|
|
Examples
|
|
--------
|
|
>>> from sklearn.linear_model import PassiveAggressiveRegressor
|
|
>>> from sklearn.datasets import make_regression
|
|
|
|
>>> X, y = make_regression(n_features=4, random_state=0)
|
|
>>> regr = PassiveAggressiveRegressor(max_iter=100, random_state=0,
|
|
... tol=1e-3)
|
|
>>> regr.fit(X, y)
|
|
PassiveAggressiveRegressor(max_iter=100, random_state=0)
|
|
>>> print(regr.coef_)
|
|
[20.48736655 34.18818427 67.59122734 87.94731329]
|
|
>>> print(regr.intercept_)
|
|
[-0.02306214]
|
|
>>> print(regr.predict([[0, 0, 0, 0]]))
|
|
[-0.02306214]
|
|
"""
|
|
|
|
_parameter_constraints: dict = {
|
|
**BaseSGDRegressor._parameter_constraints,
|
|
"loss": [StrOptions({"epsilon_insensitive", "squared_epsilon_insensitive"})],
|
|
"C": [Interval(Real, 0, None, closed="right")],
|
|
"epsilon": [Interval(Real, 0, None, closed="left")],
|
|
}
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
C=1.0,
|
|
fit_intercept=True,
|
|
max_iter=1000,
|
|
tol=1e-3,
|
|
early_stopping=False,
|
|
validation_fraction=0.1,
|
|
n_iter_no_change=5,
|
|
shuffle=True,
|
|
verbose=0,
|
|
loss="epsilon_insensitive",
|
|
epsilon=DEFAULT_EPSILON,
|
|
random_state=None,
|
|
warm_start=False,
|
|
average=False,
|
|
):
|
|
super().__init__(
|
|
penalty=None,
|
|
l1_ratio=0,
|
|
epsilon=epsilon,
|
|
eta0=1.0,
|
|
fit_intercept=fit_intercept,
|
|
max_iter=max_iter,
|
|
tol=tol,
|
|
early_stopping=early_stopping,
|
|
validation_fraction=validation_fraction,
|
|
n_iter_no_change=n_iter_no_change,
|
|
shuffle=shuffle,
|
|
verbose=verbose,
|
|
random_state=random_state,
|
|
warm_start=warm_start,
|
|
average=average,
|
|
)
|
|
self.C = C
|
|
self.loss = loss
|
|
|
|
@_fit_context(prefer_skip_nested_validation=True)
|
|
def partial_fit(self, X, y):
|
|
"""Fit linear model with Passive Aggressive algorithm.
|
|
|
|
Parameters
|
|
----------
|
|
X : {array-like, sparse matrix} of shape (n_samples, n_features)
|
|
Subset of training data.
|
|
|
|
y : numpy array of shape [n_samples]
|
|
Subset of target values.
|
|
|
|
Returns
|
|
-------
|
|
self : object
|
|
Fitted estimator.
|
|
"""
|
|
if not hasattr(self, "coef_"):
|
|
self._more_validate_params(for_partial_fit=True)
|
|
|
|
lr = "pa1" if self.loss == "epsilon_insensitive" else "pa2"
|
|
return self._partial_fit(
|
|
X,
|
|
y,
|
|
alpha=1.0,
|
|
C=self.C,
|
|
loss="epsilon_insensitive",
|
|
learning_rate=lr,
|
|
max_iter=1,
|
|
sample_weight=None,
|
|
coef_init=None,
|
|
intercept_init=None,
|
|
)
|
|
|
|
@_fit_context(prefer_skip_nested_validation=True)
|
|
def fit(self, X, y, coef_init=None, intercept_init=None):
|
|
"""Fit linear model with Passive Aggressive algorithm.
|
|
|
|
Parameters
|
|
----------
|
|
X : {array-like, sparse matrix} of shape (n_samples, n_features)
|
|
Training data.
|
|
|
|
y : numpy array of shape [n_samples]
|
|
Target values.
|
|
|
|
coef_init : array, shape = [n_features]
|
|
The initial coefficients to warm-start the optimization.
|
|
|
|
intercept_init : array, shape = [1]
|
|
The initial intercept to warm-start the optimization.
|
|
|
|
Returns
|
|
-------
|
|
self : object
|
|
Fitted estimator.
|
|
"""
|
|
self._more_validate_params()
|
|
|
|
lr = "pa1" if self.loss == "epsilon_insensitive" else "pa2"
|
|
return self._fit(
|
|
X,
|
|
y,
|
|
alpha=1.0,
|
|
C=self.C,
|
|
loss="epsilon_insensitive",
|
|
learning_rate=lr,
|
|
coef_init=coef_init,
|
|
intercept_init=intercept_init,
|
|
)
|