Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update RMSE, MSLE to handle missing values #542

Merged
merged 15 commits into from
Dec 23, 2024
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Rework validation of `FoldMask` to not fail on tail nans ([#536](https://github.com/etna-team/etna/pull/536))
- Add parameter `missing_mode` into `R2` and `MedAE` metrics ([#537](https://github.com/etna-team/etna/pull/537))
- Update `analysis.forecast.plots.plot_metric_per_segment` to handle `None` from metrics ([#540](https://github.com/etna-team/etna/pull/540))
- Add parameter `missing_mode` into `RMSE` and `MSLE` metrics ([#542](https://github.com/etna-team/etna/pull/542))
-
-
-
Expand Down
2 changes: 0 additions & 2 deletions etna/metrics/__init__.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
"""Module with metrics of forecasting quality."""

from sklearn.metrics import mean_squared_log_error as msle

from etna.metrics.base import Metric
from etna.metrics.base import MetricAggregationMode
from etna.metrics.base import MetricMissingMode
Expand Down
171 changes: 163 additions & 8 deletions etna/metrics/functional_metrics.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,10 @@
import warnings
from enum import Enum
from functools import partial
from typing import Optional
from typing import Sequence
from typing import Union

import numpy as np
from sklearn.metrics import mean_squared_error as mse_sklearn
from sklearn.metrics import mean_squared_log_error as msle
from typing_extensions import assert_never

ArrayLike = Union[float, Sequence[float], Sequence[Sequence[float]]]
Expand Down Expand Up @@ -68,6 +65,12 @@
:
A non-negative floating point value (the best value is 0.0), or an array of floating point values,
one for each individual target.

Raises
------
:
ValueError:
If the shapes of the input arrays do not match.
"""
y_true_array, y_pred_array = np.asarray(y_true), np.asarray(y_pred)

Expand Down Expand Up @@ -114,6 +117,12 @@
:
A non-negative floating point value (the best value is 0.0), or an array of floating point values,
one for each individual target.

Raises
------
:
ValueError:
If the shapes of the input arrays do not match.
"""
y_true_array, y_pred_array = np.asarray(y_true), np.asarray(y_pred)

Expand All @@ -132,10 +141,10 @@


def mape(y_true: ArrayLike, y_pred: ArrayLike, eps: float = 1e-15, multioutput: str = "joint") -> ArrayLike:
"""Mean absolute percentage error.
"""Mean absolute percentage error with missing values handling.

.. math::
MAPE(y\_true, y\_pred) = \\frac{1}{n} \\cdot \\sum_{i=1}^{n} \\frac{\\mid y\_true_i - y\_pred_i\\mid}{\\mid y\_true_i \\mid + \epsilon}
MAPE(y\_true, y\_pred) = \\frac{1}{n} \\cdot \\sum_{i=1}^{n} \\frac{\\mid y\_true_i - y\_pred_i\\mid}{\\mid y\_true_i \\mid + \epsilon}

`Scale-dependent errors <https://otexts.com/fpp3/accuracy.html#scale-dependent-errors>`_

Expand Down Expand Up @@ -166,6 +175,12 @@
:
A non-negative floating point value (the best value is 0.0), or an array of floating point values,
one for each individual target.

Raises
------
:
ValueError:
If the shapes of the input arrays do not match.
"""
y_true_array, y_pred_array = np.asarray(y_true), np.asarray(y_pred)

Expand All @@ -188,10 +203,10 @@


def smape(y_true: ArrayLike, y_pred: ArrayLike, eps: float = 1e-15, multioutput: str = "joint") -> ArrayLike:
"""Symmetric mean absolute percentage error.
"""Symmetric mean absolute percentage error with missing values handling.

.. math::
SMAPE(y\_true, y\_pred) = \\frac{2 \\cdot 100 \\%}{n} \\cdot \\sum_{i=1}^{n} \\frac{\\mid y\_true_i - y\_pred_i\\mid}{\\mid y\_true_i \\mid + \\mid y\_pred_i \\mid}
SMAPE(y\_true, y\_pred) = \\frac{2 \\cdot 100 \\%}{n} \\cdot \\sum_{i=1}^{n} \\frac{\\mid y\_true_i - y\_pred_i\\mid}{\\mid y\_true_i \\mid + \\mid y\_pred_i \\mid}

The nans are ignored during computation. If all values are nans, the result is NaN.

Expand Down Expand Up @@ -220,6 +235,12 @@
:
A non-negative floating point value (the best value is 0.0), or an array of floating point values,
one for each individual target.

Raises
------
:
ValueError:
If the shapes of the input arrays do not match.
"""
y_true_array, y_pred_array = np.asarray(y_true), np.asarray(y_pred)

Expand Down Expand Up @@ -270,6 +291,12 @@
:
A floating point value, or an array of floating point values,
one for each individual target.

Raises
------
:
ValueError:
If the shapes of the input arrays do not match.
"""
y_true_array, y_pred_array = np.asarray(y_true), np.asarray(y_pred)

Expand Down Expand Up @@ -346,6 +373,12 @@
:
A non-negative floating point value (the best value is 0.0), or an array of floating point values,
one for each individual target.

Raises
------
:
ValueError:
If the shapes of the input arrays do not match.
"""
y_true_array, y_pred_array = np.asarray(y_true), np.asarray(y_pred)

Expand Down Expand Up @@ -392,6 +425,12 @@
:
A floating point value, or an array of floating point values,
one for each individual target.

Raises
------
:
ValueError:
If the shapes of the input arrays do not match.
"""
y_true_array, y_pred_array = np.asarray(y_true), np.asarray(y_pred)

Expand Down Expand Up @@ -439,6 +478,12 @@
:
A non-negative floating point value (the best value is 0.0), or an array of floating point values,
one for each individual target.

Raises
------
:
ValueError:
If the shapes of the input arrays do not match.
"""
y_true_array, y_pred_array = np.asarray(y_true), np.asarray(y_pred)

Expand All @@ -457,7 +502,105 @@
return result


rmse = partial(mse_sklearn, squared=False)
def rmse(y_true: ArrayLike, y_pred: ArrayLike, multioutput: str = "joint") -> ArrayLike:
"""Root mean squared error with missing values handling.

.. math::
RMSE(y\_true, y\_pred) = \\sqrt\\frac{\\sum_{i=1}^{n}{(y\_true_i - y\_pred_i)^2}}{n}

The nans are ignored during computation. If all values are nans, the result is NaN.

Parameters
----------
y_true:
array-like of shape (n_samples,) or (n_samples, n_outputs)

Ground truth (correct) target values.

y_pred:
array-like of shape (n_samples,) or (n_samples, n_outputs)

Estimated target values.

multioutput:
Defines aggregating of multiple output values
(see :py:class:`~etna.metrics.functional_metrics.FunctionalMetricMultioutput`).

Returns
-------
:
A non-negative floating point value (the best value is 0.0), or an array of floating point values,
one for each individual target.

Raises
------
:
ValueError:
If the shapes of the input arrays do not match.
"""
mse_result = mse(y_true=y_true, y_pred=y_pred, multioutput=multioutput)
result = np.sqrt(mse_result)

return result # type: ignore


def msle(y_true: ArrayLike, y_pred: ArrayLike, multioutput: str = "joint") -> ArrayLike:
"""Mean squared logarithmic error with missing values handling.

.. math::
MSLE(y\_true, y\_pred) = \\frac{1}{n}\\cdot\\sum_{i=1}^{n}{(log(1 + y\_true_i) - log(1 + y\_pred_i))^2}

The nans are ignored during computation. If all values are nans, the result is NaN.

Parameters
----------
y_true:
array-like of shape (n_samples,) or (n_samples, n_outputs)

Ground truth (correct) target values.

y_pred:
array-like of shape (n_samples,) or (n_samples, n_outputs)

Estimated target values.

multioutput:
Defines aggregating of multiple output values
(see :py:class:`~etna.metrics.functional_metrics.FunctionalMetricMultioutput`).

Returns
-------
:
A non-negative floating point value (the best value is 0.0), or an array of floating point values,
one for each individual target.

Raises
------
:
ValueError:
If the shapes of the input arrays do not match.
ValueError:
If input arrays contain negative values.
"""
d-a-bunin marked this conversation as resolved.
Show resolved Hide resolved
y_true_array, y_pred_array = np.asarray(y_true), np.asarray(y_pred)

if len(y_true_array.shape) != len(y_pred_array.shape):
raise ValueError("Shapes of the labels must be the same")

Check warning on line 588 in etna/metrics/functional_metrics.py

View check run for this annotation

Codecov / codecov/patch

etna/metrics/functional_metrics.py#L588

Added line #L588 was not covered by tests

if (y_true_array < 0).any() or (y_pred_array < 0).any():
raise ValueError("Mean squared logarithmic error cannot be used when targets contain negative values.")

axis = _get_axis_by_multioutput(multioutput)

with warnings.catch_warnings():
# this helps to prevent warning in case of all nans
warnings.filterwarnings(
message="Mean of empty slice",
action="ignore",
)
result = np.nanmean((np.log1p(y_true_array) - np.log1p(y_pred_array)) ** 2, axis=axis)

return result


def wape(y_true: ArrayLike, y_pred: ArrayLike, multioutput: str = "joint") -> ArrayLike:
Expand Down Expand Up @@ -489,6 +632,12 @@
:
A non-negative floating point value (the best value is 0.0), or an array of floating point values,
one for each individual target.

Raises
------
:
ValueError:
If the shapes of the input arrays do not match.
"""
y_true_array, y_pred_array = np.asarray(y_true), np.asarray(y_pred)

Expand Down Expand Up @@ -554,6 +703,12 @@
:
A floating point value, or an array of floating point values,
one for each individual target.

Raises
------
:
ValueError:
If the shapes of the input arrays do not match.
"""
y_true_array, y_pred_array = np.asarray(y_true), np.asarray(y_pred)

Expand Down
Loading
Loading