[PYDF] Model self-evaluation

PiperOrigin-RevId: 601090814

yggdrasil_decision_forests/port/python/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## HEAD
+
+-   Added model self evaluation.
+
 ## 0.0.8 - 2023-12-19
 
 ### Features

yggdrasil_decision_forests/port/python/ydf/model/BUILD

@@ -174,11 +174,9 @@ py_test(
         # absl/testing:parameterized dep,
         # numpy dep,
         # pandas dep,
-        "@ydf_cc//yggdrasil_decision_forests/dataset:data_spec_py_proto",
         "//ydf/dataset",
         "//ydf/model/gradient_boosted_trees_model",
         "//ydf/model/random_forest_model",
-        "//ydf/utils:log",
         "//ydf/utils:test_utils",
     ],
 )

yggdrasil_decision_forests/port/python/ydf/model/generic_model.py

@@ -784,5 +784,32 @@ def input_features(self) -> Sequence[InputFeature]:
         for column_idx in self._model.input_features()
     ]
 
+  def self_evaluation(self) -> metric.Evaluation:
+    """Returns the model's self-evaluation.
+
+    Different models use different methods for self-evaluation. Notably, Random
+    Forests use OOB evaluation and Gradient Boosted Trees use evaluation on the
+    validation dataset. Therefore, self-evaluations are not comparable between
+    different model types.
+
+    Usage example:
+
+    ```python
+    import pandas as pd
+    import ydf
+
+    # Train model
+    train_ds = pd.read_csv("train.csv")
+    model = ydf.GradientBoostedTreesLearner(label="label").train(train_ds)
+
+    self_evaluation = model.self_evaluation()
+    # In an interactive Python environment, print a rich evaluation report.
+    self_evaluation
+    ```
+    """
+    raise NotImplementedError(
+        "Self-evaluation is not available for this model type."
+    )
+
 
 ModelType = TypeVar("ModelType", bound=GenericModel)

yggdrasil_decision_forests/port/python/ydf/model/gradient_boosted_trees_model/BUILD

@@ -28,6 +28,7 @@ py_library(
     srcs = ["gradient_boosted_trees_model.py"],
     deps = [
         # numpy dep,
+        "@ydf_cc//yggdrasil_decision_forests/metric:metric_py_proto",
         "//ydf/cc:ydf",
         "//ydf/metric",
         "//ydf/model/decision_forest_model",

yggdrasil_decision_forests/port/python/ydf/model/gradient_boosted_trees_model/gradient_boosted_trees_model.py

@@ -19,6 +19,7 @@
 
 import numpy.typing as npt
 
+from yggdrasil_decision_forests.metric import metric_pb2
 from ydf.cc import ydf
 from ydf.metric import metric
 from ydf.model.decision_forest_model import decision_forest_model
@@ -38,13 +39,13 @@ def initial_predictions(self) -> npt.NDArray[float]:
     """Returns the model's initial predictions (i.e. the model bias)."""
     return self._model.initial_predictions()
 
-  def validation_evaluation(self) -> metric.Evaluation:
+  def validation_evaluation(self) -> Optional[metric.Evaluation]:
     """Returns the validation evaluation of the model, if available.
 
     Gradient Boosted Trees use a validation dataset for early stopping.
 
-    If no validation evaluation been computed or has been removed, the
-    evaluation object may be missing fields.
+    Returns None if no validation evaluation been computed or it has been
+    removed from the model.
 
     Usage example:
 
@@ -61,4 +62,40 @@ def validation_evaluation(self) -> metric.Evaluation:
     validation_evaluation
     ```
     """
+    validation_evaluation_proto = self._model.validation_evaluation()
+    # There is no canonical way of checking if a proto is empty. This workaround
+    # just checks if the evaluation proto is valid.
+    if not validation_evaluation_proto.HasField("task"):
+      return None
     return metric.Evaluation(self._model.validation_evaluation())
+
+  def self_evaluation(self) -> Optional[metric.Evaluation]:
+    """Returns the model's self-evaluation.
+
+    For Gradient Boosted Trees models, the self-evaluation is the evaluation on
+    the validation dataset. Note that the validation dataset is extracted
+    automatically if not explicitly given. If the validation dataset is
+    deactivated, no self-evaluation is computed.
+
+    Different models use different methods for self-evaluation. Notably, Random
+    Forests use the last Out-Of-Bag evaluation. Therefore, self-evaluations are
+    not comparable between different model types.
+
+    Returns None if no self-evaluation has been computed.
+
+    Usage example:
+
+    ```python
+    import pandas as pd
+    import ydf
+
+    # Train model
+    train_ds = pd.read_csv("train.csv")
+    model = ydf.GradientBoostedTreesLearner(label="label").train(train_ds)
+
+    self_evaluation = model.self_evaluation()
+    # In an interactive Python environment, print a rich evaluation report.
+    self_evaluation
+    ```
+    """
+    return self.validation_evaluation()

yggdrasil_decision_forests/port/python/ydf/model/gradient_boosted_trees_model/gradient_boosted_trees_model_test.py

@@ -150,8 +150,7 @@ def test_validation_evaluation_empty(self):
         model, gradient_boosted_trees_model.GradientBoostedTreesModel
     )
     validation_evaluation = model.validation_evaluation()
-    self.assertIsNone(validation_evaluation.accuracy)
-    self.assertEqual(validation_evaluation.__str__(), "No metrics")
+    self.assertIsNone(validation_evaluation)
 
   def test_validation_evaluation_no_training_logs(self):
     validation_evaluation = self.adult_binary_class_gbdt.validation_evaluation()

yggdrasil_decision_forests/port/python/ydf/model/model_test.py

@@ -387,6 +387,29 @@ def test_multi_thread_predict(self):
         end = time.time()
         logging.info("Runtime for %s workers: %s", num_workers, end - begin)
 
+  def test_self_evaluation_gbt(self):
+    # This model is a classification model with full training logs.
+    gbt_adult_base_with_na_path = os.path.join(
+        test_utils.ydf_test_data_path(), "golden", "gbt_adult_base_with_na"
+    )
+    gbt_adult_base_with_na = model_lib.load_model(gbt_adult_base_with_na_path)
+    self_evaluation = gbt_adult_base_with_na.self_evaluation()
+    self.assertAlmostEqual(self_evaluation.accuracy, 0.8498403)
+
+  def test_self_evaluation_rf(self):
+    self_evaluation = self.adult_binary_class_rf.self_evaluation()
+    self.assertAlmostEqual(self_evaluation.loss, 0.31474323732)
+
+  def test_empty_self_evaluation_rf(self):
+    # Uplift models do not have OOB evaluations.
+    model_path = os.path.join(
+        test_utils.ydf_test_data_path(),
+        "model",
+        "sim_pte_categorical_uplift_rf",
+    )
+    model = model_lib.load_model(model_path)
+    self.assertIsNone(model.self_evaluation())
+
 
 if __name__ == "__main__":
   absltest.main()

yggdrasil_decision_forests/port/python/ydf/model/random_forest_model/BUILD

@@ -31,6 +31,7 @@ py_library(
         "//ydf/cc:ydf",
         "//ydf/metric",
         "//ydf/model/decision_forest_model",
+        "@ydf_cc//yggdrasil_decision_forests/metric:metric_py_proto",
         "@ydf_cc//yggdrasil_decision_forests/model/random_forest:random_forest_py_proto",
     ],
 )

yggdrasil_decision_forests/port/python/ydf/model/random_forest_model/random_forest_model.py

@@ -15,7 +15,8 @@
 """Definitions for Random Forest models."""
 
 import dataclasses
-from typing import Sequence
+from typing import Optional, Sequence
+from yggdrasil_decision_forests.metric import metric_pb2
 from yggdrasil_decision_forests.model.random_forest import random_forest_pb2
 from ydf.cc import ydf
 from ydf.metric import metric
@@ -68,7 +69,7 @@ def out_of_bag_evaluations(self) -> Sequence[OutOfBagEvaluation]:
     train_ds = pd.read_csv("train.csv")
     learner = ydf.RandomForestLearner(label="label",
                                       compute_oob_performances=True)
-    model = ydf.train(train_ds)
+    model = learner.train(train_ds)
 
     oob_evaluations = model.out_of_bag_evaluations()
     # In an interactive Python environment, print a rich evaluation report.
@@ -101,3 +102,41 @@ def winner_takes_all(self) -> bool:
     function is arbitrary and does not influence model inference.
     """
     return self._model.winner_takes_all()
+
+  def self_evaluation(self) -> Optional[metric.Evaluation]:
+    """Returns the model's self-evaluation.
+
+    For Random Forest models, the self-evaluation is out-of-bag evaluation on
+    the full model. Note that the Random Forest models do not use a validation
+    dataset. If out-of-bag evaluation is not enabled, no self-evaluation is
+    computed.
+
+    Different models use different methods for self-evaluation. Notably,
+    Gradient Boosted Trees use the evaluation on the validation dataset.
+    Therefore, self-evaluations are not comparable between different model
+    types.
+
+    Returns None if no self-evaluation has been computed.
+
+    Usage example:
+
+    ```python
+    import pandas as pd
+    import ydf
+
+    # Train model
+    train_ds = pd.read_csv("train.csv")
+    learner = ydf.RandomForestLearner(label="label",
+                                    compute_oob_performances=True)
+    model = learner.train(train_ds)
+
+    self_evaluation = model.self_evaluation()
+    # In an interactive Python environment, print a rich evaluation report.
+    self_evaluation
+    ```
+    """
+    oob_evaluation = self.out_of_bag_evaluations()
+    if oob_evaluation:
+      return oob_evaluation[-1].evaluation
+    # Return an empty evaluation object.
+    return None