From 152cbf4e4909758f280e1e86a6110248cc672083 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 10 Aug 2023 15:33:26 +0200 Subject: [PATCH 001/207] Added torchmetrics --- environment.yml | 1 + icu_benchmarks/models/constants.py | 5 ++++- icu_benchmarks/models/{metrics.py => custom_metrics.py} | 0 3 files changed, 5 insertions(+), 1 deletion(-) rename icu_benchmarks/models/{metrics.py => custom_metrics.py} (100%) diff --git a/environment.yml b/environment.yml index f2db5efc..3d1941d7 100644 --- a/environment.yml +++ b/environment.yml @@ -23,6 +23,7 @@ dependencies: - tensorboard=2.12.2 - tqdm=4.64.1 - pytorch-lightning=2.0.3 + - torchmetrics=1.0.3 - wandb=0.15.4 - pip=23.1 - einops=0.6.1 diff --git a/icu_benchmarks/models/constants.py b/icu_benchmarks/models/constants.py index 70ba5b65..bd6a24f6 100644 --- a/icu_benchmarks/models/constants.py +++ b/icu_benchmarks/models/constants.py @@ -14,9 +14,10 @@ mean_squared_error, # f1_score, ) +from torchmetrics.classification import BinaryFairness from enum import Enum -from icu_benchmarks.models.metrics import CalibrationCurve, BalancedAccuracy, MAE, JSD +from icu_benchmarks.models.custom_metrics import CalibrationCurve, BalancedAccuracy, MAE, JSD # TODO: revise transformation for metrics in wrappers.py in order to handle metrics that can not handle a mix of binary and @@ -30,6 +31,7 @@ class MLMetrics: "PR": average_precision_score, "PR_Curve": precision_recall_curve, "RO_Curve": roc_curve, + "Binary_Fairness": BinaryFairness(2, 'demographic_parity'), } MULTICLASS_CLASSIFICATION = { @@ -57,6 +59,7 @@ class DLMetrics: "PR": AveragePrecision, "PR_Curve": PrecisionRecallCurve, "RO_Curve": RocCurve, + "Binary_Fairness": BinaryFairness(num_groups=2, task='demographic_parity'), } MULTICLASS_CLASSIFICATION = { diff --git a/icu_benchmarks/models/metrics.py b/icu_benchmarks/models/custom_metrics.py similarity index 100% rename from icu_benchmarks/models/metrics.py rename to icu_benchmarks/models/custom_metrics.py From a8d7eaabc2789ede029b7c0eb527bc055390d65e Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 10 Aug 2023 15:33:39 +0200 Subject: [PATCH 002/207] refactoring --- icu_benchmarks/models/wrappers.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index 5fc94a12..9498fbe1 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -3,8 +3,7 @@ from typing import Dict, Any from typing import List, Optional, Union -import sklearn.metrics -from sklearn.metrics import log_loss +from sklearn.metrics import log_loss, mean_squared_error from torch.nn import MSELoss, CrossEntropyLoss from torch.nn.modules.loss import _Loss from torch.optim import Optimizer @@ -25,8 +24,8 @@ gin.config.external_configurable(torch.nn.functional.cross_entropy, module="torch.nn.functional") gin.config.external_configurable(torch.nn.functional.mse_loss, module="torch.nn.functional") -gin.config.external_configurable(sklearn.metrics.mean_squared_error, module="sklearn.metrics") -gin.config.external_configurable(sklearn.metrics.log_loss, module="sklearn.metrics") +gin.config.external_configurable(mean_squared_error, module="sklearn.metrics") +gin.config.external_configurable(log_loss, module="sklearn.metrics") @gin.configurable("BaseModule") From 63c57b59d531cb56f8e560cd7abb54a01848a57d Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 10 Aug 2023 15:34:10 +0200 Subject: [PATCH 003/207] removed obsolete experiment files --- configs/experiments/LGBM_Mortality.gin | 7 ------- configs/experiments/LSTM_Mortality.gin | 9 --------- 2 files changed, 16 deletions(-) delete mode 100644 configs/experiments/LGBM_Mortality.gin delete mode 100644 configs/experiments/LSTM_Mortality.gin diff --git a/configs/experiments/LGBM_Mortality.gin b/configs/experiments/LGBM_Mortality.gin deleted file mode 100644 index 2e5abf3b..00000000 --- a/configs/experiments/LGBM_Mortality.gin +++ /dev/null @@ -1,7 +0,0 @@ -include "configs/tasks/BinaryClassification.gin" -include "configs/models/LGBMClassifier.gin" - -model/hyperparameter.max_depth = 7 -model/hyperparameter.num_leaves = 32 -model/hyperparameter.subsample = 1 -model/hyperparameter.colsample_bytree = 0.66 diff --git a/configs/experiments/LSTM_Mortality.gin b/configs/experiments/LSTM_Mortality.gin deleted file mode 100644 index 7226f37f..00000000 --- a/configs/experiments/LSTM_Mortality.gin +++ /dev/null @@ -1,9 +0,0 @@ -include "configs/tasks/BinaryClassification.gin" -include "configs/models/LSTM.gin" - -# Optimizer params -optimizer/hyperparameter.lr = 1e-4 - -# Encoder params -model/hyperparameter.hidden_dim = 128 -model/hyperparameter.layer_dim = 1 From 5bad6a155cdfc8b3c327b61d9c77a13be27ff64e Mon Sep 17 00:00:00 2001 From: youssef mecky Date: Tue, 13 Jun 2023 14:24:40 +0200 Subject: [PATCH 004/207] type casted float64 to float32 to make it work with MPS --- icu_benchmarks/data/loader.py | 2 +- icu_benchmarks/models/wrappers.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/icu_benchmarks/data/loader.py b/icu_benchmarks/data/loader.py index 9b831aa7..d0e040e4 100644 --- a/icu_benchmarks/data/loader.py +++ b/icu_benchmarks/data/loader.py @@ -151,7 +151,7 @@ def get_data_and_labels(self) -> Tuple[np.array, np.array]: def to_tensor(self): data, labels = self.get_data_and_labels() - return from_numpy(data), from_numpy(labels) + return from_numpy(data).to(float32), from_numpy(labels).to(float32) @gin.configurable("ImputationDataset") diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index 5fc94a12..867e9874 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -135,7 +135,7 @@ def finalize_step(self, step_prefix=""): try: self.log_dict( { - f"{step_prefix}/{name}": metric.compute() + f"{step_prefix}/{name}": (np.float32(metric.compute()) if isinstance(metric.compute(), np.float64 ) else metric.compute() ) for name, metric in self.metrics[step_prefix].items() if "_Curve" not in name }, @@ -358,9 +358,9 @@ def test_step(self, dataset, _): self.set_metrics(test_label) test_pred = self.predict(test_rep) - self.log("test/loss", self.loss(test_label, test_pred), sync_dist=True) + self.log("test/loss", np.float32(self.loss(test_label, test_pred)), sync_dist=True) logging.debug(f"Test loss: {self.loss(test_label, test_pred)}") - self.log_metrics(test_label, test_pred, "test") + self.log_metrics(np.float32(test_label), np.float32(test_pred), "test") def predict(self, features): if self.run_mode == RunMode.regression: @@ -373,7 +373,7 @@ def log_metrics(self, label, pred, metric_type): self.log_dict( { - f"{metric_type}/{name}": metric(self.label_transform(label), self.output_transform(pred)) + f"{metric_type}/{name}": np.float32(metric(self.label_transform(label), self.output_transform(pred))) for name, metric in self.metrics.items() # Filter out metrics that return a tuple (e.g. precision_recall_curve) if not isinstance(metric(self.label_transform(label), self.output_transform(pred)), tuple) From 93263930c625cca08b9006eff926b0cdf494f882 Mon Sep 17 00:00:00 2001 From: Hendrik Schmidt Date: Fri, 11 Aug 2023 12:20:43 +0200 Subject: [PATCH 005/207] make reproducible argument for evaluate too --- icu_benchmarks/run_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/icu_benchmarks/run_utils.py b/icu_benchmarks/run_utils.py index 179ab1f5..4462e25e 100644 --- a/icu_benchmarks/run_utils.py +++ b/icu_benchmarks/run_utils.py @@ -51,6 +51,9 @@ def build_parser() -> ArgumentParser: general_args.add_argument( "-db", "--debug", required=False, default=False, action=BooleanOptionalAction, help="Set to load less data." ) + general_args.add_argument( + "--reproducible", required=False, default=True, action=BooleanOptionalAction, help="Make torch reproducible." + ) general_args.add_argument( "-lc", "--load_cache", @@ -78,9 +81,6 @@ def build_parser() -> ArgumentParser: # MODEL TRAINING ARGUMENTS prep_and_train = subparsers.add_parser("train", help="Preprocess features and train model.", parents=[parent_parser]) - prep_and_train.add_argument( - "--reproducible", required=False, default=True, action=BooleanOptionalAction, help="Make torch reproducible." - ) prep_and_train.add_argument("-hp", "--hyperparams", required=False, nargs="+", help="Hyperparameters for model.") prep_and_train.add_argument("--tune", default=False, action=BooleanOptionalAction, help="Find best hyperparameters.") prep_and_train.add_argument("--checkpoint", required=False, type=Path, help="Use previous checkpoint.") From c817ff709078670a1eebbfa85b5002e23b345c0d Mon Sep 17 00:00:00 2001 From: rvandewater Date: Fri, 11 Aug 2023 13:02:01 +0200 Subject: [PATCH 006/207] renamed needs_training to requires_backprop, removed needs_fit. --- icu_benchmarks/imputation/baselines.py | 33 ++++++++----------- icu_benchmarks/imputation/diffusion.py | 3 +- icu_benchmarks/imputation/mlp.py | 3 +- icu_benchmarks/imputation/np.py | 3 +- icu_benchmarks/imputation/rnn.py | 6 ++-- icu_benchmarks/imputation/simple_diffusion.py | 3 +- icu_benchmarks/models/train.py | 19 +++++------ 7 files changed, 28 insertions(+), 42 deletions(-) diff --git a/icu_benchmarks/imputation/baselines.py b/icu_benchmarks/imputation/baselines.py index 27ddf307..75b23fc5 100644 --- a/icu_benchmarks/imputation/baselines.py +++ b/icu_benchmarks/imputation/baselines.py @@ -15,8 +15,7 @@ class KNNImputation(ImputationWrapper): """Imputation using Scikit-Learn K-Nearest Neighbour.""" - needs_training = False - needs_fit = True + requires_backprop = False def __init__(self, *args, n_neighbors=2, **kwargs) -> None: super().__init__(*args, n_neighbors=n_neighbors, **kwargs) @@ -38,8 +37,7 @@ def forward(self, amputated_values, amputation_mask): class MICEImputation(ImputationWrapper): """Imputation using Scikit-Learn MICE.""" - needs_training = False - needs_fit = True + requires_backprop = False def __init__(self, *args, max_iter=100, verbose=2, imputation_order="random", random_state=0, **kwargs) -> None: super().__init__( @@ -69,8 +67,7 @@ def forward(self, amputated_values, amputation_mask): class MeanImputation(ImputationWrapper): """Mean imputation using Scikit-Learn SimpleImputer.""" - needs_training = False - needs_fit = True + requires_backprop = False def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) @@ -90,8 +87,9 @@ def forward(self, amputated_values, amputation_mask): @gin.configurable("Median") class MedianImputation(ImputationWrapper): - needs_training = False - needs_fit = True + """Median imputation using Scikit-Learn SimpleImputer.""" + + requires_backprop = False def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) @@ -113,8 +111,7 @@ def forward(self, amputated_values, amputation_mask): class ZeroImputation(ImputationWrapper): """Zero imputation using Scikit-Learn SimpleImputer.""" - needs_training = False - needs_fit = True + requires_backprop = False def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) @@ -136,8 +133,7 @@ def forward(self, amputated_values, amputation_mask): class MostFrequentImputation(ImputationWrapper): """Most frequent imputation using Scikit-Learn SimpleImputer.""" - needs_training = False - needs_fit = True + requires_backprop = False def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) @@ -159,8 +155,7 @@ def wrap_hyperimpute_model(methodName: str, configName: str) -> Type: class HyperImputeImputation(ImputationWrapper): """Imputation using HyperImpute package.""" - needs_training = False - needs_fit = True + requires_backprop = False def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) @@ -198,8 +193,7 @@ def forward(self, amputated_values, amputation_mask): class BRITSImputation(ImputationWrapper): """Bidirectional Recurrent Imputation for Time Series (BRITS) imputation using PyPots package.""" - needs_training = False - needs_fit = True + requires_backprop = False def __init__(self, *args, input_size, epochs=1, rnn_hidden_size=64, batch_size=256, **kwargs) -> None: super().__init__( @@ -234,8 +228,7 @@ def forward(self, amputated_values, amputation_mask): class SAITSImputation(ImputationWrapper): """Self-Attention based Imputation for Time Series (SAITS) imputation using PyPots package.""" - needs_training = False - needs_fit = True + requires_backprop = False def __init__(self, *args, input_size, epochs, n_layers, d_model, d_inner, n_head, d_k, d_v, dropout, **kwargs) -> None: super().__init__( @@ -284,8 +277,8 @@ def forward(self, amputated_values, amputation_mask): class AttentionImputation(ImputationWrapper): """Attention based Imputation (Transformer) imputation using PyPots package.""" - needs_training = False - needs_fit = True + # Handled within the library + requires_backprop = False def __init__(self, *args, input_size, epochs, n_layers, d_model, d_inner, n_head, d_k, d_v, dropout, **kwargs) -> None: super().__init__( diff --git a/icu_benchmarks/imputation/diffusion.py b/icu_benchmarks/imputation/diffusion.py index 2f270bd3..c841d703 100644 --- a/icu_benchmarks/imputation/diffusion.py +++ b/icu_benchmarks/imputation/diffusion.py @@ -20,8 +20,7 @@ class SimpleDiffusionModel(ImputationWrapper): """Simple Diffusion Model for Imputation. See https://arxiv.org/abs/2006.11239 for more details.""" - needs_training = True - needs_fit = False + requires_backprop = True input_size = [] diff --git a/icu_benchmarks/imputation/mlp.py b/icu_benchmarks/imputation/mlp.py index 68d98e23..e6062814 100644 --- a/icu_benchmarks/imputation/mlp.py +++ b/icu_benchmarks/imputation/mlp.py @@ -8,8 +8,7 @@ class MLPImputation(ImputationWrapper): """Imputation model based on a Multi-Layer Perceptron (MLP).""" - needs_training = True - needs_fit = False + requires_backprop = True def __init__(self, *args, input_size, num_hidden_layers=3, hidden_layer_size=10, **kwargs) -> None: super().__init__( diff --git a/icu_benchmarks/imputation/np.py b/icu_benchmarks/imputation/np.py index 6a974fe5..6f0d766d 100644 --- a/icu_benchmarks/imputation/np.py +++ b/icu_benchmarks/imputation/np.py @@ -13,8 +13,7 @@ class NPImputation(ImputationWrapper): """Imputation using Neural Processes. Implementation adapted from https://github.com/EmilienDupont/neural-processes/. Provides imputation wrapper for NeuralProcess class.""" - needs_training = True - needs_fit = False + requires_backprop = True def __init__( self, diff --git a/icu_benchmarks/imputation/rnn.py b/icu_benchmarks/imputation/rnn.py index 9f6a7213..1d514c32 100644 --- a/icu_benchmarks/imputation/rnn.py +++ b/icu_benchmarks/imputation/rnn.py @@ -10,8 +10,7 @@ class RNNImputation(ImputationWrapper): """Imputation model with Gated Recurrent Units (GRU) or Long-Short Term Memory Network (LSTM). Defaults to GRU.""" - needs_training = True - needs_fit = False + requires_backprop = True def __init__(self, *args, input_size, hidden_size=64, state_init="zero", cell="gru", **kwargs) -> None: super().__init__(*args, input_size=input_size, hidden_size=hidden_size, state_init=state_init, cell=cell, **kwargs) @@ -69,8 +68,7 @@ class BRNNImputation(ImputationWrapper): """Imputation model with Bidirectional Gated Recurrent Units (GRU) or Long-Short Term Memory Network (LSTM). Defaults to GRU.""" - needs_training = True - needs_fit = False + requires_backprop = True def __init__(self, *args, input_size, hidden_size=64, state_init="zero", dropout=0.0, cell="gru", **kwargs) -> None: super().__init__( diff --git a/icu_benchmarks/imputation/simple_diffusion.py b/icu_benchmarks/imputation/simple_diffusion.py index 3590a5a0..f2c23d10 100644 --- a/icu_benchmarks/imputation/simple_diffusion.py +++ b/icu_benchmarks/imputation/simple_diffusion.py @@ -11,8 +11,7 @@ class SimpleDiffusionModel(ImputationWrapper): """Imputation model based on a Simple Diffusion Model. Adapted from https://colab.research.google.com/drive/1sjy9odlSSy0RBVgMTgP7s99NXsqglsUL.""" - needs_training = True - needs_fit = False + requires_backprop = True input_size = [] diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 1c2a91cd..187720e3 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -108,7 +108,7 @@ def train_common( model.set_weight(weight, train_dataset) if load_weights: if source_dir.exists(): - # if not model.needs_training: + # if not model.requires_backprop: checkpoint = torch.load(source_dir / "model.ckpt") # else: model = model.load_state_dict(checkpoint["state_dict"]) @@ -128,26 +128,25 @@ def train_common( if precision == 16 or "16-mixed": torch.set_float32_matmul_precision("medium") trainer = Trainer( - max_epochs=epochs if model.needs_training else 1, + max_epochs=epochs if model.requires_backprop else 1, callbacks=callbacks, precision=precision, accelerator="auto" if not cpu else "cpu", devices=max(torch.cuda.device_count(), 1), - deterministic=reproducible, + deterministic="warn" if reproducible else False, benchmark=not reproducible, enable_progress_bar=verbose, logger=loggers, num_sanity_val_steps=0, ) - if model.needs_fit: - logging.info("Fitting model to data.") + if not model.requires_backprop: + logging.info("Training ML model.") model.fit(train_dataset, val_dataset) model.save_model(log_dir, "last") - logging.info("Fitting complete.") - - if model.needs_training: - logging.info("Training model.") + logging.info("Training complete.") + else: + logging.info("Training DL model.") trainer.fit(model, train_dataloaders=train_loader, val_dataloaders=val_loader) logging.info("Training complete.") @@ -162,7 +161,7 @@ def train_common( pin_memory=True, drop_last=True, ) - if model.needs_training + if model.requires_backprop else DataLoader([test_dataset.to_tensor()], batch_size=1) ) From 7678dd2ef4aae30d7e1815b563869e741b5201bb Mon Sep 17 00:00:00 2001 From: rvandewater Date: Fri, 11 Aug 2023 13:02:16 +0200 Subject: [PATCH 007/207] Documentation update --- docs/imputation_methods.md | 41 +++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/docs/imputation_methods.md b/docs/imputation_methods.md index 2eead00b..26e73058 100644 --- a/docs/imputation_methods.md +++ b/docs/imputation_methods.md @@ -7,28 +7,29 @@ To add another imputation model, you have to create a class that inherits from ` from icu_benchmarks.models.wrappers import ImputationWrapper import gin + @gin.configurable("newmethod") class New_Method(ImputationWrapper): - - # adjust this accordingly - needs_training = False # if true, the method is trained iteratively (like a deep learning model) - needs_fit = True # if true, it receives the complete training data to perform a fit on - - def __init__(self, *args, model_arg1, model_arg2, **kwargs): - super().__init__(*args, **kwargs) - # define your new model here - self.model = ... - - # the following method has to be implemented for all methods - def forward(self, amputated_values, amputation_mask): - imputated_values = amputated_values - ... - return imputated_values - - # implement this, if needs_fit is true, otherwise you can leave it out. - # this method receives the complete input training data to perform a fit on. - def fit(self, train_data): - ... + # adjust this accordingly + # if true, the method is trained iteratively (like a deep learning model). + # If false it receives the complete training data to perform a fit on + requires_backprop = False + + def __init__(self, *args, model_arg1, model_arg2, **kwargs): + super().__init__(*args, **kwargs) + # define your new model here + self.model = ... + + # the following method has to be implemented for all methods + def forward(self, amputated_values, amputation_mask): + imputated_values = amputated_values + ... + return imputated_values + + # implement this, if needs_fit is true, otherwise you can leave it out. + # this method receives the complete input training data to perform a fit on. + def fit(self, train_data): + ... ``` You also need to create a gin configuration file in the `configs/imputation` directory, From cd9144989c34da2906c82c95aad46e656a71e5ae Mon Sep 17 00:00:00 2001 From: Hendrik Schmidt Date: Fri, 11 Aug 2023 16:21:42 +0200 Subject: [PATCH 008/207] fix model loading for ML --- icu_benchmarks/models/train.py | 10 ++++++---- icu_benchmarks/models/wrappers.py | 6 +++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 1c2a91cd..36fcc398 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -3,6 +3,7 @@ import torch import logging import pandas as pd +import pickle from torch.optim import Adam from torch.utils.data import DataLoader from pytorch_lightning.loggers import TensorBoardLogger @@ -108,10 +109,11 @@ def train_common( model.set_weight(weight, train_dataset) if load_weights: if source_dir.exists(): - # if not model.needs_training: - checkpoint = torch.load(source_dir / "model.ckpt") - # else: - model = model.load_state_dict(checkpoint["state_dict"]) + if model.needs_training: + checkpoint = torch.load(source_dir / "model.ckpt") + model = model.load_state_dict(checkpoint) + else: + model = torch.load(source_dir / "last.ckpt") else: raise Exception(f"No weights to load at path : {source_dir}") diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index 867e9874..9853ae51 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -11,11 +11,11 @@ import inspect import gin import numpy as np +import pickle import torch from ignite.exceptions import NotComputableError from icu_benchmarks.models.constants import ImputationInit from icu_benchmarks.models.utils import create_optimizer, create_scheduler -from joblib import dump from pytorch_lightning import LightningModule from icu_benchmarks.models.constants import MLMetrics, DLMetrics @@ -390,10 +390,10 @@ def __getstate__(self) -> Dict[str, Any]: del state["output_transform"] return state - def save_model(self, save_path, file_name, file_extension=".joblib"): + def save_model(self, save_path, file_name, file_extension=".ckpt"): path = save_path / (file_name + file_extension) try: - dump(self.model, path) + torch.save(self, path) logging.info(f"Model saved to {str(path.resolve())}.") except Exception as e: logging.error(f"Cannot save model to path {str(path.resolve())}: {e}.") From cd5ee98cf649f81051e60e0c9a8adc867284c33f Mon Sep 17 00:00:00 2001 From: rvandewater Date: Mon, 14 Aug 2023 14:55:18 +0200 Subject: [PATCH 009/207] rework metric constants and added binaryfairness wrapper --- icu_benchmarks/data/loader.py | 2 +- icu_benchmarks/models/constants.py | 22 ++-- icu_benchmarks/models/custom_metrics.py | 96 ++++++++++------ icu_benchmarks/models/wrappers.py | 142 +++++++++++++++++------- 4 files changed, 179 insertions(+), 83 deletions(-) diff --git a/icu_benchmarks/data/loader.py b/icu_benchmarks/data/loader.py index 9b831aa7..7c9d1400 100644 --- a/icu_benchmarks/data/loader.py +++ b/icu_benchmarks/data/loader.py @@ -19,7 +19,6 @@ class CommonDataset(Dataset): columns in the data. grouping_segment: str, optional: The segment of the data contains the grouping column with only unique values. Defaults to Segment.outcome. Is used to calculate the number of stays in the data. """ - def __init__( self, data: dict, @@ -154,6 +153,7 @@ def to_tensor(self): return from_numpy(data), from_numpy(labels) + @gin.configurable("ImputationDataset") class ImputationDataset(CommonDataset): """Subclass of Common Dataset that contains data for imputation models.""" diff --git a/icu_benchmarks/models/constants.py b/icu_benchmarks/models/constants.py index bd6a24f6..b571e7ab 100644 --- a/icu_benchmarks/models/constants.py +++ b/icu_benchmarks/models/constants.py @@ -14,10 +14,10 @@ mean_squared_error, # f1_score, ) -from torchmetrics.classification import BinaryFairness +from torchmetrics.classification import BinaryFairness, AUROC, AveragePrecision, \ + PrecisionRecallCurve, ROC, CalibrationError, F1Score from enum import Enum - -from icu_benchmarks.models.custom_metrics import CalibrationCurve, BalancedAccuracy, MAE, JSD +from icu_benchmarks.models.custom_metrics import CalibrationCurve, BalancedAccuracy, MAE, JSD, TorchMetricsWrapper, BinaryFairnessWrapper # TODO: revise transformation for metrics in wrappers.py in order to handle metrics that can not handle a mix of binary and @@ -53,13 +53,17 @@ class MLMetrics: # TODO: add support for confusion matrix class DLMetrics: BINARY_CLASSIFICATION = { - "AUC": ROC_AUC, - "Calibration_Curve": CalibrationCurve, + "AUC": AUROC(task="binary"), + "AUC_ignite": ROC_AUC, + # "PR" : AveragePrecision(task="binary"), + # "F1": F1Score(task="binary", num_classes=2), + "Calibration_Error": CalibrationError(task="binary",n_bins=10), + # "Calibration_Curve": CalibrationCurve, # "Confusion_Matrix": ConfusionMatrix(num_classes=2), - "PR": AveragePrecision, - "PR_Curve": PrecisionRecallCurve, - "RO_Curve": RocCurve, - "Binary_Fairness": BinaryFairness(num_groups=2, task='demographic_parity'), + # "PR": TorchMetricsWrapper(AveragePrecision(task="binary")), + # "PR_Curve": PrecisionRecallCurve, + # "RO_Curve": RocCurve, + "Binary_Fairness": BinaryFairnessWrapper(num_groups=2, task='demographic_parity', group_name="sex"), } MULTICLASS_CLASSIFICATION = { diff --git a/icu_benchmarks/models/custom_metrics.py b/icu_benchmarks/models/custom_metrics.py index 280a0653..c1ad3846 100644 --- a/icu_benchmarks/models/custom_metrics.py +++ b/icu_benchmarks/models/custom_metrics.py @@ -5,10 +5,11 @@ from sklearn.metrics import balanced_accuracy_score, mean_absolute_error from sklearn.calibration import calibration_curve from scipy.spatial.distance import jensenshannon +from torchmetrics.classification import BinaryFairness """" -This file contains metrics that are not available in ignite.metrics. Specifically, it adds transformation capabilities to some -metrics. +This file contains custom metrics that are not available in ignite.metrics. +Specifically, it adds transformation capabilities to some metrics. """ @@ -29,48 +30,36 @@ def accuracy(output, target, topk=(1,)): return res -def balanced_accuracy_compute_fn(y_preds: torch.Tensor, y_targets: torch.Tensor) -> float: - y_true = y_targets.numpy() - y_pred = np.argmax(y_preds.numpy(), axis=-1) - return balanced_accuracy_score(y_true, y_pred) - - -def ece_curve_compute_fn(y_preds: torch.Tensor, y_targets: torch.Tensor) -> float: - y_true = y_targets.numpy() - y_pred = y_preds.numpy() - return calibration_curve(y_true, y_pred, n_bins=10) - - -def mae_with_invert_compute_fn(y_preds: torch.Tensor, y_targets: torch.Tensor, invert_fn=Callable) -> float: - y_true = invert_fn(y_targets.numpy().reshape(-1, 1))[:, 0] - y_pred = invert_fn(y_preds.numpy().reshape(-1, 1))[:, 0] - return mean_absolute_error(y_true, y_pred) - - -def JSD_fn(y_preds: torch.Tensor, y_targets: torch.Tensor): - return jensenshannon(abs(y_preds).flatten(), abs(y_targets).flatten()) ** 2 - - class BalancedAccuracy(EpochMetric): def __init__(self, output_transform: Callable = lambda x: x, check_compute_fn: bool = False) -> None: super(BalancedAccuracy, self).__init__( - balanced_accuracy_compute_fn, output_transform=output_transform, check_compute_fn=check_compute_fn + self.balanced_accuracy_compute, output_transform=output_transform, check_compute_fn=check_compute_fn ) + def balanced_accuracy_compute(y_preds: torch.Tensor, y_targets: torch.Tensor) -> float: + y_true = y_targets.numpy() + y_pred = np.argmax(y_preds.numpy(), axis=-1) + return balanced_accuracy_score(y_true, y_pred) + class CalibrationCurve(EpochMetric): def __init__(self, output_transform: Callable = lambda x: x, check_compute_fn: bool = False) -> None: super(CalibrationCurve, self).__init__( - ece_curve_compute_fn, output_transform=output_transform, check_compute_fn=check_compute_fn + self.ece_curve_compute_fn, output_transform=output_transform, check_compute_fn=check_compute_fn ) + def ece_curve_compute_fn(y_preds: torch.Tensor, y_targets: torch.Tensor, n_bins=10) -> float: + y_true = y_targets.numpy() + y_pred = y_preds.numpy() + return calibration_curve(y_true, y_pred, n_bins=n_bins) + class MAE(EpochMetric): def __init__( - self, - output_transform: Callable = lambda x: x, - check_compute_fn: bool = False, - invert_transform: Callable = lambda x: x, + self, + output_transform: Callable = lambda x: x, + check_compute_fn: bool = False, + invert_transform: Callable = lambda x: x, ) -> None: super(MAE, self).__init__( lambda x, y: mae_with_invert_compute_fn(x, y, invert_transform), @@ -78,15 +67,56 @@ def __init__( check_compute_fn=check_compute_fn, ) + def mae_with_invert_compute_fn(y_preds: torch.Tensor, y_targets: torch.Tensor, invert_fn=Callable) -> float: + y_true = invert_fn(y_targets.numpy().reshape(-1, 1))[:, 0] + y_pred = invert_fn(y_preds.numpy().reshape(-1, 1))[:, 0] + return mean_absolute_error(y_true, y_pred) + class JSD(EpochMetric): def __init__( - self, - output_transform: Callable = lambda x: x, - check_compute_fn: bool = False, + self, + output_transform: Callable = lambda x: x, + check_compute_fn: bool = False, ) -> None: super(JSD, self).__init__( lambda x, y: JSD_fn(x, y), output_transform=output_transform, check_compute_fn=check_compute_fn, ) + + def JSD_fn(y_preds: torch.Tensor, y_targets: torch.Tensor): + return jensenshannon(abs(y_preds).flatten(), abs(y_targets).flatten()) ** 2 + + +class TorchMetricsWrapper(): + metric = None + + def __init__(self, metric) -> None: + self.metric = metric + + def update(self, output_tuple) -> None: + self.metric.update(output_tuple[0], output_tuple[1]) + + def compute(self) -> None: + return self.metric.compute() + + def reset(self) -> None: + return self.metric.reset() + + +class BinaryFairnessWrapper(BinaryFairness): + """ + This class is a wrapper for the BinaryFairness metric from TorchMetrics. + """ + group_name = None + def __init__(self, group_name = "sex", *args, **kwargs) -> None: + self.group_name = group_name + super().__init__(*args, **kwargs) + def update(self, preds, target, data, feature_names) -> None: + """" Standard metric update function""" + groups = data[:, :, feature_names.index(self.group_name)] + group_per_id = groups[:, 0] + return super().update(preds=preds.cpu(), + target=target.cpu(), + groups=group_per_id.long().cpu()) diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index 9498fbe1..c48e0304 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -1,16 +1,19 @@ import logging from abc import ABC -from typing import Dict, Any -from typing import List, Optional, Union +from typing import Dict, Any, List, Optional, Union +import torchmetrics from sklearn.metrics import log_loss, mean_squared_error + +import torch from torch.nn import MSELoss, CrossEntropyLoss -from torch.nn.modules.loss import _Loss -from torch.optim import Optimizer +import torch.nn as nn +from torch import Tensor, FloatTensor +from torch.optim import Optimizer, Adam + import inspect import gin import numpy as np -import torch from ignite.exceptions import NotComputableError from icu_benchmarks.models.constants import ImputationInit from icu_benchmarks.models.utils import create_optimizer, create_scheduler @@ -20,9 +23,9 @@ from icu_benchmarks.models.constants import MLMetrics, DLMetrics from icu_benchmarks.contants import RunMode -gin.config.external_configurable(torch.nn.functional.nll_loss, module="torch.nn.functional") -gin.config.external_configurable(torch.nn.functional.cross_entropy, module="torch.nn.functional") -gin.config.external_configurable(torch.nn.functional.mse_loss, module="torch.nn.functional") +gin.config.external_configurable(nn.functional.nll_loss, module="torch.nn.functional") +gin.config.external_configurable(nn.functional.cross_entropy, module="torch.nn.functional") +gin.config.external_configurable(nn.functional.mse_loss, module="torch.nn.functional") gin.config.external_configurable(mean_squared_error, module="sklearn.metrics") gin.config.external_configurable(log_loss, module="sklearn.metrics") @@ -30,12 +33,14 @@ @gin.configurable("BaseModule") class BaseModule(LightningModule): - needs_training = False - needs_fit = False - + # DL type models, requires backpropagation + requires_backprop = False + # Loss function weight initialization type weight = None + # Metrics to be logged metrics = {} trained_columns = None + # Type of run mode run_mode = None def forward(self, *args, **kwargs): @@ -90,15 +95,14 @@ def check_supported_runmode(self, runmode: RunMode): @gin.configurable("DLWrapper") class DLWrapper(BaseModule, ABC): - needs_training = True - needs_fit = False + requires_backprop = True _metrics_warning_printed = set() _supported_run_modes = [RunMode.classification, RunMode.regression, RunMode.imputation] def __init__( self, loss=CrossEntropyLoss(), - optimizer=torch.optim.Adam, + optimizer=Adam, run_mode: RunMode = RunMode.classification, input_shape=None, lr: float = 0.002, @@ -107,11 +111,11 @@ def __init__( lr_factor: float = 0.99, lr_steps: Optional[List[int]] = None, epochs: int = 100, - input_size: torch.Tensor = None, + input_size: Tensor = None, initialization_method: str = "normal", **kwargs, ): - """Interface for Deep Learning models.""" + """General interface for Deep Learning (DL) models.""" super().__init__() self.save_hyperparameters(ignore=["loss", "optimizer"]) self.loss = loss @@ -162,10 +166,10 @@ def configure_optimizers(self): return {"optimizer": optimizer, "lr_scheduler": scheduler} def on_test_epoch_start(self) -> None: - self.metrics = { - step_name: {metric_name: metric() for metric_name, metric in self.set_metrics().items()} - for step_name in ["train", "val", "test"] - } + # self.metrics = { + # step_name: {metric_name: metric() for metric_name, metric in self.set_metrics().items()} + # for step_name in ["train", "val", "test"] + # } return super().on_test_epoch_start() def save_model(self, save_path, file_name, file_extension=".ckpt"): @@ -183,13 +187,47 @@ class DLPredictionWrapper(DLWrapper): _supported_run_modes = [RunMode.classification, RunMode.regression] + def __init__( + self, + loss=CrossEntropyLoss(), + optimizer=torch.optim.Adam, + run_mode: RunMode = RunMode.classification, + input_shape=None, + lr: float = 0.002, + momentum: float = 0.9, + lr_scheduler: Optional[str] = None, + lr_factor: float = 0.99, + lr_steps: Optional[List[int]] = None, + epochs: int = 100, + input_size: Tensor = None, + initialization_method: str = "normal", + **kwargs, + ): + super().__init__( + loss=loss, + optimizer=optimizer, + run_mode=run_mode, + input_shape=input_shape, + lr=lr, + momentum=momentum, + lr_scheduler=lr_scheduler, + lr_factor=lr_factor, + lr_steps=lr_steps, + epochs=epochs, + input_size=input_size, + initialization_method=initialization_method, + kwargs=kwargs, + ) + self.output_transform = None + self.loss_weights = None + def set_weight(self, weight, dataset): """Set the weight for the loss function.""" if isinstance(weight, list): - weight = torch.FloatTensor(weight).to(self.device) + weight = FloatTensor(weight).to(self.device) elif weight == "balanced": - weight = torch.FloatTensor(dataset.get_balance()).to(self.device) + weight = FloatTensor(dataset.get_balance()).to(self.device) self.loss_weights = weight def set_metrics(self, *args): @@ -223,10 +261,19 @@ def softmax_multi_output_transform(output): metrics = DLMetrics.REGRESSION else: raise ValueError(f"Run mode {self.run_mode} not supported.") + for key, value in metrics.items(): + # Torchmetrics metrics are not moved to the device by default + if isinstance(value, torchmetrics.Metric): + value.to(self.device) return metrics def step_fn(self, element, step_prefix=""): - """Perform a step in the training loop.""" + """Perform a step in the DL prediction model training loop. + + Args: + element (object): + step_prefix (str): Step type, by default: test, train, val. + """ if len(element) == 2: data, labels = element[0], element[1].to(self.device) @@ -247,24 +294,41 @@ def step_fn(self, element, step_prefix=""): else: raise Exception("Loader should return either (data, label) or (data, label, mask)") out = self(data) + + # If aux_loss is present, it is returned as a tuple if len(out) == 2 and isinstance(out, tuple): out, aux_loss = out else: aux_loss = 0 + # Get prediction and target prediction = torch.masked_select(out, mask.unsqueeze(-1)).reshape(-1, out.shape[-1]).to(self.device) target = torch.masked_select(labels, mask).to(self.device) + if prediction.shape[-1] > 1 and self.run_mode == RunMode.classification: # Classification task loss = self.loss(prediction, target.long(), weight=self.loss_weights.to(self.device)) + aux_loss - # torch.long because NLL + # Returns torch.long because negative log likelihood loss elif self.run_mode == RunMode.regression: # Regression task loss = self.loss(prediction[:, 0], target.float()) + aux_loss else: - raise ValueError(f"Run mode {self.run_mode} not supported.") + raise ValueError(f"Run mode {self.run_mode} not yet supported. Please implement it.") transformed_output = self.output_transform((prediction, target)) - for metric in self.metrics[step_prefix].values(): - metric.update(transformed_output) + + for key, value in self.metrics[step_prefix].items(): + if isinstance(value, torchmetrics.Metric): + if key == "Binary_Fairness": + if step_prefix == "train": + feature_names = self.trainer.train_dataloader.dataset.features + elif step_prefix == "val": + feature_names = self.trainer.train_dataloader.dataset.features + else: + feature_names = self.trainer.test_dataloader.dataset.features + value.update(transformed_output[0], transformed_output[1], data, feature_names) + else: + value.update(transformed_output[0], transformed_output[1]) + else: + value.update(transformed_output) self.log(f"{step_prefix}/loss", loss, on_step=False, on_epoch=True, sync_dist=True) return loss @@ -273,8 +337,7 @@ def step_fn(self, element, step_prefix=""): class MLWrapper(BaseModule, ABC): """Interface for prediction with traditional Scikit-learn-like Machine Learning models.""" - needs_training = False - needs_fit = True + requires_backprop = False _supported_run_modes = [RunMode.classification, RunMode.regression] def __init__(self, *args, run_mode=RunMode.classification, loss=log_loss, patience=10, **kwargs): @@ -413,13 +476,12 @@ def set_model_args(self, model, *args, **kwargs): class ImputationWrapper(DLWrapper): """Interface for imputation models.""" - needs_training = True - needs_fit = False + requires_backprop = True _supported_run_modes = [RunMode.imputation] def __init__( self, - loss: _Loss = MSELoss(), + loss: nn.modules.loss._Loss = MSELoss(), optimizer: Union[str, Optimizer] = "adam", runmode: RunMode = RunMode.imputation, lr: float = 0.002, @@ -427,7 +489,7 @@ def __init__( lr_scheduler: Optional[str] = None, lr_factor: float = 0.99, lr_steps: Optional[List[int]] = None, - input_size: torch.Tensor = None, + input_size: Tensor = None, initialization_method: ImputationInit = ImputationInit.NORMAL, **kwargs: str, ) -> None: @@ -446,20 +508,20 @@ def init_func(m): classname = m.__class__.__name__ if hasattr(m, "weight") and (classname.find("Conv") != -1 or classname.find("Linear") != -1): if init_type == ImputationInit.NORMAL: - torch.nn.init.normal_(m.weight.data, 0.0, gain) + nn.init.normal_(m.weight.data, 0.0, gain) elif init_type == ImputationInit.XAVIER: - torch.nn.init.xavier_normal_(m.weight.data, gain=gain) + nn.init.xavier_normal_(m.weight.data, gain=gain) elif init_type == ImputationInit.KAIMING: - torch.nn.init.kaiming_normal_(m.weight.data, a=0, mode="fan_out") + nn.init.kaiming_normal_(m.weight.data, a=0, mode="fan_out") elif init_type == ImputationInit.ORTHOGONAL: - torch.nn.init.orthogonal_(m.weight.data, gain=gain) + nn.init.orthogonal_(m.weight.data, gain=gain) else: raise NotImplementedError(f"Initialization method {init_type} is not implemented") if hasattr(m, "bias") and m.bias is not None: - torch.nn.init.constant_(m.bias.data, 0.0) + nn.init.constant_(m.bias.data, 0.0) elif classname.find("BatchNorm2d") != -1: - torch.nn.init.normal_(m.weight.data, 1.0, gain) - torch.nn.init.constant_(m.bias.data, 0.0) + nn.init.normal_(m.weight.data, 1.0, gain) + nn.init.constant_(m.bias.data, 0.0) self.apply(init_func) From 9a090384998b27194bf52c3abd3e88864ba97b6c Mon Sep 17 00:00:00 2001 From: rvandewater Date: Mon, 14 Aug 2023 16:01:45 +0200 Subject: [PATCH 010/207] feature names extraction --- icu_benchmarks/data/loader.py | 6 ++++++ icu_benchmarks/models/wrappers.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/icu_benchmarks/data/loader.py b/icu_benchmarks/data/loader.py index 7c9d1400..fca46184 100644 --- a/icu_benchmarks/data/loader.py +++ b/icu_benchmarks/data/loader.py @@ -92,6 +92,9 @@ def __getitem__(self, idx: int) -> Tuple[Tensor, Tensor, Tensor]: pad_value = 0.0 stay_id = self.outcome_df.index.unique()[idx] # [self.vars["GROUP"]] + # Get the feature names + self.features = list(self.features_df.columns) + # slice to make sure to always return a DF window = self.features_df.loc[stay_id:stay_id].to_numpy() labels = self.outcome_df.loc[stay_id:stay_id][self.vars["LABEL"]].to_numpy(dtype=float) @@ -102,6 +105,9 @@ def __getitem__(self, idx: int) -> Tuple[Tensor, Tensor, Tensor]: length_diff = self.maxlen - window.shape[0] + # Indicate padding for feature names + self.features.extend("padding" for _ in range(length_diff)) + pad_mask = np.ones(window.shape[0]) # Padding the array to fulfill size requirement diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index c48e0304..2af0a30b 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -323,7 +323,7 @@ def step_fn(self, element, step_prefix=""): elif step_prefix == "val": feature_names = self.trainer.train_dataloader.dataset.features else: - feature_names = self.trainer.test_dataloader.dataset.features + feature_names = self.trainer.test_dataloaders.dataset.features value.update(transformed_output[0], transformed_output[1], data, feature_names) else: value.update(transformed_output[0], transformed_output[1]) From 65ef59cf67c84ae07d0763cb47aec1602a9ccc4b Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 15 Aug 2023 08:35:58 +0200 Subject: [PATCH 011/207] Predict proba for ml models --- icu_benchmarks/models/ml_models.py | 8 +++++++- icu_benchmarks/models/wrappers.py | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/icu_benchmarks/models/ml_models.py b/icu_benchmarks/models/ml_models.py index e06d3fe7..3f854fce 100644 --- a/icu_benchmarks/models/ml_models.py +++ b/icu_benchmarks/models/ml_models.py @@ -1,5 +1,6 @@ import gin import lightgbm +import numpy as np from sklearn import linear_model from sklearn import ensemble from sklearn import neural_network @@ -11,13 +12,15 @@ class LGBMWrapper(MLWrapper): def fit_model(self, train_data, train_labels, val_data, val_labels): """Fitting function for LGBM models.""" + # self.model.set_params(random_state=np.random.get_state()[1][0]) + self.model.fit( train_data, train_labels, eval_set=(val_data, val_labels), verbose=True, callbacks=[ - lightgbm.early_stopping(self.hparams.patience, verbose=False), + lightgbm.early_stopping(self.hparams.patience, verbose=True), lightgbm.log_evaluation(period=-1, show_stdv=False), ], ) @@ -33,6 +36,9 @@ def __init__(self, *args, **kwargs): self.model = self.set_model_args(lightgbm.LGBMClassifier, *args, **kwargs) super().__init__(*args, **kwargs) + def predict(self, features): + """Predicts labels for the given features.""" + return self.model.predict(features) @gin.configurable class LGBMRegressor(LGBMWrapper): diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index 5fc94a12..af2693ec 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -365,8 +365,8 @@ def test_step(self, dataset, _): def predict(self, features): if self.run_mode == RunMode.regression: return self.model.predict(features) - else: - return self.model.predict(features) + else: # Classification: return probabilities + return self.model.predict_proba(features) def log_metrics(self, label, pred, metric_type): """Log metrics to the PL logs.""" From c8499cb519146d8d8b329deb1a399ffa7fdec667 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 15 Aug 2023 13:09:05 +0200 Subject: [PATCH 012/207] small changes for lgbm model --- icu_benchmarks/models/ml_models.py | 9 +++++---- icu_benchmarks/models/wrappers.py | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/icu_benchmarks/models/ml_models.py b/icu_benchmarks/models/ml_models.py index 3f854fce..f0764353 100644 --- a/icu_benchmarks/models/ml_models.py +++ b/icu_benchmarks/models/ml_models.py @@ -7,14 +7,14 @@ from sklearn import svm from icu_benchmarks.models.wrappers import MLWrapper from icu_benchmarks.contants import RunMode - +from wandb.lightgbm import wandb_callback class LGBMWrapper(MLWrapper): def fit_model(self, train_data, train_labels, val_data, val_labels): """Fitting function for LGBM models.""" - # self.model.set_params(random_state=np.random.get_state()[1][0]) + self.model.set_params(random_state=np.random.get_state()[1][0]) - self.model.fit( + self.model = self.model.fit( train_data, train_labels, eval_set=(val_data, val_labels), @@ -22,6 +22,7 @@ def fit_model(self, train_data, train_labels, val_data, val_labels): callbacks=[ lightgbm.early_stopping(self.hparams.patience, verbose=True), lightgbm.log_evaluation(period=-1, show_stdv=False), + # wandb_callback(), ], ) val_loss = list(self.model.best_score_["valid_0"].values())[0] @@ -38,7 +39,7 @@ def __init__(self, *args, **kwargs): def predict(self, features): """Predicts labels for the given features.""" - return self.model.predict(features) + return self.model.predict_proba(features) @gin.configurable class LGBMRegressor(LGBMWrapper): diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index af2693ec..c599d57a 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -321,8 +321,8 @@ def fit(self, train_dataset, val_dataset): self.set_metrics(train_label) - # if "class_weight" in self.model.get_params().keys(): # Set class weights - # self.model.set_params(class_weight=self.weight) + if "class_weight" in self.model.get_params().keys(): # Set class weights + self.model.set_params(class_weight=self.weight) val_loss = self.fit_model(train_rep, train_label, val_rep, val_label) From fcf2970db60e230da90453271ebd39946f64fd24 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 15 Aug 2023 13:16:44 +0200 Subject: [PATCH 013/207] small changes to transform --- icu_benchmarks/models/wrappers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index c599d57a..ebdf065b 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -292,8 +292,7 @@ def set_metrics(self, labels): # Binary classification if len(np.unique(labels)) == 2: # if isinstance(self.model, lightgbm.basic.Booster): - self.output_transform = lambda x: x - # self.output_transform = lambda x: x[:, 1] + self.output_transform = lambda x: x[:, 1] self.label_transform = lambda x: x self.metrics = MLMetrics.BINARY_CLASSIFICATION From 1b423d485fd943fedb2db52ee4405c99fe906138 Mon Sep 17 00:00:00 2001 From: Hendrik Schmidt Date: Fri, 11 Aug 2023 12:20:43 +0200 Subject: [PATCH 014/207] make reproducible argument for evaluate too --- icu_benchmarks/run_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/icu_benchmarks/run_utils.py b/icu_benchmarks/run_utils.py index 179ab1f5..4462e25e 100644 --- a/icu_benchmarks/run_utils.py +++ b/icu_benchmarks/run_utils.py @@ -51,6 +51,9 @@ def build_parser() -> ArgumentParser: general_args.add_argument( "-db", "--debug", required=False, default=False, action=BooleanOptionalAction, help="Set to load less data." ) + general_args.add_argument( + "--reproducible", required=False, default=True, action=BooleanOptionalAction, help="Make torch reproducible." + ) general_args.add_argument( "-lc", "--load_cache", @@ -78,9 +81,6 @@ def build_parser() -> ArgumentParser: # MODEL TRAINING ARGUMENTS prep_and_train = subparsers.add_parser("train", help="Preprocess features and train model.", parents=[parent_parser]) - prep_and_train.add_argument( - "--reproducible", required=False, default=True, action=BooleanOptionalAction, help="Make torch reproducible." - ) prep_and_train.add_argument("-hp", "--hyperparams", required=False, nargs="+", help="Hyperparameters for model.") prep_and_train.add_argument("--tune", default=False, action=BooleanOptionalAction, help="Find best hyperparameters.") prep_and_train.add_argument("--checkpoint", required=False, type=Path, help="Use previous checkpoint.") From 730c7e48135c532b6a3682d80006b3db9a90d5c5 Mon Sep 17 00:00:00 2001 From: Hendrik Schmidt Date: Fri, 11 Aug 2023 16:21:42 +0200 Subject: [PATCH 015/207] fix model loading for ML --- icu_benchmarks/models/train.py | 10 ++++++---- icu_benchmarks/models/wrappers.py | 6 +++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 1c2a91cd..36fcc398 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -3,6 +3,7 @@ import torch import logging import pandas as pd +import pickle from torch.optim import Adam from torch.utils.data import DataLoader from pytorch_lightning.loggers import TensorBoardLogger @@ -108,10 +109,11 @@ def train_common( model.set_weight(weight, train_dataset) if load_weights: if source_dir.exists(): - # if not model.needs_training: - checkpoint = torch.load(source_dir / "model.ckpt") - # else: - model = model.load_state_dict(checkpoint["state_dict"]) + if model.needs_training: + checkpoint = torch.load(source_dir / "model.ckpt") + model = model.load_state_dict(checkpoint) + else: + model = torch.load(source_dir / "last.ckpt") else: raise Exception(f"No weights to load at path : {source_dir}") diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index 5fc94a12..5d9e659e 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -11,11 +11,11 @@ import inspect import gin import numpy as np +import pickle import torch from ignite.exceptions import NotComputableError from icu_benchmarks.models.constants import ImputationInit from icu_benchmarks.models.utils import create_optimizer, create_scheduler -from joblib import dump from pytorch_lightning import LightningModule from icu_benchmarks.models.constants import MLMetrics, DLMetrics @@ -390,10 +390,10 @@ def __getstate__(self) -> Dict[str, Any]: del state["output_transform"] return state - def save_model(self, save_path, file_name, file_extension=".joblib"): + def save_model(self, save_path, file_name, file_extension=".ckpt"): path = save_path / (file_name + file_extension) try: - dump(self.model, path) + torch.save(self, path) logging.info(f"Model saved to {str(path.resolve())}.") except Exception as e: logging.error(f"Cannot save model to path {str(path.resolve())}: {e}.") From 8a5dfb3c123f26eae00362eae203513fe6cc7bf3 Mon Sep 17 00:00:00 2001 From: Hendrik Schmidt Date: Tue, 15 Aug 2023 14:16:04 +0200 Subject: [PATCH 016/207] rename checkpoint to hp_checkpoint --- icu_benchmarks/run.py | 4 ++-- icu_benchmarks/run_utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index 0f8b7859..1dbfd090 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -118,7 +118,7 @@ def main(my_args=tuple(sys.argv[1:])): gin.parse_config_file(source_dir / "train_config.gin") else: # Train - checkpoint = log_dir / args.checkpoint if args.checkpoint else None + hp_checkpoint = log_dir / args.hp_checkpoint if args.hp_checkpoint else None model_path = ( Path("configs") / ("imputation_models" if mode == RunMode.imputation else "prediction_models") / f"{model}.gin" ) @@ -136,7 +136,7 @@ def main(my_args=tuple(sys.argv[1:])): run_dir, args.seed, run_mode=mode, - checkpoint=checkpoint, + checkpoint=hp_checkpoint, debug=args.debug, generate_cache=args.generate_cache, load_cache=args.load_cache, diff --git a/icu_benchmarks/run_utils.py b/icu_benchmarks/run_utils.py index 4462e25e..2bcb026f 100644 --- a/icu_benchmarks/run_utils.py +++ b/icu_benchmarks/run_utils.py @@ -83,7 +83,7 @@ def build_parser() -> ArgumentParser: prep_and_train = subparsers.add_parser("train", help="Preprocess features and train model.", parents=[parent_parser]) prep_and_train.add_argument("-hp", "--hyperparams", required=False, nargs="+", help="Hyperparameters for model.") prep_and_train.add_argument("--tune", default=False, action=BooleanOptionalAction, help="Find best hyperparameters.") - prep_and_train.add_argument("--checkpoint", required=False, type=Path, help="Use previous checkpoint.") + prep_and_train.add_argument("--hp-checkpoint", required=False, type=Path, help="Use previous hyperparameter checkpoint.") # EVALUATION PARSER evaluate = subparsers.add_parser("evaluate", help="Evaluate trained model on data.", parents=[parent_parser]) From 0d2613b0048b0eff394146dfd18b6103fe92e720 Mon Sep 17 00:00:00 2001 From: Hendrik Schmidt Date: Tue, 15 Aug 2023 14:27:47 +0200 Subject: [PATCH 017/207] remove verbose required=False --- icu_benchmarks/run_utils.py | 61 +++++++++++-------------------------- 1 file changed, 18 insertions(+), 43 deletions(-) diff --git a/icu_benchmarks/run_utils.py b/icu_benchmarks/run_utils.py index 2bcb026f..91ae6219 100644 --- a/icu_benchmarks/run_utils.py +++ b/icu_benchmarks/run_utils.py @@ -29,61 +29,36 @@ def build_parser() -> ArgumentParser: general_args = parent_parser.add_argument_group("General arguments") general_args.add_argument("-d", "--data-dir", required=True, type=Path, help="Path to the parquet data directory.") general_args.add_argument("-t", "--task", default="BinaryClassification", required=True, help="Name of the task gin.") - general_args.add_argument("-n", "--name", required=False, help="Name of the (target) dataset.") - general_args.add_argument("-tn", "--task-name", required=False, help="Name of the task, used for naming experiments.") - general_args.add_argument("-m", "--model", default="LGBMClassifier", required=False, help="Name of the model gin.") - general_args.add_argument("-e", "--experiment", required=False, help="Name of the experiment gin.") + general_args.add_argument("-n", "--name", help="Name of the (target) dataset.") + general_args.add_argument("-tn", "--task-name", help="Name of the task, used for naming experiments.") + general_args.add_argument("-m", "--model", default="LGBMClassifier", help="Name of the model gin.") + general_args.add_argument("-e", "--experiment", help="Name of the experiment gin.") general_args.add_argument( - "-l", "--log-dir", required=False, default=Path("../yaib_logs/"), type=Path, help="Log directory with model weights." + "-l", "--log-dir", default=Path("../yaib_logs/"), type=Path, help="Log directory for model weights." ) + general_args.add_argument("-s", "--seed", default=1234, type=int, help="Random seed for processing, tuning and training.") general_args.add_argument( - "-s", "--seed", required=False, default=1234, type=int, help="Random seed for processing, tuning and training." + "-v", "--verbose", default=False, action=BooleanOptionalAction, help="Set to log verbosly. Disable for clean logs." ) + general_args.add_argument("--cpu", default=False, action=BooleanOptionalAction, help="Set to use CPU.") + general_args.add_argument("-db", "--debug", default=False, action=BooleanOptionalAction, help="Set to load less data.") + general_args.add_argument("--reproducible", default=True, action=BooleanOptionalAction, help="Make torch reproducible.") general_args.add_argument( - "-v", - "--verbose", - default=False, - required=False, - action=BooleanOptionalAction, - help="Whether to use verbose logging. Disable for clean logs.", + "-lc", "--load_cache", default=False, action=BooleanOptionalAction, help="Set to load generated data cache." ) - general_args.add_argument("--cpu", default=False, required=False, action=BooleanOptionalAction, help="Set to use CPU.") general_args.add_argument( - "-db", "--debug", required=False, default=False, action=BooleanOptionalAction, help="Set to load less data." - ) - general_args.add_argument( - "--reproducible", required=False, default=True, action=BooleanOptionalAction, help="Make torch reproducible." - ) - general_args.add_argument( - "-lc", - "--load_cache", - required=False, - default=False, - action=BooleanOptionalAction, - help="Set to load generated data cache.", - ) - general_args.add_argument( - "-gc", - "--generate_cache", - required=False, - default=False, - action=BooleanOptionalAction, - help="Set to generate data cache.", - ) - general_args.add_argument("-p", "--preprocessor", required=False, type=Path, help="Load custom preprocessor from file.") - general_args.add_argument("-pl", "--plot", required=False, action=BooleanOptionalAction, help="Generate common plots.") - general_args.add_argument( - "-wd", "--wandb-sweep", required=False, action="store_true", help="Activates wandb hyper parameter sweep." - ) - general_args.add_argument( - "-imp", "--pretrained-imputation", required=False, type=str, help="Path to pretrained imputation model." + "-gc", "--generate_cache", default=False, action=BooleanOptionalAction, help="Set to generate data cache." ) + general_args.add_argument("-p", "--preprocessor", type=Path, help="Load custom preprocessor from file.") + general_args.add_argument("-pl", "--plot", action=BooleanOptionalAction, help="Generate common plots.") + general_args.add_argument("-wd", "--wandb-sweep", action="store_true", help="Activates wandb hyper parameter sweep.") + general_args.add_argument("-imp", "--pretrained-imputation", type=str, help="Path to pretrained imputation model.") # MODEL TRAINING ARGUMENTS prep_and_train = subparsers.add_parser("train", help="Preprocess features and train model.", parents=[parent_parser]) - prep_and_train.add_argument("-hp", "--hyperparams", required=False, nargs="+", help="Hyperparameters for model.") + prep_and_train.add_argument("-hp", "--hyperparams", nargs="+", help="Hyperparameters for model.") prep_and_train.add_argument("--tune", default=False, action=BooleanOptionalAction, help="Find best hyperparameters.") - prep_and_train.add_argument("--hp-checkpoint", required=False, type=Path, help="Use previous hyperparameter checkpoint.") + prep_and_train.add_argument("--hp-checkpoint", type=Path, help="Use previous hyperparameter checkpoint.") # EVALUATION PARSER evaluate = subparsers.add_parser("evaluate", help="Evaluate trained model on data.", parents=[parent_parser]) From 4e9271bea38f10933d329c6e95a39da0aa9748d2 Mon Sep 17 00:00:00 2001 From: Hendrik Schmidt Date: Tue, 15 Aug 2023 15:04:36 +0200 Subject: [PATCH 018/207] remove commands and add support for eval and finetune - skip training when eval is set - change readme to reflect changes to command line --- README.md | 18 ++++---- icu_benchmarks/cross_validation.py | 3 ++ icu_benchmarks/models/train.py | 25 +++++------ icu_benchmarks/run.py | 12 +++--- icu_benchmarks/run_utils.py | 69 +++++++++++------------------- 5 files changed, 59 insertions(+), 68 deletions(-) diff --git a/README.md b/README.md index d001dfa4..4028b15b 100644 --- a/README.md +++ b/README.md @@ -182,17 +182,16 @@ load existing cache files. ``` -icu-benchmarks train \ +icu-benchmarks \ -d demo_data/mortality24/mimic_demo \ -n mimic_demo \ -t BinaryClassification \ -tn Mortality24 \ -m LGBMClassifier \ -hp LGBMClassifier.min_child_samples=10 \ - --generate_cache + --generate_cache \ --load_cache \ --seed 2222 \ - -s 2222 \ -l ../yaib_logs/ \ --tune ``` @@ -224,13 +223,13 @@ wandb agent > Note: You will need to have a wandb account and be logged in to run the above commands. -## Evaluate +## Evaluate or Finetune -It is possible to evaluate a model trained on another dataset. In this case, the source dataset is the demo data from MIMIC and -the target is the eICU demo: +It is possible to evaluate a model trained on another dataset and no additional training is done. +In this case, the source dataset is the demo data from MIMIC and the target is the eICU demo: ``` -icu-benchmarks evaluate \ +icu-benchmarks \ -d demo_data/mortality24/eicu_demo \ -n eicu_demo \ -t BinaryClassification \ @@ -240,10 +239,13 @@ icu-benchmarks evaluate \ --load_cache \ -s 2222 \ -l ../yaib_logs \ + --eval \ -sn mimic \ - --source-dir ../yaib_logs/mimic_demo/Mortality24/LGBMClassifier/2022-12-12T15-24-46/fold_0 + --source-dir ../yaib_logs/mimic_demo/Mortality24/LGBMClassifier/2022-12-12T15-24-46/repetition_0/fold_0 ``` +> A similar syntax is used for finetuning, where a model is loaded and then retrained. To run finetuning, replace `--eval` with `-ft`. + ## Models We provide several existing machine learning models that are commonly used for multivariate time-series data. diff --git a/icu_benchmarks/cross_validation.py b/icu_benchmarks/cross_validation.py index 89a98864..e5ae6d20 100644 --- a/icu_benchmarks/cross_validation.py +++ b/icu_benchmarks/cross_validation.py @@ -19,6 +19,7 @@ def execute_repeated_cv( data_dir: Path, log_dir: Path, seed: int, + eval_only: bool = False, load_weights: bool = False, source_dir: Path = None, cv_repetitions: int = 5, @@ -43,6 +44,7 @@ def execute_repeated_cv( data_dir: Path to the data directory. log_dir: Path to the log directory. seed: Random seed. + eval_only: Whether to only evaluate the model. load_weights: Whether to load weights from source_dir. source_dir: Path to the source directory. cv_folds: Number of folds for cross validation. @@ -92,6 +94,7 @@ def execute_repeated_cv( agg_loss += train_common( data, log_dir=repetition_fold_dir, + eval_only=eval_only, load_weights=load_weights, source_dir=source_dir, reproducible=reproducible, diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 36fcc398..fc4df4ea 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -28,6 +28,7 @@ def assure_minimum_length(dataset): def train_common( data: dict[str, pd.DataFrame], log_dir: Path, + eval_only: bool = False, load_weights: bool = False, source_dir: Path = None, reproducible: bool = True, @@ -52,6 +53,7 @@ def train_common( Args: data: Dict containing data to be trained on. log_dir: Path to directory where model output should be saved. + eval_only: If set to true, skip training and only evaluate the model. load_weights: If set to true, skip training and load weights from source_dir instead. source_dir: If set to load weights, path to directory containing trained weights. reproducible: If set to true, set torch to run reproducibly. @@ -118,9 +120,8 @@ def train_common( raise Exception(f"No weights to load at path : {source_dir}") model.set_trained_columns(train_dataset.get_feature_names()) - + loggers = [TensorBoardLogger(log_dir), JSONMetricsLogger(log_dir)] - callbacks = [ EarlyStopping(monitor="val/loss", min_delta=min_delta, patience=patience, strict=False), ModelCheckpoint(log_dir, filename="model", save_top_k=1, save_last=True), @@ -142,16 +143,16 @@ def train_common( num_sanity_val_steps=0, ) - if model.needs_fit: - logging.info("Fitting model to data.") - model.fit(train_dataset, val_dataset) - model.save_model(log_dir, "last") - logging.info("Fitting complete.") - - if model.needs_training: - logging.info("Training model.") - trainer.fit(model, train_dataloaders=train_loader, val_dataloaders=val_loader) - logging.info("Training complete.") + if not eval_only: + if model.needs_fit: + logging.info("Fitting model to data.") + model.fit(train_dataset, val_dataset) + model.save_model(log_dir, "last") + logging.info("Fitting complete.") + if model.needs_training: + logging.info("Training model.") + trainer.fit(model, train_dataloaders=train_loader, val_dataloaders=val_loader) + logging.info("Training complete.") test_dataset = dataset_class(data, split=test_on) test_dataset = assure_minimum_length(test_dataset) diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index 1dbfd090..53cc21f5 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -44,11 +44,8 @@ def main(my_args=tuple(sys.argv[1:])): verbose = args.verbose setup_logging(date_format, log_format, verbose) - # Load weights if in evaluation mode - load_weights = args.command == "evaluate" - data_dir = Path(args.data_dir) - # Get arguments + data_dir = Path(args.data_dir) name = args.name task = args.task model = args.model @@ -110,8 +107,12 @@ def main(my_args=tuple(sys.argv[1:])): except Exception as e: logging.error(f"Could not import custom preprocessor from {args.preprocessor}: {e}") + # Load pretrained model in evaluate mode or when finetuning + evaluate = args.eval + load_weights = evaluate or args.fine_tune if load_weights: - # Evaluate + if args.source_dir is None: + raise ValueError("Please specify a source directory when evaluating or finetuning.") log_dir /= f"from_{args.source_name}" run_dir = create_run_dir(log_dir) source_dir = args.source_dir @@ -150,6 +151,7 @@ def main(my_args=tuple(sys.argv[1:])): data_dir, run_dir, args.seed, + eval_only=evaluate, load_weights=load_weights, source_dir=source_dir, reproducible=reproducible, diff --git a/icu_benchmarks/run_utils.py b/icu_benchmarks/run_utils.py index 91ae6219..eafedd65 100644 --- a/icu_benchmarks/run_utils.py +++ b/icu_benchmarks/run_utils.py @@ -3,7 +3,7 @@ import torch import json -from argparse import ArgumentParser, BooleanOptionalAction +from argparse import ArgumentParser, BooleanOptionalAction as BOA from datetime import datetime, timedelta import logging from pathlib import Path @@ -22,48 +22,31 @@ def build_parser() -> ArgumentParser: """ parser = ArgumentParser(description="Benchmark lib for processing and evaluation of deep learning models on ICU data") - parent_parser = ArgumentParser(add_help=False) - subparsers = parser.add_subparsers(title="Commands", dest="command", required=True) - - # ARGUMENTS FOR ALL COMMANDS - general_args = parent_parser.add_argument_group("General arguments") - general_args.add_argument("-d", "--data-dir", required=True, type=Path, help="Path to the parquet data directory.") - general_args.add_argument("-t", "--task", default="BinaryClassification", required=True, help="Name of the task gin.") - general_args.add_argument("-n", "--name", help="Name of the (target) dataset.") - general_args.add_argument("-tn", "--task-name", help="Name of the task, used for naming experiments.") - general_args.add_argument("-m", "--model", default="LGBMClassifier", help="Name of the model gin.") - general_args.add_argument("-e", "--experiment", help="Name of the experiment gin.") - general_args.add_argument( - "-l", "--log-dir", default=Path("../yaib_logs/"), type=Path, help="Log directory for model weights." - ) - general_args.add_argument("-s", "--seed", default=1234, type=int, help="Random seed for processing, tuning and training.") - general_args.add_argument( - "-v", "--verbose", default=False, action=BooleanOptionalAction, help="Set to log verbosly. Disable for clean logs." - ) - general_args.add_argument("--cpu", default=False, action=BooleanOptionalAction, help="Set to use CPU.") - general_args.add_argument("-db", "--debug", default=False, action=BooleanOptionalAction, help="Set to load less data.") - general_args.add_argument("--reproducible", default=True, action=BooleanOptionalAction, help="Make torch reproducible.") - general_args.add_argument( - "-lc", "--load_cache", default=False, action=BooleanOptionalAction, help="Set to load generated data cache." - ) - general_args.add_argument( - "-gc", "--generate_cache", default=False, action=BooleanOptionalAction, help="Set to generate data cache." - ) - general_args.add_argument("-p", "--preprocessor", type=Path, help="Load custom preprocessor from file.") - general_args.add_argument("-pl", "--plot", action=BooleanOptionalAction, help="Generate common plots.") - general_args.add_argument("-wd", "--wandb-sweep", action="store_true", help="Activates wandb hyper parameter sweep.") - general_args.add_argument("-imp", "--pretrained-imputation", type=str, help="Path to pretrained imputation model.") - - # MODEL TRAINING ARGUMENTS - prep_and_train = subparsers.add_parser("train", help="Preprocess features and train model.", parents=[parent_parser]) - prep_and_train.add_argument("-hp", "--hyperparams", nargs="+", help="Hyperparameters for model.") - prep_and_train.add_argument("--tune", default=False, action=BooleanOptionalAction, help="Find best hyperparameters.") - prep_and_train.add_argument("--hp-checkpoint", type=Path, help="Use previous hyperparameter checkpoint.") - - # EVALUATION PARSER - evaluate = subparsers.add_parser("evaluate", help="Evaluate trained model on data.", parents=[parent_parser]) - evaluate.add_argument("-sn", "--source-name", required=True, type=Path, help="Name of the source dataset.") - evaluate.add_argument("--source-dir", required=True, type=Path, help="Directory containing gin and model weights.") + parser.add_argument("-d", "--data-dir", required=True, type=Path, help="Path to the parquet data directory.") + parser.add_argument("-t", "--task", default="BinaryClassification", required=True, help="Name of the task gin.") + parser.add_argument("-n", "--name", help="Name of the (target) dataset.") + parser.add_argument("-tn", "--task-name", help="Name of the task, used for naming experiments.") + parser.add_argument("-m", "--model", default="LGBMClassifier", help="Name of the model gin.") + parser.add_argument("-e", "--experiment", help="Name of the experiment gin.") + parser.add_argument( "-l", "--log-dir", default=Path("../yaib_logs/"), type=Path, help="Log directory for model weights.") + parser.add_argument("-s", "--seed", default=1234, type=int, help="Random seed for processing, tuning and training.") + parser.add_argument("-v", "--verbose", default=False, action=BOA, help="Set to log verbosly. Disable for clean logs.") + parser.add_argument("--cpu", default=False, action=BOA, help="Set to use CPU.") + parser.add_argument("-db", "--debug", default=False, action=BOA, help="Set to load less data.") + parser.add_argument("--reproducible", default=True, action=BOA, help="Make torch reproducible.") + parser.add_argument("-lc", "--load_cache", default=False, action=BOA, help="Set to load generated data cache.") + parser.add_argument("-gc", "--generate_cache", default=False, action=BOA, help="Set to generate data cache.") + parser.add_argument("-p", "--preprocessor", type=Path, help="Load custom preprocessor from file.") + parser.add_argument("-pl", "--plot", action=BOA, help="Generate common plots.") + parser.add_argument("-wd", "--wandb-sweep", action="store_true", help="Activates wandb hyper parameter sweep.") + parser.add_argument("-imp", "--pretrained-imputation", type=str, help="Path to pretrained imputation model.") + parser.add_argument("-hp", "--hyperparams", nargs="+", help="Hyperparameters for model.") + parser.add_argument("--tune", default=False, action=BOA, help="Find best hyperparameters.") + parser.add_argument("--hp-checkpoint", type=Path, help="Use previous hyperparameter checkpoint.") + parser.add_argument("--eval", default=False, action=BOA, help="Only evaluate model, skip training.") + parser.add_argument("-ft", "--fine-tune", default=False, action=BOA, help="Load model and finetune on data.") + parser.add_argument("-sn", "--source-name", type=Path, help="Name of the source dataset.") + parser.add_argument("--source-dir", type=Path, help="Directory containing gin and model weights.") return parser From ef25a33f07a7e253b9beb3515e17317f2026f4a8 Mon Sep 17 00:00:00 2001 From: Hendrik Schmidt Date: Tue, 15 Aug 2023 15:53:25 +0200 Subject: [PATCH 019/207] move eval flag to first position --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4028b15b..3c1aa6a0 100644 --- a/README.md +++ b/README.md @@ -230,6 +230,7 @@ In this case, the source dataset is the demo data from MIMIC and the target is t ``` icu-benchmarks \ + --eval \ -d demo_data/mortality24/eicu_demo \ -n eicu_demo \ -t BinaryClassification \ @@ -239,7 +240,6 @@ icu-benchmarks \ --load_cache \ -s 2222 \ -l ../yaib_logs \ - --eval \ -sn mimic \ --source-dir ../yaib_logs/mimic_demo/Mortality24/LGBMClassifier/2022-12-12T15-24-46/repetition_0/fold_0 ``` From 2fd51b0781bb22564664bfdc27a4e6492b1fc93f Mon Sep 17 00:00:00 2001 From: Hendrik Schmidt Date: Tue, 15 Aug 2023 15:54:30 +0200 Subject: [PATCH 020/207] fix lint --- icu_benchmarks/models/train.py | 9 ++++----- icu_benchmarks/models/wrappers.py | 1 - icu_benchmarks/run_utils.py | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index fc4df4ea..0a53323f 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -3,7 +3,6 @@ import torch import logging import pandas as pd -import pickle from torch.optim import Adam from torch.utils.data import DataLoader from pytorch_lightning.loggers import TensorBoardLogger @@ -112,15 +111,15 @@ def train_common( if load_weights: if source_dir.exists(): if model.needs_training: - checkpoint = torch.load(source_dir / "model.ckpt") - model = model.load_state_dict(checkpoint) + checkpoint = torch.load(source_dir / "model.ckpt") + model = model.load_state_dict(checkpoint) else: - model = torch.load(source_dir / "last.ckpt") + model = torch.load(source_dir / "last.ckpt") else: raise Exception(f"No weights to load at path : {source_dir}") model.set_trained_columns(train_dataset.get_feature_names()) - + loggers = [TensorBoardLogger(log_dir), JSONMetricsLogger(log_dir)] callbacks = [ EarlyStopping(monitor="val/loss", min_delta=min_delta, patience=patience, strict=False), diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index 5d9e659e..23b187ce 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -11,7 +11,6 @@ import inspect import gin import numpy as np -import pickle import torch from ignite.exceptions import NotComputableError from icu_benchmarks.models.constants import ImputationInit diff --git a/icu_benchmarks/run_utils.py b/icu_benchmarks/run_utils.py index eafedd65..73b3e6b5 100644 --- a/icu_benchmarks/run_utils.py +++ b/icu_benchmarks/run_utils.py @@ -28,7 +28,7 @@ def build_parser() -> ArgumentParser: parser.add_argument("-tn", "--task-name", help="Name of the task, used for naming experiments.") parser.add_argument("-m", "--model", default="LGBMClassifier", help="Name of the model gin.") parser.add_argument("-e", "--experiment", help="Name of the experiment gin.") - parser.add_argument( "-l", "--log-dir", default=Path("../yaib_logs/"), type=Path, help="Log directory for model weights.") + parser.add_argument("-l", "--log-dir", default=Path("../yaib_logs/"), type=Path, help="Log directory for model weights.") parser.add_argument("-s", "--seed", default=1234, type=int, help="Random seed for processing, tuning and training.") parser.add_argument("-v", "--verbose", default=False, action=BOA, help="Set to log verbosly. Disable for clean logs.") parser.add_argument("--cpu", default=False, action=BOA, help="Set to use CPU.") From 25adb0db8b254810344146d682c35222f97bd855 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 15 Aug 2023 16:00:01 +0200 Subject: [PATCH 021/207] added support for model-v1.cpkt --- icu_benchmarks/models/train.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index fc4df4ea..d80f9140 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -112,8 +112,13 @@ def train_common( if load_weights: if source_dir.exists(): if model.needs_training: - checkpoint = torch.load(source_dir / "model.ckpt") - model = model.load_state_dict(checkpoint) + if(source_dir / "model.ckpt").exists(): + checkpoint = torch.load(source_dir / "model.ckpt") + elif (source_dir / "model-v1.ckpt").exists(): + checkpoint = torch.load(source_dir / "model-v1.ckpt") + else: + return Exception(f"No weights to load at path : {source_dir}") + model = model.load_state_dict(checkpoint) else: model = torch.load(source_dir / "last.ckpt") else: From e7d3c593a0ec9507bfc5a8e5201d8da3d3528a9e Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 15 Aug 2023 17:59:02 +0200 Subject: [PATCH 022/207] Pytorch lightning syntax --- icu_benchmarks/models/train.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 34ec6e38..3e199cea 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -3,6 +3,7 @@ import torch import logging import pandas as pd +from joblib import load from torch.optim import Adam from torch.utils.data import DataLoader from pytorch_lightning.loggers import TensorBoardLogger @@ -108,18 +109,26 @@ def train_common( model = model(optimizer=optimizer, input_size=data_shape, epochs=epochs, run_mode=mode) model.set_weight(weight, train_dataset) + pl_model = True if load_weights: if source_dir.exists(): if model.needs_training: - if(source_dir / "model.ckpt").exists(): - checkpoint = torch.load(source_dir / "model.ckpt") + model_path = "" + if (source_dir / "last.ckpt").exists(): + model_path = source_dir / "last.ckpt" + elif(source_dir / "model.ckpt").exists(): + model_path = source_dir / "model.ckpt" elif (source_dir / "model-v1.ckpt").exists(): - checkpoint = torch.load(source_dir / "model-v1.ckpt") + model_path = source_dir / "model-v1.ckpt" else: return Exception(f"No weights to load at path : {source_dir}") - model = model.load_state_dict(checkpoint) + if(pl_model): + model = model.load_from_checkpoint(source_dir / "last.ckpt") + else: + checkpoint = torch.load(model_path) + model.load_state_dict(checkpoint) else: - model = torch.load(source_dir / "last.ckpt") + model = load(source_dir/"model.joblib") else: raise Exception(f"No weights to load at path : {source_dir}") From 66661a9010d48d87f5dffbbcc0671e11701d4627 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 15 Aug 2023 18:00:16 +0200 Subject: [PATCH 023/207] fixed string --- icu_benchmarks/models/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 3e199cea..d62156d7 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -123,7 +123,7 @@ def train_common( else: return Exception(f"No weights to load at path : {source_dir}") if(pl_model): - model = model.load_from_checkpoint(source_dir / "last.ckpt") + model = model.load_from_checkpoint(model_path) else: checkpoint = torch.load(model_path) model.load_state_dict(checkpoint) From fe6f6796c3d6183a4eac792186401505c2c215e0 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 15 Aug 2023 20:26:52 +0200 Subject: [PATCH 024/207] adjusted the pytorch lightning bool --- icu_benchmarks/models/train.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index d62156d7..9e399891 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -46,6 +46,7 @@ def train_common( cpu: bool = False, verbose=False, ram_cache=False, + pl_model=True, num_workers: int = min(cpu_core_count, torch.cuda.device_count() * 4 * int(torch.cuda.is_available()), 32), ): """Common wrapper to train all benchmarked models. @@ -71,6 +72,7 @@ def train_common( cpu: If set to true, run on cpu. verbose: Enable detailed logging. ram_cache: Whether to cache the data in RAM. + pl_model: Loading a pytorch lightning model. num_workers: Number of workers to use for data loading. """ @@ -109,11 +111,9 @@ def train_common( model = model(optimizer=optimizer, input_size=data_shape, epochs=epochs, run_mode=mode) model.set_weight(weight, train_dataset) - pl_model = True if load_weights: if source_dir.exists(): if model.needs_training: - model_path = "" if (source_dir / "last.ckpt").exists(): model_path = source_dir / "last.ckpt" elif(source_dir / "model.ckpt").exists(): From 2eb8c1c870de51a84177dd8bcbc2ad0c37e13fc3 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 15 Aug 2023 20:28:33 +0200 Subject: [PATCH 025/207] Linting --- icu_benchmarks/models/train.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 9e399891..d7357f6b 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -116,19 +116,19 @@ def train_common( if model.needs_training: if (source_dir / "last.ckpt").exists(): model_path = source_dir / "last.ckpt" - elif(source_dir / "model.ckpt").exists(): + elif (source_dir / "model.ckpt").exists(): model_path = source_dir / "model.ckpt" elif (source_dir / "model-v1.ckpt").exists(): model_path = source_dir / "model-v1.ckpt" else: return Exception(f"No weights to load at path : {source_dir}") - if(pl_model): + if pl_model: model = model.load_from_checkpoint(model_path) else: checkpoint = torch.load(model_path) model.load_state_dict(checkpoint) else: - model = load(source_dir/"model.joblib") + model = load(source_dir / "model.joblib") else: raise Exception(f"No weights to load at path : {source_dir}") From afe9b3276fde87d3c9ed2ba81d84b0a050e27bc0 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 16 Aug 2023 10:58:16 +0200 Subject: [PATCH 026/207] logging fix --- icu_benchmarks/cross_validation.py | 2 +- icu_benchmarks/run.py | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/icu_benchmarks/cross_validation.py b/icu_benchmarks/cross_validation.py index e5ae6d20..48090034 100644 --- a/icu_benchmarks/cross_validation.py +++ b/icu_benchmarks/cross_validation.py @@ -107,7 +107,7 @@ def execute_repeated_cv( train_time = datetime.now() - start_time log_full_line( - f"FINISHED FOLD {fold_index}| PREPROCESSING DURATION {preprocess_time}| TRAINING DURATION {train_time}", + f"FINISHED FOLD {fold_index}| PREPROCESSING DURATION {preprocess_time}| PROCEDURE DURATION {train_time}", level=logging.INFO, ) durations = {"preprocessing_duration": preprocess_time, "train_duration": train_time} diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index 53cc21f5..d34aab8d 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -145,7 +145,14 @@ def main(my_args=tuple(sys.argv[1:])): ) log_full_line(f"Logging to {run_dir.resolve()}", level=logging.INFO) - log_full_line("STARTING TRAINING", level=logging.INFO, char="=", num_newlines=3) + if evaluate: + mode_string = "STARTING EVALUATION" + elif args.fine_tune: + mode_string = "STARTING FINE TUNING" + else: + mode_string = "STARTING TRAINING" + log_full_line(mode_string, level=logging.INFO, char="=", num_newlines=3) + start_time = datetime.now() execute_repeated_cv( data_dir, From 06ee06203779f254ee70cd03e9192b5e288398e3 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 16 Aug 2023 11:04:08 +0200 Subject: [PATCH 027/207] created torchmetrics dictionary --- icu_benchmarks/models/constants.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/icu_benchmarks/models/constants.py b/icu_benchmarks/models/constants.py index b571e7ab..5d26aefb 100644 --- a/icu_benchmarks/models/constants.py +++ b/icu_benchmarks/models/constants.py @@ -26,20 +26,15 @@ class MLMetrics: BINARY_CLASSIFICATION = { "AUC": roc_auc_score, "Calibration_Curve": calibration_curve, - # "Confusion_Matrix": confusion_matrix, - # "F1": f1_score, "PR": average_precision_score, "PR_Curve": precision_recall_curve, "RO_Curve": roc_curve, - "Binary_Fairness": BinaryFairness(2, 'demographic_parity'), } MULTICLASS_CLASSIFICATION = { "Accuracy": accuracy_score, "AUC": roc_auc_score, "Balanced_Accuracy": balanced_accuracy_score, - # "Confusion_Matrix": confusion_matrix, - # "F1": f1_score, "PR": average_precision_score, } @@ -53,16 +48,18 @@ class MLMetrics: # TODO: add support for confusion matrix class DLMetrics: BINARY_CLASSIFICATION = { + "AUC": ROC_AUC, + "Calibration_Curve": CalibrationCurve, + "PR": AveragePrecision, + "PR_Curve": PrecisionRecallCurve, + "RO_Curve": RocCurve, + } + + BINARY_CLASSIFICATION_TORCHMETRICS = { "AUC": AUROC(task="binary"), - "AUC_ignite": ROC_AUC, - # "PR" : AveragePrecision(task="binary"), - # "F1": F1Score(task="binary", num_classes=2), - "Calibration_Error": CalibrationError(task="binary",n_bins=10), - # "Calibration_Curve": CalibrationCurve, - # "Confusion_Matrix": ConfusionMatrix(num_classes=2), - # "PR": TorchMetricsWrapper(AveragePrecision(task="binary")), - # "PR_Curve": PrecisionRecallCurve, - # "RO_Curve": RocCurve, + "PR": AveragePrecision(task="binary"), + "Calibration_Error": CalibrationError(task="binary", n_bins=10), + "F1": F1Score(task="binary", num_classes=2), "Binary_Fairness": BinaryFairnessWrapper(num_groups=2, task='demographic_parity', group_name="sex"), } From 27adc9cb97f69ee5be2551934d2086ba4d7f9f49 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 16 Aug 2023 11:34:07 +0200 Subject: [PATCH 028/207] removed features from dataloader --- icu_benchmarks/data/loader.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/icu_benchmarks/data/loader.py b/icu_benchmarks/data/loader.py index fca46184..55d5add5 100644 --- a/icu_benchmarks/data/loader.py +++ b/icu_benchmarks/data/loader.py @@ -92,9 +92,6 @@ def __getitem__(self, idx: int) -> Tuple[Tensor, Tensor, Tensor]: pad_value = 0.0 stay_id = self.outcome_df.index.unique()[idx] # [self.vars["GROUP"]] - # Get the feature names - self.features = list(self.features_df.columns) - # slice to make sure to always return a DF window = self.features_df.loc[stay_id:stay_id].to_numpy() labels = self.outcome_df.loc[stay_id:stay_id][self.vars["LABEL"]].to_numpy(dtype=float) @@ -104,10 +101,6 @@ def __getitem__(self, idx: int) -> Tuple[Tensor, Tensor, Tensor]: labels = np.concatenate([np.empty(window.shape[0] - 1) * np.nan, labels], axis=0) length_diff = self.maxlen - window.shape[0] - - # Indicate padding for feature names - self.features.extend("padding" for _ in range(length_diff)) - pad_mask = np.ones(window.shape[0]) # Padding the array to fulfill size requirement From 54ac7e58fbbf194264f248fbeb4dfd11a503869c Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 16 Aug 2023 11:36:11 +0200 Subject: [PATCH 029/207] typo in README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3c1aa6a0..808a0764 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,8 @@ [//]: # (TODO: add coverage once we have some tests ) -Yet another ICU benchmark (YAIB) provides a framework for doing clinical machine learning experiments on Intensive Care Unit ( -ICU) EHR data. +Yet another ICU benchmark (YAIB) provides a framework for doing clinical machine learning experiments on Intensive Care Unit +(ICU) EHR data. We support the following datasets out of the box: From e75501743c2e19fc4cd65272df510d380b932090 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 16 Aug 2023 11:40:11 +0200 Subject: [PATCH 030/207] linting and documentation --- icu_benchmarks/data/loader.py | 2 +- icu_benchmarks/imputation/sssds4.py | 2 +- icu_benchmarks/models/constants.py | 24 ++++++++++++-------- icu_benchmarks/models/custom_metrics.py | 29 +++++++++++++------------ icu_benchmarks/models/train.py | 1 - 5 files changed, 32 insertions(+), 26 deletions(-) diff --git a/icu_benchmarks/data/loader.py b/icu_benchmarks/data/loader.py index 55d5add5..0efc6c7d 100644 --- a/icu_benchmarks/data/loader.py +++ b/icu_benchmarks/data/loader.py @@ -19,6 +19,7 @@ class CommonDataset(Dataset): columns in the data. grouping_segment: str, optional: The segment of the data contains the grouping column with only unique values. Defaults to Segment.outcome. Is used to calculate the number of stays in the data. """ + def __init__( self, data: dict, @@ -152,7 +153,6 @@ def to_tensor(self): return from_numpy(data), from_numpy(labels) - @gin.configurable("ImputationDataset") class ImputationDataset(CommonDataset): """Subclass of Common Dataset that contains data for imputation models.""" diff --git a/icu_benchmarks/imputation/sssds4.py b/icu_benchmarks/imputation/sssds4.py index ce6e8c0d..d359fb09 100644 --- a/icu_benchmarks/imputation/sssds4.py +++ b/icu_benchmarks/imputation/sssds4.py @@ -298,7 +298,7 @@ def forward(self, input_data): h = self.S42(h.permute(2, 0, 1)).permute(1, 2, 0) - out = torch.tanh(h[:, :self.res_channels, :]) * torch.sigmoid(h[:, self.res_channels:, :]) + out = torch.tanh(h[:, : self.res_channels, :]) * torch.sigmoid(h[:, self.res_channels :, :]) res = self.res_conv(out) assert x.shape == res.shape diff --git a/icu_benchmarks/models/constants.py b/icu_benchmarks/models/constants.py index 5d26aefb..360cf847 100644 --- a/icu_benchmarks/models/constants.py +++ b/icu_benchmarks/models/constants.py @@ -9,19 +9,25 @@ mean_absolute_error, precision_recall_curve, roc_curve, - # confusion_matrix, r2_score, mean_squared_error, - # f1_score, ) -from torchmetrics.classification import BinaryFairness, AUROC, AveragePrecision, \ - PrecisionRecallCurve, ROC, CalibrationError, F1Score +from torchmetrics.classification import ( + AUROC, + AveragePrecision, + PrecisionRecallCurve, + CalibrationError, + F1Score, +) from enum import Enum -from icu_benchmarks.models.custom_metrics import CalibrationCurve, BalancedAccuracy, MAE, JSD, TorchMetricsWrapper, BinaryFairnessWrapper - +from icu_benchmarks.models.custom_metrics import ( + CalibrationCurve, + BalancedAccuracy, + MAE, + JSD, + BinaryFairnessWrapper, +) -# TODO: revise transformation for metrics in wrappers.py in order to handle metrics that can not handle a mix of binary and -# continuous targets class MLMetrics: BINARY_CLASSIFICATION = { "AUC": roc_auc_score, @@ -60,7 +66,7 @@ class DLMetrics: "PR": AveragePrecision(task="binary"), "Calibration_Error": CalibrationError(task="binary", n_bins=10), "F1": F1Score(task="binary", num_classes=2), - "Binary_Fairness": BinaryFairnessWrapper(num_groups=2, task='demographic_parity', group_name="sex"), + "Binary_Fairness": BinaryFairnessWrapper(num_groups=2, task="demographic_parity", group_name="sex"), } MULTICLASS_CLASSIFICATION = { diff --git a/icu_benchmarks/models/custom_metrics.py b/icu_benchmarks/models/custom_metrics.py index c1ad3846..990be02e 100644 --- a/icu_benchmarks/models/custom_metrics.py +++ b/icu_benchmarks/models/custom_metrics.py @@ -56,10 +56,10 @@ def ece_curve_compute_fn(y_preds: torch.Tensor, y_targets: torch.Tensor, n_bins= class MAE(EpochMetric): def __init__( - self, - output_transform: Callable = lambda x: x, - check_compute_fn: bool = False, - invert_transform: Callable = lambda x: x, + self, + output_transform: Callable = lambda x: x, + check_compute_fn: bool = False, + invert_transform: Callable = lambda x: x, ) -> None: super(MAE, self).__init__( lambda x, y: mae_with_invert_compute_fn(x, y, invert_transform), @@ -75,9 +75,9 @@ def mae_with_invert_compute_fn(y_preds: torch.Tensor, y_targets: torch.Tensor, i class JSD(EpochMetric): def __init__( - self, - output_transform: Callable = lambda x: x, - check_compute_fn: bool = False, + self, + output_transform: Callable = lambda x: x, + check_compute_fn: bool = False, ) -> None: super(JSD, self).__init__( lambda x, y: JSD_fn(x, y), @@ -89,7 +89,7 @@ def JSD_fn(y_preds: torch.Tensor, y_targets: torch.Tensor): return jensenshannon(abs(y_preds).flatten(), abs(y_targets).flatten()) ** 2 -class TorchMetricsWrapper(): +class TorchMetricsWrapper: metric = None def __init__(self, metric) -> None: @@ -107,16 +107,17 @@ def reset(self) -> None: class BinaryFairnessWrapper(BinaryFairness): """ - This class is a wrapper for the BinaryFairness metric from TorchMetrics. + This class is a wrapper for the BinaryFairness metric from TorchMetrics. """ + group_name = None - def __init__(self, group_name = "sex", *args, **kwargs) -> None: + + def __init__(self, group_name="sex", *args, **kwargs) -> None: self.group_name = group_name super().__init__(*args, **kwargs) + def update(self, preds, target, data, feature_names) -> None: - """" Standard metric update function""" + """ " Standard metric update function""" groups = data[:, :, feature_names.index(self.group_name)] group_per_id = groups[:, 0] - return super().update(preds=preds.cpu(), - target=target.cpu(), - groups=group_per_id.long().cpu()) + return super().update(preds=preds.cpu(), target=target.cpu(), groups=group_per_id.long().cpu()) diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index e01515c6..2ced4105 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -166,7 +166,6 @@ def train_common( model.save_model(log_dir, "last") logging.info("Training complete.") - test_dataset = dataset_class(data, split=test_on) test_dataset = assure_minimum_length(test_dataset) test_loader = ( From 13e3055dfddfb6328cdf74396f7f05df7a3ce723 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 16 Aug 2023 11:57:07 +0200 Subject: [PATCH 031/207] Changes to metrics used and some reformatting --- icu_benchmarks/models/constants.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/icu_benchmarks/models/constants.py b/icu_benchmarks/models/constants.py index 360cf847..e9a0d9c0 100644 --- a/icu_benchmarks/models/constants.py +++ b/icu_benchmarks/models/constants.py @@ -1,5 +1,5 @@ -from ignite.contrib.metrics import AveragePrecision, ROC_AUC, PrecisionRecallCurve, RocCurve -from ignite.metrics import Accuracy, RootMeanSquaredError # , ConfusionMatrix +from ignite.contrib.metrics import AveragePrecision, ROC_AUC, RocCurve, PrecisionRecallCurve +from ignite.metrics import Accuracy, RootMeanSquaredError from sklearn.calibration import calibration_curve from sklearn.metrics import ( average_precision_score, @@ -14,8 +14,8 @@ ) from torchmetrics.classification import ( AUROC, - AveragePrecision, - PrecisionRecallCurve, + AveragePrecision as TorchMetricsAveragePrecision, + PrecisionRecallCurve as TorchMetricsPrecisionRecallCurve, CalibrationError, F1Score, ) @@ -63,7 +63,8 @@ class DLMetrics: BINARY_CLASSIFICATION_TORCHMETRICS = { "AUC": AUROC(task="binary"), - "PR": AveragePrecision(task="binary"), + "PR": TorchMetricsAveragePrecision(task="binary"), + "PrecisionRecallCurve": TorchMetricsPrecisionRecallCurve(task="binary"), "Calibration_Error": CalibrationError(task="binary", n_bins=10), "F1": F1Score(task="binary", num_classes=2), "Binary_Fairness": BinaryFairnessWrapper(num_groups=2, task="demographic_parity", group_name="sex"), From 44e02e7e04e6d1436e4af4f3372164122a3966bd Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 16 Aug 2023 13:29:14 +0200 Subject: [PATCH 032/207] small refactorings --- icu_benchmarks/imputation/diffwave.py | 2 +- icu_benchmarks/imputation/sssds4.py | 2 +- icu_benchmarks/models/constants.py | 1 + icu_benchmarks/models/custom_metrics.py | 10 ++++++++++ icu_benchmarks/models/wrappers.py | 7 +------ 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/icu_benchmarks/imputation/diffwave.py b/icu_benchmarks/imputation/diffwave.py index 0945c0e6..a1f04fd8 100644 --- a/icu_benchmarks/imputation/diffwave.py +++ b/icu_benchmarks/imputation/diffwave.py @@ -301,7 +301,7 @@ def forward(self, input_data): cond = self.cond_conv(cond) h += cond - out = torch.tanh(h[:, : self.res_channels, :]) * torch.sigmoid(h[:, self.res_channels:, :]) + out = torch.tanh(h[:, : self.res_channels, :]) * torch.sigmoid(h[:, self.res_channels :, :]) res = self.res_conv(out) assert x.shape == res.shape diff --git a/icu_benchmarks/imputation/sssds4.py b/icu_benchmarks/imputation/sssds4.py index d359fb09..86264f88 100644 --- a/icu_benchmarks/imputation/sssds4.py +++ b/icu_benchmarks/imputation/sssds4.py @@ -298,7 +298,7 @@ def forward(self, input_data): h = self.S42(h.permute(2, 0, 1)).permute(1, 2, 0) - out = torch.tanh(h[:, : self.res_channels, :]) * torch.sigmoid(h[:, self.res_channels :, :]) + out = torch.tanh(h[:,: self.res_channels,:]) * torch.sigmoid(h[:, self.res_channels :,:]) res = self.res_conv(out) assert x.shape == res.shape diff --git a/icu_benchmarks/models/constants.py b/icu_benchmarks/models/constants.py index e9a0d9c0..45af8271 100644 --- a/icu_benchmarks/models/constants.py +++ b/icu_benchmarks/models/constants.py @@ -28,6 +28,7 @@ BinaryFairnessWrapper, ) + class MLMetrics: BINARY_CLASSIFICATION = { "AUC": roc_auc_score, diff --git a/icu_benchmarks/models/custom_metrics.py b/icu_benchmarks/models/custom_metrics.py index 990be02e..44315a9e 100644 --- a/icu_benchmarks/models/custom_metrics.py +++ b/icu_benchmarks/models/custom_metrics.py @@ -121,3 +121,13 @@ def update(self, preds, target, data, feature_names) -> None: groups = data[:, :, feature_names.index(self.group_name)] group_per_id = groups[:, 0] return super().update(preds=preds.cpu(), target=target.cpu(), groups=group_per_id.long().cpu()) + + def feature_helper(self, trainer, step_prefix): + """Helper function to get the feature names from the trainer""" + if step_prefix == "train": + feature_names = trainer.train_dataloader.dataset.features + elif step_prefix == "val": + feature_names = trainer.train_dataloader.dataset.features + else: + feature_names = trainer.test_dataloaders.dataset.features + return feature_names diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index be88eaa2..3906b5cd 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -317,12 +317,7 @@ def step_fn(self, element, step_prefix=""): for key, value in self.metrics[step_prefix].items(): if isinstance(value, torchmetrics.Metric): if key == "Binary_Fairness": - if step_prefix == "train": - feature_names = self.trainer.train_dataloader.dataset.features - elif step_prefix == "val": - feature_names = self.trainer.train_dataloader.dataset.features - else: - feature_names = self.trainer.test_dataloaders.dataset.features + feature_names = key.feature_helper(self.trainer) value.update(transformed_output[0], transformed_output[1], data, feature_names) else: value.update(transformed_output[0], transformed_output[1]) From a68c7611bc873d07f757da06c66ec005f077d9c3 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 16 Aug 2023 13:38:47 +0200 Subject: [PATCH 033/207] small refactorings --- icu_benchmarks/imputation/sssds4.py | 2 +- icu_benchmarks/models/ml_models.py | 19 +++++++++++-------- icu_benchmarks/models/wrappers.py | 2 +- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/icu_benchmarks/imputation/sssds4.py b/icu_benchmarks/imputation/sssds4.py index ce6e8c0d..d359fb09 100644 --- a/icu_benchmarks/imputation/sssds4.py +++ b/icu_benchmarks/imputation/sssds4.py @@ -298,7 +298,7 @@ def forward(self, input_data): h = self.S42(h.permute(2, 0, 1)).permute(1, 2, 0) - out = torch.tanh(h[:, :self.res_channels, :]) * torch.sigmoid(h[:, self.res_channels:, :]) + out = torch.tanh(h[:, : self.res_channels, :]) * torch.sigmoid(h[:, self.res_channels :, :]) res = self.res_conv(out) assert x.shape == res.shape diff --git a/icu_benchmarks/models/ml_models.py b/icu_benchmarks/models/ml_models.py index f0764353..0ff14317 100644 --- a/icu_benchmarks/models/ml_models.py +++ b/icu_benchmarks/models/ml_models.py @@ -1,6 +1,7 @@ import gin -import lightgbm +import lightgbm as lgbm import numpy as np +import wandb from sklearn import linear_model from sklearn import ensemble from sklearn import neural_network @@ -9,21 +10,22 @@ from icu_benchmarks.contants import RunMode from wandb.lightgbm import wandb_callback + class LGBMWrapper(MLWrapper): def fit_model(self, train_data, train_labels, val_data, val_labels): """Fitting function for LGBM models.""" self.model.set_params(random_state=np.random.get_state()[1][0]) + callbacks = [lgbm.early_stopping(self.hparams.patience, verbose=True), lgbm.log_evaluation(period=-1)] + + if wandb.run is not None: + callbacks.append(callbacks) self.model = self.model.fit( train_data, train_labels, eval_set=(val_data, val_labels), verbose=True, - callbacks=[ - lightgbm.early_stopping(self.hparams.patience, verbose=True), - lightgbm.log_evaluation(period=-1, show_stdv=False), - # wandb_callback(), - ], + callbacks=callbacks, ) val_loss = list(self.model.best_score_["valid_0"].values())[0] return val_loss @@ -34,19 +36,20 @@ class LGBMClassifier(LGBMWrapper): _supported_run_modes = [RunMode.classification] def __init__(self, *args, **kwargs): - self.model = self.set_model_args(lightgbm.LGBMClassifier, *args, **kwargs) + self.model = self.set_model_args(lgbm.LGBMClassifier, *args, **kwargs) super().__init__(*args, **kwargs) def predict(self, features): """Predicts labels for the given features.""" return self.model.predict_proba(features) + @gin.configurable class LGBMRegressor(LGBMWrapper): _supported_run_modes = [RunMode.regression] def __init__(self, *args, **kwargs): - self.model = self.set_model_args(lightgbm.LGBMRegressor, *args, **kwargs) + self.model = self.set_model_args(lgbm.LGBMRegressor, *args, **kwargs) super().__init__(*args, **kwargs) diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index ebdf065b..bdccca53 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -364,7 +364,7 @@ def test_step(self, dataset, _): def predict(self, features): if self.run_mode == RunMode.regression: return self.model.predict(features) - else: # Classification: return probabilities + else: # Classification: return probabilities return self.model.predict_proba(features) def log_metrics(self, label, pred, metric_type): From 9d2f0fd53a3f4a807ffe1c92ede20e6f7309c474 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 16 Aug 2023 13:50:28 +0200 Subject: [PATCH 034/207] Minor callback correction --- icu_benchmarks/models/ml_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icu_benchmarks/models/ml_models.py b/icu_benchmarks/models/ml_models.py index 0ff14317..53bd5a2c 100644 --- a/icu_benchmarks/models/ml_models.py +++ b/icu_benchmarks/models/ml_models.py @@ -18,7 +18,7 @@ def fit_model(self, train_data, train_labels, val_data, val_labels): callbacks = [lgbm.early_stopping(self.hparams.patience, verbose=True), lgbm.log_evaluation(period=-1)] if wandb.run is not None: - callbacks.append(callbacks) + callbacks.append(wandb_callback()) self.model = self.model.fit( train_data, From 8d5d162c57d36136dbca8d8ec60aac22a2f1ac02 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 16 Aug 2023 14:18:00 +0200 Subject: [PATCH 035/207] Minor formatting --- icu_benchmarks/imputation/diffwave.py | 2 +- icu_benchmarks/imputation/sssds4.py | 2 +- icu_benchmarks/models/custom_metrics.py | 4 +--- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/icu_benchmarks/imputation/diffwave.py b/icu_benchmarks/imputation/diffwave.py index a1f04fd8..147eeb37 100644 --- a/icu_benchmarks/imputation/diffwave.py +++ b/icu_benchmarks/imputation/diffwave.py @@ -301,7 +301,7 @@ def forward(self, input_data): cond = self.cond_conv(cond) h += cond - out = torch.tanh(h[:, : self.res_channels, :]) * torch.sigmoid(h[:, self.res_channels :, :]) + out = torch.tanh(h[:, :self.res_channels, :]) * torch.sigmoid(h[:, self.res_channels:, :]) res = self.res_conv(out) assert x.shape == res.shape diff --git a/icu_benchmarks/imputation/sssds4.py b/icu_benchmarks/imputation/sssds4.py index d359fb09..ce6e8c0d 100644 --- a/icu_benchmarks/imputation/sssds4.py +++ b/icu_benchmarks/imputation/sssds4.py @@ -298,7 +298,7 @@ def forward(self, input_data): h = self.S42(h.permute(2, 0, 1)).permute(1, 2, 0) - out = torch.tanh(h[:, : self.res_channels, :]) * torch.sigmoid(h[:, self.res_channels :, :]) + out = torch.tanh(h[:, :self.res_channels, :]) * torch.sigmoid(h[:, self.res_channels:, :]) res = self.res_conv(out) assert x.shape == res.shape diff --git a/icu_benchmarks/models/custom_metrics.py b/icu_benchmarks/models/custom_metrics.py index 44315a9e..437ca614 100644 --- a/icu_benchmarks/models/custom_metrics.py +++ b/icu_benchmarks/models/custom_metrics.py @@ -6,10 +6,8 @@ from sklearn.calibration import calibration_curve from scipy.spatial.distance import jensenshannon from torchmetrics.classification import BinaryFairness - """" -This file contains custom metrics that are not available in ignite.metrics. -Specifically, it adds transformation capabilities to some metrics. +This file contains custom metrics that can be added to YAIB. """ From 10826517e7ccb428f8789d7cd1e87155a287e91b Mon Sep 17 00:00:00 2001 From: Hendrik Schmidt Date: Fri, 18 Aug 2023 10:41:44 +0200 Subject: [PATCH 036/207] introduce shuffle splits for fixed train sizes --- icu_benchmarks/data/split_process_data.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index 08db9d75..aebdd2d9 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -7,7 +7,7 @@ from pathlib import Path import pickle -from sklearn.model_selection import StratifiedKFold, KFold +from sklearn.model_selection import StratifiedKFold, KFold, StratifiedShuffleSplit, ShuffleSplit from icu_benchmarks.data.preprocessor import Preprocessor, DefaultClassificationPreprocessor from icu_benchmarks.contants import RunMode @@ -111,6 +111,7 @@ def make_single_split( repetition_index: int, cv_folds: int, fold_index: int, + train_size: int = None, seed: int = 42, debug: bool = False, runmode: RunMode = RunMode.classification, @@ -125,6 +126,7 @@ def make_single_split( repetition_index: Index of the repetition to return. cv_folds: Number of folds for cross validation. fold_index: Index of the fold to return. + train_size: Fixed size of train size (including validation data). seed: Random seed. debug: Load less data if true. @@ -150,7 +152,10 @@ def make_single_split( f"The smallest amount of samples in a class is: {labels.value_counts().min()}, " f"but {cv_folds} folds are requested. Reduce the number of folds or use more data." ) - outer_cv = StratifiedKFold(cv_repetitions, shuffle=True, random_state=seed) + if train_size: + outer_cv = StratifiedShuffleSplit(cv_repetitions, train_size=train_size) + else: + outer_cv = StratifiedKFold(cv_repetitions, shuffle=True, random_state=seed) inner_cv = StratifiedKFold(cv_folds, shuffle=True, random_state=seed) dev, test = list(outer_cv.split(stays, labels))[repetition_index] @@ -158,7 +163,10 @@ def make_single_split( train, val = list(inner_cv.split(dev_stays, labels.iloc[dev]))[fold_index] else: # If there are no labels, or the task is regression, use regular k-fold. - outer_cv = KFold(cv_repetitions, shuffle=True, random_state=seed) + if train_size: + outer_cv = ShuffleSplit(cv_repetitions, train_size=train_size) + else: + outer_cv = KFold(n_splits, shuffle=True, random_state=seed) inner_cv = KFold(cv_folds, shuffle=True, random_state=seed) dev, test = list(outer_cv.split(stays))[repetition_index] From a93d36c4ae24ffe71e6cc86a82da1e0a0109862f Mon Sep 17 00:00:00 2001 From: Hendrik Schmidt Date: Fri, 18 Aug 2023 10:53:50 +0200 Subject: [PATCH 037/207] make train size arg of preprocess --- icu_benchmarks/data/split_process_data.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index aebdd2d9..8a8b1381 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -25,6 +25,7 @@ def preprocess_data( debug: bool = False, cv_repetitions: int = 5, repetition_index: int = 0, + train_size: int = None, cv_folds: int = 5, load_cache: bool = False, generate_cache: bool = False, @@ -43,6 +44,7 @@ def preprocess_data( debug: Load less data if true. cv_repetitions: Number of times to repeat cross validation. repetition_index: Index of the repetition to return. + train_size: Fixed size of train split (including validation data). cv_folds: Number of folds to use for cross validation. load_cache: Use cached preprocessed data if true. generate_cache: Generate cached preprocessed data if true. @@ -87,7 +89,16 @@ def preprocess_data( # Generate the splits logging.info("Generating splits.") data = make_single_split( - data, vars, cv_repetitions, repetition_index, cv_folds, fold_index, seed=seed, debug=debug, runmode=runmode + data, + vars, + cv_repetitions, + repetition_index, + cv_folds, + fold_index, + train_size=train_size, + seed=seed, + debug=debug, + runmode=runmode ) # Apply preprocessing @@ -109,9 +120,9 @@ def make_single_split( vars: dict[str], cv_repetitions: int, repetition_index: int, + train_size: int = None, cv_folds: int, fold_index: int, - train_size: int = None, seed: int = 42, debug: bool = False, runmode: RunMode = RunMode.classification, @@ -124,9 +135,9 @@ def make_single_split( vars: Contains the names of columns in the data. cv_repetitions: Number of times to repeat cross validation. repetition_index: Index of the repetition to return. + train_size: Fixed size of train split (including validation data). cv_folds: Number of folds for cross validation. fold_index: Index of the fold to return. - train_size: Fixed size of train size (including validation data). seed: Random seed. debug: Load less data if true. From aa1ab1ad58549b5fbfe02604b0c8e377fc38d5d3 Mon Sep 17 00:00:00 2001 From: Hendrik Schmidt Date: Fri, 18 Aug 2023 12:06:59 +0200 Subject: [PATCH 038/207] use finetune flag for train size --- icu_benchmarks/cross_validation.py | 3 +++ icu_benchmarks/data/split_process_data.py | 8 ++++---- icu_benchmarks/run.py | 4 +++- icu_benchmarks/run_utils.py | 2 +- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/icu_benchmarks/cross_validation.py b/icu_benchmarks/cross_validation.py index 48090034..94db439a 100644 --- a/icu_benchmarks/cross_validation.py +++ b/icu_benchmarks/cross_validation.py @@ -20,6 +20,7 @@ def execute_repeated_cv( log_dir: Path, seed: int, eval_only: bool = False, + train_size: int = None, load_weights: bool = False, source_dir: Path = None, cv_repetitions: int = 5, @@ -45,6 +46,7 @@ def execute_repeated_cv( log_dir: Path to the log directory. seed: Random seed. eval_only: Whether to only evaluate the model. + train_size: Fixed size of train split (including validation data). load_weights: Whether to load weights from source_dir. source_dir: Path to the source directory. cv_folds: Number of folds for cross validation. @@ -81,6 +83,7 @@ def execute_repeated_cv( generate_cache=generate_cache, cv_repetitions=cv_repetitions, repetition_index=repetition, + train_size=train_size, cv_folds=cv_folds, fold_index=fold_index, pretrained_imputation_model=pretrained_imputation_model, diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index 8a8b1381..dd5db431 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -25,8 +25,8 @@ def preprocess_data( debug: bool = False, cv_repetitions: int = 5, repetition_index: int = 0, - train_size: int = None, cv_folds: int = 5, + train_size: int = None, load_cache: bool = False, generate_cache: bool = False, fold_index: int = 0, @@ -44,8 +44,8 @@ def preprocess_data( debug: Load less data if true. cv_repetitions: Number of times to repeat cross validation. repetition_index: Index of the repetition to return. - train_size: Fixed size of train split (including validation data). cv_folds: Number of folds to use for cross validation. + train_size: Fixed size of train split (including validation data). load_cache: Use cached preprocessed data if true. generate_cache: Generate cached preprocessed data if true. fold_index: Index of the fold to return. @@ -120,9 +120,9 @@ def make_single_split( vars: dict[str], cv_repetitions: int, repetition_index: int, - train_size: int = None, cv_folds: int, fold_index: int, + train_size: int = None, seed: int = 42, debug: bool = False, runmode: RunMode = RunMode.classification, @@ -135,9 +135,9 @@ def make_single_split( vars: Contains the names of columns in the data. cv_repetitions: Number of times to repeat cross validation. repetition_index: Index of the repetition to return. - train_size: Fixed size of train split (including validation data). cv_folds: Number of folds for cross validation. fold_index: Index of the fold to return. + train_size: Fixed size of train split (including validation data). seed: Random seed. debug: Load less data if true. diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index d34aab8d..0764262a 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -109,7 +109,8 @@ def main(my_args=tuple(sys.argv[1:])): # Load pretrained model in evaluate mode or when finetuning evaluate = args.eval - load_weights = evaluate or args.fine_tune + fine_tune_size = args.fine_tune + load_weights = evaluate or fine_tune_size is not None if load_weights: if args.source_dir is None: raise ValueError("Please specify a source directory when evaluating or finetuning.") @@ -159,6 +160,7 @@ def main(my_args=tuple(sys.argv[1:])): run_dir, args.seed, eval_only=evaluate, + train_size=fine_tune_size, load_weights=load_weights, source_dir=source_dir, reproducible=reproducible, diff --git a/icu_benchmarks/run_utils.py b/icu_benchmarks/run_utils.py index 73b3e6b5..9b4446cc 100644 --- a/icu_benchmarks/run_utils.py +++ b/icu_benchmarks/run_utils.py @@ -44,7 +44,7 @@ def build_parser() -> ArgumentParser: parser.add_argument("--tune", default=False, action=BOA, help="Find best hyperparameters.") parser.add_argument("--hp-checkpoint", type=Path, help="Use previous hyperparameter checkpoint.") parser.add_argument("--eval", default=False, action=BOA, help="Only evaluate model, skip training.") - parser.add_argument("-ft", "--fine-tune", default=False, action=BOA, help="Load model and finetune on data.") + parser.add_argument("-ft", "--fine-tune", default=None, type=int, help="Finetune model with amount of train data.") parser.add_argument("-sn", "--source-name", type=Path, help="Name of the source dataset.") parser.add_argument("--source-dir", type=Path, help="Directory containing gin and model weights.") From 6b0d8512bc955202e7d07e40572502809787c27c Mon Sep 17 00:00:00 2001 From: Hendrik Schmidt Date: Fri, 18 Aug 2023 12:07:50 +0200 Subject: [PATCH 039/207] save ML with joblib again --- icu_benchmarks/models/wrappers.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index 951600dd..37441ec9 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -17,6 +17,7 @@ from ignite.exceptions import NotComputableError from icu_benchmarks.models.constants import ImputationInit from icu_benchmarks.models.utils import create_optimizer, create_scheduler +from joblib import dump from pytorch_lightning import LightningModule from icu_benchmarks.models.constants import MLMetrics, DLMetrics @@ -445,10 +446,10 @@ def __getstate__(self) -> Dict[str, Any]: del state["output_transform"] return state - def save_model(self, save_path, file_name, file_extension=".ckpt"): + def save_model(self, save_path, file_name, file_extension=".joblib"): path = save_path / (file_name + file_extension) try: - torch.save(self, path) + dump(self.model, path) logging.info(f"Model saved to {str(path.resolve())}.") except Exception as e: logging.error(f"Cannot save model to path {str(path.resolve())}: {e}.") From b726050be558bcefb52c9b2157c498c23fec9dde Mon Sep 17 00:00:00 2001 From: Hendrik Schmidt Date: Fri, 18 Aug 2023 12:08:21 +0200 Subject: [PATCH 040/207] set weights after model load --- icu_benchmarks/models/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 2ced4105..12523201 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -110,7 +110,6 @@ def train_common( data_shape = next(iter(train_loader))[0].shape model = model(optimizer=optimizer, input_size=data_shape, epochs=epochs, run_mode=mode) - model.set_weight(weight, train_dataset) if load_weights: if source_dir.exists(): if model.requires_backprop: @@ -132,6 +131,7 @@ def train_common( else: raise Exception(f"No weights to load at path : {source_dir}") + model.set_weight(weight, train_dataset) model.set_trained_columns(train_dataset.get_feature_names()) loggers = [TensorBoardLogger(log_dir), JSONMetricsLogger(log_dir)] From 2d0d1d6c0da5ea4f8fd0050192e8091fbb37a3be Mon Sep 17 00:00:00 2001 From: Hendrik Schmidt Date: Fri, 18 Aug 2023 12:18:22 +0200 Subject: [PATCH 041/207] fix variable name --- icu_benchmarks/data/split_process_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index dd5db431..ebcb20d4 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -177,7 +177,7 @@ def make_single_split( if train_size: outer_cv = ShuffleSplit(cv_repetitions, train_size=train_size) else: - outer_cv = KFold(n_splits, shuffle=True, random_state=seed) + outer_cv = KFold(cv_repetitions, shuffle=True, random_state=seed) inner_cv = KFold(cv_folds, shuffle=True, random_state=seed) dev, test = list(outer_cv.split(stays))[repetition_index] From f8a6cb1d4bea0c834a3678bab846e96e6f88caa5 Mon Sep 17 00:00:00 2001 From: Hendrik Schmidt Date: Fri, 18 Aug 2023 12:18:29 +0200 Subject: [PATCH 042/207] linting --- icu_benchmarks/data/split_process_data.py | 2 +- icu_benchmarks/models/wrappers.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index ebcb20d4..82efd847 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -7,7 +7,7 @@ from pathlib import Path import pickle -from sklearn.model_selection import StratifiedKFold, KFold, StratifiedShuffleSplit, ShuffleSplit +from sklearn.model_selection import StratifiedKFold, KFold, StratifiedShuffleSplit, ShuffleSplit from icu_benchmarks.data.preprocessor import Preprocessor, DefaultClassificationPreprocessor from icu_benchmarks.contants import RunMode diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index 37441ec9..e436a621 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -138,7 +138,9 @@ def finalize_step(self, step_prefix=""): try: self.log_dict( { - f"{step_prefix}/{name}": (np.float32(metric.compute()) if isinstance(metric.compute(), np.float64 ) else metric.compute() ) + f"{step_prefix}/{name}": (np.float32(metric.compute()) + if isinstance(metric.compute(), np.float64) + else metric.compute()) for name, metric in self.metrics[step_prefix].items() if "_Curve" not in name }, From b3f89261246e8de5025431a52785728fce69b673 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Fri, 18 Aug 2023 12:22:18 +0200 Subject: [PATCH 043/207] comment in on_test_epoch_start --- icu_benchmarks/models/wrappers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index f6feea48..7e9a82d5 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -165,10 +165,10 @@ def configure_optimizers(self): return {"optimizer": optimizer, "lr_scheduler": scheduler} def on_test_epoch_start(self) -> None: - # self.metrics = { - # step_name: {metric_name: metric() for metric_name, metric in self.set_metrics().items()} - # for step_name in ["train", "val", "test"] - # } + self.metrics = { + step_name: {metric_name: metric() for metric_name, metric in self.set_metrics().items()} + for step_name in ["train", "val", "test"] + } return super().on_test_epoch_start() def save_model(self, save_path, file_name, file_extension=".ckpt"): From b7a16797d610ad1a7ef86dabfc8ad738a5d5d08d Mon Sep 17 00:00:00 2001 From: Hendrik Schmidt Date: Fri, 18 Aug 2023 12:24:27 +0200 Subject: [PATCH 044/207] comment in code for DL eval --- icu_benchmarks/models/wrappers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index e436a621..d6c1496e 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -168,10 +168,10 @@ def configure_optimizers(self): return {"optimizer": optimizer, "lr_scheduler": scheduler} def on_test_epoch_start(self) -> None: - # self.metrics = { - # step_name: {metric_name: metric() for metric_name, metric in self.set_metrics().items()} - # for step_name in ["train", "val", "test"] - # } + self.metrics = { + step_name: {metric_name: metric() for metric_name, metric in self.set_metrics().items()} + for step_name in ["train", "val", "test"] + } return super().on_test_epoch_start() def save_model(self, save_path, file_name, file_extension=".ckpt"): From b2ca910007d660004605aacc79ecbe9cab98575a Mon Sep 17 00:00:00 2001 From: Hendrik Schmidt Date: Fri, 18 Aug 2023 14:06:23 +0200 Subject: [PATCH 045/207] use training checkpoint for finetuning --- icu_benchmarks/models/train.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 12523201..22f924eb 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -109,6 +109,12 @@ def train_common( data_shape = next(iter(train_loader))[0].shape + if model.requires_backprop: + # double number of epochs for finetuning + epochs = 2 * epochs if load_weights else epochs + else: + epochs = 1 + model = model(optimizer=optimizer, input_size=data_shape, epochs=epochs, run_mode=mode) if load_weights: if source_dir.exists(): @@ -144,7 +150,7 @@ def train_common( if precision == 16 or "16-mixed": torch.set_float32_matmul_precision("medium") trainer = Trainer( - max_epochs=epochs if model.requires_backprop else 1, + max_epochs=epochs, callbacks=callbacks, precision=precision, accelerator="auto" if not cpu else "cpu", @@ -158,7 +164,12 @@ def train_common( if not eval_only: if model.requires_backprop: logging.info("Training DL model.") - trainer.fit(model, train_dataloaders=train_loader, val_dataloaders=val_loader) + trainer.fit( + model, + train_dataloaders=train_loader, + val_dataloaders=val_loader, + ckpt_path=None if not load_weights else model_path + ) logging.info("Training complete.") else: logging.info("Training ML model.") From a45cd53d68af864f4a371eade72a1f63834838eb Mon Sep 17 00:00:00 2001 From: Hendrik Schmidt Date: Fri, 18 Aug 2023 15:14:27 +0200 Subject: [PATCH 046/207] include train size in cache --- icu_benchmarks/data/split_process_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index 82efd847..b87cc7c2 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -70,8 +70,8 @@ def preprocess_data( if isinstance(preprocessor, DefaultClassificationPreprocessor): preprocessor.set_imputation_model(pretrained_imputation_model) - hash_config = f"{preprocessor.to_cache_string()}{dumped_file_names}{dumped_vars}{debug}".encode("utf-8") - cache_filename = f"s_{seed}_r_{repetition_index}_f_{fold_index}_d_{debug}_{hashlib.md5(hash_config).hexdigest()}" + hash_config = f"{preprocessor.to_cache_string()}{dumped_file_names}{dumped_vars}".encode("utf-8") + cache_filename = f"s_{seed}_r_{repetition_index}_f_{fold_index}_t_{train_size}_d_{debug}_{hashlib.md5(hash_config).hexdigest()}" cache_file = cache_dir / cache_filename if load_cache: From d5538643581c7fd2df6b9edb5cb95286cccc284e Mon Sep 17 00:00:00 2001 From: Hendrik Schmidt Date: Fri, 18 Aug 2023 15:31:58 +0200 Subject: [PATCH 047/207] change from checkpoint fit to simple loading --- icu_benchmarks/models/train.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 22f924eb..12523201 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -109,12 +109,6 @@ def train_common( data_shape = next(iter(train_loader))[0].shape - if model.requires_backprop: - # double number of epochs for finetuning - epochs = 2 * epochs if load_weights else epochs - else: - epochs = 1 - model = model(optimizer=optimizer, input_size=data_shape, epochs=epochs, run_mode=mode) if load_weights: if source_dir.exists(): @@ -150,7 +144,7 @@ def train_common( if precision == 16 or "16-mixed": torch.set_float32_matmul_precision("medium") trainer = Trainer( - max_epochs=epochs, + max_epochs=epochs if model.requires_backprop else 1, callbacks=callbacks, precision=precision, accelerator="auto" if not cpu else "cpu", @@ -164,12 +158,7 @@ def train_common( if not eval_only: if model.requires_backprop: logging.info("Training DL model.") - trainer.fit( - model, - train_dataloaders=train_loader, - val_dataloaders=val_loader, - ckpt_path=None if not load_weights else model_path - ) + trainer.fit(model, train_dataloaders=train_loader, val_dataloaders=val_loader) logging.info("Training complete.") else: logging.info("Training ML model.") From e5c2d766a4c06c54d34a24fc61932c0f6c1fe315 Mon Sep 17 00:00:00 2001 From: Hendrik Schmidt Date: Fri, 18 Aug 2023 15:34:05 +0200 Subject: [PATCH 048/207] linting --- icu_benchmarks/data/split_process_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index b87cc7c2..d8a065cb 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -70,8 +70,8 @@ def preprocess_data( if isinstance(preprocessor, DefaultClassificationPreprocessor): preprocessor.set_imputation_model(pretrained_imputation_model) - hash_config = f"{preprocessor.to_cache_string()}{dumped_file_names}{dumped_vars}".encode("utf-8") - cache_filename = f"s_{seed}_r_{repetition_index}_f_{fold_index}_t_{train_size}_d_{debug}_{hashlib.md5(hash_config).hexdigest()}" + hash_config = hashlib.md5(f"{preprocessor.to_cache_string()}{dumped_file_names}{dumped_vars}".encode("utf-8")) + cache_filename = f"s_{seed}_r_{repetition_index}_f_{fold_index}_t_{train_size}_d_{debug}_{hash_config.hexdigest()}" cache_file = cache_dir / cache_filename if load_cache: From f6861ac6375ea083213de06407fc837fd26a9dcb Mon Sep 17 00:00:00 2001 From: rvandewater Date: Sat, 19 Aug 2023 19:49:45 +0200 Subject: [PATCH 049/207] Naming and experiment config --- experiments/experiment_finetuning.yml | 63 +++++++++++++++++++++++++++ icu_benchmarks/run.py | 5 +++ 2 files changed, 68 insertions(+) create mode 100644 experiments/experiment_finetuning.yml diff --git a/experiments/experiment_finetuning.yml b/experiments/experiment_finetuning.yml new file mode 100644 index 00000000..02a7860f --- /dev/null +++ b/experiments/experiment_finetuning.yml @@ -0,0 +1,63 @@ +command: + - ${env} + - ${program} + - -ft + - 0 + - -d + - ../data/ + - -t + - BinaryClassification + - --log-dir + - ../yaib_logs_finetune + - --tune + - --wandb-sweep + - -gc + - -lc + - -sn + - eicu + - --source-dir + - /dhc/home/robin.vandewater/projects/transfer_learning/gru_mortality/eicu +method: grid +name: yaib_regression_benchmark +parameters: + fine_tune: + values: + - 100 +# - 500 +# - 1000 +# - 2000 + - 3000 +# - 4000 + - 5000 + - 6000 + - 7000 +# - 8000 + - 9000 + - 10000 + - 15000 + - 20000 + data_dir: + values: + - ../../data/mortality24/miiv + - ../../data/mortality24/hirid +# - ../../data/mortality24/eicu + - ../../data/mortality24/aumc +# - ../../data/kf/miiv +# - ../../data/kf/hirid +# - ../../data/kf/eicu +# - ../../data/kf/aumc + model: + values: +# - ElasticNet +# - LGBMRegressor + - GRU +# - LSTM +# - TCN +# - Transformer + seed: + values: + - 1111 + use_pretrained_imputation: + values: + - None +program: icu-benchmarks diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index 0764262a..d7a362ac 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -63,8 +63,11 @@ def main(my_args=tuple(sys.argv[1:])): if args.wandb_sweep: run_name = f"{mode}_{model}_{name}" + if (args.fine_tune): + run_name += f"_{args.source_name}_fine_tune_{args.fine_tune}" set_wandb_run_name(run_name) + logging.info(f"Task mode: {mode}.") experiment = args.experiment @@ -115,6 +118,8 @@ def main(my_args=tuple(sys.argv[1:])): if args.source_dir is None: raise ValueError("Please specify a source directory when evaluating or finetuning.") log_dir /= f"from_{args.source_name}" + if args.fine_tune: + log_dir /= f"fine_tune_{args.fine_tune}" run_dir = create_run_dir(log_dir) source_dir = args.source_dir gin.parse_config_file(source_dir / "train_config.gin") From 52117f44ce0e3250cb50427e527f7d97d2d1a782 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Sat, 19 Aug 2023 20:12:43 +0200 Subject: [PATCH 050/207] Added mps option for typecasting --- icu_benchmarks/data/loader.py | 7 ++- icu_benchmarks/models/wrappers.py | 96 +++++++++++++++++-------------- 2 files changed, 58 insertions(+), 45 deletions(-) diff --git a/icu_benchmarks/data/loader.py b/icu_benchmarks/data/loader.py index 8a021ad8..72208ee3 100644 --- a/icu_benchmarks/data/loader.py +++ b/icu_benchmarks/data/loader.py @@ -26,6 +26,7 @@ def __init__( split: str = Split.train, vars: Dict[str, str] = gin.REQUIRED, grouping_segment: str = Segment.outcome, + mps: bool = False ): self.split = split self.vars = vars @@ -37,6 +38,7 @@ def __init__( # calculate basic info for the data self.num_stays = self.grouping_df.index.unique().shape[0] self.maxlen = self.features_df.groupby([self.vars["GROUP"]]).size().max() + self.mps = mps def ram_cache(self, cache: bool = True): self._cached_dataset = None @@ -150,7 +152,10 @@ def get_data_and_labels(self) -> Tuple[np.array, np.array]: def to_tensor(self): data, labels = self.get_data_and_labels() - return from_numpy(data).to(float32), from_numpy(labels).to(float32) + if self.mps: + return from_numpy(data).to(float32), from_numpy(labels).to(float32) + else: + return from_numpy(data), from_numpy(labels) @gin.configurable("ImputationDataset") diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index d6c1496e..af434d7e 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -100,20 +100,20 @@ class DLWrapper(BaseModule, ABC): _supported_run_modes = [RunMode.classification, RunMode.regression, RunMode.imputation] def __init__( - self, - loss=CrossEntropyLoss(), - optimizer=Adam, - run_mode: RunMode = RunMode.classification, - input_shape=None, - lr: float = 0.002, - momentum: float = 0.9, - lr_scheduler: Optional[str] = None, - lr_factor: float = 0.99, - lr_steps: Optional[List[int]] = None, - epochs: int = 100, - input_size: Tensor = None, - initialization_method: str = "normal", - **kwargs, + self, + loss=CrossEntropyLoss(), + optimizer=Adam, + run_mode: RunMode = RunMode.classification, + input_shape=None, + lr: float = 0.002, + momentum: float = 0.9, + lr_scheduler: Optional[str] = None, + lr_factor: float = 0.99, + lr_steps: Optional[List[int]] = None, + epochs: int = 100, + input_size: Tensor = None, + initialization_method: str = "normal", + **kwargs, ): """General interface for Deep Learning (DL) models.""" super().__init__() @@ -190,20 +190,20 @@ class DLPredictionWrapper(DLWrapper): _supported_run_modes = [RunMode.classification, RunMode.regression] def __init__( - self, - loss=CrossEntropyLoss(), - optimizer=torch.optim.Adam, - run_mode: RunMode = RunMode.classification, - input_shape=None, - lr: float = 0.002, - momentum: float = 0.9, - lr_scheduler: Optional[str] = None, - lr_factor: float = 0.99, - lr_steps: Optional[List[int]] = None, - epochs: int = 100, - input_size: Tensor = None, - initialization_method: str = "normal", - **kwargs, + self, + loss=CrossEntropyLoss(), + optimizer=torch.optim.Adam, + run_mode: RunMode = RunMode.classification, + input_shape=None, + lr: float = 0.002, + momentum: float = 0.9, + lr_scheduler: Optional[str] = None, + lr_factor: float = 0.99, + lr_steps: Optional[List[int]] = None, + epochs: int = 100, + input_size: Tensor = None, + initialization_method: str = "normal", + **kwargs, ): super().__init__( loss=loss, @@ -337,7 +337,7 @@ class MLWrapper(BaseModule, ABC): requires_backprop = False _supported_run_modes = [RunMode.classification, RunMode.regression] - def __init__(self, *args, run_mode=RunMode.classification, loss=log_loss, patience=10, **kwargs): + def __init__(self, *args, run_mode=RunMode.classification, loss=log_loss, patience=10, mps=False, **kwargs): super().__init__() self.save_hyperparameters() self.scaler = None @@ -345,6 +345,7 @@ def __init__(self, *args, run_mode=RunMode.classification, loss=log_loss, patien self.run_mode = run_mode self.loss = loss self.patience = patience + self.mps = mps def set_metrics(self, labels): if self.run_mode == RunMode.classification: @@ -416,9 +417,13 @@ def test_step(self, dataset, _): self.set_metrics(test_label) test_pred = self.predict(test_rep) - self.log("test/loss", np.float32(self.loss(test_label, test_pred)), sync_dist=True) + if self.mps: + self.log("test/loss", np.float32(self.loss(test_label, test_pred)), sync_dist=True) + self.log_metrics(np.float32(test_label), np.float32(test_pred), "test") + else: + self.log("test/loss", self.loss(test_label, test_pred), sync_dist=True) + self.log_metrics(test_label, test_pred, "test") logging.debug(f"Test loss: {self.loss(test_label, test_pred)}") - self.log_metrics(np.float32(test_label), np.float32(test_pred), "test") def predict(self, features): if self.run_mode == RunMode.regression: @@ -431,7 +436,10 @@ def log_metrics(self, label, pred, metric_type): self.log_dict( { - f"{metric_type}/{name}": np.float32(metric(self.label_transform(label), self.output_transform(pred))) + # MPS dependent type casting + f"{metric_type}/{name}": metric(self.label_transform(label), self.output_transform(pred)) if not self.mps + else metric(self.label_transform(label), self.output_transform(pred)) + # Fore very metric for name, metric in self.metrics.items() # Filter out metrics that return a tuple (e.g. precision_recall_curve) if not isinstance(metric(self.label_transform(label), self.output_transform(pred)), tuple) @@ -476,18 +484,18 @@ class ImputationWrapper(DLWrapper): _supported_run_modes = [RunMode.imputation] def __init__( - self, - loss: nn.modules.loss._Loss = MSELoss(), - optimizer: Union[str, Optimizer] = "adam", - runmode: RunMode = RunMode.imputation, - lr: float = 0.002, - momentum: float = 0.9, - lr_scheduler: Optional[str] = None, - lr_factor: float = 0.99, - lr_steps: Optional[List[int]] = None, - input_size: Tensor = None, - initialization_method: ImputationInit = ImputationInit.NORMAL, - **kwargs: str, + self, + loss: nn.modules.loss._Loss = MSELoss(), + optimizer: Union[str, Optimizer] = "adam", + runmode: RunMode = RunMode.imputation, + lr: float = 0.002, + momentum: float = 0.9, + lr_scheduler: Optional[str] = None, + lr_factor: float = 0.99, + lr_steps: Optional[List[int]] = None, + input_size: Tensor = None, + initialization_method: ImputationInit = ImputationInit.NORMAL, + **kwargs: str, ) -> None: super().__init__() self.check_supported_runmode(runmode) From 438e46714fd570c9510865355d9601df708fb265 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Sun, 20 Aug 2023 16:17:01 +0200 Subject: [PATCH 051/207] More logging and dataset names --- icu_benchmarks/data/loader.py | 6 +- icu_benchmarks/models/train.py | 109 ++++++++++++++++-------------- icu_benchmarks/models/wrappers.py | 35 ++++++++-- icu_benchmarks/run.py | 28 +++++--- 4 files changed, 112 insertions(+), 66 deletions(-) diff --git a/icu_benchmarks/data/loader.py b/icu_benchmarks/data/loader.py index 72208ee3..3c7a9280 100644 --- a/icu_benchmarks/data/loader.py +++ b/icu_benchmarks/data/loader.py @@ -26,7 +26,8 @@ def __init__( split: str = Split.train, vars: Dict[str, str] = gin.REQUIRED, grouping_segment: str = Segment.outcome, - mps: bool = False + mps: bool = False, + name: str = "", ): self.split = split self.vars = vars @@ -39,11 +40,12 @@ def __init__( self.num_stays = self.grouping_df.index.unique().shape[0] self.maxlen = self.features_df.groupby([self.vars["GROUP"]]).size().max() self.mps = mps + self.name = name def ram_cache(self, cache: bool = True): self._cached_dataset = None if cache: - logging.info("Caching dataset in ram.") + logging.info(f"Caching {self.split} dataset in ram.") self._cached_dataset = [self[i] for i in range(len(self))] def __len__(self) -> int: diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 12523201..aac279d8 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -14,10 +14,10 @@ from icu_benchmarks.models.utils import save_config_file, JSONMetricsLogger from icu_benchmarks.contants import RunMode from icu_benchmarks.data.constants import DataSplit as Split - +from icu_benchmarks.models.constants import Precision +# from finetuning_scheduler import FinetuningScheduler cpu_core_count = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else os.cpu_count() - - +from pytorch_lightning.loggers import WandbLogger def assure_minimum_length(dataset): if len(dataset) < 2: return [dataset[0], dataset[0]] @@ -26,28 +26,29 @@ def assure_minimum_length(dataset): @gin.configurable("train_common") def train_common( - data: dict[str, pd.DataFrame], - log_dir: Path, - eval_only: bool = False, - load_weights: bool = False, - source_dir: Path = None, - reproducible: bool = True, - mode: str = RunMode.classification, - model: object = gin.REQUIRED, - weight: str = None, - optimizer: type = Adam, - precision=32, - batch_size=64, - epochs=1000, - patience=20, - min_delta=1e-5, - test_on: str = Split.test, - use_wandb: bool = False, - cpu: bool = False, - verbose=False, - ram_cache=False, - pl_model=True, - num_workers: int = min(cpu_core_count, torch.cuda.device_count() * 4 * int(torch.cuda.is_available()), 32), + data: dict[str, pd.DataFrame], + log_dir: Path, + eval_only: bool = False, + load_weights: bool = False, + source_dir: Path = None, + reproducible: bool = True, + mode: str = RunMode.classification, + model: object = gin.REQUIRED, + weight: str = None, + optimizer: type = Adam, + precision=32, + batch_size=64, + epochs=1000, + patience=20, + min_delta=1e-5, + test_on: str = Split.test, + dataset_names = None, + use_wandb: bool = False, + cpu: bool = False, + verbose=False, + ram_cache=False, + pl_model=True, + num_workers: int = min(cpu_core_count, torch.cuda.device_count() * 8 * int(torch.cuda.is_available()), 32), ): """Common wrapper to train all benchmarked models. @@ -65,7 +66,7 @@ def train_common( precision: Pytorch precision to be used for training. Can be 16 or 32. batch_size: Batch size to be used for training. epochs: Number of epochs to train for. - patience: Number of epochs to wait before early stopping. + patience: Number of epochs to wait for improvement before early stopping. min_delta: Minimum change in loss to be considered an improvement. test_on: If set to "test", evaluate the model on the test set. If set to "val", evaluate on the validation set. use_wandb: If set to true, log to wandb. @@ -82,12 +83,13 @@ def train_common( logging.info(f"Logging to directory: {log_dir}.") save_config_file(log_dir) # We save the operative config before and also after training - train_dataset = dataset_class(data, split=Split.train, ram_cache=ram_cache) - val_dataset = dataset_class(data, split=Split.val, ram_cache=ram_cache) + train_dataset = dataset_class(data, split=Split.train, ram_cache=ram_cache, name=dataset_names["train"]) + val_dataset = dataset_class(data, split=Split.val, ram_cache=ram_cache, name=dataset_names["val"]) train_dataset, val_dataset = assure_minimum_length(train_dataset), assure_minimum_length(val_dataset) batch_size = min(batch_size, len(train_dataset), len(val_dataset)) - logging.debug(f"Training on {len(train_dataset)} samples and validating on {len(val_dataset)} samples.") + logging.info(f"Training on {train_dataset.name} with {len(train_dataset)} samples and validating on {val_dataset.name} with" + f" {len(val_dataset)} samples.") logging.info(f"Using {num_workers} workers for data loading.") train_loader = DataLoader( @@ -111,33 +113,18 @@ def train_common( model = model(optimizer=optimizer, input_size=data_shape, epochs=epochs, run_mode=mode) if load_weights: - if source_dir.exists(): - if model.requires_backprop: - if (source_dir / "last.ckpt").exists(): - model_path = source_dir / "last.ckpt" - elif (source_dir / "model.ckpt").exists(): - model_path = source_dir / "model.ckpt" - elif (source_dir / "model-v1.ckpt").exists(): - model_path = source_dir / "model-v1.ckpt" - else: - return Exception(f"No weights to load at path : {source_dir}") - if pl_model: - model = model.load_from_checkpoint(model_path) - else: - checkpoint = torch.load(model_path) - model.load_state_dict(checkpoint) - else: - model = load(source_dir / "model.joblib") - else: - raise Exception(f"No weights to load at path : {source_dir}") + model = load_model(model, source_dir, pl_model) model.set_weight(weight, train_dataset) model.set_trained_columns(train_dataset.get_feature_names()) loggers = [TensorBoardLogger(log_dir), JSONMetricsLogger(log_dir)] + if use_wandb: + loggers.append(WandbLogger(save_dir=log_dir)) callbacks = [ EarlyStopping(monitor="val/loss", min_delta=min_delta, patience=patience, strict=False), ModelCheckpoint(log_dir, filename="model", save_top_k=1, save_last=True), + #FinetuningScheduler() ] if verbose: callbacks.append(TQDMProgressBar(refresh_rate=min(100, len(train_loader) // 2))) @@ -166,8 +153,9 @@ def train_common( model.save_model(log_dir, "last") logging.info("Training complete.") - test_dataset = dataset_class(data, split=test_on) + test_dataset = dataset_class(data, split=test_on, name=dataset_names["test"]) test_dataset = assure_minimum_length(test_dataset) + logging.info(f"Testing on {test_dataset.name} with {len(test_dataset)} samples.") test_loader = ( DataLoader( test_dataset, @@ -185,3 +173,26 @@ def train_common( test_loss = trainer.test(model, dataloaders=test_loader, verbose=verbose)[0]["test/loss"] save_config_file(log_dir) return test_loss + +def load_model(model, source_dir, pl_model=True): + if source_dir.exists(): + if model.requires_backprop: + if (source_dir / "last.ckpt").exists(): + model_path = source_dir / "last.ckpt" + elif (source_dir / "model.ckpt").exists(): + model_path = source_dir / "model.ckpt" + elif (source_dir / "model-v1.ckpt").exists(): + model_path = source_dir / "model-v1.ckpt" + else: + return Exception(f"No weights to load at path : {source_dir}") + if pl_model: + model = model.load_from_checkpoint(model_path) + else: + checkpoint = torch.load(model_path) + model.load_state_dict(checkpoint) + else: + model = load(source_dir / "model.joblib") + else: + raise Exception(f"No weights to load at path : {source_dir}") + logging.info(f"Loaded {type(model)} model from {source_dir}") + return model diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index af434d7e..b8d74c64 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -120,9 +120,19 @@ def __init__( self.save_hyperparameters(ignore=["loss", "optimizer"]) self.loss = loss self.optimizer = optimizer - self.scaler = None self.check_supported_runmode(run_mode) self.run_mode = run_mode + self.input_shape = input_shape + self.lr = lr + self.momentum = momentum + self.lr_scheduler = lr_scheduler + self.lr_factor = lr_factor + self.lr_steps = lr_steps + self.epochs = epochs + self.input_size = input_size + self.initialization_method = initialization_method + self.scaler = None + logging.info(f"Learning rate: {self.lr}, learning factor {self.lr_factor}, learning steps {self.lr_steps}") def on_fit_start(self): self.metrics = { @@ -487,7 +497,7 @@ def __init__( self, loss: nn.modules.loss._Loss = MSELoss(), optimizer: Union[str, Optimizer] = "adam", - runmode: RunMode = RunMode.imputation, + run_mode: RunMode = RunMode.imputation, lr: float = 0.002, momentum: float = 0.9, lr_scheduler: Optional[str] = None, @@ -495,11 +505,26 @@ def __init__( lr_steps: Optional[List[int]] = None, input_size: Tensor = None, initialization_method: ImputationInit = ImputationInit.NORMAL, + epochs=100, **kwargs: str, ) -> None: - super().__init__() - self.check_supported_runmode(runmode) - self.run_mode = runmode + + super().__init__( + loss=loss, + optimizer=optimizer, + run_mode=run_mode, + lr=lr, + momentum=momentum, + lr_scheduler=lr_scheduler, + lr_factor=lr_factor, + lr_steps=lr_steps, + epochs=epochs, + input_size=input_size, + initialization_method=initialization_method, + kwargs=kwargs, + ) + self.check_supported_runmode(run_mode) + self.run_mode = run_mode self.save_hyperparameters(ignore=["loss", "optimizer"]) self.loss = loss self.optimizer = optimizer diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index d7a362ac..5f6c52d8 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -61,13 +61,19 @@ def main(my_args=tuple(sys.argv[1:])): mode = get_mode() + evaluate = args.eval + fine_tune_size = args.fine_tune + load_weights = evaluate or fine_tune_size is not None + if args.wandb_sweep: run_name = f"{mode}_{model}_{name}" - if (args.fine_tune): - run_name += f"_{args.source_name}_fine_tune_{args.fine_tune}" + if load_weights: + if args.fine_tune: + run_name += f"_source_{args.source_name}_fine-tune_{args.fine_tune}_samples" + else: + run_name += f"_source_{args.source_name}" set_wandb_run_name(run_name) - logging.info(f"Task mode: {mode}.") experiment = args.experiment @@ -111,23 +117,25 @@ def main(my_args=tuple(sys.argv[1:])): logging.error(f"Could not import custom preprocessor from {args.preprocessor}: {e}") # Load pretrained model in evaluate mode or when finetuning - evaluate = args.eval - fine_tune_size = args.fine_tune - load_weights = evaluate or fine_tune_size is not None + if load_weights: if args.source_dir is None: - raise ValueError("Please specify a source directory when evaluating or finetuning.") - log_dir /= f"from_{args.source_name}" + raise ValueError("Please specify a source directory when evaluating or fine-tuning.") + log_dir /= f"_from_{args.source_name}" + gin.bind_parameter("train_common.dataset_names", {"train": args.source_name, "val": args.source_name, "test": args.name}) if args.fine_tune: log_dir /= f"fine_tune_{args.fine_tune}" + gin.bind_parameter("train_common.dataset_names", {"train": args.name, "val": args.name, "test": args.name}) run_dir = create_run_dir(log_dir) source_dir = args.source_dir + logging.info(f"Will load weights from {source_dir} and bind train gin-config. Note: this might override your config.") gin.parse_config_file(source_dir / "train_config.gin") else: - # Train + # Normal train and evaluate + gin.bind_parameter("train_common.dataset_names", {"train": args.name, "val": args.name, "test": args.name}) hp_checkpoint = log_dir / args.hp_checkpoint if args.hp_checkpoint else None model_path = ( - Path("configs") / ("imputation_models" if mode == RunMode.imputation else "prediction_models") / f"{model}.gin" + Path("configs") / ("imputation_models" if mode == RunMode.imputation else "prediction_models") / f"{model}.gin" ) gin_config_files = ( [Path(f"configs/experiments/{args.experiment}.gin")] From bda617d1b7b795fea29016b126f1ae4708e35161 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Sun, 20 Aug 2023 16:57:32 +0200 Subject: [PATCH 052/207] different checkpoint loading --- icu_benchmarks/models/train.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index aac279d8..9d11877c 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -14,7 +14,7 @@ from icu_benchmarks.models.utils import save_config_file, JSONMetricsLogger from icu_benchmarks.contants import RunMode from icu_benchmarks.data.constants import DataSplit as Split -from icu_benchmarks.models.constants import Precision +from icu_benchmarks.models.dl_models import GRUNet # from finetuning_scheduler import FinetuningScheduler cpu_core_count = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else os.cpu_count() from pytorch_lightning.loggers import WandbLogger @@ -111,9 +111,10 @@ def train_common( data_shape = next(iter(train_loader))[0].shape - model = model(optimizer=optimizer, input_size=data_shape, epochs=epochs, run_mode=mode) + # model = model(optimizer=optimizer, input_size=data_shape, epochs=epochs, run_mode=mode) if load_weights: - model = load_model(model, source_dir, pl_model) + model = GRUNet.load_from_checkpoint(source_dir/"last.ckpt") + # model = load_model(model, source_dir, pl_model) model.set_weight(weight, train_dataset) model.set_trained_columns(train_dataset.get_feature_names()) @@ -130,6 +131,7 @@ def train_common( callbacks.append(TQDMProgressBar(refresh_rate=min(100, len(train_loader) // 2))) if precision == 16 or "16-mixed": torch.set_float32_matmul_precision("medium") + trainer = Trainer( max_epochs=epochs if model.requires_backprop else 1, callbacks=callbacks, @@ -141,6 +143,7 @@ def train_common( enable_progress_bar=verbose, logger=loggers, num_sanity_val_steps=0, + ) if not eval_only: if model.requires_backprop: From f1a05f9876241126ea5997fc3209b1526dfa5b1e Mon Sep 17 00:00:00 2001 From: rvandewater Date: Mon, 21 Aug 2023 12:21:44 +0200 Subject: [PATCH 053/207] Checkpoint loading via fit --- icu_benchmarks/models/train.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 9d11877c..0b786f8c 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -111,7 +111,7 @@ def train_common( data_shape = next(iter(train_loader))[0].shape - # model = model(optimizer=optimizer, input_size=data_shape, epochs=epochs, run_mode=mode) + model = model(optimizer=optimizer, input_size=data_shape, epochs=epochs, run_mode=mode) if load_weights: model = GRUNet.load_from_checkpoint(source_dir/"last.ckpt") # model = load_model(model, source_dir, pl_model) @@ -148,7 +148,10 @@ def train_common( if not eval_only: if model.requires_backprop: logging.info("Training DL model.") - trainer.fit(model, train_dataloaders=train_loader, val_dataloaders=val_loader) + if load_weights: + trainer.fit(model, train_dataloaders=train_loader, val_dataloaders=val_loader, ckpt_path=source_dir/"last.ckpt") + else: + trainer.fit(model, train_dataloaders=train_loader, val_dataloaders=val_loader) logging.info("Training complete.") else: logging.info("Training ML model.") From d268d7fbbab45861c4906364e0451bf2592d0ac1 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 22 Aug 2023 10:42:14 +0200 Subject: [PATCH 054/207] logging and sanity validation --- icu_benchmarks/cross_validation.py | 3 ++- icu_benchmarks/models/train.py | 12 ++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/icu_benchmarks/cross_validation.py b/icu_benchmarks/cross_validation.py index 94db439a..773339e8 100644 --- a/icu_benchmarks/cross_validation.py +++ b/icu_benchmarks/cross_validation.py @@ -70,10 +70,11 @@ def execute_repeated_cv( if not cv_folds_to_train: cv_folds_to_train = cv_folds agg_loss = 0 - + logging.info(f"Starting nested CV with {cv_repetitions_to_train} repetitions of {cv_folds_to_train} folds") seed_everything(seed, reproducible) for repetition in range(cv_repetitions_to_train): for fold_index in range(cv_folds_to_train): + start_time = datetime.now() data = preprocess_data( data_dir, diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 0b786f8c..84e36b5a 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -8,7 +8,7 @@ from torch.utils.data import DataLoader from pytorch_lightning.loggers import TensorBoardLogger from pytorch_lightning import Trainer -from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint, TQDMProgressBar +from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint, TQDMProgressBar, LearningRateMonitor from pathlib import Path from icu_benchmarks.data.loader import PredictionDataset, ImputationDataset from icu_benchmarks.models.utils import save_config_file, JSONMetricsLogger @@ -123,8 +123,9 @@ def train_common( if use_wandb: loggers.append(WandbLogger(save_dir=log_dir)) callbacks = [ - EarlyStopping(monitor="val/loss", min_delta=min_delta, patience=patience, strict=False), + EarlyStopping(monitor="val/loss", min_delta=min_delta, patience=patience*3, strict=False, verbose=verbose), ModelCheckpoint(log_dir, filename="model", save_top_k=1, save_last=True), + LearningRateMonitor(logging_interval="step"), #FinetuningScheduler() ] if verbose: @@ -137,12 +138,13 @@ def train_common( callbacks=callbacks, precision=precision, accelerator="auto" if not cpu else "cpu", - devices=max(torch.cuda.device_count(), 1), + devices=max(torch.cuda.device_count(), 1) if not cpu else -1, deterministic="warn" if reproducible else False, benchmark=not reproducible, enable_progress_bar=verbose, logger=loggers, - num_sanity_val_steps=0, + num_sanity_val_steps=-1, + log_every_n_steps=5, ) if not eval_only: @@ -176,6 +178,8 @@ def train_common( ) model.set_weight("balanced", train_dataset) + + # test_loss = trainer.test(model, dataloaders=test_loader, verbose=verbose, ckpt_path= source_dir/"last.ckpt" if eval_only else None)[0]["test/loss"] test_loss = trainer.test(model, dataloaders=test_loader, verbose=verbose)[0]["test/loss"] save_config_file(log_dir) return test_loss From 59a27ca3d45a87e97a4ea687614f44dfddae152c Mon Sep 17 00:00:00 2001 From: prockenschaub Date: Tue, 22 Aug 2023 11:03:08 +0200 Subject: [PATCH 055/207] use quick ffill and nafill(0) --- icu_benchmarks/data/preprocessor.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/icu_benchmarks/data/preprocessor.py b/icu_benchmarks/data/preprocessor.py index c924980a..5e5ddf0e 100644 --- a/icu_benchmarks/data/preprocessor.py +++ b/icu_benchmarks/data/preprocessor.py @@ -5,7 +5,7 @@ import pandas as pd from recipys.recipe import Recipe from recipys.selector import all_numeric_predictors, all_outcomes, has_type, all_of -from recipys.step import StepScale, StepImputeFill, StepSklearn, StepHistorical, Accumulator, StepImputeModel +from recipys.step import StepScale, StepImputeFastForwardFill, StepImputeFastZeroFill, StepSklearn, StepHistorical, Accumulator, StepImputeModel from sklearn.impute import SimpleImputer, MissingIndicator from sklearn.preprocessing import LabelEncoder, FunctionTransformer, MinMaxScaler @@ -92,7 +92,7 @@ def _process_static(self, data, vars): if self.scaling: sta_rec.add_step(StepScale()) - sta_rec.add_step(StepImputeFill(sel=all_numeric_predictors(), value=0)) + sta_rec.add_step(StepImputeFastZeroFill(sel=all_numeric_predictors())) sta_rec.add_step(StepSklearn(SimpleImputer(missing_values=None, strategy="most_frequent"), sel=has_type("object"))) sta_rec.add_step(StepSklearn(LabelEncoder(), sel=has_type("object"), columnwise=True)) @@ -122,8 +122,8 @@ def _process_dynamic(self, data, vars): if self.imputation_model is not None: dyn_rec.add_step(StepImputeModel(model=self.model_impute, sel=all_of(vars[Segment.dynamic]))) dyn_rec.add_step(StepSklearn(MissingIndicator(), sel=all_of(vars[Segment.dynamic]), in_place=False)) - dyn_rec.add_step(StepImputeFill(method="ffill")) - dyn_rec.add_step(StepImputeFill(value=0)) + dyn_rec.add_step(StepImputeFastForwardFill()) + dyn_rec.add_step(StepImputeFastZeroFill()) if self.generate_features: dyn_rec = self._dynamic_feature_generation(dyn_rec, all_of(vars[Segment.dynamic])) data = apply_recipe_to_splits(dyn_rec, data, Segment.dynamic) From 230eb3d6d682f3d989081f93f83ad77326fac622 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 22 Aug 2023 11:17:59 +0200 Subject: [PATCH 056/207] model loading adjustment --- icu_benchmarks/models/train.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 84e36b5a..a855c90c 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -187,12 +187,12 @@ def train_common( def load_model(model, source_dir, pl_model=True): if source_dir.exists(): if model.requires_backprop: - if (source_dir / "last.ckpt").exists(): - model_path = source_dir / "last.ckpt" - elif (source_dir / "model.ckpt").exists(): + if (source_dir / "model.ckpt").exists(): model_path = source_dir / "model.ckpt" elif (source_dir / "model-v1.ckpt").exists(): model_path = source_dir / "model-v1.ckpt" + elif (source_dir / "last.ckpt").exists(): + model_path = source_dir / "last.ckpt" else: return Exception(f"No weights to load at path : {source_dir}") if pl_model: @@ -201,8 +201,9 @@ def load_model(model, source_dir, pl_model=True): checkpoint = torch.load(model_path) model.load_state_dict(checkpoint) else: - model = load(source_dir / "model.joblib") + model_path = source_dir / "model.joblib" + model = load(model_path) else: raise Exception(f"No weights to load at path : {source_dir}") - logging.info(f"Loaded {type(model)} model from {source_dir}") + logging.info(f"Loaded {type(model)} model from {model_path}") return model From 76d52907021f1d6be4f2f88534f3e0683b14fac7 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 22 Aug 2023 11:20:45 +0200 Subject: [PATCH 057/207] added experiment for "small set training" --- experiments/experiment_small_set_training.yml | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 experiments/experiment_small_set_training.yml diff --git a/experiments/experiment_small_set_training.yml b/experiments/experiment_small_set_training.yml new file mode 100644 index 00000000..d37d294f --- /dev/null +++ b/experiments/experiment_small_set_training.yml @@ -0,0 +1,60 @@ +command: + - ${env} + - ${program} + - -ft + - 0 + - -d + - ../data/ + - -t + - BinaryClassification + - --log-dir + - ../yaib_logs_small_set_training + - --tune + - --wandb-sweep + - -gc + - -lc + - --verbose +method: grid +name: yaib_regression_benchmark +parameters: + fine_tune: + values: + - 100 + - 500 + - 1000 + - 2000 +# - 3000 +## - 4000 +# - 5000 +# - 6000 +# - 7000 +## - 8000 +# - 9000 +# - 10000 +# - 15000 +# - 20000 + data_dir: + values: +# - ../../data/mortality24/miiv + - ../../data/mortality24/hirid +# - ../../data/mortality24/eicu +# - ../../data/mortality24/aumc +# - ../../data/kf/miiv +# - ../../data/kf/hirid +# - ../../data/kf/eicu +# - ../../data/kf/aumc + model: + values: +# - ElasticNet +# - LGBMRegressor + - GRU +# - LSTM +# - TCN +# - Transformer + seed: + values: + - 1111 + use_pretrained_imputation: + values: + - None +program: icu-benchmarks From afc106270cc5296d8746371f2f42e5ae1d76502b Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 22 Aug 2023 13:02:32 +0200 Subject: [PATCH 058/207] conflicting run dir --- icu_benchmarks/run_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/icu_benchmarks/run_utils.py b/icu_benchmarks/run_utils.py index 9b4446cc..427a25d9 100644 --- a/icu_benchmarks/run_utils.py +++ b/icu_benchmarks/run_utils.py @@ -64,7 +64,10 @@ def create_run_dir(log_dir: Path, randomly_searched_params: str = None) -> Path: Returns: Path to the created run log directory. """ - log_dir_run = log_dir / str(datetime.now().strftime("%Y-%m-%dT%H-%M-%S")) + if not (log_dir/ str(datetime.now().strftime("%Y-%m-%dT%H-%M-%S"))).exists() : + log_dir_run = log_dir / str(datetime.now().strftime("%Y-%m-%dT%H-%M-%S")) + else: + log_dir_run = log_dir / str(datetime.now().strftime("%Y-%m-%dT%H-%M-%S.%f")) log_dir_run.mkdir(parents=True) if randomly_searched_params: (log_dir_run / randomly_searched_params).touch() From 3f55507d12cf2b1a351b89726a636e9b51ae2433 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 22 Aug 2023 13:30:04 +0200 Subject: [PATCH 059/207] Added custom train size independent of fine-tuning. Log dir clashing fixed. Minor changes to model loading. Added small-set training experiment --- experiments/experiment_small_set_training.yml | 2 ++ icu_benchmarks/models/train.py | 19 ++++++++++++------- icu_benchmarks/run_utils.py | 4 ++-- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/experiments/experiment_small_set_training.yml b/experiments/experiment_small_set_training.yml index d37d294f..46559a76 100644 --- a/experiments/experiment_small_set_training.yml +++ b/experiments/experiment_small_set_training.yml @@ -14,6 +14,8 @@ command: - -gc - -lc - --verbose + - --source-dir + - /dhc/home/robin.vandewater/projects/transfer_learning/gru_mortality/hirid method: grid name: yaib_regression_benchmark parameters: diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index a855c90c..74a82574 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -15,9 +15,12 @@ from icu_benchmarks.contants import RunMode from icu_benchmarks.data.constants import DataSplit as Split from icu_benchmarks.models.dl_models import GRUNet + # from finetuning_scheduler import FinetuningScheduler cpu_core_count = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else os.cpu_count() from pytorch_lightning.loggers import WandbLogger + + def assure_minimum_length(dataset): if len(dataset) < 2: return [dataset[0], dataset[0]] @@ -42,7 +45,7 @@ def train_common( patience=20, min_delta=1e-5, test_on: str = Split.test, - dataset_names = None, + dataset_names=None, use_wandb: bool = False, cpu: bool = False, verbose=False, @@ -88,8 +91,9 @@ def train_common( train_dataset, val_dataset = assure_minimum_length(train_dataset), assure_minimum_length(val_dataset) batch_size = min(batch_size, len(train_dataset), len(val_dataset)) - logging.info(f"Training on {train_dataset.name} with {len(train_dataset)} samples and validating on {val_dataset.name} with" - f" {len(val_dataset)} samples.") + logging.info( + f"Training on {train_dataset.name} with {len(train_dataset)} samples and validating on {val_dataset.name} with" + f" {len(val_dataset)} samples.") logging.info(f"Using {num_workers} workers for data loading.") train_loader = DataLoader( @@ -113,7 +117,7 @@ def train_common( model = model(optimizer=optimizer, input_size=data_shape, epochs=epochs, run_mode=mode) if load_weights: - model = GRUNet.load_from_checkpoint(source_dir/"last.ckpt") + model = GRUNet.load_from_checkpoint(source_dir / "model.ckpt") # model = load_model(model, source_dir, pl_model) model.set_weight(weight, train_dataset) @@ -123,10 +127,10 @@ def train_common( if use_wandb: loggers.append(WandbLogger(save_dir=log_dir)) callbacks = [ - EarlyStopping(monitor="val/loss", min_delta=min_delta, patience=patience*3, strict=False, verbose=verbose), + EarlyStopping(monitor="val/loss", min_delta=min_delta, patience=patience * 3, strict=False, verbose=verbose), ModelCheckpoint(log_dir, filename="model", save_top_k=1, save_last=True), LearningRateMonitor(logging_interval="step"), - #FinetuningScheduler() + # FinetuningScheduler() ] if verbose: callbacks.append(TQDMProgressBar(refresh_rate=min(100, len(train_loader) // 2))) @@ -151,7 +155,7 @@ def train_common( if model.requires_backprop: logging.info("Training DL model.") if load_weights: - trainer.fit(model, train_dataloaders=train_loader, val_dataloaders=val_loader, ckpt_path=source_dir/"last.ckpt") + trainer.fit(model, train_dataloaders=train_loader, val_dataloaders=val_loader, ckpt_path=source_dir/"model.ckpt") else: trainer.fit(model, train_dataloaders=train_loader, val_dataloaders=val_loader) logging.info("Training complete.") @@ -184,6 +188,7 @@ def train_common( save_config_file(log_dir) return test_loss + def load_model(model, source_dir, pl_model=True): if source_dir.exists(): if model.requires_backprop: diff --git a/icu_benchmarks/run_utils.py b/icu_benchmarks/run_utils.py index 427a25d9..85f67324 100644 --- a/icu_benchmarks/run_utils.py +++ b/icu_benchmarks/run_utils.py @@ -47,7 +47,7 @@ def build_parser() -> ArgumentParser: parser.add_argument("-ft", "--fine-tune", default=None, type=int, help="Finetune model with amount of train data.") parser.add_argument("-sn", "--source-name", type=Path, help="Name of the source dataset.") parser.add_argument("--source-dir", type=Path, help="Directory containing gin and model weights.") - + parser.add_argument("-sa", "--samples", type=int, default=None, help="Number of samples to use for evaluation.") return parser @@ -64,7 +64,7 @@ def create_run_dir(log_dir: Path, randomly_searched_params: str = None) -> Path: Returns: Path to the created run log directory. """ - if not (log_dir/ str(datetime.now().strftime("%Y-%m-%dT%H-%M-%S"))).exists() : + if not (log_dir/ str(datetime.now().strftime("%Y-%m-%dT%H-%M-%S"))).exists(): log_dir_run = log_dir / str(datetime.now().strftime("%Y-%m-%dT%H-%M-%S")) else: log_dir_run = log_dir / str(datetime.now().strftime("%Y-%m-%dT%H-%M-%S.%f")) From 6886e1476b77af5211cf1a21e001313c77e76999 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 22 Aug 2023 13:30:51 +0200 Subject: [PATCH 060/207] Added custom train size independent of fine-tuning. --- icu_benchmarks/run.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index 5f6c52d8..d6d5e3a6 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -43,7 +43,7 @@ def main(my_args=tuple(sys.argv[1:])): date_format = "%Y-%m-%d %H:%M:%S" verbose = args.verbose setup_logging(date_format, log_format, verbose) - + verbose = True # Get arguments data_dir = Path(args.data_dir) name = args.name @@ -62,8 +62,10 @@ def main(my_args=tuple(sys.argv[1:])): mode = get_mode() evaluate = args.eval - fine_tune_size = args.fine_tune - load_weights = evaluate or fine_tune_size is not None + + # Set train size to fine tune size if fine tune is set, else use custom train size + train_size = args.fine_tune if args.fine_tune is not None else args.samples + load_weights = evaluate or args.fine_tune is not None if args.wandb_sweep: run_name = f"{mode}_{model}_{name}" @@ -117,7 +119,7 @@ def main(my_args=tuple(sys.argv[1:])): logging.error(f"Could not import custom preprocessor from {args.preprocessor}: {e}") # Load pretrained model in evaluate mode or when finetuning - + load_weights = False if load_weights: if args.source_dir is None: raise ValueError("Please specify a source directory when evaluating or fine-tuning.") @@ -130,6 +132,11 @@ def main(my_args=tuple(sys.argv[1:])): source_dir = args.source_dir logging.info(f"Will load weights from {source_dir} and bind train gin-config. Note: this might override your config.") gin.parse_config_file(source_dir / "train_config.gin") + elif args.samples and args.source_dir is not None: # Train model with limited samples + gin.parse_config_file(args.source_dir / "train_config.gin") + log_dir /= f"samples_{args.fine_tune}" + gin.bind_parameter("train_common.dataset_names", {"train": args.name, "val": args.name, "test": args.name}) + run_dir = create_run_dir(log_dir) else: # Normal train and evaluate gin.bind_parameter("train_common.dataset_names", {"train": args.name, "val": args.name, "test": args.name}) @@ -155,7 +162,7 @@ def main(my_args=tuple(sys.argv[1:])): debug=args.debug, generate_cache=args.generate_cache, load_cache=args.load_cache, - verbose=args.verbose, + verbose=verbose, ) log_full_line(f"Logging to {run_dir.resolve()}", level=logging.INFO) @@ -173,7 +180,7 @@ def main(my_args=tuple(sys.argv[1:])): run_dir, args.seed, eval_only=evaluate, - train_size=fine_tune_size, + train_size=train_size, load_weights=load_weights, source_dir=source_dir, reproducible=reproducible, From 4f7095b53e50beed161f2816a310db0778cf0e0a Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 22 Aug 2023 14:04:13 +0200 Subject: [PATCH 061/207] wandb changes --- experiments/experiment_small_set_training.yml | 6 +++--- icu_benchmarks/run.py | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/experiments/experiment_small_set_training.yml b/experiments/experiment_small_set_training.yml index 46559a76..6dddd1b7 100644 --- a/experiments/experiment_small_set_training.yml +++ b/experiments/experiment_small_set_training.yml @@ -1,7 +1,7 @@ command: - ${env} - ${program} - - -ft + - --samples - 0 - -d - ../data/ @@ -12,14 +12,14 @@ command: - --tune - --wandb-sweep - -gc - - -lc +# - -lc - --verbose - --source-dir - /dhc/home/robin.vandewater/projects/transfer_learning/gru_mortality/hirid method: grid name: yaib_regression_benchmark parameters: - fine_tune: + samples: values: - 100 - 500 diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index d6d5e3a6..720827e3 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -74,6 +74,8 @@ def main(my_args=tuple(sys.argv[1:])): run_name += f"_source_{args.source_name}_fine-tune_{args.fine_tune}_samples" else: run_name += f"_source_{args.source_name}" + if args.samples: + run_name += f"_train_size_{args.samples}" set_wandb_run_name(run_name) logging.info(f"Task mode: {mode}.") From 99253db3a682ff823fb5c9cd4efcfc403e61f4fb Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 22 Aug 2023 14:22:01 +0200 Subject: [PATCH 062/207] logging --- icu_benchmarks/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index 720827e3..1569a2fb 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -134,7 +134,8 @@ def main(my_args=tuple(sys.argv[1:])): source_dir = args.source_dir logging.info(f"Will load weights from {source_dir} and bind train gin-config. Note: this might override your config.") gin.parse_config_file(source_dir / "train_config.gin") - elif args.samples and args.source_dir is not None: # Train model with limited samples + elif args.samples and args.source_dir is not None: # Train model with limited samples and bind existing config + logging.info("Binding train gin-config. Note: this might override your config.") gin.parse_config_file(args.source_dir / "train_config.gin") log_dir /= f"samples_{args.fine_tune}" gin.bind_parameter("train_common.dataset_names", {"train": args.name, "val": args.name, "test": args.name}) From 1614cc320a0581539b6c2fa6ed8e74974221bbc2 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 22 Aug 2023 14:47:20 +0200 Subject: [PATCH 063/207] load weights bool left over --- icu_benchmarks/run.py | 1 - 1 file changed, 1 deletion(-) diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index d6d5e3a6..b0faa19f 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -119,7 +119,6 @@ def main(my_args=tuple(sys.argv[1:])): logging.error(f"Could not import custom preprocessor from {args.preprocessor}: {e}") # Load pretrained model in evaluate mode or when finetuning - load_weights = False if load_weights: if args.source_dir is None: raise ValueError("Please specify a source directory when evaluating or fine-tuning.") From 9767ced56489579e550be0dfe21210930007f8a3 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 22 Aug 2023 15:03:08 +0200 Subject: [PATCH 064/207] cpu fix --- icu_benchmarks/models/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 74a82574..2876b0c3 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -142,7 +142,7 @@ def train_common( callbacks=callbacks, precision=precision, accelerator="auto" if not cpu else "cpu", - devices=max(torch.cuda.device_count(), 1) if not cpu else -1, + devices=max(torch.cuda.device_count(), 1), deterministic="warn" if reproducible else False, benchmark=not reproducible, enable_progress_bar=verbose, From 5fb281071dbf0f27c80fd15290913a1c4cb22d79 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 22 Aug 2023 17:54:28 +0200 Subject: [PATCH 065/207] preprocessor experiment --- icu_benchmarks/data/preprocessor.py | 16 ++++++++++++++++ icu_benchmarks/models/train.py | 7 ++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/icu_benchmarks/data/preprocessor.py b/icu_benchmarks/data/preprocessor.py index 5e5ddf0e..6cfd7d54 100644 --- a/icu_benchmarks/data/preprocessor.py +++ b/icu_benchmarks/data/preprocessor.py @@ -1,3 +1,5 @@ +import pickle + import torch import logging @@ -273,6 +275,20 @@ def apply_recipe_to_splits(recipe: Recipe, data: dict[dict[pd.DataFrame]], type: Returns: Transformed features divided into 'train', 'val', and 'test'. """ + + # cache_file = "data/" + # cache_file_name = f"eicu_{Split.train}_{type}" + # cache_file += cache_file_name + # # # with open(cache_file, "wb") as f: + # # # pickle.dump(data, f, pickle.HIGHEST_PROTOCOL) + # # # logging.info(f"Cached data in {cache_file}.") + # with open(cache_file, "rb") as f: + # logging.info(f"Loading cached data from {cache_file}.") + # eicu_train = pickle.load(f) + # eicu_train = recipe.prep() + # data[Split.train][type] = recipe.bake(data[Split.train][type]) + # data[Split.val][type] = recipe.bake(data[Split.val][type]) + # data[Split.test][type] = recipe.bake(data[Split.test][type]) data[Split.train][type] = recipe.prep() data[Split.val][type] = recipe.bake(data[Split.val][type]) data[Split.test][type] = recipe.bake(data[Split.test][type]) diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 2876b0c3..373a5bc8 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -91,9 +91,10 @@ def train_common( train_dataset, val_dataset = assure_minimum_length(train_dataset), assure_minimum_length(val_dataset) batch_size = min(batch_size, len(train_dataset), len(val_dataset)) - logging.info( - f"Training on {train_dataset.name} with {len(train_dataset)} samples and validating on {val_dataset.name} with" - f" {len(val_dataset)} samples.") + if not eval_only: + logging.info( + f"Training on {train_dataset.name} with {len(train_dataset)} samples and validating on {val_dataset.name} with" + f" {len(val_dataset)} samples.") logging.info(f"Using {num_workers} workers for data loading.") train_loader = DataLoader( From 44944bad2febdc18c85f46d8a3c4dc17f03f98b1 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 23 Aug 2023 15:07:59 +0200 Subject: [PATCH 066/207] rudimentary caching in preprocessor.py --- icu_benchmarks/data/preprocessor.py | 84 +++++++++++++++++++---------- 1 file changed, 55 insertions(+), 29 deletions(-) diff --git a/icu_benchmarks/data/preprocessor.py b/icu_benchmarks/data/preprocessor.py index 6cfd7d54..e153e66d 100644 --- a/icu_benchmarks/data/preprocessor.py +++ b/icu_benchmarks/data/preprocessor.py @@ -1,3 +1,4 @@ +import copy import pickle import torch @@ -7,7 +8,9 @@ import pandas as pd from recipys.recipe import Recipe from recipys.selector import all_numeric_predictors, all_outcomes, has_type, all_of -from recipys.step import StepScale, StepImputeFastForwardFill, StepImputeFastZeroFill, StepSklearn, StepHistorical, Accumulator, StepImputeModel +from recipys.step import StepScale, StepImputeFastForwardFill, StepImputeFastZeroFill, StepSklearn, StepHistorical, \ + Accumulator, StepImputeModel, StepImputeFill + from sklearn.impute import SimpleImputer, MissingIndicator from sklearn.preprocessing import LabelEncoder, FunctionTransformer, MinMaxScaler @@ -57,6 +60,7 @@ def apply(self, data, vars) -> dict[dict[pd.DataFrame]]: Preprocessed data. """ logging.info("Preprocessing dynamic features.") + # input caching here data = self._process_dynamic(data, vars) if self.use_static_features: logging.info("Preprocessing static features.") @@ -94,6 +98,7 @@ def _process_static(self, data, vars): if self.scaling: sta_rec.add_step(StepScale()) + # sta_rec.add_step(StepImputeFastZeroFill(sel=all_numeric_predictors())) sta_rec.add_step(StepImputeFastZeroFill(sel=all_numeric_predictors())) sta_rec.add_step(StepSklearn(SimpleImputer(missing_values=None, strategy="most_frequent"), sel=has_type("object"))) sta_rec.add_step(StepSklearn(LabelEncoder(), sel=has_type("object"), columnwise=True)) @@ -126,6 +131,8 @@ def _process_dynamic(self, data, vars): dyn_rec.add_step(StepSklearn(MissingIndicator(), sel=all_of(vars[Segment.dynamic]), in_place=False)) dyn_rec.add_step(StepImputeFastForwardFill()) dyn_rec.add_step(StepImputeFastZeroFill()) + # dyn_rec.add_step(StepImputeFill(method="ffill")) + # dyn_rec.add_step(StepImputeFill(value=0)) if self.generate_features: dyn_rec = self._dynamic_feature_generation(dyn_rec, all_of(vars[Segment.dynamic])) data = apply_recipe_to_splits(dyn_rec, data, Segment.dynamic) @@ -141,8 +148,8 @@ def _dynamic_feature_generation(self, data, dynamic_vars): def to_cache_string(self): return ( - super().to_cache_string() - + f"_classification_{self.generate_features}_{self.scaling}_{self.imputation_model.__class__.__name__}" + super().to_cache_string() + + f"_classification_{self.generate_features}_{self.scaling}_{self.imputation_model.__class__.__name__}" ) @@ -150,12 +157,12 @@ def to_cache_string(self): class DefaultRegressionPreprocessor(DefaultClassificationPreprocessor): # Override base classification preprocessor def __init__( - self, - generate_features: bool = True, - scaling: bool = True, - use_static_features: bool = True, - outcome_max=None, - outcome_min=None, + self, + generate_features: bool = True, + scaling: bool = True, + use_static_features: bool = True, + outcome_max=None, + outcome_min=None, ): """ Args: @@ -209,10 +216,10 @@ def _process_outcome(self, data, vars, split): @gin.configurable("base_imputation_preprocessor") class DefaultImputationPreprocessor(Preprocessor): def __init__( - self, - scaling: bool = True, - use_static_features: bool = True, - filter_missing_values: bool = True, + self, + scaling: bool = True, + use_static_features: bool = True, + filter_missing_values: bool = True, ): """Preprocesses data for imputation. @@ -276,20 +283,39 @@ def apply_recipe_to_splits(recipe: Recipe, data: dict[dict[pd.DataFrame]], type: Transformed features divided into 'train', 'val', and 'test'. """ - # cache_file = "data/" - # cache_file_name = f"eicu_{Split.train}_{type}" - # cache_file += cache_file_name - # # # with open(cache_file, "wb") as f: - # # # pickle.dump(data, f, pickle.HIGHEST_PROTOCOL) - # # # logging.info(f"Cached data in {cache_file}.") - # with open(cache_file, "rb") as f: - # logging.info(f"Loading cached data from {cache_file}.") - # eicu_train = pickle.load(f) - # eicu_train = recipe.prep() - # data[Split.train][type] = recipe.bake(data[Split.train][type]) - # data[Split.val][type] = recipe.bake(data[Split.val][type]) - # data[Split.test][type] = recipe.bake(data[Split.test][type]) - data[Split.train][type] = recipe.prep() - data[Split.val][type] = recipe.bake(data[Split.val][type]) - data[Split.test][type] = recipe.bake(data[Split.test][type]) + cache_file = "data/" + cache_file_name = f"eicu_{Split.train}_{type}" + cache_file += cache_file_name + restore_preproc = False + cache_preproc = False + + if restore_preproc: + recipe = restore_recipe(cache_file) + data[Split.train][type] = recipe.bake(data[Split.train][type]) + data[Split.val][type] = recipe.bake(data[Split.val][type]) + data[Split.test][type] = recipe.bake(data[Split.test][type]) + elif cache_preproc: + data[Split.train][type] = recipe.prep() + cache_recipe(recipe, cache_file) + data[Split.val][type] = recipe.bake(data[Split.val][type]) + data[Split.test][type] = recipe.bake(data[Split.test][type]) + else: + data[Split.train][type] = recipe.prep() + data[Split.val][type] = recipe.bake(data[Split.val][type]) + data[Split.test][type] = recipe.bake(data[Split.test][type]) return data + + +def cache_recipe(recipe: Recipe, cache_file: str) -> None: + recipe_cache = copy.deepcopy(recipe) + recipe_cache.cache() + logging.info(f"Cached recipe in {cache_file}.") + with open(cache_file, "wb") as f: + pickle.dump(recipe_cache, f, pickle.HIGHEST_PROTOCOL) + + +def restore_recipe(cache_file: str) -> Recipe: + with open(cache_file, "rb") as f: + logging.info(f"Loading cached recipe from {cache_file}.") + recipe = pickle.load(f) + return recipe From 49c25e3d7df9768af3b5d67f24decead9c228752 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 23 Aug 2023 21:19:59 +0200 Subject: [PATCH 067/207] experiment definition --- icu_benchmarks/models/train.py | 22 ++++++++++++---------- icu_benchmarks/models/utils.py | 2 +- icu_benchmarks/models/wrappers.py | 29 +++++++++++++++++++++-------- 3 files changed, 34 insertions(+), 19 deletions(-) diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 373a5bc8..5fa97854 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -8,10 +8,11 @@ from torch.utils.data import DataLoader from pytorch_lightning.loggers import TensorBoardLogger from pytorch_lightning import Trainer -from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint, TQDMProgressBar, LearningRateMonitor +from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint, TQDMProgressBar, LearningRateMonitor, \ + LearningRateFinder from pathlib import Path from icu_benchmarks.data.loader import PredictionDataset, ImputationDataset -from icu_benchmarks.models.utils import save_config_file, JSONMetricsLogger +from icu_benchmarks.models.utils import save_config_file, JSONMetricsLogger, create_optimizer from icu_benchmarks.contants import RunMode from icu_benchmarks.data.constants import DataSplit as Split from icu_benchmarks.models.dl_models import GRUNet @@ -118,23 +119,25 @@ def train_common( model = model(optimizer=optimizer, input_size=data_shape, epochs=epochs, run_mode=mode) if load_weights: - model = GRUNet.load_from_checkpoint(source_dir / "model.ckpt") - # model = load_model(model, source_dir, pl_model) + model = model.load_from_checkpoint(source_dir / "model.ckpt")#optimizer=create_optimizer("adam", model, 0.1, 1)) + #model = load_model(model, source_dir, pl_model) model.set_weight(weight, train_dataset) model.set_trained_columns(train_dataset.get_feature_names()) - + model.optimizer = create_optimizer("adam", model, 0.00001, 1) + model.hparams.lr_scheduler = "exponential" loggers = [TensorBoardLogger(log_dir), JSONMetricsLogger(log_dir)] if use_wandb: loggers.append(WandbLogger(save_dir=log_dir)) callbacks = [ - EarlyStopping(monitor="val/loss", min_delta=min_delta, patience=patience * 3, strict=False, verbose=verbose), + EarlyStopping(monitor="val/loss", min_delta=min_delta, patience=patience * 5, strict=False, verbose=verbose), ModelCheckpoint(log_dir, filename="model", save_top_k=1, save_last=True), LearningRateMonitor(logging_interval="step"), + # LearningRateFinder() # FinetuningScheduler() ] - if verbose: - callbacks.append(TQDMProgressBar(refresh_rate=min(100, len(train_loader) // 2))) + # if verbose: + # callbacks.append(TQDMProgressBar(refresh_rate=min(100, len(train_loader) // 2))) if precision == 16 or "16-mixed": torch.set_float32_matmul_precision("medium") @@ -150,13 +153,12 @@ def train_common( logger=loggers, num_sanity_val_steps=-1, log_every_n_steps=5, - ) if not eval_only: if model.requires_backprop: logging.info("Training DL model.") if load_weights: - trainer.fit(model, train_dataloaders=train_loader, val_dataloaders=val_loader, ckpt_path=source_dir/"model.ckpt") + trainer.fit(model, train_dataloaders=train_loader, val_dataloaders=val_loader) #, ckpt_path=source_dir/"model.ckpt") else: trainer.fit(model, train_dataloaders=train_loader, val_dataloaders=val_loader) logging.info("Training complete.") diff --git a/icu_benchmarks/models/utils.py b/icu_benchmarks/models/utils.py index 53bfcae2..6c944ae7 100644 --- a/icu_benchmarks/models/utils.py +++ b/icu_benchmarks/models/utils.py @@ -23,7 +23,7 @@ def save_config_file(log_dir): f.write(gin.operative_config_str()) -def create_optimizer(name: str, model: Module, lr: float, momentum: float) -> Optimizer: +def create_optimizer(name: str, model: Module, lr: float, momentum: float = 0) -> Optimizer: """creates the specified optimizer with the given parameters Args: diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index b8d74c64..acfbb650 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -102,7 +102,7 @@ class DLWrapper(BaseModule, ABC): def __init__( self, loss=CrossEntropyLoss(), - optimizer=Adam, + optimizer=None, run_mode: RunMode = RunMode.classification, input_shape=None, lr: float = 0.002, @@ -144,6 +144,15 @@ def on_fit_start(self): } return super().on_fit_start() + def on_train_start(self): + self.metrics = { + step_name: { + metric_name: (metric() if isinstance(metric, type) else metric) + for metric_name, metric in self.set_metrics().items() + } + for step_name in ["train", "val", "test"] + } + return super().on_train_start() def finalize_step(self, step_prefix=""): try: self.log_dict( @@ -166,16 +175,21 @@ def finalize_step(self, step_prefix=""): def configure_optimizers(self): if isinstance(self.optimizer, str): - optimizer = create_optimizer(self.optimizer, self, self.hparams.lr, self.hparams.momentum) - else: + optimizer = create_optimizer(self.optimizer, 0.5, self.hparams.momentum) + elif self.optimizer is None: optimizer = self.optimizer(self.parameters()) + else: + # Already set + optimizer = self.optimizer if self.hparams.lr_scheduler is None or self.hparams.lr_scheduler == "": return optimizer scheduler = create_scheduler( self.hparams.lr_scheduler, optimizer, self.hparams.lr_factor, self.hparams.lr_steps, self.hparams.epochs ) - return {"optimizer": optimizer, "lr_scheduler": scheduler} + optimizers = {"optimizer": optimizer, "lr_scheduler": scheduler} + logging.info(f"Using: {optimizers}") + return optimizers def on_test_epoch_start(self) -> None: self.metrics = { @@ -205,10 +219,10 @@ def __init__( optimizer=torch.optim.Adam, run_mode: RunMode = RunMode.classification, input_shape=None, - lr: float = 0.002, + lr: float = 0.00002, momentum: float = 0.9, lr_scheduler: Optional[str] = None, - lr_factor: float = 0.99, + lr_factor: float = 0.099, lr_steps: Optional[List[int]] = None, epochs: int = 100, input_size: Tensor = None, @@ -233,6 +247,7 @@ def __init__( self.output_transform = None self.loss_weights = None + def set_weight(self, weight, dataset): """Set the weight for the loss function.""" @@ -244,7 +259,6 @@ def set_weight(self, weight, dataset): def set_metrics(self, *args): """Set the evaluation metrics for the prediction model.""" - def softmax_binary_output_transform(output): with torch.no_grad(): y_pred, y = output @@ -256,7 +270,6 @@ def softmax_multi_output_transform(output): y_pred, y = output y_pred = torch.softmax(y_pred, dim=1) return y_pred, y - # Output transform is not applied for contrib metrics, so we do our own. if self.run_mode == RunMode.classification: # Binary classification From b72dad9a212f733193b9bfbc80bc9e3b1d3d50db Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 23 Aug 2023 21:20:09 +0200 Subject: [PATCH 068/207] experiment definition --- experiments/benchmark_classification.yml | 42 ++++++++++--------- experiments/experiment_finetuning.yml | 26 ++++++------ experiments/experiment_small_set_training.yml | 17 +++----- 3 files changed, 41 insertions(+), 44 deletions(-) diff --git a/experiments/benchmark_classification.yml b/experiments/benchmark_classification.yml index 61763ffa..28ba2d61 100644 --- a/experiments/benchmark_classification.yml +++ b/experiments/benchmark_classification.yml @@ -7,36 +7,38 @@ command: - -t - BinaryClassification - --log-dir - - ../yaib_logs + - ../yaib_logs_mortality24 - --tune - --wandb-sweep - - -gc - - -lc + - --hp-checkpoint + - /dhc/home/robin.vandewater/projects/yaib_logs_mortality24/hirid/BinaryClassification/GRU/2023-08-16T16-33-19 +# - -gc +# - -lc method: grid name: yaib_classification_benchmark parameters: data_dir: values: - - ../data/mortality24/miiv - - ../data/mortality24/hirid - - ../data/mortality24/eicu - - ../data/mortality24/aumc - - ../data/aki/miiv - - ../data/aki/hirid - - ../data/aki/eicu - - ../data/aki/aumc - - ../data/sepsis/miiv - - ../data/sepsis/hirid - - ../data/sepsis/eicu - - ../data/sepsis/aumc +# - ../data/mortality24/miiv + - ../../data/mortality24/hirid +# - ../data/mortality24/eicu +# - ../data/mortality24/aumc +# - ../../data/aki/miiv +# - ../../data/aki/hirid +# - ../../data/aki/eicu +# - ../../data/aki/aumc +# - ../../data/sepsis/miiv +# - ../../data/sepsis/hirid +# - ../../data/sepsis/eicu +# - ../../data/sepsis/aumc model: values: - - LogisticRegression - - LGBMClassifier +# - LogisticRegression +# - LGBMClassifier - GRU - - LSTM - - TCN - - Transformer +# - LSTM +# - TCN +# - Transformer seed: values: - 1111 diff --git a/experiments/experiment_finetuning.yml b/experiments/experiment_finetuning.yml index 02a7860f..7a410179 100644 --- a/experiments/experiment_finetuning.yml +++ b/experiments/experiment_finetuning.yml @@ -12,36 +12,38 @@ command: - --tune - --wandb-sweep - -gc - - -lc +# - -lc - -sn - eicu - --source-dir - /dhc/home/robin.vandewater/projects/transfer_learning/gru_mortality/eicu + - --verbose method: grid name: yaib_regression_benchmark parameters: fine_tune: values: - - 100 +# - 100 # - 500 # - 1000 # - 2000 - - 3000 +## - 3000 # - 4000 - - 5000 - - 6000 - - 7000 +## - 5000 +# - 6000 +## - 7000 # - 8000 - - 9000 - - 10000 - - 15000 - - 20000 +## - 9000 +# - 10000 + - 12000 +# - 15000 +# - 20000 data_dir: values: - - ../../data/mortality24/miiv +# - ../../data/mortality24/miiv - ../../data/mortality24/hirid # - ../../data/mortality24/eicu - - ../../data/mortality24/aumc +# - ../../data/mortality24/aumc # - ../../data/kf/miiv # - ../../data/kf/hirid # - ../../data/kf/eicu diff --git a/experiments/experiment_small_set_training.yml b/experiments/experiment_small_set_training.yml index 6dddd1b7..821e5e85 100644 --- a/experiments/experiment_small_set_training.yml +++ b/experiments/experiment_small_set_training.yml @@ -11,8 +11,6 @@ command: - ../yaib_logs_small_set_training - --tune - --wandb-sweep - - -gc -# - -lc - --verbose - --source-dir - /dhc/home/robin.vandewater/projects/transfer_learning/gru_mortality/hirid @@ -25,16 +23,11 @@ parameters: - 500 - 1000 - 2000 -# - 3000 -## - 4000 -# - 5000 -# - 6000 -# - 7000 -## - 8000 -# - 9000 -# - 10000 -# - 15000 -# - 20000 + - 4000 + - 6000 + - 8000 + - 10000 + - 12000 data_dir: values: # - ../../data/mortality24/miiv From ee9267196d8ab0fb4f60e8423999dda984508d73 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 24 Aug 2023 16:04:56 +0200 Subject: [PATCH 069/207] cleanup train.py --- icu_benchmarks/models/train.py | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 5fa97854..94db6dfb 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -8,14 +8,12 @@ from torch.utils.data import DataLoader from pytorch_lightning.loggers import TensorBoardLogger from pytorch_lightning import Trainer -from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint, TQDMProgressBar, LearningRateMonitor, \ - LearningRateFinder +from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint, TQDMProgressBar, LearningRateMonitor from pathlib import Path from icu_benchmarks.data.loader import PredictionDataset, ImputationDataset -from icu_benchmarks.models.utils import save_config_file, JSONMetricsLogger, create_optimizer +from icu_benchmarks.models.utils import save_config_file, JSONMetricsLogger from icu_benchmarks.contants import RunMode from icu_benchmarks.data.constants import DataSplit as Split -from icu_benchmarks.models.dl_models import GRUNet # from finetuning_scheduler import FinetuningScheduler cpu_core_count = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else os.cpu_count() @@ -118,26 +116,22 @@ def train_common( data_shape = next(iter(train_loader))[0].shape model = model(optimizer=optimizer, input_size=data_shape, epochs=epochs, run_mode=mode) + if load_weights: - model = model.load_from_checkpoint(source_dir / "model.ckpt")#optimizer=create_optimizer("adam", model, 0.1, 1)) - #model = load_model(model, source_dir, pl_model) + model = load_model(model, source_dir, pl_model=pl_model) model.set_weight(weight, train_dataset) model.set_trained_columns(train_dataset.get_feature_names()) - model.optimizer = create_optimizer("adam", model, 0.00001, 1) - model.hparams.lr_scheduler = "exponential" loggers = [TensorBoardLogger(log_dir), JSONMetricsLogger(log_dir)] if use_wandb: loggers.append(WandbLogger(save_dir=log_dir)) callbacks = [ - EarlyStopping(monitor="val/loss", min_delta=min_delta, patience=patience * 5, strict=False, verbose=verbose), + EarlyStopping(monitor="val/loss", min_delta=min_delta, patience=patience, strict=False, verbose=verbose), ModelCheckpoint(log_dir, filename="model", save_top_k=1, save_last=True), LearningRateMonitor(logging_interval="step"), - # LearningRateFinder() - # FinetuningScheduler() ] - # if verbose: - # callbacks.append(TQDMProgressBar(refresh_rate=min(100, len(train_loader) // 2))) + if verbose: + callbacks.append(TQDMProgressBar(refresh_rate=min(100, len(train_loader) // 2))) if precision == 16 or "16-mixed": torch.set_float32_matmul_precision("medium") @@ -158,7 +152,7 @@ def train_common( if model.requires_backprop: logging.info("Training DL model.") if load_weights: - trainer.fit(model, train_dataloaders=train_loader, val_dataloaders=val_loader) #, ckpt_path=source_dir/"model.ckpt") + trainer.fit(model, train_dataloaders=train_loader, val_dataloaders=val_loader) else: trainer.fit(model, train_dataloaders=train_loader, val_dataloaders=val_loader) logging.info("Training complete.") @@ -185,8 +179,6 @@ def train_common( ) model.set_weight("balanced", train_dataset) - - # test_loss = trainer.test(model, dataloaders=test_loader, verbose=verbose, ckpt_path= source_dir/"last.ckpt" if eval_only else None)[0]["test/loss"] test_loss = trainer.test(model, dataloaders=test_loader, verbose=verbose)[0]["test/loss"] save_config_file(log_dir) return test_loss @@ -207,7 +199,7 @@ def load_model(model, source_dir, pl_model=True): model = model.load_from_checkpoint(model_path) else: checkpoint = torch.load(model_path) - model.load_state_dict(checkpoint) + model.load_from_checkpoint(checkpoint) else: model_path = source_dir / "model.joblib" model = load(model_path) From 378aad4bd0054d9c0bd6f139cd7132a15a8fecc9 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 24 Aug 2023 16:40:29 +0200 Subject: [PATCH 070/207] Cleaned up recipe caching --- icu_benchmarks/data/preprocessor.py | 100 +++++++++++++--------- icu_benchmarks/data/split_process_data.py | 6 +- 2 files changed, 63 insertions(+), 43 deletions(-) diff --git a/icu_benchmarks/data/preprocessor.py b/icu_benchmarks/data/preprocessor.py index e153e66d..82380d8d 100644 --- a/icu_benchmarks/data/preprocessor.py +++ b/icu_benchmarks/data/preprocessor.py @@ -8,8 +8,16 @@ import pandas as pd from recipys.recipe import Recipe from recipys.selector import all_numeric_predictors, all_outcomes, has_type, all_of -from recipys.step import StepScale, StepImputeFastForwardFill, StepImputeFastZeroFill, StepSklearn, StepHistorical, \ - Accumulator, StepImputeModel, StepImputeFill +from recipys.step import ( + StepScale, + StepImputeFastForwardFill, + StepImputeFastZeroFill, + StepSklearn, + StepHistorical, + Accumulator, + StepImputeModel, + StepImputeFill, +) from sklearn.impute import SimpleImputer, MissingIndicator from sklearn.preprocessing import LabelEncoder, FunctionTransformer, MinMaxScaler @@ -22,7 +30,7 @@ class Preprocessor: @abc.abstractmethod - def apply(self, data, vars): + def apply(self, data, vars, save_cache=False, load_cache=None): return data @abc.abstractmethod @@ -37,7 +45,14 @@ def set_imputation_model(self, imputation_model): @gin.configurable("base_classification_preprocessor") class DefaultClassificationPreprocessor(Preprocessor): - def __init__(self, generate_features: bool = True, scaling: bool = True, use_static_features: bool = True): + def __init__( + self, + generate_features: bool = True, + scaling: bool = True, + use_static_features: bool = True, + save_cache=None, + load_cache=None, + ): """ Args: generate_features: Generate features for dynamic data. @@ -50,6 +65,8 @@ def __init__(self, generate_features: bool = True, scaling: bool = True, use_sta self.scaling = scaling self.use_static_features = use_static_features self.imputation_model = None + self.save_cache = save_cache + self.load_cache = load_cache def apply(self, data, vars) -> dict[dict[pd.DataFrame]]: """ @@ -60,7 +77,7 @@ def apply(self, data, vars) -> dict[dict[pd.DataFrame]]: Preprocessed data. """ logging.info("Preprocessing dynamic features.") - # input caching here + data = self._process_dynamic(data, vars) if self.use_static_features: logging.info("Preprocessing static features.") @@ -98,12 +115,11 @@ def _process_static(self, data, vars): if self.scaling: sta_rec.add_step(StepScale()) - # sta_rec.add_step(StepImputeFastZeroFill(sel=all_numeric_predictors())) sta_rec.add_step(StepImputeFastZeroFill(sel=all_numeric_predictors())) sta_rec.add_step(StepSklearn(SimpleImputer(missing_values=None, strategy="most_frequent"), sel=has_type("object"))) sta_rec.add_step(StepSklearn(LabelEncoder(), sel=has_type("object"), columnwise=True)) - data = apply_recipe_to_splits(sta_rec, data, Segment.static) + data = apply_recipe_to_splits(sta_rec, data, Segment.static, self.save_cache, self.load_cache) return data @@ -131,11 +147,9 @@ def _process_dynamic(self, data, vars): dyn_rec.add_step(StepSklearn(MissingIndicator(), sel=all_of(vars[Segment.dynamic]), in_place=False)) dyn_rec.add_step(StepImputeFastForwardFill()) dyn_rec.add_step(StepImputeFastZeroFill()) - # dyn_rec.add_step(StepImputeFill(method="ffill")) - # dyn_rec.add_step(StepImputeFill(value=0)) if self.generate_features: dyn_rec = self._dynamic_feature_generation(dyn_rec, all_of(vars[Segment.dynamic])) - data = apply_recipe_to_splits(dyn_rec, data, Segment.dynamic) + data = apply_recipe_to_splits(dyn_rec, data, Segment.dynamic, self.save_cache, self.load_cache) return data def _dynamic_feature_generation(self, data, dynamic_vars): @@ -148,8 +162,8 @@ def _dynamic_feature_generation(self, data, dynamic_vars): def to_cache_string(self): return ( - super().to_cache_string() - + f"_classification_{self.generate_features}_{self.scaling}_{self.imputation_model.__class__.__name__}" + super().to_cache_string() + + f"_classification_{self.generate_features}_{self.scaling}_{self.imputation_model.__class__.__name__}" ) @@ -157,12 +171,12 @@ def to_cache_string(self): class DefaultRegressionPreprocessor(DefaultClassificationPreprocessor): # Override base classification preprocessor def __init__( - self, - generate_features: bool = True, - scaling: bool = True, - use_static_features: bool = True, - outcome_max=None, - outcome_min=None, + self, + generate_features: bool = True, + scaling: bool = True, + use_static_features: bool = True, + outcome_max=None, + outcome_min=None, ): """ Args: @@ -216,10 +230,10 @@ def _process_outcome(self, data, vars, split): @gin.configurable("base_imputation_preprocessor") class DefaultImputationPreprocessor(Preprocessor): def __init__( - self, - scaling: bool = True, - use_static_features: bool = True, - filter_missing_values: bool = True, + self, + scaling: bool = True, + use_static_features: bool = True, + filter_missing_values: bool = True, ): """Preprocesses data for imputation. @@ -245,7 +259,7 @@ def apply(self, data, vars): dyn_rec = Recipe(data[Split.train][Segment.dynamic], [], vars[Segment.dynamic], vars["GROUP"], vars["SEQUENCE"]) if self.scaling: dyn_rec.add_step(StepScale()) - data = apply_recipe_to_splits(dyn_rec, data, Segment.dynamic) + data = apply_recipe_to_splits(dyn_rec, data, Segment.dynamic, self.save_cache, self.load_cache) data[Split.train][Segment.features] = ( data[Split.train].pop(Segment.dynamic).loc[:, vars[Segment.dynamic] + [vars["GROUP"], vars["SEQUENCE"]]] @@ -271,10 +285,14 @@ def _process_dynamic_data(self, data, vars): @staticmethod -def apply_recipe_to_splits(recipe: Recipe, data: dict[dict[pd.DataFrame]], type: str) -> dict[dict[pd.DataFrame]]: +def apply_recipe_to_splits( + recipe: Recipe, data: dict[dict[pd.DataFrame]], type: str, save_cache=None, load_cache=None +) -> dict[dict[pd.DataFrame]]: """Fits and transforms the training features, then transforms the validation and test features with the recipe. Args: + load_cache: Load recipe from cache, for e.g. transfer learning. + save_cache: Save recipe to cache, for e.g. transfer learning. recipe: Object containing info about the features and steps. data: Dict containing 'train', 'val', and 'test' and types of features per split. type: Whether to apply recipe to dynamic features, static features or outcomes. @@ -283,35 +301,35 @@ def apply_recipe_to_splits(recipe: Recipe, data: dict[dict[pd.DataFrame]], type: Transformed features divided into 'train', 'val', and 'test'. """ - cache_file = "data/" - cache_file_name = f"eicu_{Split.train}_{type}" - cache_file += cache_file_name - restore_preproc = False - cache_preproc = False - - if restore_preproc: - recipe = restore_recipe(cache_file) + if load_cache: + # Load existing recipe + recipe = restore_recipe(load_cache) data[Split.train][type] = recipe.bake(data[Split.train][type]) - data[Split.val][type] = recipe.bake(data[Split.val][type]) - data[Split.test][type] = recipe.bake(data[Split.test][type]) - elif cache_preproc: + elif save_cache: + # Save prepped recipe data[Split.train][type] = recipe.prep() - cache_recipe(recipe, cache_file) - data[Split.val][type] = recipe.bake(data[Split.val][type]) - data[Split.test][type] = recipe.bake(data[Split.test][type]) + cache_recipe(recipe, save_cache) else: + # No saving or loading of existing cache data[Split.train][type] = recipe.prep() - data[Split.val][type] = recipe.bake(data[Split.val][type]) - data[Split.test][type] = recipe.bake(data[Split.test][type]) + + data[Split.val][type] = recipe.bake(data[Split.val][type]) + data[Split.test][type] = recipe.bake(data[Split.test][type]) return data def cache_recipe(recipe: Recipe, cache_file: str) -> None: recipe_cache = copy.deepcopy(recipe) recipe_cache.cache() - logging.info(f"Cached recipe in {cache_file}.") + # with open(cache_file, "wb") as f: + # pickle.dump(recipe_cache, f, pickle.HIGHEST_PROTOCOL) + if not (cache_file/"..").exists(): + (cache_file/"..").mkdir() + cache_file.touch() with open(cache_file, "wb") as f: pickle.dump(recipe_cache, f, pickle.HIGHEST_PROTOCOL) + logging.info(f"Cached recipe in {cache_file}.") + def restore_recipe(cache_file: str) -> Recipe: diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index d8a065cb..c866d5f3 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -65,13 +65,15 @@ def preprocess_data( dumped_file_names = json.dumps(file_names, sort_keys=True) dumped_vars = json.dumps(vars, sort_keys=True) + cache_filename = f"s_{seed}_r_{repetition_index}_f_{fold_index}_t_{train_size}_d_{debug}" + logging.log(logging.INFO, f"Using preprocessor: {preprocessor.__name__}") - preprocessor = preprocessor(use_static_features=use_static) + preprocessor = preprocessor(use_static_features=use_static, save_cache=data_dir / "preproc"/ (cache_filename+"_recipe")) if isinstance(preprocessor, DefaultClassificationPreprocessor): preprocessor.set_imputation_model(pretrained_imputation_model) hash_config = hashlib.md5(f"{preprocessor.to_cache_string()}{dumped_file_names}{dumped_vars}".encode("utf-8")) - cache_filename = f"s_{seed}_r_{repetition_index}_f_{fold_index}_t_{train_size}_d_{debug}_{hash_config.hexdigest()}" + cache_filename += f"_{hash_config.hexdigest()}" cache_file = cache_dir / cache_filename if load_cache: From 668e1c7be1984f667c9411ea70818f9744ed0c9a Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 24 Aug 2023 16:45:45 +0200 Subject: [PATCH 071/207] Optimizer check --- icu_benchmarks/models/train.py | 1 - icu_benchmarks/models/wrappers.py | 10 ++++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 94db6dfb..dd043f15 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -15,7 +15,6 @@ from icu_benchmarks.contants import RunMode from icu_benchmarks.data.constants import DataSplit as Split -# from finetuning_scheduler import FinetuningScheduler cpu_core_count = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else os.cpu_count() from pytorch_lightning.loggers import WandbLogger diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index acfbb650..17bd9520 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -174,13 +174,15 @@ def finalize_step(self, step_prefix=""): pass def configure_optimizers(self): + """Configure optimizers and learning rate schedulers.""" + if isinstance(self.optimizer, str): - optimizer = create_optimizer(self.optimizer, 0.5, self.hparams.momentum) - elif self.optimizer is None: - optimizer = self.optimizer(self.parameters()) - else: + optimizer = create_optimizer(self.optimizer, self.lr, self.hparams.momentum) + elif isinstance(self.optimizer, Optimizer): # Already set optimizer = self.optimizer + else: + optimizer = self.optimizer(self.parameters()) if self.hparams.lr_scheduler is None or self.hparams.lr_scheduler == "": return optimizer From 85db1bcd634fcece24387a21fc40ad77372bf369 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 24 Aug 2023 16:47:23 +0200 Subject: [PATCH 072/207] cleanup --- icu_benchmarks/models/wrappers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index 17bd9520..8a8aaccb 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -132,7 +132,6 @@ def __init__( self.input_size = input_size self.initialization_method = initialization_method self.scaler = None - logging.info(f"Learning rate: {self.lr}, learning factor {self.lr_factor}, learning steps {self.lr_steps}") def on_fit_start(self): self.metrics = { @@ -272,6 +271,7 @@ def softmax_multi_output_transform(output): y_pred, y = output y_pred = torch.softmax(y_pred, dim=1) return y_pred, y + # Output transform is not applied for contrib metrics, so we do our own. if self.run_mode == RunMode.classification: # Binary classification From acc6a37c72a3121e41118e09ce1552557cd94a35 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 24 Aug 2023 16:51:39 +0200 Subject: [PATCH 073/207] cleanup wandb experiment yml's --- experiments/benchmark_classification.yml | 42 +++++++++--------- experiments/experiment_finetuning.yml | 44 ++++++------------- experiments/experiment_small_set_training.yml | 19 +++----- 3 files changed, 39 insertions(+), 66 deletions(-) diff --git a/experiments/benchmark_classification.yml b/experiments/benchmark_classification.yml index 28ba2d61..61763ffa 100644 --- a/experiments/benchmark_classification.yml +++ b/experiments/benchmark_classification.yml @@ -7,38 +7,36 @@ command: - -t - BinaryClassification - --log-dir - - ../yaib_logs_mortality24 + - ../yaib_logs - --tune - --wandb-sweep - - --hp-checkpoint - - /dhc/home/robin.vandewater/projects/yaib_logs_mortality24/hirid/BinaryClassification/GRU/2023-08-16T16-33-19 -# - -gc -# - -lc + - -gc + - -lc method: grid name: yaib_classification_benchmark parameters: data_dir: values: -# - ../data/mortality24/miiv - - ../../data/mortality24/hirid -# - ../data/mortality24/eicu -# - ../data/mortality24/aumc -# - ../../data/aki/miiv -# - ../../data/aki/hirid -# - ../../data/aki/eicu -# - ../../data/aki/aumc -# - ../../data/sepsis/miiv -# - ../../data/sepsis/hirid -# - ../../data/sepsis/eicu -# - ../../data/sepsis/aumc + - ../data/mortality24/miiv + - ../data/mortality24/hirid + - ../data/mortality24/eicu + - ../data/mortality24/aumc + - ../data/aki/miiv + - ../data/aki/hirid + - ../data/aki/eicu + - ../data/aki/aumc + - ../data/sepsis/miiv + - ../data/sepsis/hirid + - ../data/sepsis/eicu + - ../data/sepsis/aumc model: values: -# - LogisticRegression -# - LGBMClassifier + - LogisticRegression + - LGBMClassifier - GRU -# - LSTM -# - TCN -# - Transformer + - LSTM + - TCN + - Transformer seed: values: - 1111 diff --git a/experiments/experiment_finetuning.yml b/experiments/experiment_finetuning.yml index 7a410179..4e31c23c 100644 --- a/experiments/experiment_finetuning.yml +++ b/experiments/experiment_finetuning.yml @@ -12,50 +12,34 @@ command: - --tune - --wandb-sweep - -gc -# - -lc + - -lc - -sn - eicu - --source-dir - /dhc/home/robin.vandewater/projects/transfer_learning/gru_mortality/eicu - - --verbose method: grid -name: yaib_regression_benchmark +name: yaib_finetuning_benchmark parameters: fine_tune: values: -# - 100 -# - 500 -# - 1000 -# - 2000 -## - 3000 -# - 4000 -## - 5000 -# - 6000 -## - 7000 -# - 8000 -## - 9000 -# - 10000 + - 100 + - 500 + - 1000 + - 2000 + - 4000 + - 6000 + - 8000 + - 10000 - 12000 -# - 15000 -# - 20000 data_dir: values: -# - ../../data/mortality24/miiv - - ../../data/mortality24/hirid -# - ../../data/mortality24/eicu -# - ../../data/mortality24/aumc -# - ../../data/kf/miiv -# - ../../data/kf/hirid -# - ../../data/kf/eicu -# - ../../data/kf/aumc + - ../data/mortality24/miiv + - ../data/mortality24/hirid + - ../data/mortality24/eicu + - ../data/mortality24/aumc model: values: -# - ElasticNet -# - LGBMRegressor - GRU -# - LSTM -# - TCN -# - Transformer seed: values: - 1111 diff --git a/experiments/experiment_small_set_training.yml b/experiments/experiment_small_set_training.yml index 821e5e85..f8aadb73 100644 --- a/experiments/experiment_small_set_training.yml +++ b/experiments/experiment_small_set_training.yml @@ -1,3 +1,4 @@ +# This experiment trains models with progressively more samples to see how the performance changes command: - ${env} - ${program} @@ -11,11 +12,10 @@ command: - ../yaib_logs_small_set_training - --tune - --wandb-sweep - - --verbose - --source-dir - /dhc/home/robin.vandewater/projects/transfer_learning/gru_mortality/hirid method: grid -name: yaib_regression_benchmark +name: yaib_samples_benchmark parameters: samples: values: @@ -30,22 +30,13 @@ parameters: - 12000 data_dir: values: -# - ../../data/mortality24/miiv + - ../../data/mortality24/miiv - ../../data/mortality24/hirid -# - ../../data/mortality24/eicu -# - ../../data/mortality24/aumc -# - ../../data/kf/miiv -# - ../../data/kf/hirid -# - ../../data/kf/eicu -# - ../../data/kf/aumc + - ../../data/mortality24/eicu + - ../../data/mortality24/aumc model: values: -# - ElasticNet -# - LGBMRegressor - GRU -# - LSTM -# - TCN -# - Transformer seed: values: - 1111 From 151bc15ef00ce0dec30c3ee4de82b2ad8b9db1ad Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 24 Aug 2023 16:55:58 +0200 Subject: [PATCH 074/207] cleanup preprocessing and caching --- icu_benchmarks/data/preprocessor.py | 21 +++++++++++---------- icu_benchmarks/data/split_process_data.py | 4 ++-- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/icu_benchmarks/data/preprocessor.py b/icu_benchmarks/data/preprocessor.py index 82380d8d..ec15d3fe 100644 --- a/icu_benchmarks/data/preprocessor.py +++ b/icu_benchmarks/data/preprocessor.py @@ -16,7 +16,6 @@ StepHistorical, Accumulator, StepImputeModel, - StepImputeFill, ) from sklearn.impute import SimpleImputer, MissingIndicator @@ -319,21 +318,23 @@ def apply_recipe_to_splits( def cache_recipe(recipe: Recipe, cache_file: str) -> None: + """Cache recipe to make it available for e.g. transfer learning.""" recipe_cache = copy.deepcopy(recipe) recipe_cache.cache() - # with open(cache_file, "wb") as f: - # pickle.dump(recipe_cache, f, pickle.HIGHEST_PROTOCOL) - if not (cache_file/"..").exists(): - (cache_file/"..").mkdir() + if not (cache_file / "..").exists(): + (cache_file / "..").mkdir() cache_file.touch() with open(cache_file, "wb") as f: pickle.dump(recipe_cache, f, pickle.HIGHEST_PROTOCOL) logging.info(f"Cached recipe in {cache_file}.") - def restore_recipe(cache_file: str) -> Recipe: - with open(cache_file, "rb") as f: - logging.info(f"Loading cached recipe from {cache_file}.") - recipe = pickle.load(f) - return recipe + """Restore recipe from cache to use for e.g. transfer learning.""" + if cache_file.exists(): + with open(cache_file, "rb") as f: + logging.info(f"Loading cached recipe from {cache_file}.") + recipe = pickle.load(f) + return recipe + else: + raise FileNotFoundError(f"Cache file {cache_file} not found.") diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index c866d5f3..8b049c3a 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -68,7 +68,7 @@ def preprocess_data( cache_filename = f"s_{seed}_r_{repetition_index}_f_{fold_index}_t_{train_size}_d_{debug}" logging.log(logging.INFO, f"Using preprocessor: {preprocessor.__name__}") - preprocessor = preprocessor(use_static_features=use_static, save_cache=data_dir / "preproc"/ (cache_filename+"_recipe")) + preprocessor = preprocessor(use_static_features=use_static, save_cache=data_dir / "preproc" / (cache_filename + "_recipe")) if isinstance(preprocessor, DefaultClassificationPreprocessor): preprocessor.set_imputation_model(pretrained_imputation_model) @@ -100,7 +100,7 @@ def preprocess_data( train_size=train_size, seed=seed, debug=debug, - runmode=runmode + runmode=runmode, ) # Apply preprocessing From 5ea33e3f375f839290e6cae4692c8b947caa9691 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 24 Aug 2023 16:57:27 +0200 Subject: [PATCH 075/207] got rid of useless if statement in train.py --- icu_benchmarks/models/train.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index dd043f15..d70a0920 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -150,10 +150,7 @@ def train_common( if not eval_only: if model.requires_backprop: logging.info("Training DL model.") - if load_weights: - trainer.fit(model, train_dataloaders=train_loader, val_dataloaders=val_loader) - else: - trainer.fit(model, train_dataloaders=train_loader, val_dataloaders=val_loader) + trainer.fit(model, train_dataloaders=train_loader, val_dataloaders=val_loader) logging.info("Training complete.") else: logging.info("Training ML model.") From 3fb4f1919207d520433f55bb657899b9258810a3 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 24 Aug 2023 16:59:33 +0200 Subject: [PATCH 076/207] restore default values --- icu_benchmarks/models/wrappers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index 8a8aaccb..530ef2c1 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -102,7 +102,7 @@ class DLWrapper(BaseModule, ABC): def __init__( self, loss=CrossEntropyLoss(), - optimizer=None, + optimizer=Adam, run_mode: RunMode = RunMode.classification, input_shape=None, lr: float = 0.002, @@ -220,10 +220,10 @@ def __init__( optimizer=torch.optim.Adam, run_mode: RunMode = RunMode.classification, input_shape=None, - lr: float = 0.00002, + lr: float = 0.002, momentum: float = 0.9, lr_scheduler: Optional[str] = None, - lr_factor: float = 0.099, + lr_factor: float = 0.99, lr_steps: Optional[List[int]] = None, epochs: int = 100, input_size: Tensor = None, From e33d3387e442003a28f823f34c6477c8742e8b34 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 24 Aug 2023 17:02:31 +0200 Subject: [PATCH 077/207] linting --- icu_benchmarks/cross_validation.py | 1 - icu_benchmarks/models/custom_metrics.py | 1 + icu_benchmarks/models/train.py | 49 ++++++------- icu_benchmarks/models/wrappers.py | 95 +++++++++++++------------ icu_benchmarks/run.py | 8 ++- icu_benchmarks/run_utils.py | 2 +- 6 files changed, 80 insertions(+), 76 deletions(-) diff --git a/icu_benchmarks/cross_validation.py b/icu_benchmarks/cross_validation.py index 773339e8..cc2329f9 100644 --- a/icu_benchmarks/cross_validation.py +++ b/icu_benchmarks/cross_validation.py @@ -74,7 +74,6 @@ def execute_repeated_cv( seed_everything(seed, reproducible) for repetition in range(cv_repetitions_to_train): for fold_index in range(cv_folds_to_train): - start_time = datetime.now() data = preprocess_data( data_dir, diff --git a/icu_benchmarks/models/custom_metrics.py b/icu_benchmarks/models/custom_metrics.py index 437ca614..ddb5d37e 100644 --- a/icu_benchmarks/models/custom_metrics.py +++ b/icu_benchmarks/models/custom_metrics.py @@ -6,6 +6,7 @@ from sklearn.calibration import calibration_curve from scipy.spatial.distance import jensenshannon from torchmetrics.classification import BinaryFairness + """" This file contains custom metrics that can be added to YAIB. """ diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index d70a0920..83ab204f 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -27,29 +27,29 @@ def assure_minimum_length(dataset): @gin.configurable("train_common") def train_common( - data: dict[str, pd.DataFrame], - log_dir: Path, - eval_only: bool = False, - load_weights: bool = False, - source_dir: Path = None, - reproducible: bool = True, - mode: str = RunMode.classification, - model: object = gin.REQUIRED, - weight: str = None, - optimizer: type = Adam, - precision=32, - batch_size=64, - epochs=1000, - patience=20, - min_delta=1e-5, - test_on: str = Split.test, - dataset_names=None, - use_wandb: bool = False, - cpu: bool = False, - verbose=False, - ram_cache=False, - pl_model=True, - num_workers: int = min(cpu_core_count, torch.cuda.device_count() * 8 * int(torch.cuda.is_available()), 32), + data: dict[str, pd.DataFrame], + log_dir: Path, + eval_only: bool = False, + load_weights: bool = False, + source_dir: Path = None, + reproducible: bool = True, + mode: str = RunMode.classification, + model: object = gin.REQUIRED, + weight: str = None, + optimizer: type = Adam, + precision=32, + batch_size=64, + epochs=1000, + patience=20, + min_delta=1e-5, + test_on: str = Split.test, + dataset_names=None, + use_wandb: bool = False, + cpu: bool = False, + verbose=False, + ram_cache=False, + pl_model=True, + num_workers: int = min(cpu_core_count, torch.cuda.device_count() * 8 * int(torch.cuda.is_available()), 32), ): """Common wrapper to train all benchmarked models. @@ -92,7 +92,8 @@ def train_common( if not eval_only: logging.info( f"Training on {train_dataset.name} with {len(train_dataset)} samples and validating on {val_dataset.name} with" - f" {len(val_dataset)} samples.") + f" {len(val_dataset)} samples." + ) logging.info(f"Using {num_workers} workers for data loading.") train_loader = DataLoader( diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index 530ef2c1..e8595a70 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -100,20 +100,20 @@ class DLWrapper(BaseModule, ABC): _supported_run_modes = [RunMode.classification, RunMode.regression, RunMode.imputation] def __init__( - self, - loss=CrossEntropyLoss(), - optimizer=Adam, - run_mode: RunMode = RunMode.classification, - input_shape=None, - lr: float = 0.002, - momentum: float = 0.9, - lr_scheduler: Optional[str] = None, - lr_factor: float = 0.99, - lr_steps: Optional[List[int]] = None, - epochs: int = 100, - input_size: Tensor = None, - initialization_method: str = "normal", - **kwargs, + self, + loss=CrossEntropyLoss(), + optimizer=Adam, + run_mode: RunMode = RunMode.classification, + input_shape=None, + lr: float = 0.002, + momentum: float = 0.9, + lr_scheduler: Optional[str] = None, + lr_factor: float = 0.99, + lr_steps: Optional[List[int]] = None, + epochs: int = 100, + input_size: Tensor = None, + initialization_method: str = "normal", + **kwargs, ): """General interface for Deep Learning (DL) models.""" super().__init__() @@ -152,13 +152,14 @@ def on_train_start(self): for step_name in ["train", "val", "test"] } return super().on_train_start() + def finalize_step(self, step_prefix=""): try: self.log_dict( { - f"{step_prefix}/{name}": (np.float32(metric.compute()) - if isinstance(metric.compute(), np.float64) - else metric.compute()) + f"{step_prefix}/{name}": ( + np.float32(metric.compute()) if isinstance(metric.compute(), np.float64) else metric.compute() + ) for name, metric in self.metrics[step_prefix].items() if "_Curve" not in name }, @@ -215,20 +216,20 @@ class DLPredictionWrapper(DLWrapper): _supported_run_modes = [RunMode.classification, RunMode.regression] def __init__( - self, - loss=CrossEntropyLoss(), - optimizer=torch.optim.Adam, - run_mode: RunMode = RunMode.classification, - input_shape=None, - lr: float = 0.002, - momentum: float = 0.9, - lr_scheduler: Optional[str] = None, - lr_factor: float = 0.99, - lr_steps: Optional[List[int]] = None, - epochs: int = 100, - input_size: Tensor = None, - initialization_method: str = "normal", - **kwargs, + self, + loss=CrossEntropyLoss(), + optimizer=torch.optim.Adam, + run_mode: RunMode = RunMode.classification, + input_shape=None, + lr: float = 0.002, + momentum: float = 0.9, + lr_scheduler: Optional[str] = None, + lr_factor: float = 0.99, + lr_steps: Optional[List[int]] = None, + epochs: int = 100, + input_size: Tensor = None, + initialization_method: str = "normal", + **kwargs, ): super().__init__( loss=loss, @@ -248,7 +249,6 @@ def __init__( self.output_transform = None self.loss_weights = None - def set_weight(self, weight, dataset): """Set the weight for the loss function.""" @@ -260,6 +260,7 @@ def set_weight(self, weight, dataset): def set_metrics(self, *args): """Set the evaluation metrics for the prediction model.""" + def softmax_binary_output_transform(output): with torch.no_grad(): y_pred, y = output @@ -462,7 +463,8 @@ def log_metrics(self, label, pred, metric_type): self.log_dict( { # MPS dependent type casting - f"{metric_type}/{name}": metric(self.label_transform(label), self.output_transform(pred)) if not self.mps + f"{metric_type}/{name}": metric(self.label_transform(label), self.output_transform(pred)) + if not self.mps else metric(self.label_transform(label), self.output_transform(pred)) # Fore very metric for name, metric in self.metrics.items() @@ -509,21 +511,20 @@ class ImputationWrapper(DLWrapper): _supported_run_modes = [RunMode.imputation] def __init__( - self, - loss: nn.modules.loss._Loss = MSELoss(), - optimizer: Union[str, Optimizer] = "adam", - run_mode: RunMode = RunMode.imputation, - lr: float = 0.002, - momentum: float = 0.9, - lr_scheduler: Optional[str] = None, - lr_factor: float = 0.99, - lr_steps: Optional[List[int]] = None, - input_size: Tensor = None, - initialization_method: ImputationInit = ImputationInit.NORMAL, - epochs=100, - **kwargs: str, + self, + loss: nn.modules.loss._Loss = MSELoss(), + optimizer: Union[str, Optimizer] = "adam", + run_mode: RunMode = RunMode.imputation, + lr: float = 0.002, + momentum: float = 0.9, + lr_scheduler: Optional[str] = None, + lr_factor: float = 0.99, + lr_steps: Optional[List[int]] = None, + input_size: Tensor = None, + initialization_method: ImputationInit = ImputationInit.NORMAL, + epochs=100, + **kwargs: str, ) -> None: - super().__init__( loss=loss, optimizer=optimizer, diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index e315d391..6a498a6a 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -125,7 +125,9 @@ def main(my_args=tuple(sys.argv[1:])): if args.source_dir is None: raise ValueError("Please specify a source directory when evaluating or fine-tuning.") log_dir /= f"_from_{args.source_name}" - gin.bind_parameter("train_common.dataset_names", {"train": args.source_name, "val": args.source_name, "test": args.name}) + gin.bind_parameter( + "train_common.dataset_names", {"train": args.source_name, "val": args.source_name, "test": args.name} + ) if args.fine_tune: log_dir /= f"fine_tune_{args.fine_tune}" gin.bind_parameter("train_common.dataset_names", {"train": args.name, "val": args.name, "test": args.name}) @@ -133,7 +135,7 @@ def main(my_args=tuple(sys.argv[1:])): source_dir = args.source_dir logging.info(f"Will load weights from {source_dir} and bind train gin-config. Note: this might override your config.") gin.parse_config_file(source_dir / "train_config.gin") - elif args.samples and args.source_dir is not None: # Train model with limited samples and bind existing config + elif args.samples and args.source_dir is not None: # Train model with limited samples and bind existing config logging.info("Binding train gin-config. Note: this might override your config.") gin.parse_config_file(args.source_dir / "train_config.gin") log_dir /= f"samples_{args.fine_tune}" @@ -144,7 +146,7 @@ def main(my_args=tuple(sys.argv[1:])): gin.bind_parameter("train_common.dataset_names", {"train": args.name, "val": args.name, "test": args.name}) hp_checkpoint = log_dir / args.hp_checkpoint if args.hp_checkpoint else None model_path = ( - Path("configs") / ("imputation_models" if mode == RunMode.imputation else "prediction_models") / f"{model}.gin" + Path("configs") / ("imputation_models" if mode == RunMode.imputation else "prediction_models") / f"{model}.gin" ) gin_config_files = ( [Path(f"configs/experiments/{args.experiment}.gin")] diff --git a/icu_benchmarks/run_utils.py b/icu_benchmarks/run_utils.py index 85f67324..f00510ec 100644 --- a/icu_benchmarks/run_utils.py +++ b/icu_benchmarks/run_utils.py @@ -64,7 +64,7 @@ def create_run_dir(log_dir: Path, randomly_searched_params: str = None) -> Path: Returns: Path to the created run log directory. """ - if not (log_dir/ str(datetime.now().strftime("%Y-%m-%dT%H-%M-%S"))).exists(): + if not (log_dir / str(datetime.now().strftime("%Y-%m-%dT%H-%M-%S"))).exists(): log_dir_run = log_dir / str(datetime.now().strftime("%Y-%m-%dT%H-%M-%S")) else: log_dir_run = log_dir / str(datetime.now().strftime("%Y-%m-%dT%H-%M-%S.%f")) From effeb4606318a436ea599ff424653b7857d0fbea Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 24 Aug 2023 17:18:19 +0200 Subject: [PATCH 078/207] Minor tweaks to ordering --- icu_benchmarks/data/preprocessor.py | 10 +++++++--- icu_benchmarks/models/train.py | 3 +-- icu_benchmarks/models/utils.py | 2 ++ icu_benchmarks/run.py | 13 ++----------- icu_benchmarks/run_utils.py | 16 ++++++++++++++++ 5 files changed, 28 insertions(+), 16 deletions(-) diff --git a/icu_benchmarks/data/preprocessor.py b/icu_benchmarks/data/preprocessor.py index ec15d3fe..be004d8e 100644 --- a/icu_benchmarks/data/preprocessor.py +++ b/icu_benchmarks/data/preprocessor.py @@ -176,6 +176,8 @@ def __init__( use_static_features: bool = True, outcome_max=None, outcome_min=None, + save_cache=None, + load_cache=None, ): """ Args: @@ -184,10 +186,12 @@ def __init__( use_static_features: Use static features. max_range: Maximum value in outcome. min_range: Minimum value in outcome. + save_cache: Save recipe cache. + load_cache: Load recipe cache. Returns: Preprocessed data. """ - super().__init__(generate_features, scaling, use_static_features) + super().__init__(generate_features, scaling, use_static_features, save_cache, load_cache) self.outcome_max = outcome_max self.outcome_min = outcome_min @@ -300,11 +304,11 @@ def apply_recipe_to_splits( Transformed features divided into 'train', 'val', and 'test'. """ - if load_cache: + if isinstance(load_cache, str): # Load existing recipe recipe = restore_recipe(load_cache) data[Split.train][type] = recipe.bake(data[Split.train][type]) - elif save_cache: + elif isinstance(save_cache, str): # Save prepped recipe data[Split.train][type] = recipe.prep() cache_recipe(recipe, save_cache) diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 83ab204f..d99e929f 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -6,7 +6,7 @@ from joblib import load from torch.optim import Adam from torch.utils.data import DataLoader -from pytorch_lightning.loggers import TensorBoardLogger +from pytorch_lightning.loggers import TensorBoardLogger, WandbLogger from pytorch_lightning import Trainer from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint, TQDMProgressBar, LearningRateMonitor from pathlib import Path @@ -16,7 +16,6 @@ from icu_benchmarks.data.constants import DataSplit as Split cpu_core_count = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else os.cpu_count() -from pytorch_lightning.loggers import WandbLogger def assure_minimum_length(dataset): diff --git a/icu_benchmarks/models/utils.py b/icu_benchmarks/models/utils.py index 6c944ae7..ca49bc1f 100644 --- a/icu_benchmarks/models/utils.py +++ b/icu_benchmarks/models/utils.py @@ -17,6 +17,7 @@ from torch.optim.lr_scheduler import _LRScheduler, CosineAnnealingLR, MultiStepLR, ExponentialLR + def save_config_file(log_dir): config_path = log_dir / "train_config.gin" with config_path.open("w") as f: @@ -188,3 +189,4 @@ def version(self): @rank_zero_only def log_hyperparams(self, params): pass + diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index 6a498a6a..47cdb836 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -5,7 +5,6 @@ import logging import sys from pathlib import Path -import importlib.util import torch.cuda @@ -20,6 +19,7 @@ log_full_line, load_pretrained_imputation_model, setup_logging, + import_preprocessor ) from icu_benchmarks.contants import RunMode @@ -109,16 +109,7 @@ def main(my_args=tuple(sys.argv[1:])): log_full_line(f"Logging to {log_dir}.", logging.INFO) if args.preprocessor: - # Import custom supplied preprocessor - log_full_line(f"Importing custom preprocessor from {args.preprocessor}.", logging.INFO) - try: - spec = importlib.util.spec_from_file_location("CustomPreprocessor", args.preprocessor) - module = importlib.util.module_from_spec(spec) - sys.modules["preprocessor"] = module - spec.loader.exec_module(module) - gin.bind_parameter("preprocess.preprocessor", module.CustomPreprocessor) - except Exception as e: - logging.error(f"Could not import custom preprocessor from {args.preprocessor}: {e}") + import_preprocessor(args.preprocessor) # Load pretrained model in evaluate mode or when finetuning if load_weights: diff --git a/icu_benchmarks/run_utils.py b/icu_benchmarks/run_utils.py index f00510ec..bd853011 100644 --- a/icu_benchmarks/run_utils.py +++ b/icu_benchmarks/run_utils.py @@ -1,6 +1,9 @@ +import importlib +import sys import warnings from math import sqrt +import gin import torch import json from argparse import ArgumentParser, BooleanOptionalAction as BOA @@ -14,6 +17,19 @@ from icu_benchmarks.wandb_utils import wandb_log +def import_preprocessor(preprocessor_path: str): + # Import custom supplied preprocessor + log_full_line(f"Importing custom preprocessor from {preprocessor_path}.", logging.INFO) + try: + spec = importlib.util.spec_from_file_location("CustomPreprocessor", preprocessor_path) + module = importlib.util.module_from_spec(spec) + sys.modules["preprocessor"] = module + spec.loader.exec_module(module) + gin.bind_parameter("preprocess.preprocessor", module.CustomPreprocessor) + except Exception as e: + logging.error(f"Could not import custom preprocessor from {preprocessor_path}: {e}") + + def build_parser() -> ArgumentParser: """Builds an ArgumentParser for the command line. From 8317b8b59254c383235ec29e9afba8666855f64b Mon Sep 17 00:00:00 2001 From: rvandewater Date: Fri, 25 Aug 2023 13:54:43 +0200 Subject: [PATCH 079/207] Cleanup of run.py --- icu_benchmarks/models/utils.py | 2 -- icu_benchmarks/run.py | 64 +++++++++++++--------------------- icu_benchmarks/run_utils.py | 34 ++++++++++-------- icu_benchmarks/wandb_utils.py | 14 ++++++-- 4 files changed, 56 insertions(+), 58 deletions(-) diff --git a/icu_benchmarks/models/utils.py b/icu_benchmarks/models/utils.py index ca49bc1f..6c944ae7 100644 --- a/icu_benchmarks/models/utils.py +++ b/icu_benchmarks/models/utils.py @@ -17,7 +17,6 @@ from torch.optim.lr_scheduler import _LRScheduler, CosineAnnealingLR, MultiStepLR, ExponentialLR - def save_config_file(log_dir): config_path = log_dir / "train_config.gin" with config_path.open("w") as f: @@ -189,4 +188,3 @@ def version(self): @rank_zero_only def log_hyperparams(self, params): pass - diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index 47cdb836..14b8ee6a 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -8,7 +8,7 @@ import torch.cuda -from icu_benchmarks.wandb_utils import update_wandb_config, apply_wandb_sweep, set_wandb_run_name +from icu_benchmarks.wandb_utils import update_wandb_config, apply_wandb_sweep, set_wandb_experiment_name from icu_benchmarks.tuning.hyperparameters import choose_and_bind_hyperparameters from scripts.plotting.utils import plot_aggregated_results from icu_benchmarks.cross_validation import execute_repeated_cv @@ -19,7 +19,8 @@ log_full_line, load_pretrained_imputation_model, setup_logging, - import_preprocessor + import_preprocessor, + name_datasets, ) from icu_benchmarks.contants import RunMode @@ -33,56 +34,40 @@ def get_mode(mode: gin.REQUIRED): def main(my_args=tuple(sys.argv[1:])): args, _ = build_parser().parse_known_args(my_args) - - # Set arguments for wandb sweep - if args.wandb_sweep: - args = apply_wandb_sweep(args) - # Initialize loggers log_format = "%(asctime)s - %(levelname)s - %(name)s : %(message)s" date_format = "%Y-%m-%d %H:%M:%S" verbose = args.verbose setup_logging(date_format, log_format, verbose) - verbose = True # Get arguments data_dir = Path(args.data_dir) name = args.name task = args.task model = args.model reproducible = args.reproducible - - # Set experiment name - if name is None: - name = data_dir.name - logging.info(f"Running experiment {name}.") + evaluate = args.eval + experiment = args.experiment # Load task config gin.parse_config_file(f"configs/tasks/{task}.gin") - mode = get_mode() + # Set arguments for wandb sweep + if args.wandb_sweep: + args = apply_wandb_sweep(args) + set_wandb_experiment_name(args, mode) - evaluate = args.eval + # Set experiment name + if name is None: + name = data_dir.name + logging.info(f"Running experiment {name}.") + logging.info(f"Task mode: {mode}.") # Set train size to fine tune size if fine tune is set, else use custom train size - train_size = args.fine_tune if args.fine_tune is not None else args.samples + train_size = args.fine_tune if args.fine_tune is not None else args.samples if args.samples is not None else None + # Whether to load weights from a previous run load_weights = evaluate or args.fine_tune is not None - if args.wandb_sweep: - run_name = f"{mode}_{model}_{name}" - if load_weights: - if args.fine_tune: - run_name += f"_source_{args.source_name}_fine-tune_{args.fine_tune}_samples" - else: - run_name += f"_source_{args.source_name}" - if args.samples: - run_name += f"_train_size_{args.samples}" - set_wandb_run_name(run_name) - - logging.info(f"Task mode: {mode}.") - experiment = args.experiment - pretrained_imputation_model = load_pretrained_imputation_model(args.pretrained_imputation) - # Log imputation model to wandb update_wandb_config( { @@ -91,13 +76,16 @@ def main(my_args=tuple(sys.argv[1:])): else "None" } ) - source_dir = None + log_dir_name = args.log_dir / name log_dir = ( (log_dir_name / experiment) if experiment else (log_dir_name / (args.task_name if args.task_name is not None else args.task) / model) ) + log_full_line(f"Logging to {log_dir}.", logging.INFO) + + # Check cuda availability if torch.cuda.is_available(): for name in range(0, torch.cuda.device_count()): log_full_line(f"Available GPU {name}: {torch.cuda.get_device_name(name)}", level=logging.INFO) @@ -106,8 +94,6 @@ def main(my_args=tuple(sys.argv[1:])): "No GPUs available: please check your device and Torch,Cuda installation if unintended.", level=logging.WARNING ) - log_full_line(f"Logging to {log_dir}.", logging.INFO) - if args.preprocessor: import_preprocessor(args.preprocessor) @@ -116,12 +102,10 @@ def main(my_args=tuple(sys.argv[1:])): if args.source_dir is None: raise ValueError("Please specify a source directory when evaluating or fine-tuning.") log_dir /= f"_from_{args.source_name}" - gin.bind_parameter( - "train_common.dataset_names", {"train": args.source_name, "val": args.source_name, "test": args.name} - ) + name_datasets(args.source_name, args.source_name, args.name) if args.fine_tune: log_dir /= f"fine_tune_{args.fine_tune}" - gin.bind_parameter("train_common.dataset_names", {"train": args.name, "val": args.name, "test": args.name}) + name_datasets(args.name, args.name, args.name) run_dir = create_run_dir(log_dir) source_dir = args.source_dir logging.info(f"Will load weights from {source_dir} and bind train gin-config. Note: this might override your config.") @@ -130,11 +114,11 @@ def main(my_args=tuple(sys.argv[1:])): logging.info("Binding train gin-config. Note: this might override your config.") gin.parse_config_file(args.source_dir / "train_config.gin") log_dir /= f"samples_{args.fine_tune}" - gin.bind_parameter("train_common.dataset_names", {"train": args.name, "val": args.name, "test": args.name}) + name_datasets(args.name, args.name, args.name) run_dir = create_run_dir(log_dir) else: # Normal train and evaluate - gin.bind_parameter("train_common.dataset_names", {"train": args.name, "val": args.name, "test": args.name}) + name_datasets(args.name, args.name, args.name) hp_checkpoint = log_dir / args.hp_checkpoint if args.hp_checkpoint else None model_path = ( Path("configs") / ("imputation_models" if mode == RunMode.imputation else "prediction_models") / f"{model}.gin" diff --git a/icu_benchmarks/run_utils.py b/icu_benchmarks/run_utils.py index bd853011..f1587269 100644 --- a/icu_benchmarks/run_utils.py +++ b/icu_benchmarks/run_utils.py @@ -17,26 +17,13 @@ from icu_benchmarks.wandb_utils import wandb_log -def import_preprocessor(preprocessor_path: str): - # Import custom supplied preprocessor - log_full_line(f"Importing custom preprocessor from {preprocessor_path}.", logging.INFO) - try: - spec = importlib.util.spec_from_file_location("CustomPreprocessor", preprocessor_path) - module = importlib.util.module_from_spec(spec) - sys.modules["preprocessor"] = module - spec.loader.exec_module(module) - gin.bind_parameter("preprocess.preprocessor", module.CustomPreprocessor) - except Exception as e: - logging.error(f"Could not import custom preprocessor from {preprocessor_path}: {e}") - - def build_parser() -> ArgumentParser: """Builds an ArgumentParser for the command line. Returns: The configured ArgumentParser. """ - parser = ArgumentParser(description="Benchmark lib for processing and evaluation of deep learning models on ICU data") + parser = ArgumentParser(description="Framework for benchmarking ML/DL models on ICU data") parser.add_argument("-d", "--data-dir", required=True, type=Path, help="Path to the parquet data directory.") parser.add_argument("-t", "--task", default="BinaryClassification", required=True, help="Name of the task gin.") @@ -90,6 +77,19 @@ def create_run_dir(log_dir: Path, randomly_searched_params: str = None) -> Path: return log_dir_run +def import_preprocessor(preprocessor_path: str): + # Import custom supplied preprocessor + log_full_line(f"Importing custom preprocessor from {preprocessor_path}.", logging.INFO) + try: + spec = importlib.util.spec_from_file_location("CustomPreprocessor", preprocessor_path) + module = importlib.util.module_from_spec(spec) + sys.modules["preprocessor"] = module + spec.loader.exec_module(module) + gin.bind_parameter("preprocess.preprocessor", module.CustomPreprocessor) + except Exception as e: + logging.error(f"Could not import custom preprocessor from {preprocessor_path}: {e}") + + def aggregate_results(log_dir: Path, execution_time: timedelta = None): """Aggregates results from all folds and writes to JSON file. @@ -156,6 +156,12 @@ def aggregate_results(log_dir: Path, execution_time: timedelta = None): wandb_log(json.loads(json.dumps(accumulated_metrics, cls=JsonResultLoggingEncoder))) +def name_datasets(train="default", val="default", test="default"): + """Names the datasets for logging (optional).""" + gin.bind_parameter("train_common.dataset_names", {"train": train, "val": val, "test": test}) + + + def log_full_line(msg: str, level: int = logging.INFO, char: str = "-", num_newlines: int = 0): """Logs a full line of a given character with a message centered. diff --git a/icu_benchmarks/wandb_utils.py b/icu_benchmarks/wandb_utils.py index be8bac7d..41aca287 100644 --- a/icu_benchmarks/wandb_utils.py +++ b/icu_benchmarks/wandb_utils.py @@ -49,12 +49,22 @@ def wandb_log(log_dict): wandb.log(log_dict) -def set_wandb_run_name(run_name): +def set_wandb_experiment_name(args, mode): """stores the run name in wandb config Args: - run_name (str): name of the current run + args (Namespace): parsed arguments + mode (RunMode): run mode """ + run_name = f"{mode}_{args.model}_{args.name}" + + if args.fine_tune: + run_name += f"_source_{args.source_name}_fine-tune_{args.fine_tune}_samples" + elif args.eval: + run_name += f"_source_{args.source_name}" + elif args.samples: + run_name += f"_train_size_{args.samples}_samples" + if wandb_running(): wandb.config.update({"run-name": run_name}) wandb.run.name = run_name From be4b54c6af523328b5da97e891f77cbadb7dda0c Mon Sep 17 00:00:00 2001 From: rvandewater Date: Fri, 25 Aug 2023 16:05:08 +0200 Subject: [PATCH 080/207] Pooled datasets --- icu_benchmarks/data/pooling.py | 49 ++++++++++++++++++ icu_benchmarks/data/split_process_data.py | 63 +++++++++++++---------- icu_benchmarks/run.py | 2 +- 3 files changed, 86 insertions(+), 28 deletions(-) create mode 100644 icu_benchmarks/data/pooling.py diff --git a/icu_benchmarks/data/pooling.py b/icu_benchmarks/data/pooling.py new file mode 100644 index 00000000..8c3361fb --- /dev/null +++ b/icu_benchmarks/data/pooling.py @@ -0,0 +1,49 @@ +import copy +import pickle +import torch +import logging +import gin +import pandas as pd +from sklearn.model_selection import StratifiedKFold, train_test_split +from .constants import DataSplit as Split, DataSegment as Segment, VarType as Var + + +def pool_datasets(datasets={}, samples=10000, vars = [], seed=42,shuffle=True, stratify=None, **kwargs): + """ + Pool datasets into a single dataset. + Args: + datasets: list of datasets to pool + Returns: + pooled dataset + """ + if len(datasets) == 0: + raise ValueError("No datasets supplied.") + pooled_data = {Segment.static:[], Segment.dynamic:[], Segment.outcome:[]} + id = vars[Var.group] + int_id = 0 + for key, value in datasets.items(): + if samples: + int_id += 1 + num_classes = 2 + repeated_digit = str(int_id) * 4 + samples_per_class = int(samples / num_classes) + # outcome = value[Segment.outcome].groupby(vars["LABEL"], group_keys=False).sample(samples_per_class,random_state=seed) + outcome = value[Segment.outcome] + outcome = train_test_split(outcome, stratify=outcome[vars["LABEL"]], shuffle=shuffle, random_state=seed, train_size=samples)[0] + stays = pd.Series(outcome[id].unique()) + static = value[Segment.static] + dynamic = value[Segment.dynamic] + static = static.loc[static[id].isin(stays)] + dynamic = dynamic.loc[dynamic[id].isin(stays)] + outcome[id] = outcome[id].map(lambda x: int(str(x) + repeated_digit)) + static[id] = static[id].map(lambda x: int(str(x) + repeated_digit)) + dynamic[id] = dynamic[id].map(lambda x: int(str(x) + repeated_digit)) + pooled_data[Segment.static].append(static) + pooled_data[Segment.dynamic].append(dynamic) + pooled_data[Segment.outcome].append(outcome) + # value[Segment.static] = [value[Segment.static].loc[stays] + # value[Segment.dynamic] = value[Segment.dynamic].loc[stays] + # stays = stays.sample(samples=, random_state=seed) + for key, value in pooled_data.items(): + pooled_data[key] = pd.concat(value, ignore_index=True) + return pooled_data diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index 8b049c3a..abba6d6a 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -12,26 +12,27 @@ from icu_benchmarks.data.preprocessor import Preprocessor, DefaultClassificationPreprocessor from icu_benchmarks.contants import RunMode from .constants import DataSplit as Split, DataSegment as Segment, VarType as Var +from .pooling import pool_datasets @gin.configurable("preprocess") def preprocess_data( - data_dir: Path, - file_names: dict[str] = gin.REQUIRED, - preprocessor: Preprocessor = DefaultClassificationPreprocessor, - use_static: bool = True, - vars: dict[str] = gin.REQUIRED, - seed: int = 42, - debug: bool = False, - cv_repetitions: int = 5, - repetition_index: int = 0, - cv_folds: int = 5, - train_size: int = None, - load_cache: bool = False, - generate_cache: bool = False, - fold_index: int = 0, - pretrained_imputation_model: str = None, - runmode: RunMode = RunMode.classification, + data_dir: Path, + file_names: dict[str] = gin.REQUIRED, + preprocessor: Preprocessor = DefaultClassificationPreprocessor, + use_static: bool = True, + vars: dict[str] = gin.REQUIRED, + seed: int = 42, + debug: bool = False, + cv_repetitions: int = 5, + repetition_index: int = 0, + cv_folds: int = 5, + train_size: int = None, + load_cache: bool = False, + generate_cache: bool = False, + fold_index: int = 0, + pretrained_imputation_model: str = None, + runmode: RunMode = RunMode.classification, ) -> dict[dict[pd.DataFrame]]: """Perform loading, splitting, imputing and normalising of task data. @@ -86,7 +87,15 @@ def preprocess_data( # Read parquet files into pandas dataframes and remove the parquet file from memory logging.info(f"Loading data from directory {data_dir.absolute()}") - data = {f: pq.read_table(data_dir / file_names[f]).to_pandas(self_destruct=True) for f in file_names.keys()} + pooling = True + if pooling: + data = {} + for folder in data_dir.iterdir(): + if folder.is_dir(): + data[folder.name] = {f: pq.read_table(folder / file_names[f]).to_pandas(self_destruct=True) for f in file_names.keys()} + data = pool_datasets(datasets=data, samples=50, vars=vars, shuffle=True, stratify=None) + else: + data = {f: pq.read_table(data_dir / file_names[f]).to_pandas(self_destruct=True) for f in file_names.keys()} # Generate the splits logging.info("Generating splits.") @@ -118,16 +127,16 @@ def preprocess_data( def make_single_split( - data: dict[pd.DataFrame], - vars: dict[str], - cv_repetitions: int, - repetition_index: int, - cv_folds: int, - fold_index: int, - train_size: int = None, - seed: int = 42, - debug: bool = False, - runmode: RunMode = RunMode.classification, + data: dict[pd.DataFrame], + vars: dict[str], + cv_repetitions: int, + repetition_index: int, + cv_folds: int, + fold_index: int, + train_size: int = None, + seed: int = 42, + debug: bool = False, + runmode: RunMode = RunMode.classification, ) -> dict[dict[pd.DataFrame]]: """Randomly split the data into training, validation, and test set. diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index 14b8ee6a..f15725b4 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -47,7 +47,7 @@ def main(my_args=tuple(sys.argv[1:])): reproducible = args.reproducible evaluate = args.eval experiment = args.experiment - + source_dir = args.source_dir # Load task config gin.parse_config_file(f"configs/tasks/{task}.gin") mode = get_mode() From 9885c69102ef743b1f7b33232fef42ca920cb4ea Mon Sep 17 00:00:00 2001 From: rvandewater Date: Fri, 25 Aug 2023 17:34:09 +0200 Subject: [PATCH 081/207] Pooled datasets saving --- icu_benchmarks/data/pooling.py | 3 --- icu_benchmarks/data/split_process_data.py | 21 +++++++++++++++++++-- icu_benchmarks/run_utils.py | 1 + 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/icu_benchmarks/data/pooling.py b/icu_benchmarks/data/pooling.py index 8c3361fb..350c227b 100644 --- a/icu_benchmarks/data/pooling.py +++ b/icu_benchmarks/data/pooling.py @@ -41,9 +41,6 @@ def pool_datasets(datasets={}, samples=10000, vars = [], seed=42,shuffle=True, s pooled_data[Segment.static].append(static) pooled_data[Segment.dynamic].append(dynamic) pooled_data[Segment.outcome].append(outcome) - # value[Segment.static] = [value[Segment.static].loc[stays] - # value[Segment.dynamic] = value[Segment.dynamic].loc[stays] - # stays = stays.sample(samples=, random_state=seed) for key, value in pooled_data.items(): pooled_data[key] = pd.concat(value, ignore_index=True) return pooled_data diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index abba6d6a..db468469 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -88,15 +88,32 @@ def preprocess_data( # Read parquet files into pandas dataframes and remove the parquet file from memory logging.info(f"Loading data from directory {data_dir.absolute()}") pooling = True + + datasets = ["eicu", "hirid", "miiv"] + # datasets = ["eicu", "hirid", "aumc"] + # datasets = ["eicu", "aumc", "miiv"] + # datasets = ["aumc", "hirid", "miiv"] + + if pooling: data = {} for folder in data_dir.iterdir(): if folder.is_dir(): - data[folder.name] = {f: pq.read_table(folder / file_names[f]).to_pandas(self_destruct=True) for f in file_names.keys()} - data = pool_datasets(datasets=data, samples=50, vars=vars, shuffle=True, stratify=None) + if folder.name in datasets: + data[folder.name] = {f: pq.read_table(folder / file_names[f]).to_pandas(self_destruct=True) for f in file_names.keys()} + data = pool_datasets(datasets=data, samples=10000, vars=vars, shuffle=True, stratify=None) else: data = {f: pq.read_table(data_dir / file_names[f]).to_pandas(self_destruct=True) for f in file_names.keys()} + cache_file = "_".join(datasets) + cache_dir = Path(r'C:\Users\Robin\Downloads\mortality24_eicu_hirid_aumc\cache') + cache_dir = cache_dir / cache_file + if not cache_dir.exists(): + cache_dir.mkdir() + for key, value in data.items(): + value.to_parquet(cache_dir / Path( key + ".parquet")) + caching(cache_dir, cache_file, data, load_cache) + # Generate the splits logging.info("Generating splits.") data = make_single_split( diff --git a/icu_benchmarks/run_utils.py b/icu_benchmarks/run_utils.py index f1587269..69fe0944 100644 --- a/icu_benchmarks/run_utils.py +++ b/icu_benchmarks/run_utils.py @@ -47,6 +47,7 @@ def build_parser() -> ArgumentParser: parser.add_argument("--tune", default=False, action=BOA, help="Find best hyperparameters.") parser.add_argument("--hp-checkpoint", type=Path, help="Use previous hyperparameter checkpoint.") parser.add_argument("--eval", default=False, action=BOA, help="Only evaluate model, skip training.") + # parser.add_argument("--no-eval", default=False, action=BOA, help="Only train model") parser.add_argument("-ft", "--fine-tune", default=None, type=int, help="Finetune model with amount of train data.") parser.add_argument("-sn", "--source-name", type=Path, help="Name of the source dataset.") parser.add_argument("--source-dir", type=Path, help="Directory containing gin and model weights.") From 47dedd41fe231ee3661b1b9551b71ff4378635eb Mon Sep 17 00:00:00 2001 From: rvandewater Date: Fri, 25 Aug 2023 21:55:33 +0200 Subject: [PATCH 082/207] Added pooling.py for pooling datasets --- experiments/experiment_pooled.yml | 36 ++++++++++++++ icu_benchmarks/data/pooling.py | 83 +++++++++++++++++++++---------- 2 files changed, 92 insertions(+), 27 deletions(-) create mode 100644 experiments/experiment_pooled.yml diff --git a/experiments/experiment_pooled.yml b/experiments/experiment_pooled.yml new file mode 100644 index 00000000..7f34ac12 --- /dev/null +++ b/experiments/experiment_pooled.yml @@ -0,0 +1,36 @@ +# This experiment trains models with progressively more samples to see how the performance changes +command: + - ${env} + - ${program} + - --samples + - 0 + - -d + - ../data/ + - -t + - BinaryClassification + - --log-dir + - ../yaib_logs_pooled_training + - --tune + - --wandb-sweep +method: grid +name: yaib_samples_benchmark +parameters: + samples: + values: + - 30000 + data_dir: + values: + - ../../data/pooled_mortality24_10000/eicu_aumc_miiv + - ../../data/pooled_mortality24_10000/aumc_hirid_miiv + - ../../data/pooled_mortality24_10000/eicu_hirid_miiv +# - ../../data/pooled_mortality24_10000/eicu_hirid_aumc + model: + values: + - GRU + seed: + values: + - 1111 + use_pretrained_imputation: + values: + - None +program: icu-benchmarks diff --git a/icu_benchmarks/data/pooling.py b/icu_benchmarks/data/pooling.py index 350c227b..ac4d26e8 100644 --- a/icu_benchmarks/data/pooling.py +++ b/icu_benchmarks/data/pooling.py @@ -1,14 +1,41 @@ -import copy -import pickle -import torch +from pathlib import Path import logging -import gin import pandas as pd -from sklearn.model_selection import StratifiedKFold, train_test_split -from .constants import DataSplit as Split, DataSegment as Segment, VarType as Var +from sklearn.model_selection import train_test_split +from .constants import DataSegment as Segment, VarType as Var +import pyarrow.parquet as pq -def pool_datasets(datasets={}, samples=10000, vars = [], seed=42,shuffle=True, stratify=None, **kwargs): +class PooledDataset: + eicu_hirid_miiv = ["eicu", "hirid", "miiv"] + eicu_hirid_aumc = ["eicu", "hirid", "aumc"] + eicu_aumc_miiv = ["eicu", "aumc", "miiv"] + aumc_hirid_miiv = ["aumc", "hirid", "miiv"] + + +def generate_pooled_data(data_dir, vars, datasets, file_names, samples, seed, shuffle, stratify): + data = {} + for folder in data_dir.iterdir(): + if folder.is_dir(): + if folder.name in datasets: + data[folder.name] = { + f: pq.read_table(folder / file_names[f]).to_pandas(self_destruct=True) for f in file_names.keys() + } + data = pool_datasets(datasets=data, samples=10000, vars=vars, shuffle=True, stratify=None) + save_pooled_data(data_dir, data, datasets) + + +def save_pooled_data(data_dir, data, datasets): + save_folder = "_".join(datasets) + save_dir = data_dir / save_folder + if not save_dir.exists(): + save_dir.mkdir() + for key, value in data.items(): + value.to_parquet(save_dir / Path(key + ".parquet")) + logging.info(f"Saved pooled data at {save_dir}") + + +def pool_datasets(datasets={}, samples=10000, vars=[], seed=42, shuffle=True, stratify=None, **kwargs): """ Pool datasets into a single dataset. Args: @@ -18,29 +45,31 @@ def pool_datasets(datasets={}, samples=10000, vars = [], seed=42,shuffle=True, s """ if len(datasets) == 0: raise ValueError("No datasets supplied.") - pooled_data = {Segment.static:[], Segment.dynamic:[], Segment.outcome:[]} + pooled_data = {Segment.static: [], Segment.dynamic: [], Segment.outcome: []} id = vars[Var.group] int_id = 0 for key, value in datasets.items(): - if samples: - int_id += 1 - num_classes = 2 - repeated_digit = str(int_id) * 4 - samples_per_class = int(samples / num_classes) - # outcome = value[Segment.outcome].groupby(vars["LABEL"], group_keys=False).sample(samples_per_class,random_state=seed) - outcome = value[Segment.outcome] - outcome = train_test_split(outcome, stratify=outcome[vars["LABEL"]], shuffle=shuffle, random_state=seed, train_size=samples)[0] - stays = pd.Series(outcome[id].unique()) - static = value[Segment.static] - dynamic = value[Segment.dynamic] - static = static.loc[static[id].isin(stays)] - dynamic = dynamic.loc[dynamic[id].isin(stays)] - outcome[id] = outcome[id].map(lambda x: int(str(x) + repeated_digit)) - static[id] = static[id].map(lambda x: int(str(x) + repeated_digit)) - dynamic[id] = dynamic[id].map(lambda x: int(str(x) + repeated_digit)) - pooled_data[Segment.static].append(static) - pooled_data[Segment.dynamic].append(dynamic) - pooled_data[Segment.outcome].append(outcome) + int_id += 1 + repeated_digit = str(int_id) * 4 + # outcome = value[Segment.outcome].groupby(vars["LABEL"], group_keys=False).sample(samples_per_class,random_state=seed) + outcome = value[Segment.outcome] + outcome = train_test_split( + outcome, stratify=outcome[Var.label], shuffle=shuffle, random_state=seed, train_size=samples + )[0] + stays = pd.Series(outcome[id].unique()) + static = value[Segment.static] + dynamic = value[Segment.dynamic] + static = static.loc[static[id].isin(stays)] + dynamic = dynamic.loc[dynamic[id].isin(stays)] + # Preventing id clashing + outcome[id] = outcome[id].map(lambda x: int(str(x) + repeated_digit)) + static[id] = static[id].map(lambda x: int(str(x) + repeated_digit)) + dynamic[id] = dynamic[id].map(lambda x: int(str(x) + repeated_digit)) + # Adding to pooled data + pooled_data[Segment.static].append(static) + pooled_data[Segment.dynamic].append(dynamic) + pooled_data[Segment.outcome].append(outcome) + # Add each datatype together for key, value in pooled_data.items(): pooled_data[key] = pd.concat(value, ignore_index=True) return pooled_data From d6aeb35f1b06f079fd8f97691d69556589031d37 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Fri, 25 Aug 2023 22:07:44 +0200 Subject: [PATCH 083/207] Added train_only flag for training on the full dataset without cross-validation --- icu_benchmarks/cross_validation.py | 8 ++ icu_benchmarks/data/split_process_data.py | 116 ++++++++++++++-------- icu_benchmarks/run.py | 8 +- icu_benchmarks/run_utils.py | 2 +- 4 files changed, 89 insertions(+), 45 deletions(-) diff --git a/icu_benchmarks/cross_validation.py b/icu_benchmarks/cross_validation.py index cc2329f9..f0f3f3b9 100644 --- a/icu_benchmarks/cross_validation.py +++ b/icu_benchmarks/cross_validation.py @@ -37,11 +37,13 @@ def execute_repeated_cv( cpu: bool = False, verbose: bool = False, wandb: bool = False, + train_only: bool = False ) -> float: """Preprocesses data and trains a model for each fold. Args: + wandb: Use wandb for logging. data_dir: Path to the data directory. log_dir: Path to the log directory. seed: Random seed. @@ -72,6 +74,10 @@ def execute_repeated_cv( agg_loss = 0 logging.info(f"Starting nested CV with {cv_repetitions_to_train} repetitions of {cv_folds_to_train} folds") seed_everything(seed, reproducible) + if train_only: + logging.info("Will train full model without cross validation") + cv_repetitions_to_train = 1 + cv_folds_to_train = 1 for repetition in range(cv_repetitions_to_train): for fold_index in range(cv_folds_to_train): start_time = datetime.now() @@ -88,6 +94,7 @@ def execute_repeated_cv( fold_index=fold_index, pretrained_imputation_model=pretrained_imputation_model, runmode=mode, + train_only=train_only ) repetition_fold_dir = log_dir / f"repetition_{repetition}" / f"fold_{fold_index}" @@ -106,6 +113,7 @@ def execute_repeated_cv( cpu=cpu, verbose=verbose, use_wandb=wandb, + train_only=train_only ) train_time = datetime.now() - start_time diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index db468469..fc150669 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -1,3 +1,4 @@ +import copy import logging import gin import json @@ -7,12 +8,11 @@ from pathlib import Path import pickle -from sklearn.model_selection import StratifiedKFold, KFold, StratifiedShuffleSplit, ShuffleSplit +from sklearn.model_selection import StratifiedKFold, KFold, StratifiedShuffleSplit, ShuffleSplit, train_test_split from icu_benchmarks.data.preprocessor import Preprocessor, DefaultClassificationPreprocessor from icu_benchmarks.contants import RunMode from .constants import DataSplit as Split, DataSegment as Segment, VarType as Var -from .pooling import pool_datasets @gin.configurable("preprocess") @@ -32,6 +32,7 @@ def preprocess_data( generate_cache: bool = False, fold_index: int = 0, pretrained_imputation_model: str = None, + full_train: bool = False, runmode: RunMode = RunMode.classification, ) -> dict[dict[pd.DataFrame]]: """Perform loading, splitting, imputing and normalising of task data. @@ -87,47 +88,24 @@ def preprocess_data( # Read parquet files into pandas dataframes and remove the parquet file from memory logging.info(f"Loading data from directory {data_dir.absolute()}") - pooling = True - - datasets = ["eicu", "hirid", "miiv"] - # datasets = ["eicu", "hirid", "aumc"] - # datasets = ["eicu", "aumc", "miiv"] - # datasets = ["aumc", "hirid", "miiv"] - - - if pooling: - data = {} - for folder in data_dir.iterdir(): - if folder.is_dir(): - if folder.name in datasets: - data[folder.name] = {f: pq.read_table(folder / file_names[f]).to_pandas(self_destruct=True) for f in file_names.keys()} - data = pool_datasets(datasets=data, samples=10000, vars=vars, shuffle=True, stratify=None) - else: - data = {f: pq.read_table(data_dir / file_names[f]).to_pandas(self_destruct=True) for f in file_names.keys()} - - cache_file = "_".join(datasets) - cache_dir = Path(r'C:\Users\Robin\Downloads\mortality24_eicu_hirid_aumc\cache') - cache_dir = cache_dir / cache_file - if not cache_dir.exists(): - cache_dir.mkdir() - for key, value in data.items(): - value.to_parquet(cache_dir / Path( key + ".parquet")) - caching(cache_dir, cache_file, data, load_cache) - + data = {f: pq.read_table(data_dir / file_names[f]).to_pandas(self_destruct=True) for f in file_names.keys()} # Generate the splits logging.info("Generating splits.") - data = make_single_split( - data, - vars, - cv_repetitions, - repetition_index, - cv_folds, - fold_index, - train_size=train_size, - seed=seed, - debug=debug, - runmode=runmode, - ) + if not full_train: + data = make_single_split( + data, + vars, + cv_repetitions, + repetition_index, + cv_folds, + fold_index, + train_size=train_size, + seed=seed, + debug=debug, + runmode=runmode, + ) + else: + data = make_train_val(data, vars, train_size=0.8, seed=seed, debug=debug) # Apply preprocessing data = preprocessor.apply(data, vars) @@ -142,6 +120,62 @@ def preprocess_data( return data +def make_train_val( + data: dict[pd.DataFrame], + vars: dict[str], + train_size=0.8, + seed: int = 42, + debug: bool = False, +) -> dict[dict[pd.DataFrame]]: + """Randomly split the data into training and validation sets for fitting a full model. + + Args: + data: dictionary containing data divided int OUTCOME, STATIC, and DYNAMIC. + vars: Contains the names of columns in the data. + train_size: Fixed size of train split (including validation data). + seed: Random seed. + debug: Load less data if true. + Returns: + Input data divided into 'train', 'val', and 'test'. + """ + # ID variable + id = vars[Var.group] + + # Get stay IDs from outcome segment + stays = pd.Series(data[Segment.outcome][id].unique(), name=id) + + if debug: + # Only use 1% of the data + stays = stays.sample(frac=0.01, random_state=seed) + + # If there are labels, and the task is classification, use stratified k-fold + if Var.label in vars: + # Get labels from outcome data (takes the highest value (or True) in case seq2seq classification) + labels = data[Segment.outcome].groupby(id).max()[vars[Var.label]].reset_index(drop=True) + if train_size: + train_val = StratifiedShuffleSplit(train_size=train_size, random_state=seed, n_splits=1) + train, val = list(train_val.split(stays, labels))[0] + else: + # If there are no labels, use random split + train, val = train_test_split(stays, train_size=train_size, random_state=seed) + + split = { + Split.train: stays.iloc[train], + Split.val: stays.iloc[val] + } + + data_split = {} + + for fold in split.keys(): # Loop through splits (train / val / test) + # Loop through segments (DYNAMIC / STATIC / OUTCOME) + # set sort to true to make sure that IDs are reordered after scrambling earlier + data_split[fold] = { + data_type: data[data_type].merge(split[fold], on=id, how="right", sort=True) for data_type in data.keys() + } + # Maintain compatibility with test split + data_split[Split.test] = copy.deepcopy(data_split[Split.val]) + return data_split + def make_single_split( data: dict[pd.DataFrame], diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index f15725b4..73ce8228 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -33,7 +33,11 @@ def get_mode(mode: gin.REQUIRED): def main(my_args=tuple(sys.argv[1:])): + args, _ = build_parser().parse_known_args(my_args) + if args.wandb_sweep: + args = apply_wandb_sweep(args) + set_wandb_experiment_name(args, "run") # Initialize loggers log_format = "%(asctime)s - %(levelname)s - %(name)s : %(message)s" date_format = "%Y-%m-%d %H:%M:%S" @@ -52,9 +56,7 @@ def main(my_args=tuple(sys.argv[1:])): gin.parse_config_file(f"configs/tasks/{task}.gin") mode = get_mode() # Set arguments for wandb sweep - if args.wandb_sweep: - args = apply_wandb_sweep(args) - set_wandb_experiment_name(args, mode) + # Set experiment name if name is None: diff --git a/icu_benchmarks/run_utils.py b/icu_benchmarks/run_utils.py index 69fe0944..8a2fb961 100644 --- a/icu_benchmarks/run_utils.py +++ b/icu_benchmarks/run_utils.py @@ -47,7 +47,7 @@ def build_parser() -> ArgumentParser: parser.add_argument("--tune", default=False, action=BOA, help="Find best hyperparameters.") parser.add_argument("--hp-checkpoint", type=Path, help="Use previous hyperparameter checkpoint.") parser.add_argument("--eval", default=False, action=BOA, help="Only evaluate model, skip training.") - # parser.add_argument("--no-eval", default=False, action=BOA, help="Only train model") + parser.add_argument("--full-train", default=False, action=BOA, help="Only train model") parser.add_argument("-ft", "--fine-tune", default=None, type=int, help="Finetune model with amount of train data.") parser.add_argument("-sn", "--source-name", type=Path, help="Name of the source dataset.") parser.add_argument("--source-dir", type=Path, help="Directory containing gin and model weights.") From 8aac28268a82c54242f4b0dc2ac7a48f03e3d5f8 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Fri, 25 Aug 2023 22:08:08 +0200 Subject: [PATCH 084/207] Train only for train.py --- icu_benchmarks/models/train.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index d99e929f..0035b419 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -48,6 +48,7 @@ def train_common( verbose=False, ram_cache=False, pl_model=True, + train_only=False, num_workers: int = min(cpu_core_count, torch.cuda.device_count() * 8 * int(torch.cuda.is_available()), 32), ): """Common wrapper to train all benchmarked models. @@ -157,7 +158,8 @@ def train_common( model.fit(train_dataset, val_dataset) model.save_model(log_dir, "last") logging.info("Training complete.") - + if train_only: + return 0 test_dataset = dataset_class(data, split=test_on, name=dataset_names["test"]) test_dataset = assure_minimum_length(test_dataset) logging.info(f"Testing on {test_dataset.name} with {len(test_dataset)} samples.") From 64009d740443d198fba38e91fd9eafd78a83a8c0 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Fri, 25 Aug 2023 22:08:32 +0200 Subject: [PATCH 085/207] naming fix for wandb_utils.py --- icu_benchmarks/wandb_utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/icu_benchmarks/wandb_utils.py b/icu_benchmarks/wandb_utils.py index 41aca287..57ad269b 100644 --- a/icu_benchmarks/wandb_utils.py +++ b/icu_benchmarks/wandb_utils.py @@ -1,5 +1,7 @@ from argparse import Namespace import logging +from pathlib import Path + import wandb @@ -56,6 +58,9 @@ def set_wandb_experiment_name(args, mode): args (Namespace): parsed arguments mode (RunMode): run mode """ + if args.name is None: + data_dir = Path(args.data_dir) + args.name = data_dir.name run_name = f"{mode}_{args.model}_{args.name}" if args.fine_tune: From a8e1f9b72b6e6ab2126d6bf815cf5f8061d25e24 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Fri, 25 Aug 2023 22:27:14 +0200 Subject: [PATCH 086/207] full-train fixes --- experiments/experiment_full_training.yml | 35 ++++++++++++++++++++++++ icu_benchmarks/cross_validation.py | 15 ++++++---- icu_benchmarks/run.py | 1 + icu_benchmarks/wandb_utils.py | 2 ++ 4 files changed, 47 insertions(+), 6 deletions(-) create mode 100644 experiments/experiment_full_training.yml diff --git a/experiments/experiment_full_training.yml b/experiments/experiment_full_training.yml new file mode 100644 index 00000000..fdda808b --- /dev/null +++ b/experiments/experiment_full_training.yml @@ -0,0 +1,35 @@ +# This experiment trains models with progressively more samples to see how the performance changes +command: + - ${env} + - ${program} + - --full-train + - -d + - ../data/ + - -t + - BinaryClassification + - --log-dir + - ../yaib_logs_pooled_training + - --tune + - --wandb-sweep + - --tune + - --hp-checkpoint + - /dhc/home/robin.vandewater/projects/yaib_logs_pooled_training/eicu_hirid_aumc/BinaryClassification/GRU/2023-08-25T17-59-10 +method: grid +name: yaib_full_benchmark +parameters: + data_dir: + values: +# - ../../data/pooled_mortality24_10000/eicu_aumc_miiv +# - ../../data/pooled_mortality24_10000/aumc_hirid_miiv +# - ../../data/pooled_mortality24_10000/eicu_hirid_miiv + - ../../data/pooled_mortality24_10000/eicu_hirid_aumc + model: + values: + - GRU + seed: + values: + - 1111 + use_pretrained_imputation: + values: + - None +program: icu-benchmarks diff --git a/icu_benchmarks/cross_validation.py b/icu_benchmarks/cross_validation.py index f0f3f3b9..a69bd005 100644 --- a/icu_benchmarks/cross_validation.py +++ b/icu_benchmarks/cross_validation.py @@ -37,7 +37,7 @@ def execute_repeated_cv( cpu: bool = False, verbose: bool = False, wandb: bool = False, - train_only: bool = False + full_train: bool = False ) -> float: """Preprocesses data and trains a model for each fold. @@ -72,12 +72,15 @@ def execute_repeated_cv( if not cv_folds_to_train: cv_folds_to_train = cv_folds agg_loss = 0 - logging.info(f"Starting nested CV with {cv_repetitions_to_train} repetitions of {cv_folds_to_train} folds") seed_everything(seed, reproducible) - if train_only: - logging.info("Will train full model without cross validation") + if full_train: + logging.info("Will train full model without cross validation.") cv_repetitions_to_train = 1 cv_folds_to_train = 1 + + else: + logging.info(f"Starting nested CV with {cv_repetitions_to_train} repetitions of {cv_folds_to_train} folds.") + for repetition in range(cv_repetitions_to_train): for fold_index in range(cv_folds_to_train): start_time = datetime.now() @@ -94,7 +97,7 @@ def execute_repeated_cv( fold_index=fold_index, pretrained_imputation_model=pretrained_imputation_model, runmode=mode, - train_only=train_only + full_train=full_train ) repetition_fold_dir = log_dir / f"repetition_{repetition}" / f"fold_{fold_index}" @@ -113,7 +116,7 @@ def execute_repeated_cv( cpu=cpu, verbose=verbose, use_wandb=wandb, - train_only=train_only + train_only=full_train ) train_time = datetime.now() - start_time diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index 73ce8228..6c0d0eb6 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -173,6 +173,7 @@ def main(my_args=tuple(sys.argv[1:])): pretrained_imputation_model=pretrained_imputation_model, cpu=args.cpu, wandb=args.wandb_sweep, + full_train=args.full_train ) log_full_line("FINISHED TRAINING", level=logging.INFO, char="=", num_newlines=3) diff --git a/icu_benchmarks/wandb_utils.py b/icu_benchmarks/wandb_utils.py index 57ad269b..48a2035c 100644 --- a/icu_benchmarks/wandb_utils.py +++ b/icu_benchmarks/wandb_utils.py @@ -69,6 +69,8 @@ def set_wandb_experiment_name(args, mode): run_name += f"_source_{args.source_name}" elif args.samples: run_name += f"_train_size_{args.samples}_samples" + elif args.full_train: + run_name += f"_full_training" if wandb_running(): wandb.config.update({"run-name": run_name}) From 1505db8a25ee42db937010b0e87b68fb158b4797 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Sat, 26 Aug 2023 12:23:36 +0200 Subject: [PATCH 087/207] experiment configs --- experiments/experiment_eval_pooled.yml | 51 ++++++++++++++++++++++++ experiments/experiment_full_training.yml | 13 +++--- experiments/experiment_pooled.yml | 23 ++++++----- icu_benchmarks/models/train.py | 5 ++- 4 files changed, 75 insertions(+), 17 deletions(-) create mode 100644 experiments/experiment_eval_pooled.yml diff --git a/experiments/experiment_eval_pooled.yml b/experiments/experiment_eval_pooled.yml new file mode 100644 index 00000000..0ce4ca9a --- /dev/null +++ b/experiments/experiment_eval_pooled.yml @@ -0,0 +1,51 @@ +command: + - ${env} + - ${program} + - --eval + - -d + - ../data/ + - -t + - BinaryClassification + - --log-dir + - ../yaib_logs_pooled + - --tune + - --wandb-sweep +# - -gc +# - -lc + - -sn +# - eicu_hirid_aumc + - eicu_hirid_miiv + - --source-dir + - /dhc/home/robin.vandewater/data/pooled_mortality24_10000/aumc_hirid_miiv/BinaryClassification/GRU/2023-08-26T11-41-24/repetition_0/fold_0 +# - /dhc/home/robin.vandewater/data/pooled_mortality24_10000/eicu_aumc_miiv/BinaryClassification/GRU/2023-08-26T11-40-36/repetition_0/fold_0 +# - /dhc/home/robin.vandewater/data/pooled_mortality24_10000/eicu_hirid_miiv/BinaryClassification/GRU/2023-08-26T11-41-25/repetition_0/fold_0 +# - /dhc/home/robin.vandewater/projects/yaib_logs_pooled_training/eicu_hirid_aumc/BinaryClassification/GRU/2023-08-25T22-24-02/repetition_0/fold_0 + - +method: grid +name: yaib_pooled_eval +parameters: + data_dir: + values: +# - ../../data/mortality24/miiv +# - ../../data/mortality24/hirid + - ../../data/mortality24/eicu +# - ../../data/mortality24/aumc +# - ../../data/kf/miiv +# - ../../data/kf/hirid +# - ../../data/kf/eicu +# - ../../data/kf/aumc + model: + values: +# - ElasticNet +# - LGBMRegressor + - GRU +# - LSTM +# - TCN +# - Transformer + seed: + values: + - 1111 + use_pretrained_imputation: + values: + - None +program: icu-benchmarks \ No newline at end of file diff --git a/experiments/experiment_full_training.yml b/experiments/experiment_full_training.yml index fdda808b..ebe6267f 100644 --- a/experiments/experiment_full_training.yml +++ b/experiments/experiment_full_training.yml @@ -8,21 +8,22 @@ command: - -t - BinaryClassification - --log-dir - - ../yaib_logs_pooled_training +# - ../yaib_logs_pooled_training + - ../../data/pooled_mortality24_10000 - --tune - --wandb-sweep - --tune - --hp-checkpoint - - /dhc/home/robin.vandewater/projects/yaib_logs_pooled_training/eicu_hirid_aumc/BinaryClassification/GRU/2023-08-25T17-59-10 + - test method: grid name: yaib_full_benchmark parameters: data_dir: values: -# - ../../data/pooled_mortality24_10000/eicu_aumc_miiv -# - ../../data/pooled_mortality24_10000/aumc_hirid_miiv -# - ../../data/pooled_mortality24_10000/eicu_hirid_miiv - - ../../data/pooled_mortality24_10000/eicu_hirid_aumc + - ../../data/pooled_mortality24_10000/eicu_aumc_miiv + - ../../data/pooled_mortality24_10000/aumc_hirid_miiv + - ../../data/pooled_mortality24_10000/eicu_hirid_miiv +# - ../../data/pooled_mortality24_10000/eicu_hirid_aumc model: values: - GRU diff --git a/experiments/experiment_pooled.yml b/experiments/experiment_pooled.yml index 7f34ac12..1f636cc9 100644 --- a/experiments/experiment_pooled.yml +++ b/experiments/experiment_pooled.yml @@ -2,28 +2,31 @@ command: - ${env} - ${program} - - --samples - - 0 +# - --samples +# - 0 - -d - ../data/ - -t - BinaryClassification - --log-dir +# - ../../data/pooled_mortality24_10000 - ../yaib_logs_pooled_training - --tune - --wandb-sweep + - --hp-checkpoint + - test method: grid -name: yaib_samples_benchmark +name: yaib_pooled_benchmark parameters: - samples: - values: - - 30000 +# samples: +# values: +# - 30000 data_dir: values: - - ../../data/pooled_mortality24_10000/eicu_aumc_miiv - - ../../data/pooled_mortality24_10000/aumc_hirid_miiv - - ../../data/pooled_mortality24_10000/eicu_hirid_miiv -# - ../../data/pooled_mortality24_10000/eicu_hirid_aumc +# - ../../data/pooled_mortality24_10000/eicu_aumc_miiv +# - ../../data/pooled_mortality24_10000/aumc_hirid_miiv +# - ../../data/pooled_mortality24_10000/eicu_hirid_miiv + - ../../data/pooled_mortality24_10000/eicu_hirid_aumc model: values: - GRU diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 0035b419..7b0e7cfb 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -115,10 +115,11 @@ def train_common( data_shape = next(iter(train_loader))[0].shape - model = model(optimizer=optimizer, input_size=data_shape, epochs=epochs, run_mode=mode) if load_weights: model = load_model(model, source_dir, pl_model=pl_model) + else: + model = model(optimizer=optimizer, input_size=data_shape, epochs=epochs, run_mode=mode) model.set_weight(weight, train_dataset) model.set_trained_columns(train_dataset.get_feature_names()) @@ -159,6 +160,8 @@ def train_common( model.save_model(log_dir, "last") logging.info("Training complete.") if train_only: + logging.info("Finished training full model.") + save_config_file(log_dir) return 0 test_dataset = dataset_class(data, split=test_on, name=dataset_names["test"]) test_dataset = assure_minimum_length(test_dataset) From 15ad68efae6ed381c78b0e11bae14188a15e69aa Mon Sep 17 00:00:00 2001 From: rvandewater Date: Sat, 26 Aug 2023 13:06:38 +0200 Subject: [PATCH 088/207] small tweaks to pooling.py --- experiments/experiment_pooled.yml | 3 ++- icu_benchmarks/data/pooling.py | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/experiments/experiment_pooled.yml b/experiments/experiment_pooled.yml index 1f636cc9..ec312f90 100644 --- a/experiments/experiment_pooled.yml +++ b/experiments/experiment_pooled.yml @@ -26,7 +26,8 @@ parameters: # - ../../data/pooled_mortality24_10000/eicu_aumc_miiv # - ../../data/pooled_mortality24_10000/aumc_hirid_miiv # - ../../data/pooled_mortality24_10000/eicu_hirid_miiv - - ../../data/pooled_mortality24_10000/eicu_hirid_aumc +# - ../../data/pooled_mortality24_10000/eicu_hirid_aumc + - ../../data/pooled_mortality24_10000/aumc_hirid_eicu_miiv model: values: - GRU diff --git a/icu_benchmarks/data/pooling.py b/icu_benchmarks/data/pooling.py index ac4d26e8..c05149e5 100644 --- a/icu_benchmarks/data/pooling.py +++ b/icu_benchmarks/data/pooling.py @@ -11,9 +11,10 @@ class PooledDataset: eicu_hirid_aumc = ["eicu", "hirid", "aumc"] eicu_aumc_miiv = ["eicu", "aumc", "miiv"] aumc_hirid_miiv = ["aumc", "hirid", "miiv"] + eicu_hirid_aumc_miiv = ["eicu", "hirid", "aumc", "miiv"] -def generate_pooled_data(data_dir, vars, datasets, file_names, samples, seed, shuffle, stratify): +def generate_pooled_data(data_dir, vars, datasets, file_names, samples=10000, seed=42, shuffle=False, stratify=None): data = {} for folder in data_dir.iterdir(): if folder.is_dir(): @@ -21,7 +22,7 @@ def generate_pooled_data(data_dir, vars, datasets, file_names, samples, seed, sh data[folder.name] = { f: pq.read_table(folder / file_names[f]).to_pandas(self_destruct=True) for f in file_names.keys() } - data = pool_datasets(datasets=data, samples=10000, vars=vars, shuffle=True, stratify=None) + data = pool_datasets(datasets=data, samples=samples, vars=vars, shuffle=True, stratify=None) save_pooled_data(data_dir, data, datasets) @@ -54,7 +55,7 @@ def pool_datasets(datasets={}, samples=10000, vars=[], seed=42, shuffle=True, st # outcome = value[Segment.outcome].groupby(vars["LABEL"], group_keys=False).sample(samples_per_class,random_state=seed) outcome = value[Segment.outcome] outcome = train_test_split( - outcome, stratify=outcome[Var.label], shuffle=shuffle, random_state=seed, train_size=samples + outcome, stratify=outcome[vars[Var.label]], shuffle=shuffle, random_state=seed, train_size=samples )[0] stays = pd.Series(outcome[id].unique()) static = value[Segment.static] From 3955da355ce82191441fdf2a13585b3a8c224906 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Mon, 28 Aug 2023 12:58:20 +0200 Subject: [PATCH 089/207] pooling adjustments and fixes --- configs/tasks/Regression.gin | 2 +- experiments/experiment_eval_pooled.yml | 31 +++++++--- experiments/experiment_full_training.yml | 20 +++++-- experiments/experiment_pooled.yml | 22 +++++-- icu_benchmarks/data/pooling.py | 71 ++++++++++++++++++----- icu_benchmarks/data/split_process_data.py | 16 +++-- 6 files changed, 120 insertions(+), 42 deletions(-) diff --git a/configs/tasks/Regression.gin b/configs/tasks/Regression.gin index 5cf3f8d9..40c788e0 100644 --- a/configs/tasks/Regression.gin +++ b/configs/tasks/Regression.gin @@ -25,7 +25,7 @@ preprocess.use_static = True # SPECIFYING REGRESSION OUTCOME SCALING base_regression_preprocessor.outcome_min = 0 -base_regression_preprocessor.outcome_max = 15 +base_regression_preprocessor.outcome_max = 168 # SELECTING DATASET PredictionDataset.vars = %vars diff --git a/experiments/experiment_eval_pooled.yml b/experiments/experiment_eval_pooled.yml index 0ce4ca9a..44abb0cb 100644 --- a/experiments/experiment_eval_pooled.yml +++ b/experiments/experiment_eval_pooled.yml @@ -5,43 +5,56 @@ command: - -d - ../data/ - -t - - BinaryClassification +# - BinaryClassification + - Regression - --log-dir - ../yaib_logs_pooled - - --tune +# - --tune - --wandb-sweep # - -gc # - -lc - - -sn +# - -sn # - eicu_hirid_aumc - - eicu_hirid_miiv +# - eicu_hirid_miiv - --source-dir - - /dhc/home/robin.vandewater/data/pooled_mortality24_10000/aumc_hirid_miiv/BinaryClassification/GRU/2023-08-26T11-41-24/repetition_0/fold_0 +# - /dhc/home/robin.vandewater/projects/yaib_logs_pooled_los/aumc_eicu_miiv_10000/Regression/Transformer/2023-08-28T11-06-03/repetition_0/fold_0 +# - /dhc/home/robin.vandewater/projects/yaib_logs_pooled_los/aumc_hirid_miiv_10000/Regression/Transformer/2023-08-28T11-06-05/repetition_0/fold_0 +# - /dhc/home/robin.vandewater/projects/yaib_logs_pooled_los/aumc_hirid_eicu_10000/Regression/Transformer/2023-08-28T11-03-18/repetition_0/fold_0 +# - /dhc/home/robin.vandewater/projects/yaib_logs_pooled_training/aumc_hirid_eicu_miiv/BinaryClassification/GRU/2023-08-27T12-10-12/repetition_0/fold_0 + - /dhc/home/robin.vandewater/projects/yaib_logs_pooled_los/hirid_eicu_miiv_10000/Regression/Transformer/2023-08-28T11-06-03/repetition_0/fold_0 + #- /dhc/home/robin.vandewater/data/pooled_mortality24_10000/aumc_hirid_miiv/BinaryClassification/GRU/2023-08-26T11-41-24/repetition_0/fold_0 # - /dhc/home/robin.vandewater/data/pooled_mortality24_10000/eicu_aumc_miiv/BinaryClassification/GRU/2023-08-26T11-40-36/repetition_0/fold_0 # - /dhc/home/robin.vandewater/data/pooled_mortality24_10000/eicu_hirid_miiv/BinaryClassification/GRU/2023-08-26T11-41-25/repetition_0/fold_0 # - /dhc/home/robin.vandewater/projects/yaib_logs_pooled_training/eicu_hirid_aumc/BinaryClassification/GRU/2023-08-25T22-24-02/repetition_0/fold_0 - - method: grid name: yaib_pooled_eval parameters: data_dir: values: +# - ../../data/mortality24/miiv_42045 +# - ../../data/mortality24/hirid_2859 +# - ../../data/mortality24/eicu_103382 +# - ../../data/mortality24/aumc_535 # - ../../data/mortality24/miiv # - ../../data/mortality24/hirid - - ../../data/mortality24/eicu +# - ../../data/mortality24/eicu # - ../../data/mortality24/aumc # - ../../data/kf/miiv # - ../../data/kf/hirid # - ../../data/kf/eicu # - ../../data/kf/aumc +# - ../../data/los/miiv +# - ../../data/los/hirid +# - ../../data/los/eicu + - ../../data/los/aumc model: values: # - ElasticNet # - LGBMRegressor - - GRU +# - GRU # - LSTM # - TCN -# - Transformer + - Transformer seed: values: - 1111 diff --git a/experiments/experiment_full_training.yml b/experiments/experiment_full_training.yml index ebe6267f..f29e0226 100644 --- a/experiments/experiment_full_training.yml +++ b/experiments/experiment_full_training.yml @@ -6,10 +6,11 @@ command: - -d - ../data/ - -t - - BinaryClassification +# - BinaryClassification + - Regression - --log-dir # - ../yaib_logs_pooled_training - - ../../data/pooled_mortality24_10000 + - ../yaib_logs_pooled_los - --tune - --wandb-sweep - --tune @@ -20,13 +21,20 @@ name: yaib_full_benchmark parameters: data_dir: values: - - ../../data/pooled_mortality24_10000/eicu_aumc_miiv - - ../../data/pooled_mortality24_10000/aumc_hirid_miiv - - ../../data/pooled_mortality24_10000/eicu_hirid_miiv +# - ../../data/pooled_mortality24_10000/eicu_aumc_miiv +# - ../../data/pooled_mortality24_10000/aumc_hirid_miiv +# - ../../data/pooled_mortality24_10000/eicu_hirid_miiv # - ../../data/pooled_mortality24_10000/eicu_hirid_aumc +# - ../../data/pooled_mortality24_10000/aumc_hirid_eicu_miiv + - ../../data/los/aumc_hirid_eicu_10000 + - ../../data/los/hirid_eicu_miiv_10000 + - ../../data/los/aumc_eicu_miiv_10000 + - ../../data/los/aumc_hirid_miiv_10000 + - ../../data/los/aumc_hirid_eicu_miiv_10000 model: values: - - GRU + #- GRU + - Transformer seed: values: - 1111 diff --git a/experiments/experiment_pooled.yml b/experiments/experiment_pooled.yml index ec312f90..616e9766 100644 --- a/experiments/experiment_pooled.yml +++ b/experiments/experiment_pooled.yml @@ -7,14 +7,17 @@ command: - -d - ../data/ - -t - - BinaryClassification +# - BinaryClassification + - Regression - --log-dir # - ../../data/pooled_mortality24_10000 - - ../yaib_logs_pooled_training + - ../yaib_logs_pooled_los - --tune - --wandb-sweep - --hp-checkpoint - test + - -gc + - -lc method: grid name: yaib_pooled_benchmark parameters: @@ -23,14 +26,25 @@ parameters: # - 30000 data_dir: values: +# - ../../data/kf/pooled_datasets_6000/hirid_eicu_miiv +# - ../../data/kf/pooled_datasets_6000/aumc_hirid_eicu +# - ../../data/kf/pooled_datasets_6000/aumc_eicu_miiv +# - ../../data/kf/pooled_datasets_6000/aumc_hirid_miiv +# - ../../data/kf/pooled_datasets_6000/aumc_hirid_eicu_miiv + - ../../data/los/aumc_hirid_eicu_10000 + - ../../data/los/hirid_eicu_miiv_10000 + - ../../data/los/aumc_eicu_miiv_10000 + - ../../data/los/aumc_hirid_miiv_10000 + - ../../data/los/aumc_hirid_eicu_miiv_10000 # - ../../data/pooled_mortality24_10000/eicu_aumc_miiv # - ../../data/pooled_mortality24_10000/aumc_hirid_miiv # - ../../data/pooled_mortality24_10000/eicu_hirid_miiv # - ../../data/pooled_mortality24_10000/eicu_hirid_aumc - - ../../data/pooled_mortality24_10000/aumc_hirid_eicu_miiv +# - ../../data/pooled_mortality24_10000/aumc_hirid_eicu_miiv model: values: - - GRU +# - GRU + - Transformer seed: values: - 1111 diff --git a/icu_benchmarks/data/pooling.py b/icu_benchmarks/data/pooling.py index c05149e5..18454e3d 100644 --- a/icu_benchmarks/data/pooling.py +++ b/icu_benchmarks/data/pooling.py @@ -3,18 +3,19 @@ import pandas as pd from sklearn.model_selection import train_test_split from .constants import DataSegment as Segment, VarType as Var +from icu_benchmarks.contants import RunMode import pyarrow.parquet as pq class PooledDataset: - eicu_hirid_miiv = ["eicu", "hirid", "miiv"] - eicu_hirid_aumc = ["eicu", "hirid", "aumc"] - eicu_aumc_miiv = ["eicu", "aumc", "miiv"] + hirid_eicu_miiv = ["hirid", "eicu", "miiv"] + aumc_hirid_eicu = ["aumc", "hirid","eicu"] + aumc_eicu_miiv = ["aumc", "eicu", "miiv"] aumc_hirid_miiv = ["aumc", "hirid", "miiv"] - eicu_hirid_aumc_miiv = ["eicu", "hirid", "aumc", "miiv"] + aumc_hirid_eicu_miiv = ["aumc","hirid", "eicu", "miiv"] -def generate_pooled_data(data_dir, vars, datasets, file_names, samples=10000, seed=42, shuffle=False, stratify=None): +def generate_pooled_data(data_dir, vars, datasets, file_names, samples=10000, seed=42, shuffle=False, stratify=None, runmode=RunMode.classification): data = {} for folder in data_dir.iterdir(): if folder.is_dir(): @@ -22,21 +23,23 @@ def generate_pooled_data(data_dir, vars, datasets, file_names, samples=10000, se data[folder.name] = { f: pq.read_table(folder / file_names[f]).to_pandas(self_destruct=True) for f in file_names.keys() } - data = pool_datasets(datasets=data, samples=samples, vars=vars, shuffle=True, stratify=None) - save_pooled_data(data_dir, data, datasets) + data = pool_datasets(datasets=data, samples=samples, vars=vars, shuffle=True, stratify=None, runmode=runmode) + save_pooled_data(data_dir, data, datasets,file_names, samples=samples) -def save_pooled_data(data_dir, data, datasets): +def save_pooled_data(data_dir, data, datasets, file_names, samples=10000): save_folder = "_".join(datasets) + save_folder += f"_{samples}" save_dir = data_dir / save_folder if not save_dir.exists(): save_dir.mkdir() + # filenames = ["sta", "dyn", "outc"] for key, value in data.items(): - value.to_parquet(save_dir / Path(key + ".parquet")) + value.to_parquet(save_dir / Path(file_names[key])) logging.info(f"Saved pooled data at {save_dir}") -def pool_datasets(datasets={}, samples=10000, vars=[], seed=42, shuffle=True, stratify=None, **kwargs): +def pool_datasets(datasets={}, samples=10000, vars=[], seed=42, shuffle=True, runmode = RunMode.classification, stratify=None, **kwargs): """ Pool datasets into a single dataset. Args: @@ -54,14 +57,50 @@ def pool_datasets(datasets={}, samples=10000, vars=[], seed=42, shuffle=True, st repeated_digit = str(int_id) * 4 # outcome = value[Segment.outcome].groupby(vars["LABEL"], group_keys=False).sample(samples_per_class,random_state=seed) outcome = value[Segment.outcome] - outcome = train_test_split( - outcome, stratify=outcome[vars[Var.label]], shuffle=shuffle, random_state=seed, train_size=samples - )[0] - stays = pd.Series(outcome[id].unique()) static = value[Segment.static] dynamic = value[Segment.dynamic] - static = static.loc[static[id].isin(stays)] - dynamic = dynamic.loc[dynamic[id].isin(stays)] + # Get unique stay IDs from outcome segment + stays = pd.Series(outcome[id].unique()) + + # if(len(stays) is len(outcome[id]): + if runmode is RunMode.classification: + # If we have more outcomes than stays, check max label value per stay id + labels = outcome.groupby(id).max()[vars[Var.label]].reset_index(drop=True) + # if pd.Series(outcome[id].unique()) is outcome[id]): + selected_stays = train_test_split(stays, stratify=labels, shuffle=shuffle, random_state=seed, train_size=samples) + else: + selected_stays = train_test_split(stays, shuffle=shuffle, random_state=seed, train_size=samples) + # Select only stays that are in the selected_stays + save_test = True + # Save test sets to test on without leakage + if save_test: + data_dir = Path(r'C:\Users\Robin\Documents\Git\YAIB\data\YAIB_Datasets\data\mortality24') + select = selected_stays[1] + # if(runmode is RunMode.classification): + # select=train_test_split(stays, stratify=labels, shuffle=shuffle, random_state=seed, train_size=samples)[0] + # else: + # select = train_test_split(select, shuffle=shuffle, random_state=seed, train_size=samples)[0] + outcome = outcome.loc[outcome[id].isin(select)] + static = static.loc[static[id].isin(select)] + dynamic = dynamic.loc[dynamic[id].isin(select)] + # Preventing id clashing + outcome[id] = outcome[id].map(lambda x: int(str(x) + repeated_digit)) + static[id] = static[id].map(lambda x: int(str(x) + repeated_digit)) + dynamic[id] = dynamic[id].map(lambda x: int(str(x) + repeated_digit)) + save_folder = key + save_folder += f"_{len(select)}" + save_dir = data_dir / save_folder + if not save_dir.exists(): + save_dir.mkdir() + # filenames = ["sta", "dyn", "outc"] + outcome.to_parquet(save_dir / Path("outc.parquet")) + static.to_parquet(save_dir / Path("sta.parquet")) + dynamic.to_parquet(save_dir / Path("dyn.parquet")) + logging.info(f"Saved train data at {save_dir}") + selected_stays = selected_stays[0] + outcome = outcome.loc[outcome[id].isin(selected_stays)] + static = static.loc[static[id].isin(selected_stays)] + dynamic = dynamic.loc[dynamic[id].isin(selected_stays)] # Preventing id clashing outcome[id] = outcome[id].map(lambda x: int(str(x) + repeated_digit)) static[id] = static[id].map(lambda x: int(str(x) + repeated_digit)) diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index fc150669..4ae16b5a 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -13,8 +13,7 @@ from icu_benchmarks.data.preprocessor import Preprocessor, DefaultClassificationPreprocessor from icu_benchmarks.contants import RunMode from .constants import DataSplit as Split, DataSegment as Segment, VarType as Var - - +from .pooling import PooledDataset, generate_pooled_data @gin.configurable("preprocess") def preprocess_data( data_dir: Path, @@ -77,7 +76,9 @@ def preprocess_data( hash_config = hashlib.md5(f"{preprocessor.to_cache_string()}{dumped_file_names}{dumped_vars}".encode("utf-8")) cache_filename += f"_{hash_config.hexdigest()}" cache_file = cache_dir / cache_filename - + # PooledDataset.aumc_eicu_miiv, PooledDataset.hirid_eicu_miiv, PooledDataset.aumc_hirid_eicu, PooledDataset.aumc_hirid_miiv, + # for item in [PooledDataset.aumc_hirid_eicu_miiv]: + # generate_pooled_data(Path(r'C:\Users\Robin\Documents\Git\YAIB\data\YAIB_Datasets\data\mortality24'), vars=vars, datasets=item, file_names=file_names, seed=seed, runmode=runmode) if load_cache: if cache_file.exists(): with open(cache_file, "rb") as f: @@ -105,7 +106,7 @@ def preprocess_data( runmode=runmode, ) else: - data = make_train_val(data, vars, train_size=0.8, seed=seed, debug=debug) + data = make_train_val(data, vars, train_size=0.8, seed=seed, debug=debug, runmode=runmode) # Apply preprocessing data = preprocessor.apply(data, vars) @@ -126,6 +127,7 @@ def make_train_val( train_size=0.8, seed: int = 42, debug: bool = False, + runmode: RunMode = RunMode.classification, ) -> dict[dict[pd.DataFrame]]: """Randomly split the data into training and validation sets for fitting a full model. @@ -149,7 +151,7 @@ def make_train_val( stays = stays.sample(frac=0.01, random_state=seed) # If there are labels, and the task is classification, use stratified k-fold - if Var.label in vars: + if Var.label in vars and runmode is RunMode.classification: # Get labels from outcome data (takes the highest value (or True) in case seq2seq classification) labels = data[Segment.outcome].groupby(id).max()[vars[Var.label]].reset_index(drop=True) if train_size: @@ -157,7 +159,9 @@ def make_train_val( train, val = list(train_val.split(stays, labels))[0] else: # If there are no labels, use random split - train, val = train_test_split(stays, train_size=train_size, random_state=seed) + train_val = ShuffleSplit(train_size=train_size, random_state=seed) + train, val = list(train_val.split(stays))[0] + split = { Split.train: stays.iloc[train], From aa85fd6b1ace58f412567f221b0bb8c81ec76b66 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 30 Aug 2023 15:11:50 +0200 Subject: [PATCH 090/207] cleanup --- configs/tasks/Regression.gin | 2 +- experiments/experiment_eval_pooled.yml | 69 ++++++++++------------- experiments/experiment_full_training.yml | 44 ++++++++------- experiments/experiment_pooled.yml | 57 ++++++++++--------- icu_benchmarks/data/split_process_data.py | 6 +- 5 files changed, 89 insertions(+), 89 deletions(-) diff --git a/configs/tasks/Regression.gin b/configs/tasks/Regression.gin index 40c788e0..5cf3f8d9 100644 --- a/configs/tasks/Regression.gin +++ b/configs/tasks/Regression.gin @@ -25,7 +25,7 @@ preprocess.use_static = True # SPECIFYING REGRESSION OUTCOME SCALING base_regression_preprocessor.outcome_min = 0 -base_regression_preprocessor.outcome_max = 168 +base_regression_preprocessor.outcome_max = 15 # SELECTING DATASET PredictionDataset.vars = %vars diff --git a/experiments/experiment_eval_pooled.yml b/experiments/experiment_eval_pooled.yml index 44abb0cb..5ecff272 100644 --- a/experiments/experiment_eval_pooled.yml +++ b/experiments/experiment_eval_pooled.yml @@ -1,3 +1,4 @@ +# This experiment evaluates the pooled models trained in a previous experiment. command: - ${env} - ${program} @@ -5,55 +6,45 @@ command: - -d - ../data/ - -t -# - BinaryClassification - - Regression + - BinaryClassification +# - Regression - --log-dir - - ../yaib_logs_pooled -# - --tune + - ../yaib_logs - --wandb-sweep -# - -gc -# - -lc -# - -sn -# - eicu_hirid_aumc -# - eicu_hirid_miiv + - -gc + - -lc - --source-dir -# - /dhc/home/robin.vandewater/projects/yaib_logs_pooled_los/aumc_eicu_miiv_10000/Regression/Transformer/2023-08-28T11-06-03/repetition_0/fold_0 -# - /dhc/home/robin.vandewater/projects/yaib_logs_pooled_los/aumc_hirid_miiv_10000/Regression/Transformer/2023-08-28T11-06-05/repetition_0/fold_0 -# - /dhc/home/robin.vandewater/projects/yaib_logs_pooled_los/aumc_hirid_eicu_10000/Regression/Transformer/2023-08-28T11-03-18/repetition_0/fold_0 -# - /dhc/home/robin.vandewater/projects/yaib_logs_pooled_training/aumc_hirid_eicu_miiv/BinaryClassification/GRU/2023-08-27T12-10-12/repetition_0/fold_0 - - /dhc/home/robin.vandewater/projects/yaib_logs_pooled_los/hirid_eicu_miiv_10000/Regression/Transformer/2023-08-28T11-06-03/repetition_0/fold_0 - #- /dhc/home/robin.vandewater/data/pooled_mortality24_10000/aumc_hirid_miiv/BinaryClassification/GRU/2023-08-26T11-41-24/repetition_0/fold_0 -# - /dhc/home/robin.vandewater/data/pooled_mortality24_10000/eicu_aumc_miiv/BinaryClassification/GRU/2023-08-26T11-40-36/repetition_0/fold_0 -# - /dhc/home/robin.vandewater/data/pooled_mortality24_10000/eicu_hirid_miiv/BinaryClassification/GRU/2023-08-26T11-41-25/repetition_0/fold_0 -# - /dhc/home/robin.vandewater/projects/yaib_logs_pooled_training/eicu_hirid_aumc/BinaryClassification/GRU/2023-08-25T22-24-02/repetition_0/fold_0 + - path/to/pooled_model method: grid name: yaib_pooled_eval parameters: data_dir: values: -# - ../../data/mortality24/miiv_42045 -# - ../../data/mortality24/hirid_2859 -# - ../../data/mortality24/eicu_103382 -# - ../../data/mortality24/aumc_535 -# - ../../data/mortality24/miiv -# - ../../data/mortality24/hirid -# - ../../data/mortality24/eicu -# - ../../data/mortality24/aumc -# - ../../data/kf/miiv -# - ../../data/kf/hirid -# - ../../data/kf/eicu -# - ../../data/kf/aumc -# - ../../data/los/miiv -# - ../../data/los/hirid -# - ../../data/los/eicu - - ../../data/los/aumc + - ../data/mortality24/miiv + - ../data/mortality24/hirid + - ../data/mortality24/eicu + - ../data/mortality24/aumc + - ../data/aki/miiv + - ../data/aki/hirid + - ../data/aki/eicu + - ../data/aki/aumc + - ../data/sepsis/miiv + - ../data/sepsis/hirid + - ../data/sepsis/eicu + - ../data/sepsis/aumc + - ../data/los/miiv + - ../data/los/hirid + - ../data/los/eicu + - ../data/los/aumc + - ../data/kf/miiv + - ../data/kf/hirid + - ../data/kf/eicu + - ../data/kf/aumc model: values: -# - ElasticNet -# - LGBMRegressor -# - GRU -# - LSTM -# - TCN + - GRU + - LSTM + - TCN - Transformer seed: values: diff --git a/experiments/experiment_full_training.yml b/experiments/experiment_full_training.yml index f29e0226..b43efda9 100644 --- a/experiments/experiment_full_training.yml +++ b/experiments/experiment_full_training.yml @@ -1,4 +1,4 @@ -# This experiment trains models with progressively more samples to see how the performance changes +# This experiment trains a production-ready model with a full dataset (no cross-validation). command: - ${env} - ${program} @@ -6,35 +6,41 @@ command: - -d - ../data/ - -t -# - BinaryClassification - - Regression + - BinaryClassification +# - Regression - --log-dir -# - ../yaib_logs_pooled_training - - ../yaib_logs_pooled_los + - ../yaib_logs - --tune - --wandb-sweep - --tune - - --hp-checkpoint - - test method: grid name: yaib_full_benchmark parameters: data_dir: values: -# - ../../data/pooled_mortality24_10000/eicu_aumc_miiv -# - ../../data/pooled_mortality24_10000/aumc_hirid_miiv -# - ../../data/pooled_mortality24_10000/eicu_hirid_miiv -# - ../../data/pooled_mortality24_10000/eicu_hirid_aumc -# - ../../data/pooled_mortality24_10000/aumc_hirid_eicu_miiv - - ../../data/los/aumc_hirid_eicu_10000 - - ../../data/los/hirid_eicu_miiv_10000 - - ../../data/los/aumc_eicu_miiv_10000 - - ../../data/los/aumc_hirid_miiv_10000 - - ../../data/los/aumc_hirid_eicu_miiv_10000 + - ../data/mortality24/miiv + - ../data/mortality24/hirid + - ../data/mortality24/eicu + - ../data/mortality24/aumc + - ../data/aki/miiv + - ../data/aki/hirid + - ../data/aki/eicu + - ../data/aki/aumc + - ../data/sepsis/miiv + - ../data/sepsis/hirid + - ../data/sepsis/eicu + - ../data/sepsis/aumc + - ../data/los/miiv + - ../data/los/hirid + - ../data/los/eicu + - ../data/los/aumc + - ../data/kf/miiv + - ../data/kf/hirid + - ../data/kf/eicu + - ../data/kf/aumc model: values: - #- GRU - - Transformer + - GRU seed: values: - 1111 diff --git a/experiments/experiment_pooled.yml b/experiments/experiment_pooled.yml index 616e9766..8cec29c9 100644 --- a/experiments/experiment_pooled.yml +++ b/experiments/experiment_pooled.yml @@ -1,17 +1,14 @@ -# This experiment trains models with progressively more samples to see how the performance changes +# This experiment trains pooled models with pooled data. Note that you have to replace BinaryClassification with Regression for the appropriate dataset. command: - ${env} - ${program} -# - --samples -# - 0 - -d - ../data/ - -t -# - BinaryClassification - - Regression + - BinaryClassification + # - Regression - --log-dir -# - ../../data/pooled_mortality24_10000 - - ../yaib_logs_pooled_los + - ../yaib_logs_pooled - --tune - --wandb-sweep - --hp-checkpoint @@ -21,29 +18,37 @@ command: method: grid name: yaib_pooled_benchmark parameters: -# samples: -# values: -# - 30000 data_dir: values: -# - ../../data/kf/pooled_datasets_6000/hirid_eicu_miiv -# - ../../data/kf/pooled_datasets_6000/aumc_hirid_eicu -# - ../../data/kf/pooled_datasets_6000/aumc_eicu_miiv -# - ../../data/kf/pooled_datasets_6000/aumc_hirid_miiv -# - ../../data/kf/pooled_datasets_6000/aumc_hirid_eicu_miiv - - ../../data/los/aumc_hirid_eicu_10000 - - ../../data/los/hirid_eicu_miiv_10000 - - ../../data/los/aumc_eicu_miiv_10000 - - ../../data/los/aumc_hirid_miiv_10000 - - ../../data/los/aumc_hirid_eicu_miiv_10000 -# - ../../data/pooled_mortality24_10000/eicu_aumc_miiv -# - ../../data/pooled_mortality24_10000/aumc_hirid_miiv -# - ../../data/pooled_mortality24_10000/eicu_hirid_miiv -# - ../../data/pooled_mortality24_10000/eicu_hirid_aumc -# - ../../data/pooled_mortality24_10000/aumc_hirid_eicu_miiv + - ../data/mortality24/aumc_hirid_eicu_10000 + - ../data/mortality24/hirid_eicu_miiv_10000 + - ../data/mortality24/aumc_eicu_miiv_10000 + - ../data/mortality24/aumc_hirid_miiv_10000 + - ../data/mortality24/aumc_hirid_eicu_miiv_10000 + - ../data/aki/aumc_hirid_eicu_10000 + - ../data/aki/hirid_eicu_miiv_10000 + - ../data/aki/aumc_eicu_miiv_10000 + - ../data/aki/aumc_hirid_miiv_10000 + - ../data/aki/aumc_hirid_eicu_miiv_10000 + - ../data/sepsis/aumc_hirid_eicu_10000 + - ../data/sepsis/hirid_eicu_miiv_10000 + - ../data/sepsis/aumc_eicu_miiv_10000 + - ../data/sepsis/aumc_hirid_miiv_10000 + - ../data/sepsis/aumc_hirid_eicu_miiv_10000 + - ../data/kf/aumc_hirid_eicu_10000 + - ../data/kf/hirid_eicu_miiv_10000 + - ../data/kf/aumc_eicu_miiv_10000 + - ../data/kf/aumc_hirid_miiv_10000 + - ../data/kf/aumc_hirid_eicu_miiv_10000 + - ../data/los/aumc_hirid_eicu_10000 + - ../data/los/hirid_eicu_miiv_10000 + - ../data/los/aumc_eicu_miiv_10000 + - ../data/los/aumc_hirid_miiv_10000 + - ../data/los/aumc_hirid_eicu_miiv_10000 + model: values: -# - GRU + # - GRU - Transformer seed: values: diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index 4ae16b5a..aa4d89e0 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -13,7 +13,7 @@ from icu_benchmarks.data.preprocessor import Preprocessor, DefaultClassificationPreprocessor from icu_benchmarks.contants import RunMode from .constants import DataSplit as Split, DataSegment as Segment, VarType as Var -from .pooling import PooledDataset, generate_pooled_data + @gin.configurable("preprocess") def preprocess_data( data_dir: Path, @@ -76,9 +76,7 @@ def preprocess_data( hash_config = hashlib.md5(f"{preprocessor.to_cache_string()}{dumped_file_names}{dumped_vars}".encode("utf-8")) cache_filename += f"_{hash_config.hexdigest()}" cache_file = cache_dir / cache_filename - # PooledDataset.aumc_eicu_miiv, PooledDataset.hirid_eicu_miiv, PooledDataset.aumc_hirid_eicu, PooledDataset.aumc_hirid_miiv, - # for item in [PooledDataset.aumc_hirid_eicu_miiv]: - # generate_pooled_data(Path(r'C:\Users\Robin\Documents\Git\YAIB\data\YAIB_Datasets\data\mortality24'), vars=vars, datasets=item, file_names=file_names, seed=seed, runmode=runmode) + if load_cache: if cache_file.exists(): with open(cache_file, "rb") as f: From 47166fb9afb610b9e21bb2726164d568c489e61a Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 30 Aug 2023 15:33:49 +0200 Subject: [PATCH 091/207] Made pooling into class object --- icu_benchmarks/data/pooling.py | 266 ++++++++++++++++++++++----------- 1 file changed, 175 insertions(+), 91 deletions(-) diff --git a/icu_benchmarks/data/pooling.py b/icu_benchmarks/data/pooling.py index 18454e3d..23cb370d 100644 --- a/icu_benchmarks/data/pooling.py +++ b/icu_benchmarks/data/pooling.py @@ -9,107 +9,191 @@ class PooledDataset: hirid_eicu_miiv = ["hirid", "eicu", "miiv"] - aumc_hirid_eicu = ["aumc", "hirid","eicu"] + aumc_hirid_eicu = ["aumc", "hirid", "eicu"] aumc_eicu_miiv = ["aumc", "eicu", "miiv"] aumc_hirid_miiv = ["aumc", "hirid", "miiv"] - aumc_hirid_eicu_miiv = ["aumc","hirid", "eicu", "miiv"] + aumc_hirid_eicu_miiv = ["aumc", "hirid", "eicu", "miiv"] -def generate_pooled_data(data_dir, vars, datasets, file_names, samples=10000, seed=42, shuffle=False, stratify=None, runmode=RunMode.classification): - data = {} - for folder in data_dir.iterdir(): - if folder.is_dir(): - if folder.name in datasets: - data[folder.name] = { - f: pq.read_table(folder / file_names[f]).to_pandas(self_destruct=True) for f in file_names.keys() - } - data = pool_datasets(datasets=data, samples=samples, vars=vars, shuffle=True, stratify=None, runmode=runmode) - save_pooled_data(data_dir, data, datasets,file_names, samples=samples) +class PooledData: + def __init__(self, + data_dir, + vars, + datasets, + file_names, + shuffle=False, + stratify=None, + runmode=RunMode.classification, + save_test=True, + ): + """ + Generate pooled data from existing datasets. + Args: + data_dir: Where to read the data from + vars: Variables dictionary + datasets: Which datasets to pool + file_names: Which files to read from + samples: Amount of samples to pool + seed: Random seed + shuffle: Whether to shuffle data + stratify: Stratify data + runmode: Which task runmode + save_test: Save left over test data to test on without leakage + """ + self.data_dir = data_dir + self.vars = vars + self.datasets = datasets + self.file_names = file_names + self.shuffle = shuffle + self.stratify = stratify + self.runmode = runmode + self.save_test = save_test + def generate( + self, + datasets, + samples=10000, + seed=42, + ): + """ + Generate pooled data from existing datasets. + Args: + datasets: Which datasets to pool + samples: Amount of samples to pool + seed: Random seed + """ + data = {} + for folder in self.data_dir.iterdir(): + if folder.is_dir(): + if folder.name in datasets: + data[folder.name] = { + f: pq.read_table(folder / self.file_names[f]).to_pandas(self_destruct=True) for f in + self.file_names.keys() + } + data = self._pool_datasets( + datasets=data, + samples=samples, + vars=vars, + shuffle=self.shuffle, + stratify=self.stratify, + seed=seed, + runmode=self.runmode, + data_dir=self.data_dir, + save_test=self.save_test, + ) + self._save_pooled_data(self.data_dir, data, datasets, self.file_names, samples=samples) -def save_pooled_data(data_dir, data, datasets, file_names, samples=10000): - save_folder = "_".join(datasets) - save_folder += f"_{samples}" - save_dir = data_dir / save_folder - if not save_dir.exists(): - save_dir.mkdir() - # filenames = ["sta", "dyn", "outc"] - for key, value in data.items(): - value.to_parquet(save_dir / Path(file_names[key])) - logging.info(f"Saved pooled data at {save_dir}") + def _save_pooled_data(self, data_dir, data, datasets, file_names, samples=10000): + """ + Save pooled data to disk. + Args: + data_dir: Directory to save the data + data: Data to save + datasets: Which datasets were pooled + file_names: The file names to save to + samples: + """ + save_folder = "_".join(datasets) + save_folder += f"_{samples}" + save_dir = data_dir / save_folder + if not save_dir.exists(): + save_dir.mkdir() + for key, value in data.items(): + value.to_parquet(save_dir / Path(file_names[key])) + logging.info(f"Saved pooled data at {save_dir}") + def _pool_datasets( + self, + datasets={}, + samples=10000, + vars=[], + seed=42, + shuffle=True, + runmode=RunMode.classification, + data_dir=Path("data"), + save_test=True, + ): + """ + Pool datasets into a single dataset. + Args: + datasets: list of datasets to pool + samples: Amount of samples + vars: The variables dictionary + seed: Random seed + shuffle: Shuffle samples + runmode: Runmode + data_dir: Where to save the data + save_test: If true, save test data to test on without leakage + Returns: + pooled dataset + """ + if len(datasets) == 0: + raise ValueError("No datasets supplied.") + pooled_data = {Segment.static: [], Segment.dynamic: [], Segment.outcome: []} + id = vars[Var.group] + int_id = 0 + for key, value in datasets.items(): + int_id += 1 + # Preventing id clashing + repeated_digit = str(int_id) * 4 + outcome = value[Segment.outcome] + static = value[Segment.static] + dynamic = value[Segment.dynamic] + # Get unique stay IDs from outcome segment + stays = pd.Series(outcome[id].unique()) -def pool_datasets(datasets={}, samples=10000, vars=[], seed=42, shuffle=True, runmode = RunMode.classification, stratify=None, **kwargs): - """ - Pool datasets into a single dataset. - Args: - datasets: list of datasets to pool - Returns: - pooled dataset - """ - if len(datasets) == 0: - raise ValueError("No datasets supplied.") - pooled_data = {Segment.static: [], Segment.dynamic: [], Segment.outcome: []} - id = vars[Var.group] - int_id = 0 - for key, value in datasets.items(): - int_id += 1 - repeated_digit = str(int_id) * 4 - # outcome = value[Segment.outcome].groupby(vars["LABEL"], group_keys=False).sample(samples_per_class,random_state=seed) - outcome = value[Segment.outcome] - static = value[Segment.static] - dynamic = value[Segment.dynamic] - # Get unique stay IDs from outcome segment - stays = pd.Series(outcome[id].unique()) + if runmode is RunMode.classification: + # If we have more outcomes than stays, check max label value per stay id + labels = outcome.groupby(id).max()[vars[Var.label]].reset_index(drop=True) + # if pd.Series(outcome[id].unique()) is outcome[id]): + selected_stays = train_test_split(stays, stratify=labels, shuffle=shuffle, random_state=seed, + train_size=samples) + else: + selected_stays = train_test_split(stays, shuffle=shuffle, random_state=seed, train_size=samples) + # Select only stays that are in the selected_stays + # Save test sets to test on without leakage + if save_test: + select = selected_stays[1] + outcome, static, dynamic = self._select_stays( + outcome=outcome, static=static, dynamic=dynamic, select=select, repeated_digit=repeated_digit + ) + save_folder = key + save_folder += f"_test_{len(select)}" + save_dir = data_dir / save_folder + if not save_dir.exists(): + save_dir.mkdir() + outcome.to_parquet(save_dir / Path("outc.parquet")) + static.to_parquet(save_dir / Path("sta.parquet")) + dynamic.to_parquet(save_dir / Path("dyn.parquet")) + logging.info(f"Saved train data at {save_dir}") + selected_stays = selected_stays[0] + outcome, static, dynamic = self._select_stays( + outcome=outcome, static=static, dynamic=dynamic, select=selected_stays, repeated_digit=repeated_digit + ) + # Adding to pooled data + pooled_data[Segment.static].append(static) + pooled_data[Segment.dynamic].append(dynamic) + pooled_data[Segment.outcome].append(outcome) + # Add each datatype together + for key, value in pooled_data.items(): + pooled_data[key] = pd.concat(value, ignore_index=True) + return pooled_data - # if(len(stays) is len(outcome[id]): - if runmode is RunMode.classification: - # If we have more outcomes than stays, check max label value per stay id - labels = outcome.groupby(id).max()[vars[Var.label]].reset_index(drop=True) - # if pd.Series(outcome[id].unique()) is outcome[id]): - selected_stays = train_test_split(stays, stratify=labels, shuffle=shuffle, random_state=seed, train_size=samples) - else: - selected_stays = train_test_split(stays, shuffle=shuffle, random_state=seed, train_size=samples) - # Select only stays that are in the selected_stays - save_test = True - # Save test sets to test on without leakage - if save_test: - data_dir = Path(r'C:\Users\Robin\Documents\Git\YAIB\data\YAIB_Datasets\data\mortality24') - select = selected_stays[1] - # if(runmode is RunMode.classification): - # select=train_test_split(stays, stratify=labels, shuffle=shuffle, random_state=seed, train_size=samples)[0] - # else: - # select = train_test_split(select, shuffle=shuffle, random_state=seed, train_size=samples)[0] - outcome = outcome.loc[outcome[id].isin(select)] - static = static.loc[static[id].isin(select)] - dynamic = dynamic.loc[dynamic[id].isin(select)] - # Preventing id clashing - outcome[id] = outcome[id].map(lambda x: int(str(x) + repeated_digit)) - static[id] = static[id].map(lambda x: int(str(x) + repeated_digit)) - dynamic[id] = dynamic[id].map(lambda x: int(str(x) + repeated_digit)) - save_folder = key - save_folder += f"_{len(select)}" - save_dir = data_dir / save_folder - if not save_dir.exists(): - save_dir.mkdir() - # filenames = ["sta", "dyn", "outc"] - outcome.to_parquet(save_dir / Path("outc.parquet")) - static.to_parquet(save_dir / Path("sta.parquet")) - dynamic.to_parquet(save_dir / Path("dyn.parquet")) - logging.info(f"Saved train data at {save_dir}") - selected_stays = selected_stays[0] - outcome = outcome.loc[outcome[id].isin(selected_stays)] - static = static.loc[static[id].isin(selected_stays)] - dynamic = dynamic.loc[dynamic[id].isin(selected_stays)] + def _select_stays(self, outcome, static, dynamic, select, repeated_digit=1): + """Selects stays for outcome, static, dynamic dataframes. + + Args: + outcome: Outcome dataframe + static: Static dataframe + dynamic: Dynamic dataframe + select: Stay IDs to select + repeated_digit: Digit to repeat for ID clashing + """ + outcome = outcome.loc[outcome[id].isin(select)] + static = static.loc[static[id].isin(select)] + dynamic = dynamic.loc[dynamic[id].isin(select)] # Preventing id clashing outcome[id] = outcome[id].map(lambda x: int(str(x) + repeated_digit)) static[id] = static[id].map(lambda x: int(str(x) + repeated_digit)) dynamic[id] = dynamic[id].map(lambda x: int(str(x) + repeated_digit)) - # Adding to pooled data - pooled_data[Segment.static].append(static) - pooled_data[Segment.dynamic].append(dynamic) - pooled_data[Segment.outcome].append(outcome) - # Add each datatype together - for key, value in pooled_data.items(): - pooled_data[key] = pd.concat(value, ignore_index=True) - return pooled_data + return outcome, static, dynamic From ee468ed58bc74017ebe2769a7443f8c6fa6c389a Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 31 Aug 2023 11:17:32 +0200 Subject: [PATCH 092/207] linting and debug mode bug-fix --- icu_benchmarks/cross_validation.py | 5 +- icu_benchmarks/data/split_process_data.py | 88 ++++++++++++----------- icu_benchmarks/run.py | 9 +-- icu_benchmarks/run_utils.py | 1 - icu_benchmarks/wandb_utils.py | 2 +- 5 files changed, 52 insertions(+), 53 deletions(-) diff --git a/icu_benchmarks/cross_validation.py b/icu_benchmarks/cross_validation.py index a69bd005..34eb5c84 100644 --- a/icu_benchmarks/cross_validation.py +++ b/icu_benchmarks/cross_validation.py @@ -83,6 +83,9 @@ def execute_repeated_cv( for repetition in range(cv_repetitions_to_train): for fold_index in range(cv_folds_to_train): + repetition_fold_dir = log_dir / f"repetition_{repetition}" / f"fold_{fold_index}" + repetition_fold_dir.mkdir(parents=True, exist_ok=True) + start_time = datetime.now() data = preprocess_data( data_dir, @@ -100,8 +103,6 @@ def execute_repeated_cv( full_train=full_train ) - repetition_fold_dir = log_dir / f"repetition_{repetition}" / f"fold_{fold_index}" - repetition_fold_dir.mkdir(parents=True, exist_ok=True) preprocess_time = datetime.now() - start_time start_time = datetime.now() agg_loss += train_common( diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index aa4d89e0..9b93ac74 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -8,35 +8,39 @@ from pathlib import Path import pickle -from sklearn.model_selection import StratifiedKFold, KFold, StratifiedShuffleSplit, ShuffleSplit, train_test_split +from sklearn.model_selection import StratifiedKFold, KFold, StratifiedShuffleSplit, ShuffleSplit from icu_benchmarks.data.preprocessor import Preprocessor, DefaultClassificationPreprocessor from icu_benchmarks.contants import RunMode from .constants import DataSplit as Split, DataSegment as Segment, VarType as Var + @gin.configurable("preprocess") def preprocess_data( - data_dir: Path, - file_names: dict[str] = gin.REQUIRED, - preprocessor: Preprocessor = DefaultClassificationPreprocessor, - use_static: bool = True, - vars: dict[str] = gin.REQUIRED, - seed: int = 42, - debug: bool = False, - cv_repetitions: int = 5, - repetition_index: int = 0, - cv_folds: int = 5, - train_size: int = None, - load_cache: bool = False, - generate_cache: bool = False, - fold_index: int = 0, - pretrained_imputation_model: str = None, - full_train: bool = False, - runmode: RunMode = RunMode.classification, + data_dir: Path, + file_names: dict[str] = gin.REQUIRED, + preprocessor: Preprocessor = DefaultClassificationPreprocessor, + use_static: bool = True, + vars: dict[str] = gin.REQUIRED, + seed: int = 42, + debug: bool = False, + cv_repetitions: int = 5, + repetition_index: int = 0, + cv_folds: int = 5, + train_size: int = None, + load_cache: bool = False, + generate_cache: bool = False, + fold_index: int = 0, + pretrained_imputation_model: str = None, + full_train: bool = False, + runmode: RunMode = RunMode.classification, ) -> dict[dict[pd.DataFrame]]: """Perform loading, splitting, imputing and normalising of task data. Args: + use_static: Whether to use static features (for DL models). + full_train: Whether to use all data for training/validation. + runmode: Run mode. Can be one of the values of RunMode preprocessor: Define the preprocessor. data_dir: Path to the directory holding the data. file_names: Contains the parquet file names in data_dir. @@ -104,6 +108,7 @@ def preprocess_data( runmode=runmode, ) else: + # If full train is set, we use all data for training/validation data = make_train_val(data, vars, train_size=0.8, seed=seed, debug=debug, runmode=runmode) # Apply preprocessing @@ -119,13 +124,14 @@ def preprocess_data( return data + def make_train_val( - data: dict[pd.DataFrame], - vars: dict[str], - train_size=0.8, - seed: int = 42, - debug: bool = False, - runmode: RunMode = RunMode.classification, + data: dict[pd.DataFrame], + vars: dict[str], + train_size=0.8, + seed: int = 42, + debug: bool = False, + runmode: RunMode = RunMode.classification, ) -> dict[dict[pd.DataFrame]]: """Randomly split the data into training and validation sets for fitting a full model. @@ -160,11 +166,7 @@ def make_train_val( train_val = ShuffleSplit(train_size=train_size, random_state=seed) train, val = list(train_val.split(stays))[0] - - split = { - Split.train: stays.iloc[train], - Split.val: stays.iloc[val] - } + split = {Split.train: stays.iloc[train], Split.val: stays.iloc[val]} data_split = {} @@ -180,16 +182,16 @@ def make_train_val( def make_single_split( - data: dict[pd.DataFrame], - vars: dict[str], - cv_repetitions: int, - repetition_index: int, - cv_folds: int, - fold_index: int, - train_size: int = None, - seed: int = 42, - debug: bool = False, - runmode: RunMode = RunMode.classification, + data: dict[pd.DataFrame], + vars: dict[str], + cv_repetitions: int, + repetition_index: int, + cv_folds: int, + fold_index: int, + train_size: int = None, + seed: int = 42, + debug: bool = False, + runmode: RunMode = RunMode.classification, ) -> dict[dict[pd.DataFrame]]: """Randomly split the data into training, validation, and test set. @@ -211,12 +213,14 @@ def make_single_split( # ID variable id = vars[Var.group] + if debug: + # Only use 1% of the data + logging.info("Using only 1% of the data for debugging. Note that this might lead to errors for small datasets.") + data[Segment.outcome] = data[Segment.outcome].sample(frac=0.01, random_state=seed) # Get stay IDs from outcome segment stays = pd.Series(data[Segment.outcome][id].unique(), name=id) - if debug: - # Only use 1% of the data - stays = stays.sample(frac=0.01, random_state=seed) + # If there are labels, and the task is classification, use stratified k-fold if Var.label in vars and runmode is RunMode.classification: diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index 6c0d0eb6..cf98bebd 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -1,13 +1,10 @@ # -*- coding: utf-8 -*- from datetime import datetime - import gin import logging import sys from pathlib import Path - import torch.cuda - from icu_benchmarks.wandb_utils import update_wandb_config, apply_wandb_sweep, set_wandb_experiment_name from icu_benchmarks.tuning.hyperparameters import choose_and_bind_hyperparameters from scripts.plotting.utils import plot_aggregated_results @@ -33,7 +30,6 @@ def get_mode(mode: gin.REQUIRED): def main(my_args=tuple(sys.argv[1:])): - args, _ = build_parser().parse_known_args(my_args) if args.wandb_sweep: args = apply_wandb_sweep(args) @@ -57,7 +53,6 @@ def main(my_args=tuple(sys.argv[1:])): mode = get_mode() # Set arguments for wandb sweep - # Set experiment name if name is None: name = data_dir.name @@ -123,7 +118,7 @@ def main(my_args=tuple(sys.argv[1:])): name_datasets(args.name, args.name, args.name) hp_checkpoint = log_dir / args.hp_checkpoint if args.hp_checkpoint else None model_path = ( - Path("configs") / ("imputation_models" if mode == RunMode.imputation else "prediction_models") / f"{model}.gin" + Path("configs") / ("imputation_models" if mode == RunMode.imputation else "prediction_models") / f"{model}.gin" ) gin_config_files = ( [Path(f"configs/experiments/{args.experiment}.gin")] @@ -173,7 +168,7 @@ def main(my_args=tuple(sys.argv[1:])): pretrained_imputation_model=pretrained_imputation_model, cpu=args.cpu, wandb=args.wandb_sweep, - full_train=args.full_train + full_train=args.full_train, ) log_full_line("FINISHED TRAINING", level=logging.INFO, char="=", num_newlines=3) diff --git a/icu_benchmarks/run_utils.py b/icu_benchmarks/run_utils.py index 8a2fb961..65e570d8 100644 --- a/icu_benchmarks/run_utils.py +++ b/icu_benchmarks/run_utils.py @@ -162,7 +162,6 @@ def name_datasets(train="default", val="default", test="default"): gin.bind_parameter("train_common.dataset_names", {"train": train, "val": val, "test": test}) - def log_full_line(msg: str, level: int = logging.INFO, char: str = "-", num_newlines: int = 0): """Logs a full line of a given character with a message centered. diff --git a/icu_benchmarks/wandb_utils.py b/icu_benchmarks/wandb_utils.py index 48a2035c..54089462 100644 --- a/icu_benchmarks/wandb_utils.py +++ b/icu_benchmarks/wandb_utils.py @@ -70,7 +70,7 @@ def set_wandb_experiment_name(args, mode): elif args.samples: run_name += f"_train_size_{args.samples}_samples" elif args.full_train: - run_name += f"_full_training" + run_name += "_full_training" if wandb_running(): wandb.config.update({"run-name": run_name}) From 59a889f5a24794dcb13f93c87c1b452f74c84a87 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 31 Aug 2023 11:22:18 +0200 Subject: [PATCH 093/207] linting --- icu_benchmarks/data/split_process_data.py | 2 -- icu_benchmarks/models/train.py | 1 - 2 files changed, 3 deletions(-) diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index 9b93ac74..131c92fd 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -220,8 +220,6 @@ def make_single_split( # Get stay IDs from outcome segment stays = pd.Series(data[Segment.outcome][id].unique(), name=id) - - # If there are labels, and the task is classification, use stratified k-fold if Var.label in vars and runmode is RunMode.classification: # Get labels from outcome data (takes the highest value (or True) in case seq2seq classification) diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 7b0e7cfb..db7aabda 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -115,7 +115,6 @@ def train_common( data_shape = next(iter(train_loader))[0].shape - if load_weights: model = load_model(model, source_dir, pl_model=pl_model) else: From 0907e84090e0fc5300a69abfb3cc4f99509f4aca Mon Sep 17 00:00:00 2001 From: rvandewater Date: Fri, 1 Sep 2023 09:32:02 +0200 Subject: [PATCH 094/207] Documentation --- experiments/experiment_eval_pooled.yml | 1 + experiments/experiment_finetuning.yml | 3 ++- experiments/experiment_pooled.yml | 3 ++- icu_benchmarks/cross_validation.py | 1 + icu_benchmarks/data/preprocessor.py | 2 ++ 5 files changed, 8 insertions(+), 2 deletions(-) diff --git a/experiments/experiment_eval_pooled.yml b/experiments/experiment_eval_pooled.yml index 5ecff272..e432e779 100644 --- a/experiments/experiment_eval_pooled.yml +++ b/experiments/experiment_eval_pooled.yml @@ -7,6 +7,7 @@ command: - ../data/ - -t - BinaryClassification +# Manually set for regression tasks # - Regression - --log-dir - ../yaib_logs diff --git a/experiments/experiment_finetuning.yml b/experiments/experiment_finetuning.yml index 4e31c23c..db3a5ceb 100644 --- a/experiments/experiment_finetuning.yml +++ b/experiments/experiment_finetuning.yml @@ -1,3 +1,4 @@ +# Finetuning setup for model command: - ${env} - ${program} @@ -16,7 +17,7 @@ command: - -sn - eicu - --source-dir - - /dhc/home/robin.vandewater/projects/transfer_learning/gru_mortality/eicu + - path/to/model/to/finetune method: grid name: yaib_finetuning_benchmark parameters: diff --git a/experiments/experiment_pooled.yml b/experiments/experiment_pooled.yml index 8cec29c9..17f3f64d 100644 --- a/experiments/experiment_pooled.yml +++ b/experiments/experiment_pooled.yml @@ -6,13 +6,14 @@ command: - ../data/ - -t - BinaryClassification + # Manually set for regression tasks # - Regression - --log-dir - ../yaib_logs_pooled - --tune - --wandb-sweep - --hp-checkpoint - - test + - path/to/checkpoint - -gc - -lc method: grid diff --git a/icu_benchmarks/cross_validation.py b/icu_benchmarks/cross_validation.py index 34eb5c84..f5a62594 100644 --- a/icu_benchmarks/cross_validation.py +++ b/icu_benchmarks/cross_validation.py @@ -43,6 +43,7 @@ def execute_repeated_cv( Args: + full_train: Use the full data for training instead of held out test splits. wandb: Use wandb for logging. data_dir: Path to the data directory. log_dir: Path to the log directory. diff --git a/icu_benchmarks/data/preprocessor.py b/icu_benchmarks/data/preprocessor.py index be004d8e..69b90f5b 100644 --- a/icu_benchmarks/data/preprocessor.py +++ b/icu_benchmarks/data/preprocessor.py @@ -57,6 +57,8 @@ def __init__( generate_features: Generate features for dynamic data. scaling: Scaling of dynamic and static data. use_static_features: Use static features. + save_cache: Save recipe cache from this path. + load_cache: Load recipe cache from this path. Returns: Preprocessed data. """ From 92409101ea9397554893df01bbca62536132d9bc Mon Sep 17 00:00:00 2001 From: rvandewater Date: Fri, 1 Sep 2023 09:36:17 +0200 Subject: [PATCH 095/207] Renamed full-train to complete-train --- icu_benchmarks/cross_validation.py | 10 +++++----- icu_benchmarks/data/split_process_data.py | 6 +++--- icu_benchmarks/run.py | 2 +- icu_benchmarks/run_utils.py | 2 +- icu_benchmarks/wandb_utils.py | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/icu_benchmarks/cross_validation.py b/icu_benchmarks/cross_validation.py index f5a62594..95e44e1a 100644 --- a/icu_benchmarks/cross_validation.py +++ b/icu_benchmarks/cross_validation.py @@ -37,13 +37,13 @@ def execute_repeated_cv( cpu: bool = False, verbose: bool = False, wandb: bool = False, - full_train: bool = False + complete_train: bool = False ) -> float: """Preprocesses data and trains a model for each fold. Args: - full_train: Use the full data for training instead of held out test splits. + complete_train: Use the full data for training instead of held out test splits. wandb: Use wandb for logging. data_dir: Path to the data directory. log_dir: Path to the log directory. @@ -74,7 +74,7 @@ def execute_repeated_cv( cv_folds_to_train = cv_folds agg_loss = 0 seed_everything(seed, reproducible) - if full_train: + if complete_train: logging.info("Will train full model without cross validation.") cv_repetitions_to_train = 1 cv_folds_to_train = 1 @@ -101,7 +101,7 @@ def execute_repeated_cv( fold_index=fold_index, pretrained_imputation_model=pretrained_imputation_model, runmode=mode, - full_train=full_train + complete_train=complete_train ) preprocess_time = datetime.now() - start_time @@ -118,7 +118,7 @@ def execute_repeated_cv( cpu=cpu, verbose=verbose, use_wandb=wandb, - train_only=full_train + train_only=complete_train ) train_time = datetime.now() - start_time diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index 131c92fd..ee14d90f 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -32,14 +32,14 @@ def preprocess_data( generate_cache: bool = False, fold_index: int = 0, pretrained_imputation_model: str = None, - full_train: bool = False, + complete_train: bool = False, runmode: RunMode = RunMode.classification, ) -> dict[dict[pd.DataFrame]]: """Perform loading, splitting, imputing and normalising of task data. Args: use_static: Whether to use static features (for DL models). - full_train: Whether to use all data for training/validation. + complete_train: Whether to use all data for training/validation. runmode: Run mode. Can be one of the values of RunMode preprocessor: Define the preprocessor. data_dir: Path to the directory holding the data. @@ -94,7 +94,7 @@ def preprocess_data( data = {f: pq.read_table(data_dir / file_names[f]).to_pandas(self_destruct=True) for f in file_names.keys()} # Generate the splits logging.info("Generating splits.") - if not full_train: + if not complete_train: data = make_single_split( data, vars, diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index cf98bebd..3d596ccb 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -168,7 +168,7 @@ def main(my_args=tuple(sys.argv[1:])): pretrained_imputation_model=pretrained_imputation_model, cpu=args.cpu, wandb=args.wandb_sweep, - full_train=args.full_train, + complete_train=args.complete_train, ) log_full_line("FINISHED TRAINING", level=logging.INFO, char="=", num_newlines=3) diff --git a/icu_benchmarks/run_utils.py b/icu_benchmarks/run_utils.py index 65e570d8..85b676ac 100644 --- a/icu_benchmarks/run_utils.py +++ b/icu_benchmarks/run_utils.py @@ -47,7 +47,7 @@ def build_parser() -> ArgumentParser: parser.add_argument("--tune", default=False, action=BOA, help="Find best hyperparameters.") parser.add_argument("--hp-checkpoint", type=Path, help="Use previous hyperparameter checkpoint.") parser.add_argument("--eval", default=False, action=BOA, help="Only evaluate model, skip training.") - parser.add_argument("--full-train", default=False, action=BOA, help="Only train model") + parser.add_argument("--complete-train", default=False, action=BOA, help="Use all data to train model, skip testing.") parser.add_argument("-ft", "--fine-tune", default=None, type=int, help="Finetune model with amount of train data.") parser.add_argument("-sn", "--source-name", type=Path, help="Name of the source dataset.") parser.add_argument("--source-dir", type=Path, help="Directory containing gin and model weights.") diff --git a/icu_benchmarks/wandb_utils.py b/icu_benchmarks/wandb_utils.py index 54089462..2736f39e 100644 --- a/icu_benchmarks/wandb_utils.py +++ b/icu_benchmarks/wandb_utils.py @@ -69,8 +69,8 @@ def set_wandb_experiment_name(args, mode): run_name += f"_source_{args.source_name}" elif args.samples: run_name += f"_train_size_{args.samples}_samples" - elif args.full_train: - run_name += "_full_training" + elif args.complete_train: + run_name += "_complete_training" if wandb_running(): wandb.config.update({"run-name": run_name}) From e1784f95c32db77bc73d21380010b1032b81d3b4 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Fri, 1 Sep 2023 09:40:02 +0200 Subject: [PATCH 096/207] removed left over documentation --- icu_benchmarks/data/pooling.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/icu_benchmarks/data/pooling.py b/icu_benchmarks/data/pooling.py index 23cb370d..67becebf 100644 --- a/icu_benchmarks/data/pooling.py +++ b/icu_benchmarks/data/pooling.py @@ -33,8 +33,6 @@ def __init__(self, vars: Variables dictionary datasets: Which datasets to pool file_names: Which files to read from - samples: Amount of samples to pool - seed: Random seed shuffle: Whether to shuffle data stratify: Stratify data runmode: Which task runmode From 428ba9a05830d9e2c6e5513c8e6b979f9ca9d0dd Mon Sep 17 00:00:00 2001 From: rvandewater Date: Fri, 1 Sep 2023 09:41:22 +0200 Subject: [PATCH 097/207] documentation --- icu_benchmarks/data/pooling.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icu_benchmarks/data/pooling.py b/icu_benchmarks/data/pooling.py index 67becebf..7eeb2949 100644 --- a/icu_benchmarks/data/pooling.py +++ b/icu_benchmarks/data/pooling.py @@ -89,7 +89,7 @@ def _save_pooled_data(self, data_dir, data, datasets, file_names, samples=10000) data: Data to save datasets: Which datasets were pooled file_names: The file names to save to - samples: + samples: Amount of samples to save """ save_folder = "_".join(datasets) save_folder += f"_{samples}" From 274ad49096ef9a650d5938a28ed011bf0e478e0d Mon Sep 17 00:00:00 2001 From: Robin van de Water Date: Thu, 7 Sep 2023 14:50:54 +0200 Subject: [PATCH 098/207] Hotfix: Update environment.yml New version of recipies added --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 3d1941d7..1124300f 100644 --- a/environment.yml +++ b/environment.yml @@ -29,7 +29,7 @@ dependencies: - einops=0.6.1 - hydra-core=1.3 - pip: - - recipies==0.1.2 + - recipies==0.1.3 # Fixed version because of NumPy incompatibility and stale development status. - scikit-optimize-fix==0.9.1 - hydra-submitit-launcher==1.2.0 From bdd20d73d30c12f359b69564185029ca9c235d0a Mon Sep 17 00:00:00 2001 From: prockenschaub Date: Tue, 30 Jan 2024 10:03:04 +0100 Subject: [PATCH 099/207] rename kf to kidney_function --- .gitignore | 3 ++- .../{kf => kidney_function}/eicu_demo/attrition.csv | 0 .../{kf => kidney_function}/eicu_demo/dyn.parquet | Bin .../{kf => kidney_function}/eicu_demo/outc.parquet | Bin .../{kf => kidney_function}/eicu_demo/sta.parquet | Bin .../mimic_demo/attrition.csv | 0 .../{kf => kidney_function}/mimic_demo/dyn.parquet | Bin .../{kf => kidney_function}/mimic_demo/outc.parquet | Bin .../{kf => kidney_function}/mimic_demo/sta.parquet | Bin experiments/benchmark_regression.yml | 8 ++++---- experiments/demo_benchmark_regression.yml | 4 ++-- experiments/experiment_eval_pooled.yml | 8 ++++---- experiments/experiment_full_training.yml | 8 ++++---- experiments/experiment_pooled.yml | 10 +++++----- 14 files changed, 21 insertions(+), 20 deletions(-) rename demo_data/{kf => kidney_function}/eicu_demo/attrition.csv (100%) rename demo_data/{kf => kidney_function}/eicu_demo/dyn.parquet (100%) rename demo_data/{kf => kidney_function}/eicu_demo/outc.parquet (100%) rename demo_data/{kf => kidney_function}/eicu_demo/sta.parquet (100%) rename demo_data/{kf => kidney_function}/mimic_demo/attrition.csv (100%) rename demo_data/{kf => kidney_function}/mimic_demo/dyn.parquet (100%) rename demo_data/{kf => kidney_function}/mimic_demo/outc.parquet (100%) rename demo_data/{kf => kidney_function}/mimic_demo/sta.parquet (100%) diff --git a/.gitignore b/.gitignore index 92bd0a44..a0ab243e 100644 --- a/.gitignore +++ b/.gitignore @@ -127,4 +127,5 @@ wandb/ .vscode/launch.json yaib_logs/ *.ckpt -*.csv \ No newline at end of file +*.csv +!demo_data/*/*/attrition.csv \ No newline at end of file diff --git a/demo_data/kf/eicu_demo/attrition.csv b/demo_data/kidney_function/eicu_demo/attrition.csv similarity index 100% rename from demo_data/kf/eicu_demo/attrition.csv rename to demo_data/kidney_function/eicu_demo/attrition.csv diff --git a/demo_data/kf/eicu_demo/dyn.parquet b/demo_data/kidney_function/eicu_demo/dyn.parquet similarity index 100% rename from demo_data/kf/eicu_demo/dyn.parquet rename to demo_data/kidney_function/eicu_demo/dyn.parquet diff --git a/demo_data/kf/eicu_demo/outc.parquet b/demo_data/kidney_function/eicu_demo/outc.parquet similarity index 100% rename from demo_data/kf/eicu_demo/outc.parquet rename to demo_data/kidney_function/eicu_demo/outc.parquet diff --git a/demo_data/kf/eicu_demo/sta.parquet b/demo_data/kidney_function/eicu_demo/sta.parquet similarity index 100% rename from demo_data/kf/eicu_demo/sta.parquet rename to demo_data/kidney_function/eicu_demo/sta.parquet diff --git a/demo_data/kf/mimic_demo/attrition.csv b/demo_data/kidney_function/mimic_demo/attrition.csv similarity index 100% rename from demo_data/kf/mimic_demo/attrition.csv rename to demo_data/kidney_function/mimic_demo/attrition.csv diff --git a/demo_data/kf/mimic_demo/dyn.parquet b/demo_data/kidney_function/mimic_demo/dyn.parquet similarity index 100% rename from demo_data/kf/mimic_demo/dyn.parquet rename to demo_data/kidney_function/mimic_demo/dyn.parquet diff --git a/demo_data/kf/mimic_demo/outc.parquet b/demo_data/kidney_function/mimic_demo/outc.parquet similarity index 100% rename from demo_data/kf/mimic_demo/outc.parquet rename to demo_data/kidney_function/mimic_demo/outc.parquet diff --git a/demo_data/kf/mimic_demo/sta.parquet b/demo_data/kidney_function/mimic_demo/sta.parquet similarity index 100% rename from demo_data/kf/mimic_demo/sta.parquet rename to demo_data/kidney_function/mimic_demo/sta.parquet diff --git a/experiments/benchmark_regression.yml b/experiments/benchmark_regression.yml index 8aa8d13e..1f9176f3 100644 --- a/experiments/benchmark_regression.yml +++ b/experiments/benchmark_regression.yml @@ -21,10 +21,10 @@ parameters: - ../data/los/hirid - ../data/los/eicu - ../data/los/aumc - - ../data/kf/miiv - - ../data/kf/hirid - - ../data/kf/eicu - - ../data/kf/aumc + - ../data/kidney_function/miiv + - ../data/kidney_function/hirid + - ../data/kidney_function/eicu + - ../data/kidney_function/aumc model: values: - ElasticNet diff --git a/experiments/demo_benchmark_regression.yml b/experiments/demo_benchmark_regression.yml index 3b678371..17d25d38 100644 --- a/experiments/demo_benchmark_regression.yml +++ b/experiments/demo_benchmark_regression.yml @@ -19,8 +19,8 @@ parameters: values: - demo_data/los/eicu_demo - demo_data/los/mimic_demo - - demo_data/kf/eicu_demo - - demo_data/kf/mimic_demo + - demo_data/kidney_function/eicu_demo + - demo_data/kidney_function/mimic_demo model: values: - ElasticNet diff --git a/experiments/experiment_eval_pooled.yml b/experiments/experiment_eval_pooled.yml index e432e779..a71da879 100644 --- a/experiments/experiment_eval_pooled.yml +++ b/experiments/experiment_eval_pooled.yml @@ -37,10 +37,10 @@ parameters: - ../data/los/hirid - ../data/los/eicu - ../data/los/aumc - - ../data/kf/miiv - - ../data/kf/hirid - - ../data/kf/eicu - - ../data/kf/aumc + - ../data/kidney_function/miiv + - ../data/kidney_function/hirid + - ../data/kidney_function/eicu + - ../data/kidney_function/aumc model: values: - GRU diff --git a/experiments/experiment_full_training.yml b/experiments/experiment_full_training.yml index b43efda9..309b128e 100644 --- a/experiments/experiment_full_training.yml +++ b/experiments/experiment_full_training.yml @@ -34,10 +34,10 @@ parameters: - ../data/los/hirid - ../data/los/eicu - ../data/los/aumc - - ../data/kf/miiv - - ../data/kf/hirid - - ../data/kf/eicu - - ../data/kf/aumc + - ../data/kidney_function/miiv + - ../data/kidney_function/hirid + - ../data/kidney_function/eicu + - ../data/kidney_function/aumc model: values: - GRU diff --git a/experiments/experiment_pooled.yml b/experiments/experiment_pooled.yml index 17f3f64d..f194dfba 100644 --- a/experiments/experiment_pooled.yml +++ b/experiments/experiment_pooled.yml @@ -36,11 +36,11 @@ parameters: - ../data/sepsis/aumc_eicu_miiv_10000 - ../data/sepsis/aumc_hirid_miiv_10000 - ../data/sepsis/aumc_hirid_eicu_miiv_10000 - - ../data/kf/aumc_hirid_eicu_10000 - - ../data/kf/hirid_eicu_miiv_10000 - - ../data/kf/aumc_eicu_miiv_10000 - - ../data/kf/aumc_hirid_miiv_10000 - - ../data/kf/aumc_hirid_eicu_miiv_10000 + - ../data/kidney_function/aumc_hirid_eicu_10000 + - ../data/kidney_function/hirid_eicu_miiv_10000 + - ../data/kidney_function/aumc_eicu_miiv_10000 + - ../data/kidney_function/aumc_hirid_miiv_10000 + - ../data/kidney_function/aumc_hirid_eicu_miiv_10000 - ../data/los/aumc_hirid_eicu_10000 - ../data/los/hirid_eicu_miiv_10000 - ../data/los/aumc_eicu_miiv_10000 From 281354196439f3c60c750d6cceed34fa3d52caf7 Mon Sep 17 00:00:00 2001 From: Robin van de Water Date: Thu, 11 Apr 2024 12:30:05 +0200 Subject: [PATCH 100/207] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 808a0764..b61af6a9 100644 --- a/README.md +++ b/README.md @@ -277,6 +277,8 @@ We appreciate contributions to the project. Please read the [contribution guidel request. # Acknowledgements +Robin van de Water is funded by the “Gemeinsamer Bundesausschuss (G-BA) Innovationsausschuss” in the framework of “CASSANDRA - Clinical ASSist AND aleRt Algorithms” this project has been developed under this funding. +(project number 01VSF20015). We would like to acknowledge the work of Alisher Turubayev, Anna Shopova, Fabian Lange, Mahmut Kamalak, Paul Mattes, and Victoria Ayvasky for adding Pytorch Lightning, Weights and Biases compatibility, and several optional imputation methods to a later version of the benchmark repository. We do not own any of the datasets used in this benchmark. This project uses heavily adapted components of the [HiRID benchmark](https://github.com/ratschlab/HIRID-ICU-Benchmark/). We thank the authors for providing this codebase and From d1c0dde69efe0d933295c486483d2135afc09d49 Mon Sep 17 00:00:00 2001 From: Robin van de Water Date: Thu, 11 Apr 2024 12:35:37 +0200 Subject: [PATCH 101/207] Update README.md --- README.md | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index b61af6a9..5700f243 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ We provide five common tasks for clinical prediction by default: | 5 | Length of Stay (LoS) | Hourly (within 7D) | Regression | New tasks can be easily added. -For the purposes of getting started right away, we include the eICU and MIMIC-III demo datasets in our repository. +To get started right away, we include the eICU and MIMIC-III demo datasets in our repository. The following repositories may be relevant as well: @@ -51,27 +51,25 @@ The following repositories may be relevant as well: - [YAIB-models](https://github.com/rvandewater/YAIB-models): Pretrained models for YAIB. - [ReciPys](https://github.com/rvandewater/ReciPys): Preprocessing package for YAIB pipelines. -For all YAIB related repositories, please see: https://github.com/stars/rvandewater/lists/yaib. +For all YAIB-related repositories, please see: https://github.com/stars/rvandewater/lists/yaib. # 📄Paper -To reproduce the benchmarks in our paper, we refer to: the [ML reproducibility document](PAPER.md). +To reproduce the benchmarks in our paper, we refer to the [ML reproducibility document](PAPER.md). If you use this code in your research, please cite the following publication: ``` -@article{vandewaterYetAnotherICUBenchmark2023, - title = {Yet Another ICU Benchmark: A Flexible Multi-Center Framework for Clinical ML}, - shorttitle = {Yet Another ICU Benchmark}, - url = {http://arxiv.org/abs/2306.05109}, - language = {en}, - urldate = {2023-06-09}, - publisher = {arXiv}, - author = {Robin van de Water and Hendrik Schmidt and Paul Elbers and Patrick Thoral and Bert Arnrich and Patrick Rockenschaub}, - month = jun, - year = {2023}, - note = {arXiv:2306.05109 [cs]}, - keywords = {Computer Science - Machine Learning}, +@inproceedings{vandewaterYetAnotherICUBenchmark2024, + title = {Yet Another ICU Benchmark: A Flexible Multi-Center Framework for Clinical ML}, + shorttitle = {Yet Another ICU Benchmark}, + booktitle = {The Twelfth International Conference on Learning Representations}, + author = {van de Water, Robin and Schmidt, Hendrik Nils Aurel and Elbers, Paul and Thoral, Patrick and Arnrich, Bert and Rockenschaub, Patrick}, + year = {2024}, + month = oct, + urldate = {2024-02-19}, + langid = {english}, } + ``` This paper can also be found on arxiv [2306.05109](https://arxiv.org/abs/2306.05109) @@ -277,7 +275,7 @@ We appreciate contributions to the project. Please read the [contribution guidel request. # Acknowledgements -Robin van de Water is funded by the “Gemeinsamer Bundesausschuss (G-BA) Innovationsausschuss” in the framework of “CASSANDRA - Clinical ASSist AND aleRt Algorithms” this project has been developed under this funding. +This project has been developed partially under the funding of “Gemeinsamer Bundesausschuss (G-BA) Innovationsausschuss” in the framework of “CASSANDRA - Clinical ASSist AND aleRt Algorithms”. (project number 01VSF20015). We would like to acknowledge the work of Alisher Turubayev, Anna Shopova, Fabian Lange, Mahmut Kamalak, Paul Mattes, and Victoria Ayvasky for adding Pytorch Lightning, Weights and Biases compatibility, and several optional imputation methods to a later version of the benchmark repository. We do not own any of the datasets used in this benchmark. This project uses heavily adapted components of From 8a9efb8e6b609f2fc2ef05d99daab6c741c05608 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 18 Jun 2024 12:11:50 +0200 Subject: [PATCH 102/207] Cass changes --- docs/adding_model/RNN.gin | 18 ++ docs/adding_model/instructions.md | 190 ++++++++++++++++++++++ docs/adding_model/rnn.py | 30 ++++ environment.yml | 3 +- experiments/benchmark_cass.yml | 39 +++++ experiments/charhpc_wandb_sweep.sh | 14 ++ experiments/slurm_base_char_sc.sh | 44 +++++ icu_benchmarks/data/loader.py | 1 + icu_benchmarks/data/preprocessor.py | 5 +- icu_benchmarks/data/split_process_data.py | 14 +- icu_benchmarks/models/ml_models.py | 2 +- 11 files changed, 352 insertions(+), 8 deletions(-) create mode 100644 docs/adding_model/RNN.gin create mode 100644 docs/adding_model/instructions.md create mode 100644 docs/adding_model/rnn.py create mode 100644 experiments/benchmark_cass.yml create mode 100644 experiments/charhpc_wandb_sweep.sh create mode 100644 experiments/slurm_base_char_sc.sh diff --git a/docs/adding_model/RNN.gin b/docs/adding_model/RNN.gin new file mode 100644 index 00000000..531aeff6 --- /dev/null +++ b/docs/adding_model/RNN.gin @@ -0,0 +1,18 @@ +# Settings for Recurrent Neural Network (RNN) models. + +# Common settings for DL models +include "configs/prediction_models/common/DLCommon.gin" + +# Train params +train_common.model = @RNNet + +# Optimizer params +optimizer/hyperparameter.class_to_tune = @Adam +optimizer/hyperparameter.weight_decay = 1e-6 +optimizer/hyperparameter.lr = (1e-5, 3e-4) + +# Encoder params +model/hyperparameter.class_to_tune = @RNNet +model/hyperparameter.num_classes = %NUM_CLASSES +model/hyperparameter.hidden_dim = (32, 256, "log-uniform", 2) +model/hyperparameter.layer_dim = (1, 3) diff --git a/docs/adding_model/instructions.md b/docs/adding_model/instructions.md new file mode 100644 index 00000000..0896fedd --- /dev/null +++ b/docs/adding_model/instructions.md @@ -0,0 +1,190 @@ +# Adding new models to YAIB +## Example +We refer to the page [adding a new model](https://github.com/rvandewater/YAIB/wiki/Adding-a-new-model) for detailed instructions on adding new models. +We allow prediction models to be easily added and integrated into a Pytorch Lightning module. This +incorporates advanced logging and debugging capabilities, as well as +built-in parallelism. Our interface derives from the [`BaseModule`](https://lightning.ai/docs/pytorch/stable/common/lightning_module.html). + +Adding a model consists of three steps: +1. Add a model through the existing `MLPredictionWrapper` or `DLPredictionWrapper`. +2. Add a GIN config file to bind hyperparameters. +3. Execute YAIB using a simple command. + +This folder contains everything you need to add a model to YAIB. +Putting the `RNN.gin` file in `configs/prediction_models` and the `rnn.py` file into icu_benchmarks/models allows you to run the model fully. + +``` +icu-benchmarks train \ + -d demo_data/mortality24/mimic_demo \ # Insert cohort dataset here + -n mimic_demo \ + -t BinaryClassification \ # Insert task name here + -tn Mortality24 \ + --log-dir ../yaib_logs/ \ + -m RNN \ # Insert model here + -s 2222 \ + -l ../yaib_logs/ \ + --tune +``` +# Adding more models +## Regular ML +For standard Scikit-Learn type models (e.g., LGBM), one can +simply wrap `MLPredictionWrapper` the function with minimal code +overhead. Many ML (and some DL) models can be incorporated this way, requiring minimal code additions. See below. + +``` {#code:ml-model-definition frame="single" style="pycharm" caption="\\textit{Example ML model definition}" label="code:ml-model-definition" columns="fullflexible"} +@gin.configurable +class RFClassifier(MLWrapper): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.model = self.model_args() + + @gin.configurable(module="RFClassifier") + def model_args(self, *args, **kwargs): + return RandomForestClassifier(*args, **kwargs) +``` +## Adding DL models +It is relatively straightforward to add new Pytorch models to YAIB. We first provide a standard RNN-model which needs no extra components. Then, we show the implementation of the Temporal Fusion Transformer model. + +### Standard RNN-model +The definition of dl models can be done by creating a subclass from the +`DLPredictionWrapper`, inherits the standard methods needed for +training dl learning models. Pytorch Lightning significantly reduces the code +overhead. + + +``` {#code:dl-model-definition frame="single" style="pycharm" caption="\\textit{Example DL model definition}" label="code:dl-model-definition" columns="fullflexible"} +@gin.configurable +class RNNet(DLPredictionWrapper): + """Torch standard RNN model""" + + def __init__(self, input_size, hidden_dim, layer_dim, num_classes, *args, **kwargs): + super().__init__( + input_size=input_size, hidden_dim=hidden_dim, layer_dim=layer_dim, num_classes=num_classes, *args, **kwargs + ) + self.hidden_dim = hidden_dim + self.layer_dim = layer_dim + self.rnn = nn.RNN(input_size[2], hidden_dim, layer_dim, batch_first=True) + self.logit = nn.Linear(hidden_dim, num_classes) + + def init_hidden(self, x): + h0 = x.new_zeros(self.layer_dim, x.size(0), self.hidden_dim) + return h0 + + def forward(self, x): + h0 = self.init_hidden(x) + out, hn = self.rnn(x, h0) + pred = self.logit(out) + return pred +``` +### Adding a SOTA model: Temporal Fusion Transformer +There are two main questions when you want to add a more complex model: + +* _Do you want to manually define the model or use an existing library?_ This might require adapting the `DLPredictionWrapper`. +* _Does the model expect the data to be in a certain format?_ This might require adapting the `PredictionDataset`. + +By adapting, we mean creating a new subclass that inherits most functionality to avoid code duplication, is future-proof, and follows good coding practices. + +First, you can add modules to `models/layers.py` to use them for your model. +``` {#code:building blocks frame="single" style="pycharm" caption="\\textit{Example building block}" label="code: layers" columns="fullflexible"} +class StaticCovariateEncoder(nn.Module): + """ + Network to produce 4 context vectors to enrich static variables + Variable selection Network --> GRNs + """ + + def __init__(self, num_static_vars, hidden, dropout): + super().__init__() + self.vsn = VariableSelectionNetwork(hidden, dropout, num_static_vars) + self.context_grns = nn.ModuleList([GRN(hidden, hidden, dropout=dropout) for _ in range(4)]) + + def forward(self, x: Tensor) -> Tuple[Tensor, Tensor, Tensor, Tensor]: + variable_ctx, sparse_weights = self.vsn(x) + + # Context vectors: + # variable selection context + # enrichment context + # state_c context + # state_h context + cs, ce, ch, cc = [m(variable_ctx) for m in self.context_grns] + + return cs, ce, ch, cc +``` +Note that we can create modules out of modules as well. + +### Adapting the `DLPredictionWrapper` +The next step is to use the building blocks defined in layers.py or modules from an existing library to add to the model in `models/dl_models.py`. In this In this case, we use the Pytorch-forecasting library (https://github.com/jdb78/pytorch-forecasting): + +``` {#code:dl-model-definition frame="single" style="pycharm" caption="\\textit{Example DL model definition}" label="code:dl-model-definition" columns="fullflexible"} +class TFTpytorch(DLPredictionWrapper): + + supported_run_modes = [RunMode.classification, RunMode.regression] + + def __init__(self, dataset, hidden, dropout, n_heads, dropout_att, lr, optimizer, num_classes, *args, **kwargs): + super().__init__(lr=lr, optimizer=optimizer, *args, **kwargs) + self.model = TemporalFusionTransformer.from_dataset( + dataset=dataset) + self.logit = nn.Linear(7, num_classes) + + + def forward(self, x): + out = self.model(x) + pred = self.logit(out["prediction"]) + return pred +``` + +### Adapting the `PredictionDataset` +Some models require an adjusted dataloader to facilitate, for example, explainability methods. In this case, changes need to be made to the `data/loader.py` file to ensure the data loader returns the data in the correct format. +This can be done by creating a class that inherits from PredictionDataset and editing the get_item method. +``` {#code:dataset frame="single" style="pycharm" caption="\\textit{Example custom dataset definition}" label="code: dataset" columns="fullflexible"} +@gin.configurable("PredictionDatasetTFT") +class PredictionDatasetTFT(PredictionDataset): + def __init__(self, *args, ram_cache: bool = True, **kwargs): + super().__init__(*args, ram_cache=True, **kwargs) + +def __getitem__(self, idx: int) -> Tuple[Tensor, Tensor, Tensor]: + """Function to sample from the data split of choice. Used for TFT. + The data needs to be given to the model in the following order + [static categorical, static continuous,known categorical,known continuous, observed categorical, observed continuous,target,id] +``` +Then, you must check `models/wrapper.py`, particularly the step_fn method, to ensure the data is correctly transferred to the device. + +## Adding the model config GIN file +To define hyperparameters for each model in a standardized manner, we use GIN-config. We need to specify a GIN file to bind the parameters to train and optimize this model from a choice of hyperparameters. Note that we can use modifiers for the optimizer (e.g, Adam optimizer) and ranges that we can specify in rounded brackets "()". Square brackets, "[]", result in a random choice where the variable is uniformly sampled. +``` +# Hyperparameters for TFT model. + +# Common settings for DL models +include "configs/prediction_models/common/DLCommon.gin" + +# Optimizer params +train_common.model = @TFT + +optimizer/hyperparameter.class_to_tune = @Adam +optimizer/hyperparameter.weight_decay = 1e-6 +optimizer/hyperparameter.lr = (1e-5, 3e-4) + +# Encoder params +model/hyperparameter.class_to_tune = @TFT +model/hyperparameter.encoder_length = 24 +model/hyperparameter.hidden = 256 +model/hyperparameter.num_classes = %NUM_CLASSES +model/hyperparameter.dropout = (0.0, 0.4) +model/hyperparameter.dropout_att = (0.0, 0.4) +model/hyperparameter.n_heads =4 +model/hyperparameter.example_length=25 +``` +## Training the model +After these steps, your model should be trainable with the following command: + +``` +icu-benchmarks train \ + -d demo_data/mortality24/mimic_demo \ # Insert cohort dataset here + -n mimic_demo \ + -t BinaryClassification \ # Insert task name here + -tn Mortality24 \ + --log-dir ../yaib_logs/ \ + -m TFT \ # Insert model here + -s 2222 \ + -l ../yaib_logs/ \ + --tune +``` diff --git a/docs/adding_model/rnn.py b/docs/adding_model/rnn.py new file mode 100644 index 00000000..5500e672 --- /dev/null +++ b/docs/adding_model/rnn.py @@ -0,0 +1,30 @@ +import gin +import torch.nn as nn +from icu_benchmarks.contants import RunMode +from icu_benchmarks.models.wrappers import DLPredictionWrapper + + +@gin.configurable +class RNNet(DLPredictionWrapper): + """Torch standard RNN model""" + + _supported_run_modes = [RunMode.classification, RunMode.regression] + + def __init__(self, input_size, hidden_dim, layer_dim, num_classes, *args, **kwargs): + super().__init__( + input_size=input_size, hidden_dim=hidden_dim, layer_dim=layer_dim, num_classes=num_classes, *args, **kwargs + ) + self.hidden_dim = hidden_dim + self.layer_dim = layer_dim + self.rnn = nn.RNN(input_size[2], hidden_dim, layer_dim, batch_first=True) + self.logit = nn.Linear(hidden_dim, num_classes) + + def init_hidden(self, x): + h0 = x.new_zeros(self.layer_dim, x.size(0), self.hidden_dim) + return h0 + + def forward(self, x): + h0 = self.init_hidden(x) + out, hn = self.rnn(x, h0) + pred = self.logit(out) + return pred diff --git a/environment.yml b/environment.yml index 1124300f..0c417e44 100644 --- a/environment.yml +++ b/environment.yml @@ -1,4 +1,4 @@ -name: yaib +name: yaib_benchmark channels: - pytorch - nvidia @@ -28,6 +28,7 @@ dependencies: - pip=23.1 - einops=0.6.1 - hydra-core=1.3 + - polars=0.20.31 - pip: - recipies==0.1.3 # Fixed version because of NumPy incompatibility and stale development status. diff --git a/experiments/benchmark_cass.yml b/experiments/benchmark_cass.yml new file mode 100644 index 00000000..e828ba3b --- /dev/null +++ b/experiments/benchmark_cass.yml @@ -0,0 +1,39 @@ +command: + - ${env} + - ${program} + - train + - -d + - ../data/ + - -t + - CassClassification + - --log-dir + - /sc-projects/sc-proj-cc08-cassandra/RW_Prospective/yaib_logs + - --tune + - --wandb-sweep + - -tn + - SSI + - -n + - cass +# - -gc +# - -lc +method: grid +name: yaib_classification_benchmark +parameters: + data_dir: + values: + - /sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/notes + model: + values: +# - LogisticRegression + - LGBMClassifier +# - GRU +# - LSTM +# - TCN +# - Transformer + seed: + values: + - 1111 + use_pretrained_imputation: + values: + - None +program: icu-benchmarks \ No newline at end of file diff --git a/experiments/charhpc_wandb_sweep.sh b/experiments/charhpc_wandb_sweep.sh new file mode 100644 index 00000000..8c29aa02 --- /dev/null +++ b/experiments/charhpc_wandb_sweep.sh @@ -0,0 +1,14 @@ +#!/bin/bash +#SBATCH --job-name=yaib_experiment +#SBATCH --partition=gpu,pgpu,agpu # -p +#SBATCH --cpus-per-task=8 # -c +#SBATCH --mem=100gb +#SBATCH --output=logs/classification_%a_%j.log # %j is job id +#SBATCH --gpus=1 +#SBATCH --time=12:00:00 + +eval "$(conda shell.bash hook)" +conda activate yaib_benchmark +wandb agent --count 1 cassandra_hpi/cassandra/"$1" + + diff --git a/experiments/slurm_base_char_sc.sh b/experiments/slurm_base_char_sc.sh new file mode 100644 index 00000000..6693b339 --- /dev/null +++ b/experiments/slurm_base_char_sc.sh @@ -0,0 +1,44 @@ +#!/bin/bash +#SBATCH --job-name=default +#SBATCH --mail-type=ALL +#SBATCH --mail-user=[INSERT:EMAIL] +#SBATCH --partition=gpu # -p +#SBATCH --cpus-per-task=4 # -c +#SBATCH --mem=48gb +#SBATCH --gpus=1 +#SBATCH --output=%x_%a_%j.log # %x is job-name, %j is job id, %a is array id +#SBATCH --array=0-3 + +# Submit with e.g. --export=TASK_NAME=mortality24,MODEL_NAME=LGBMClassifier +# Basic experiment variables, please exchange [INSERT] for your experiment parameters + +TASK=BinaryClassification # BinaryClassification +YAIB_PATH=/home/vandewrp/projects/YAIB #/dhc/home/robin.vandewater/projects/yaib +EXPERIMENT_PATH=../yaib_logs/${TASK_NAME}_experiment +DATASET_ROOT_PATH= /sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format #data/YAIB_Datasets/data + +echo "This is a SLURM job named" $SLURM_JOB_NAME "with array id" $SLURM_ARRAY_TASK_ID "and job id" $SLURM_JOB_ID +echo "Resources allocated: " $SLURM_CPUS_PER_TASK "CPUs, " $SLURM_MEM_PER_NODE "GB RAM, " $SLURM_GPUS_PER_NODE "GPUs" +echi "Task type:" ${TASK} +echo "Task: " ${TASK_NAME} +echo "Model: "${MODEL_NAME} +echo "Dataset: "${DATASETS[$SLURM_ARRAY_TASK_ID]} +echo "Experiment path: "${EXPERIMENT_PATH} + +cd ${YAIB_PATH} + +eval "$(conda shell.bash hook)" +conda activate yaib + + + +icu-benchmarks train \ + -d ${DATASET_ROOT_PATH} \ + -n ${DATASETS} \ + -t ${TASK} \ + -tn ${TASK_NAME} \ + -m ${MODEL_NAME} \ + -c \ + -s 1111 \ + -l ${EXPERIMENT_PATH} \ + --tune \ No newline at end of file diff --git a/icu_benchmarks/data/loader.py b/icu_benchmarks/data/loader.py index 3c7a9280..20fb4f27 100644 --- a/icu_benchmarks/data/loader.py +++ b/icu_benchmarks/data/loader.py @@ -32,6 +32,7 @@ def __init__( self.split = split self.vars = vars self.grouping_df = data[split][grouping_segment].set_index(self.vars["GROUP"]) + # logging.info(f"data split: {data[split]}") self.features_df = ( data[split][Segment.features].set_index(self.vars["GROUP"]).drop(labels=self.vars["SEQUENCE"], axis=1) ) diff --git a/icu_benchmarks/data/preprocessor.py b/icu_benchmarks/data/preprocessor.py index 69b90f5b..73cef52d 100644 --- a/icu_benchmarks/data/preprocessor.py +++ b/icu_benchmarks/data/preprocessor.py @@ -117,8 +117,9 @@ def _process_static(self, data, vars): sta_rec.add_step(StepScale()) sta_rec.add_step(StepImputeFastZeroFill(sel=all_numeric_predictors())) - sta_rec.add_step(StepSklearn(SimpleImputer(missing_values=None, strategy="most_frequent"), sel=has_type("object"))) - sta_rec.add_step(StepSklearn(LabelEncoder(), sel=has_type("object"), columnwise=True)) + if len(data[Split.train][Segment.static].select_dtypes(include=["object"]).columns) > 0: + sta_rec.add_step(StepSklearn(SimpleImputer(missing_values=None, strategy="most_frequent"), sel=has_type("object"))) + sta_rec.add_step(StepSklearn(LabelEncoder(), sel=has_type("object"), columnwise=True)) data = apply_recipe_to_splits(sta_rec, data, Segment.static, self.save_cache, self.load_cache) diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index ee14d90f..5831a457 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -94,6 +94,7 @@ def preprocess_data( data = {f: pq.read_table(data_dir / file_names[f]).to_pandas(self_destruct=True) for f in file_names.keys()} # Generate the splits logging.info("Generating splits.") + # complete_train = True if not complete_train: data = make_single_split( data, @@ -108,12 +109,17 @@ def preprocess_data( runmode=runmode, ) else: + data = data # If full train is set, we use all data for training/validation - data = make_train_val(data, vars, train_size=0.8, seed=seed, debug=debug, runmode=runmode) + # data = make_train_val(data, vars, train_size=None, seed=seed, debug=debug, runmode=runmode) # Apply preprocessing + # data = {Split.train: data, Split.val: data, Split.test: data} data = preprocessor.apply(data, vars) + # data[Split.train][Segment.dynamic].to_parquet(data_dir / "preprocessed.parquet") + # data[Split.train][Segment.outcome].to_parquet(data_dir / "outcome.parquet") + # Generate cache if generate_cache: caching(cache_dir, cache_file, data, load_cache) @@ -155,12 +161,12 @@ def make_train_val( stays = stays.sample(frac=0.01, random_state=seed) # If there are labels, and the task is classification, use stratified k-fold - if Var.label in vars and runmode is RunMode.classification: + if Var.label in vars and runmode is RunMode.classification : # Get labels from outcome data (takes the highest value (or True) in case seq2seq classification) labels = data[Segment.outcome].groupby(id).max()[vars[Var.label]].reset_index(drop=True) if train_size: train_val = StratifiedShuffleSplit(train_size=train_size, random_state=seed, n_splits=1) - train, val = list(train_val.split(stays, labels))[0] + train, val = list(train_val.split(stays, labels))[0] else: # If there are no labels, use random split train_val = ShuffleSplit(train_size=train_size, random_state=seed) @@ -263,7 +269,7 @@ def make_single_split( data_split[fold] = { data_type: data[data_type].merge(split[fold], on=id, how="right", sort=True) for data_type in data.keys() } - + # logging.info(f"Data split: {data_split}") return data_split diff --git a/icu_benchmarks/models/ml_models.py b/icu_benchmarks/models/ml_models.py index 53bd5a2c..5e52921b 100644 --- a/icu_benchmarks/models/ml_models.py +++ b/icu_benchmarks/models/ml_models.py @@ -8,7 +8,7 @@ from sklearn import svm from icu_benchmarks.models.wrappers import MLWrapper from icu_benchmarks.contants import RunMode -from wandb.lightgbm import wandb_callback +from wandb.integration.lightgbm import wandb_callback class LGBMWrapper(MLWrapper): From d5b80cd52cc591cf2dc7efbccc5b9d736d8dbf94 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 18 Jun 2024 15:03:47 +0200 Subject: [PATCH 103/207] Start implementation of optuna hp-tuning --- environment.yml | 3 +- icu_benchmarks/models/ml_models.py | 2 +- icu_benchmarks/run.py | 4 +- icu_benchmarks/tuning/gin_utils.py | 12 ++ icu_benchmarks/tuning/hyperparameters.py | 231 +++++++++++++++++++++-- icu_benchmarks/wandb_utils.py | 4 +- 6 files changed, 232 insertions(+), 24 deletions(-) diff --git a/environment.yml b/environment.yml index 1124300f..6bb9b07a 100644 --- a/environment.yml +++ b/environment.yml @@ -24,10 +24,11 @@ dependencies: - tqdm=4.64.1 - pytorch-lightning=2.0.3 - torchmetrics=1.0.3 - - wandb=0.15.4 + - wandb=0.17.2 - pip=23.1 - einops=0.6.1 - hydra-core=1.3 + - optuna=3.6.1 - pip: - recipies==0.1.3 # Fixed version because of NumPy incompatibility and stale development status. diff --git a/icu_benchmarks/models/ml_models.py b/icu_benchmarks/models/ml_models.py index 53bd5a2c..5e52921b 100644 --- a/icu_benchmarks/models/ml_models.py +++ b/icu_benchmarks/models/ml_models.py @@ -8,7 +8,7 @@ from sklearn import svm from icu_benchmarks.models.wrappers import MLWrapper from icu_benchmarks.contants import RunMode -from wandb.lightgbm import wandb_callback +from wandb.integration.lightgbm import wandb_callback class LGBMWrapper(MLWrapper): diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index 3d596ccb..8ba067ae 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -6,7 +6,7 @@ from pathlib import Path import torch.cuda from icu_benchmarks.wandb_utils import update_wandb_config, apply_wandb_sweep, set_wandb_experiment_name -from icu_benchmarks.tuning.hyperparameters import choose_and_bind_hyperparameters +from icu_benchmarks.tuning.hyperparameters import choose_and_bind_hyperparameters, choose_and_bind_hyperparameters_optuna from scripts.plotting.utils import plot_aggregated_results from icu_benchmarks.cross_validation import execute_repeated_cv from icu_benchmarks.run_utils import ( @@ -128,7 +128,7 @@ def main(my_args=tuple(sys.argv[1:])): gin.parse_config_files_and_bindings(gin_config_files, args.hyperparams, finalize_config=False) log_full_line(f"Data directory: {data_dir.resolve()}", level=logging.INFO) run_dir = create_run_dir(log_dir) - choose_and_bind_hyperparameters( + choose_and_bind_hyperparameters_optuna( args.tune, data_dir, run_dir, diff --git a/icu_benchmarks/tuning/gin_utils.py b/icu_benchmarks/tuning/gin_utils.py index c88d3a0f..94709d21 100644 --- a/icu_benchmarks/tuning/gin_utils.py +++ b/icu_benchmarks/tuning/gin_utils.py @@ -44,3 +44,15 @@ def bind_gin_params(hyperparams_names: list[str], hyperparams_values: list): gin.bind_parameter(param, value) logging.info(f"{param} = {value}") wandb_log({param: value}) + +def bind_gin_params(hyperparams: dict[str, any]): + """Binds hyperparameters to gin config and logs them. + + Args: + + """ + logging.info("Binding Hyperparameters:") + for param, value in hyperparams.items(): + gin.bind_parameter(param, value) + logging.info(f"{param} = {value}") + wandb_log({param: value}) \ No newline at end of file diff --git a/icu_benchmarks/tuning/hyperparameters.py b/icu_benchmarks/tuning/hyperparameters.py index 212a7753..b9db6a12 100644 --- a/icu_benchmarks/tuning/hyperparameters.py +++ b/icu_benchmarks/tuning/hyperparameters.py @@ -6,7 +6,7 @@ from pathlib import Path from skopt import gp_minimize import tempfile - +import optuna from icu_benchmarks.models.utils import JsonResultLoggingEncoder, log_table_row, Align from icu_benchmarks.cross_validation import execute_repeated_cv from icu_benchmarks.run_utils import log_full_line @@ -18,24 +18,24 @@ logging.addLevelName(25, "TUNE") -@gin.configurable("tune_hyperparameters") +@gin.configurable("tune_hyperparameters_deprecated") def choose_and_bind_hyperparameters( - do_tune: bool, - data_dir: Path, - log_dir: Path, - seed: int, - run_mode: RunMode = RunMode.classification, - checkpoint: str = None, - scopes: list[str] = [], - n_initial_points: int = 3, - n_calls: int = 20, - folds_to_tune_on: int = None, - checkpoint_file: str = "hyperparameter_tuning_logs.json", - generate_cache: bool = False, - load_cache: bool = False, - debug: bool = False, - verbose: bool = False, - wandb: bool = False, + do_tune: bool, + data_dir: Path, + log_dir: Path, + seed: int, + run_mode: RunMode = RunMode.classification, + checkpoint: str = None, + scopes: list[str] = [], + n_initial_points: int = 3, + n_calls: int = 20, + folds_to_tune_on: int = None, + checkpoint_file: str = "hyperparameter_tuning_logs.json", + generate_cache: bool = False, + load_cache: bool = False, + debug: bool = False, + verbose: bool = False, + wandb: bool = False, ): """Choose hyperparameters to tune and bind them to gin. @@ -169,6 +169,201 @@ def tune_step_callback(res): bind_gin_params(hyperparams_names, res.x) +@gin.configurable("tune_hyperparameters") +def choose_and_bind_hyperparameters_optuna( + do_tune: bool, + data_dir: Path, + log_dir: Path, + seed: int, + run_mode: RunMode = RunMode.classification, + checkpoint: str = None, + scopes: list[str] = [], + n_initial_points: int = 3, + n_calls: int = 20, + folds_to_tune_on: int = None, + checkpoint_file: str = "hyperparameter_tuning_logs.db", + generate_cache: bool = False, + load_cache: bool = False, + debug: bool = False, + verbose: bool = False, + wandb: bool = False, +): + """Choose hyperparameters to tune and bind them to gin. + + Args: + wandb: Whether we use wandb or not. + load_cache: Load cached data if available. + generate_cache: Generate cache data. + do_tune: Whether to tune hyperparameters or not. + data_dir: Path to the data directory. + log_dir: Path to the log directory. + seed: Random seed. + run_mode: The run mode of the experiment. + checkpoint: Name of the checkpoint run to load previously explored hyperparameters from. + scopes: List of gin scopes to search for hyperparameters to tune. + n_initial_points: Number of initial points to explore. + n_calls: Number of iterations to optimize the hyperparameters. + folds_to_tune_on: Number of folds to tune on. + checkpoint_file: Name of the checkpoint file. + debug: Whether to load less data. + verbose: Set to true to increase log output. + + Raises: + ValueError: If checkpoint is not None and the checkpoint does not exist. + """ + hyperparams = {} + + if len(scopes) == 0 or folds_to_tune_on is None: + logging.warning("No scopes and/or folds to tune on, skipping tuning.") + return + + # Collect hyperparameters. + hyperparams_bounds, hyperparams_names = collect_bound_hyperparameters(hyperparams, scopes) + sampler = optuna.samplers.GPSampler() + + if do_tune and not hyperparams_bounds: + logging.info("No hyperparameters to tune, skipping tuning.") + return + + # Attempt checkpoint loading + configuration, evaluation = None, None + if checkpoint: + checkpoint_path = checkpoint / checkpoint_file + if not checkpoint_path.exists(): + logging.warning(f"Hyperparameter checkpoint {checkpoint_path} does not exist.") + logging.info("Attempting to find latest checkpoint file.") + checkpoint_path = find_checkpoint(log_dir.parent, checkpoint_file) + # Check if we found a checkpoint file + if checkpoint_path: + n_calls, configuration, evaluation = load_checkpoint(checkpoint_path, n_calls) + # Check if we surpassed maximum tuning iterations + if n_calls <= 0: + logging.log(TUNE, "No more hyperparameter tuning iterations left, skipping tuning.") + logging.info("Training with these hyperparameters:") + bind_gin_params(hyperparams_names, configuration[np.argmin(evaluation)]) # bind best hyperparameters + return + else: + logging.warning("No checkpoint file found, starting from scratch.") + + # Function that trains the model with the given hyperparameters. + + + header = ["ITERATION"] + hyperparams_names + ["LOSS AT ITERATION"] + + def tune_step_callback(study: optuna.study.Study, trial: optuna.trial.FrozenTrial): + + # with open(log_dir / checkpoint_file, "w") as f: + # data = { + # "x_iters": res.x_iters, + # "func_vals": res.func_vals, + # } + # f.write(json.dumps(data, cls=JsonResultLoggingEncoder)) + table_cells = [str(len(study.trials)), study.trials[-1].params, study.trials[-1].value] + highlight = study.trials[-1] == study.best_trial # highlight if best so far + log_table_row(header, TUNE) + log_table_row(table_cells, TUNE, align=Align.RIGHT, header=header, highlight=highlight) + wandb_log({"hp-iteration": len(study.trials)}) + + if do_tune: + log_full_line("STARTING TUNING", level=TUNE, char="=") + logging.log( + TUNE, + f"Applying {sampler} from {n_initial_points} points in {n_calls} " + f"iterations on {folds_to_tune_on} folds.", + ) + log_table_row(header, TUNE) + else: + logging.log(TUNE, "Hyperparameter tuning disabled") + if configuration: + # We have loaded a checkpoint, use the best hyperparameters. + logging.info("Training with the best hyperparameters from loaded checkpoint:") + bind_gin_params(hyperparams_names, configuration[np.argmin(evaluation)]) + else: + logging.log(TUNE, "Choosing hyperparameters randomly from bounds.") + n_initial_points = 1 + n_calls = 1 + + # Call gaussian process. To choose a random set of hyperparameters this functions is also called. + def bind_params_and_train(hyperparams): + with tempfile.TemporaryDirectory(dir=log_dir) as temp_dir: + bind_gin_params(hyperparams) + if not do_tune: + return 0 + return execute_repeated_cv( + data_dir, + Path(temp_dir), + seed, + mode=run_mode, + cv_repetitions_to_train=1, + cv_folds_to_train=1,#folds_to_tune_on, + generate_cache=generate_cache, + load_cache=load_cache, + test_on="val", + debug=debug, + verbose=verbose, + wandb=wandb, + ) + def objective(trail, hyperparams_bounds, hyperparams_names): + hyperparams = {} + logging.info(f"Bounds: {hyperparams_bounds}, Names: {hyperparams_names}") + for name, value in zip(hyperparams_names, hyperparams_bounds): + if isinstance(value, tuple): + if isinstance(value[0], int) and isinstance(value[1], int): + hyperparams[name] = trail.suggest_int(name, value[0], value[1]) + else: + hyperparams[name] = trail.suggest_float(name, value[0], value[1]) + # if len(value) == 2: + # if isinstance(value[0],int) and isinstance(value[1], int): + # hyperparams[name] = trail.suggest_int(name, value[0], value[1]) + # else: + # hyperparams[name] = trail.suggest_float(name, value[0], value[1]) + # else: + # if isinstance(value[0],int) and isinstance(value[1], int): + # hyperparams[name] = trail.suggest_int(name, value[0], value[1], log= lambda: True if value[2] == "log-uniform" else False) + # else: + # hyperparams[name] = trail.suggest_float(name, value[0], value[1], log= lambda: True if value[2] == "log-uniform" else False) + + # hyperparams[name] = trail.suggest_float(name, value[0], value[1], log= lambda:True if len(value>2): True if value[2] == "log-uniform" else False else False) + # hyperparams[name] = trail.suggest_float(name, value[0], value[1], log=if len(value)>2: if value[2] == "log-uniform" else: False else: False) + else: + hyperparams[name] = trail.suggest_categorical(name, value) + return bind_params_and_train(hyperparams) + + + study = optuna.create_study( + sampler=sampler, + storage="sqlite:///"+str(log_dir / checkpoint_file), + study_name=str(data_dir)+str(seed), + ) + # func = bind_params_and_train(configuration) + # hyperparams_bounds, hyperparams_names + # objective(study, hyperparams_bounds, hyperparams_names) + res = study.optimize(lambda trail: objective(trail, hyperparams_bounds, hyperparams_names), n_trials=n_calls, + callbacks=[tune_step_callback]) + + # res = gp_minimize( + # bind_params_and_train, + # hyperparams_bounds, + # x0=configuration, + # y0=evaluation, + # n_calls=n_calls, + # n_initial_points=n_initial_points, + # random_state=seed, + # noise=1e-10, # The models are deterministic, but noise is needed for the gp to work. + # callback=tune_step_callback if do_tune else None, + # ) + logging.disable(level=NOTSET) + + if do_tune: + log_full_line("FINISHED TUNING", level=TUNE, char="=", num_newlines=4) + + logging.info("Training with these hyperparameters:") + bind_gin_params(hyperparams_names, res.x) + + + + + def collect_bound_hyperparameters(hyperparams, scopes): for scope in scopes: with gin.config_scope(scope): diff --git a/icu_benchmarks/wandb_utils.py b/icu_benchmarks/wandb_utils.py index 2736f39e..7f826e81 100644 --- a/icu_benchmarks/wandb_utils.py +++ b/icu_benchmarks/wandb_utils.py @@ -18,7 +18,7 @@ def update_wandb_config(config: dict) -> None: """ logging.debug(f"Updating Wandb config: {config}") if wandb_running(): - wandb.config.update(config) + wandb.config.update(config, allow_val_change=True) def apply_wandb_sweep(args: Namespace) -> Namespace: @@ -30,7 +30,7 @@ def apply_wandb_sweep(args: Namespace) -> Namespace: Returns: Namespace: arguments with sweep configuration applied (some are applied via hyperparams) """ - wandb.init() + wandb.init(allow_val_change=True) sweep_config = wandb.config args.__dict__.update(sweep_config) if args.hyperparams is None: From a832ec22a0aac6347d000c8bf4ea6cf506c92a54 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 19 Jun 2024 10:54:15 +0200 Subject: [PATCH 104/207] requirements and gin changes --- configs/prediction_models/LGBMClassifier.gin | 2 +- configs/prediction_models/common/MLTuning.gin | 4 +- environment.yml | 36 +------ icu_benchmarks/tuning/hyperparameters.py | 94 +++++++++---------- requirements.txt | 31 ++++++ 5 files changed, 82 insertions(+), 85 deletions(-) create mode 100644 requirements.txt diff --git a/configs/prediction_models/LGBMClassifier.gin b/configs/prediction_models/LGBMClassifier.gin index b7bfec9a..f29f40cc 100644 --- a/configs/prediction_models/LGBMClassifier.gin +++ b/configs/prediction_models/LGBMClassifier.gin @@ -11,6 +11,6 @@ model/hyperparameter.colsample_bytree = (0.33, 1.0) model/hyperparameter.max_depth = (3, 7) model/hyperparameter.min_child_samples = 1000 model/hyperparameter.n_estimators = 100000 -model/hyperparameter.num_leaves = (8, 128, "log-uniform", 2) +model/hyperparameter.num_leaves = (8, 128, "log", 2) model/hyperparameter.subsample = (0.33, 1.0) model/hyperparameter.subsample_freq = 1 diff --git a/configs/prediction_models/common/MLTuning.gin b/configs/prediction_models/common/MLTuning.gin index c582a02d..92fa0f0c 100644 --- a/configs/prediction_models/common/MLTuning.gin +++ b/configs/prediction_models/common/MLTuning.gin @@ -1,5 +1,5 @@ # Hyperparameter tuner settings for classical Machine Learning. tune_hyperparameters.scopes = ["model"] tune_hyperparameters.n_initial_points = 10 -tune_hyperparameters.n_calls = 50 -tune_hyperparameters.folds_to_tune_on = 3 \ No newline at end of file +tune_hyperparameters.n_calls = 3 +tune_hyperparameters.folds_to_tune_on = 1 \ No newline at end of file diff --git a/environment.yml b/environment.yml index 6bb9b07a..8bca14db 100644 --- a/environment.yml +++ b/environment.yml @@ -1,4 +1,4 @@ -name: yaib +name: yaib_bench channels: - pytorch - nvidia @@ -6,34 +6,8 @@ channels: - anaconda dependencies: - python=3.10 - - black=23.3.0 - - coverage=7.2.3 - - flake8=5.0.4 - - matplotlib=3.7.1 - - gin-config=0.5.0 - - ignite=0.4.11 - - pytorch=2.0.1 - - pytorch-cuda=11.8 - - lightgbm=3.3.5 - - numpy=1.24.3 - - pandas=2.0.0 - - pyarrow=11.0.0 - - pytest=7.3.1 - - scikit-learn=1.2.2 - - tensorboard=2.12.2 - - tqdm=4.64.1 - - pytorch-lightning=2.0.3 - - torchmetrics=1.0.3 - - wandb=0.17.2 - - pip=23.1 - - einops=0.6.1 - - hydra-core=1.3 - - optuna=3.6.1 - - pip: - - recipies==0.1.3 - # Fixed version because of NumPy incompatibility and stale development status. - - scikit-optimize-fix==0.9.1 - - hydra-submitit-launcher==1.2.0 -# Note: versioning of Pytorch might be dependent on compatible CUDA version. -# Please check yourself if your Pytorch installation supports cuda (for gpu acceleration) + - pip=24.0 +# - pip: +# - -r requirements.txt + diff --git a/icu_benchmarks/tuning/hyperparameters.py b/icu_benchmarks/tuning/hyperparameters.py index b9db6a12..22e907d9 100644 --- a/icu_benchmarks/tuning/hyperparameters.py +++ b/icu_benchmarks/tuning/hyperparameters.py @@ -7,6 +7,8 @@ from skopt import gp_minimize import tempfile import optuna +import wandb +from optuna.integration.wandb import WeightsAndBiasesCallback from icu_benchmarks.models.utils import JsonResultLoggingEncoder, log_table_row, Align from icu_benchmarks.cross_validation import execute_repeated_cv from icu_benchmarks.run_utils import log_full_line @@ -180,6 +182,7 @@ def choose_and_bind_hyperparameters_optuna( scopes: list[str] = [], n_initial_points: int = 3, n_calls: int = 20, + sampler=optuna.samplers.GPSampler, folds_to_tune_on: int = None, checkpoint_file: str = "hyperparameter_tuning_logs.db", generate_cache: bool = False, @@ -188,9 +191,10 @@ def choose_and_bind_hyperparameters_optuna( verbose: bool = False, wandb: bool = False, ): - """Choose hyperparameters to tune and bind them to gin. + """Choose hyperparameters to tune and bind them to gin. Uses Optuna for hyperparameter optimization. Args: + sampler: wandb: Whether we use wandb or not. load_cache: Load cached data if available. generate_cache: Generate cache data. @@ -219,7 +223,6 @@ def choose_and_bind_hyperparameters_optuna( # Collect hyperparameters. hyperparams_bounds, hyperparams_names = collect_bound_hyperparameters(hyperparams, scopes) - sampler = optuna.samplers.GPSampler() if do_tune and not hyperparams_bounds: logging.info("No hyperparameters to tune, skipping tuning.") @@ -236,18 +239,17 @@ def choose_and_bind_hyperparameters_optuna( # Check if we found a checkpoint file if checkpoint_path: n_calls, configuration, evaluation = load_checkpoint(checkpoint_path, n_calls) - # Check if we surpassed maximum tuning iterations - if n_calls <= 0: - logging.log(TUNE, "No more hyperparameter tuning iterations left, skipping tuning.") - logging.info("Training with these hyperparameters:") - bind_gin_params(hyperparams_names, configuration[np.argmin(evaluation)]) # bind best hyperparameters - return + # # Check if we surpassed maximum tuning iterations + # if n_calls <= 0: + # logging.log(TUNE, "No more hyperparameter tuning iterations left, skipping tuning.") + # logging.info("Training with these hyperparameters:") + # bind_gin_params(hyperparams_names, configuration[np.argmin(evaluation)]) # bind best hyperparameters + # return else: logging.warning("No checkpoint file found, starting from scratch.") # Function that trains the model with the given hyperparameters. - header = ["ITERATION"] + hyperparams_names + ["LOSS AT ITERATION"] def tune_step_callback(study: optuna.study.Study, trial: optuna.trial.FrozenTrial): @@ -258,7 +260,7 @@ def tune_step_callback(study: optuna.study.Study, trial: optuna.trial.FrozenTria # "func_vals": res.func_vals, # } # f.write(json.dumps(data, cls=JsonResultLoggingEncoder)) - table_cells = [str(len(study.trials)), study.trials[-1].params, study.trials[-1].value] + table_cells = [str(len(study.trials)), *list(study.trials[-1].params.values()), study.trials[-1].value] highlight = study.trials[-1] == study.best_trial # highlight if best so far log_table_row(header, TUNE) log_table_row(table_cells, TUNE, align=Align.RIGHT, header=header, highlight=highlight) @@ -289,13 +291,13 @@ def bind_params_and_train(hyperparams): bind_gin_params(hyperparams) if not do_tune: return 0 - return execute_repeated_cv( + score = execute_repeated_cv( data_dir, Path(temp_dir), seed, mode=run_mode, cv_repetitions_to_train=1, - cv_folds_to_train=1,#folds_to_tune_on, + cv_folds_to_train=folds_to_tune_on, generate_cache=generate_cache, load_cache=load_cache, test_on="val", @@ -303,65 +305,47 @@ def bind_params_and_train(hyperparams): verbose=verbose, wandb=wandb, ) + logging.info(f"Score: {score}") + return score + def objective(trail, hyperparams_bounds, hyperparams_names): + # Optuna objective function hyperparams = {} logging.info(f"Bounds: {hyperparams_bounds}, Names: {hyperparams_names}") for name, value in zip(hyperparams_names, hyperparams_bounds): if isinstance(value, tuple): if isinstance(value[0], int) and isinstance(value[1], int): - hyperparams[name] = trail.suggest_int(name, value[0], value[1]) + hyperparams[name] = trail.suggest_int(name, value[0], value[1], + log=len(value) > 2 and value[2] == "log") else: - hyperparams[name] = trail.suggest_float(name, value[0], value[1]) - # if len(value) == 2: - # if isinstance(value[0],int) and isinstance(value[1], int): - # hyperparams[name] = trail.suggest_int(name, value[0], value[1]) - # else: - # hyperparams[name] = trail.suggest_float(name, value[0], value[1]) - # else: - # if isinstance(value[0],int) and isinstance(value[1], int): - # hyperparams[name] = trail.suggest_int(name, value[0], value[1], log= lambda: True if value[2] == "log-uniform" else False) - # else: - # hyperparams[name] = trail.suggest_float(name, value[0], value[1], log= lambda: True if value[2] == "log-uniform" else False) - - # hyperparams[name] = trail.suggest_float(name, value[0], value[1], log= lambda:True if len(value>2): True if value[2] == "log-uniform" else False else False) - # hyperparams[name] = trail.suggest_float(name, value[0], value[1], log=if len(value)>2: if value[2] == "log-uniform" else: False else: False) + hyperparams[name] = trail.suggest_float(name, value[0], value[1], + log=len(value) > 2 and value[2] == "log") else: hyperparams[name] = trail.suggest_categorical(name, value) return bind_params_and_train(hyperparams) + if isinstance(sampler, optuna.samplers.GPSampler): + sampler = sampler(seed=seed, n_startup_trials=n_initial_points, deterministic_objective=True) + else: + sampler = sampler(seed=seed) study = optuna.create_study( sampler=sampler, - storage="sqlite:///"+str(log_dir / checkpoint_file), - study_name=str(data_dir)+str(seed), + storage="sqlite:///" + str(log_dir / checkpoint_file), + study_name=str(data_dir) + str(seed), ) - # func = bind_params_and_train(configuration) - # hyperparams_bounds, hyperparams_names - # objective(study, hyperparams_bounds, hyperparams_names) - res = study.optimize(lambda trail: objective(trail, hyperparams_bounds, hyperparams_names), n_trials=n_calls, - callbacks=[tune_step_callback]) - - # res = gp_minimize( - # bind_params_and_train, - # hyperparams_bounds, - # x0=configuration, - # y0=evaluation, - # n_calls=n_calls, - # n_initial_points=n_initial_points, - # random_state=seed, - # noise=1e-10, # The models are deterministic, but noise is needed for the gp to work. - # callback=tune_step_callback if do_tune else None, - # ) + callbacks = [tune_step_callback()] + if wandb: + callbacks.append(WeightsAndBiasesCallback()) + study.optimize(lambda trail: objective(trail, hyperparams_bounds, hyperparams_names), n_trials=n_calls, + callbacks=callbacks, ) logging.disable(level=NOTSET) if do_tune: log_full_line("FINISHED TUNING", level=TUNE, char="=", num_newlines=4) logging.info("Training with these hyperparameters:") - bind_gin_params(hyperparams_names, res.x) - - - + bind_gin_params(study.best_params) def collect_bound_hyperparameters(hyperparams, scopes): @@ -372,7 +356,15 @@ def collect_bound_hyperparameters(hyperparams, scopes): hyperparams_bounds = list(hyperparams.values()) return hyperparams_bounds, hyperparams_names - +def load_optuna_checkpoint(checkpoint_path, n_calls): + logging.info(f"Loading checkpoint at {checkpoint_path}") + with open(checkpoint_path, "r") as f: + data = json.loads(f.read()) + x0 = data["x_iters"] + y0 = data["func_vals"] + n_calls -= len(x0) + logging.log(TUNE, f"Checkpoint contains {len(x0)} points.") + return n_calls, x0, y0 def load_checkpoint(checkpoint_path, n_calls): logging.info(f"Loading checkpoint at {checkpoint_path}") with open(checkpoint_path, "r") as f: diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..1cb55236 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,31 @@ +--extra-index-url https://download.pytorch.org/whl/cu118 +black==24.2.0 +coverage==7.2.3 +flake8==5.0.4 +matplotlib==3.7.1 +gin-config==0.5.0 +pytorch-ignite==0.5.0.post2 +# Note: versioning of Pytorch might be dependent on compatible CUDA version. +# Please check yourself if your Pytorch installation supports cuda (for gpu acceleration) +torch==2.3.1 +lightning==2.3.0 +torchmetrics==1.0.3 +#pytorch-cuda==11.8 +lightgbm==3.3.5 +numpy==1.24.3 +pandas==2.0.0 +pyarrow==11.0.0 +pytest==7.3.1 +scikit-learn==1.2.2 +tensorboard==2.12.2 +tqdm==4.64.1 +einops==0.6.1 +hydra-core==1.3 +optuna==3.6.1 +optuna-integration==3.6.0 +wandb==0.17.2 +recipies==0.1.3 +#Fixed version because of NumPy incompatibility and stale development status. +scikit-optimize-fix==0.9.1 +hydra-submitit-launcher==1.2.0 +pytest-runner==6.0.1 \ No newline at end of file From 4acbd2a108eb718958d5cdc394d53b3bd205b940 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 19 Jun 2024 11:14:20 +0200 Subject: [PATCH 105/207] small corrections --- environment.yml | 5 ----- icu_benchmarks/tuning/hyperparameters.py | 1 - 2 files changed, 6 deletions(-) diff --git a/environment.yml b/environment.yml index 8bca14db..55cb458d 100644 --- a/environment.yml +++ b/environment.yml @@ -1,9 +1,4 @@ name: yaib_bench -channels: - - pytorch - - nvidia - - conda-forge - - anaconda dependencies: - python=3.10 - pip=24.0 diff --git a/icu_benchmarks/tuning/hyperparameters.py b/icu_benchmarks/tuning/hyperparameters.py index 22e907d9..5f57de9f 100644 --- a/icu_benchmarks/tuning/hyperparameters.py +++ b/icu_benchmarks/tuning/hyperparameters.py @@ -7,7 +7,6 @@ from skopt import gp_minimize import tempfile import optuna -import wandb from optuna.integration.wandb import WeightsAndBiasesCallback from icu_benchmarks.models.utils import JsonResultLoggingEncoder, log_table_row, Align from icu_benchmarks.cross_validation import execute_repeated_cv From 50dfe350787bcd9b89f7a7e4da69aa6c99aa05f4 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 19 Jun 2024 12:37:12 +0200 Subject: [PATCH 106/207] setup corrected --- icu_benchmarks/tuning/hyperparameters.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/icu_benchmarks/tuning/hyperparameters.py b/icu_benchmarks/tuning/hyperparameters.py index 5f57de9f..f4517a16 100644 --- a/icu_benchmarks/tuning/hyperparameters.py +++ b/icu_benchmarks/tuning/hyperparameters.py @@ -333,7 +333,7 @@ def objective(trail, hyperparams_bounds, hyperparams_names): storage="sqlite:///" + str(log_dir / checkpoint_file), study_name=str(data_dir) + str(seed), ) - callbacks = [tune_step_callback()] + callbacks = [tune_step_callback] if wandb: callbacks.append(WeightsAndBiasesCallback()) study.optimize(lambda trail: objective(trail, hyperparams_bounds, hyperparams_names), n_trials=n_calls, diff --git a/setup.py b/setup.py index c4edbb99..384562ea 100644 --- a/setup.py +++ b/setup.py @@ -71,7 +71,7 @@ def parse_environment_yml(): keywords="benchmark mimic-iii eicu hirid clinical-ml machine-learning benchmark time-series mimic-iv patient-monitoring " "amsterdamumcdb clinical-data ehr icu ricu pyicu", name="yaib", - packages=find_packages(include=["icu_benchmarks"]), + packages=find_packages(), setup_requires=setup_requirements, test_suite="tests", tests_require=[], From e3955a163210fbabacfc5042fc237ba5c7cbf20d Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 19 Jun 2024 13:35:23 +0200 Subject: [PATCH 107/207] temporary logging --- icu_benchmarks/data/preprocessor.py | 3 +++ icu_benchmarks/models/train.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/icu_benchmarks/data/preprocessor.py b/icu_benchmarks/data/preprocessor.py index 73cef52d..0c3fb190 100644 --- a/icu_benchmarks/data/preprocessor.py +++ b/icu_benchmarks/data/preprocessor.py @@ -109,6 +109,9 @@ def apply(self, data, vars) -> dict[dict[pd.DataFrame]]: data[Split.train][Segment.features] = data[Split.train].pop(Segment.dynamic) data[Split.val][Segment.features] = data[Split.val].pop(Segment.dynamic) data[Split.test][Segment.features] = data[Split.test].pop(Segment.dynamic) + + logging.info(data[Split.train][Segment.features].head()) + logging.info(f"Generate features: {self.generate_features}") return data def _process_static(self, data, vars): diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index db7aabda..321c30cb 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -37,7 +37,7 @@ def train_common( weight: str = None, optimizer: type = Adam, precision=32, - batch_size=64, + batch_size=1, epochs=1000, patience=20, min_delta=1e-5, @@ -49,7 +49,7 @@ def train_common( ram_cache=False, pl_model=True, train_only=False, - num_workers: int = min(cpu_core_count, torch.cuda.device_count() * 8 * int(torch.cuda.is_available()), 32), + num_workers: int = 1 #min(cpu_core_count, torch.cuda.device_count() * 8 * int(torch.cuda.is_available()), 32), ): """Common wrapper to train all benchmarked models. From ac6d0cccb299af640ffa720dab5458e9a35a92aa Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 19 Jun 2024 15:05:24 +0200 Subject: [PATCH 108/207] wandb callback --- icu_benchmarks/run.py | 1 + icu_benchmarks/tuning/hyperparameters.py | 51 ++++++++++++------------ icu_benchmarks/wandb_utils.py | 3 +- 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index 8ba067ae..c80d29d9 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -139,6 +139,7 @@ def main(my_args=tuple(sys.argv[1:])): generate_cache=args.generate_cache, load_cache=args.load_cache, verbose=verbose, + wandb=args.wandb_sweep, ) log_full_line(f"Logging to {run_dir.resolve()}", level=logging.INFO) diff --git a/icu_benchmarks/tuning/hyperparameters.py b/icu_benchmarks/tuning/hyperparameters.py index f4517a16..14b38123 100644 --- a/icu_benchmarks/tuning/hyperparameters.py +++ b/icu_benchmarks/tuning/hyperparameters.py @@ -230,35 +230,29 @@ def choose_and_bind_hyperparameters_optuna( # Attempt checkpoint loading configuration, evaluation = None, None if checkpoint: - checkpoint_path = checkpoint / checkpoint_file - if not checkpoint_path.exists(): - logging.warning(f"Hyperparameter checkpoint {checkpoint_path} does not exist.") - logging.info("Attempting to find latest checkpoint file.") - checkpoint_path = find_checkpoint(log_dir.parent, checkpoint_file) - # Check if we found a checkpoint file - if checkpoint_path: - n_calls, configuration, evaluation = load_checkpoint(checkpoint_path, n_calls) - # # Check if we surpassed maximum tuning iterations - # if n_calls <= 0: - # logging.log(TUNE, "No more hyperparameter tuning iterations left, skipping tuning.") - # logging.info("Training with these hyperparameters:") - # bind_gin_params(hyperparams_names, configuration[np.argmin(evaluation)]) # bind best hyperparameters - # return - else: - logging.warning("No checkpoint file found, starting from scratch.") + return NotImplementedError("Checkpoint loading is not implemented for Optuna yet.") + # checkpoint_path = checkpoint / checkpoint_file + # if not checkpoint_path.exists(): + # logging.warning(f"Hyperparameter checkpoint {checkpoint_path} does not exist.") + # logging.info("Attempting to find latest checkpoint file.") + # checkpoint_path = find_checkpoint(log_dir.parent, checkpoint_file) + # # Check if we found a checkpoint file + # if checkpoint_path: + # n_calls, configuration, evaluation = load_checkpoint(checkpoint_path, n_calls) + # # # Check if we surpassed maximum tuning iterations + # # if n_calls <= 0: + # # logging.log(TUNE, "No more hyperparameter tuning iterations left, skipping tuning.") + # # logging.info("Training with these hyperparameters:") + # # bind_gin_params(hyperparams_names, configuration[np.argmin(evaluation)]) # bind best hyperparameters + # # return + # else: + # logging.warning("No checkpoint file found, starting from scratch.") # Function that trains the model with the given hyperparameters. header = ["ITERATION"] + hyperparams_names + ["LOSS AT ITERATION"] def tune_step_callback(study: optuna.study.Study, trial: optuna.trial.FrozenTrial): - - # with open(log_dir / checkpoint_file, "w") as f: - # data = { - # "x_iters": res.x_iters, - # "func_vals": res.func_vals, - # } - # f.write(json.dumps(data, cls=JsonResultLoggingEncoder)) table_cells = [str(len(study.trials)), *list(study.trials[-1].params.values()), study.trials[-1].value] highlight = study.trials[-1] == study.best_trial # highlight if best so far log_table_row(header, TUNE) @@ -307,6 +301,7 @@ def bind_params_and_train(hyperparams): logging.info(f"Score: {score}") return score + # Optuna objective function def objective(trail, hyperparams_bounds, hyperparams_names): # Optuna objective function hyperparams = {} @@ -328,6 +323,7 @@ def objective(trail, hyperparams_bounds, hyperparams_names): else: sampler = sampler(seed=seed) + # Optuna study study = optuna.create_study( sampler=sampler, storage="sqlite:///" + str(log_dir / checkpoint_file), @@ -335,9 +331,14 @@ def objective(trail, hyperparams_bounds, hyperparams_names): ) callbacks = [tune_step_callback] if wandb: - callbacks.append(WeightsAndBiasesCallback()) + wandb_kwargs = { + "config": {"sampler": sampler}, + } + wandbc = WeightsAndBiasesCallback(metric_name="loss",wandb_kwargs=wandb_kwargs) + callbacks.append(wandbc) + logging.info(f"Starting Optuna study with {n_calls} trials and callbacks: {callbacks}.") study.optimize(lambda trail: objective(trail, hyperparams_bounds, hyperparams_names), n_trials=n_calls, - callbacks=callbacks, ) + callbacks=callbacks, gc_after_trial = True) logging.disable(level=NOTSET) if do_tune: diff --git a/icu_benchmarks/wandb_utils.py b/icu_benchmarks/wandb_utils.py index 7f826e81..40347dd4 100644 --- a/icu_benchmarks/wandb_utils.py +++ b/icu_benchmarks/wandb_utils.py @@ -74,5 +74,4 @@ def set_wandb_experiment_name(args, mode): if wandb_running(): wandb.config.update({"run-name": run_name}) - wandb.run.name = run_name - wandb.run.save() + wandb.run.name = run_name \ No newline at end of file From a78ca88c65e35da81812f012e85bc9af27eb5695 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 19 Jun 2024 15:32:45 +0200 Subject: [PATCH 109/207] docs --- icu_benchmarks/tuning/hyperparameters.py | 1 + 1 file changed, 1 insertion(+) diff --git a/icu_benchmarks/tuning/hyperparameters.py b/icu_benchmarks/tuning/hyperparameters.py index 14b38123..fcb2293f 100644 --- a/icu_benchmarks/tuning/hyperparameters.py +++ b/icu_benchmarks/tuning/hyperparameters.py @@ -328,6 +328,7 @@ def objective(trail, hyperparams_bounds, hyperparams_names): sampler=sampler, storage="sqlite:///" + str(log_dir / checkpoint_file), study_name=str(data_dir) + str(seed), + pruner=optuna.pruners.HyperbandPruner(), ) callbacks = [tune_step_callback] if wandb: From 029f4cb6ff505dff87be0af70cd0ad03e03e9d67 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 19 Jun 2024 15:33:42 +0200 Subject: [PATCH 110/207] added flake8 for ci --- environment.yml | 1 + icu_benchmarks/tuning/gin_utils.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/environment.yml b/environment.yml index 55cb458d..da49ea2b 100644 --- a/environment.yml +++ b/environment.yml @@ -2,6 +2,7 @@ name: yaib_bench dependencies: - python=3.10 - pip=24.0 + - flake8=7.1.0 # - pip: # - -r requirements.txt diff --git a/icu_benchmarks/tuning/gin_utils.py b/icu_benchmarks/tuning/gin_utils.py index 94709d21..96437baf 100644 --- a/icu_benchmarks/tuning/gin_utils.py +++ b/icu_benchmarks/tuning/gin_utils.py @@ -45,14 +45,15 @@ def bind_gin_params(hyperparams_names: list[str], hyperparams_values: list): logging.info(f"{param} = {value}") wandb_log({param: value}) + def bind_gin_params(hyperparams: dict[str, any]): - """Binds hyperparameters to gin config and logs them. + """Binds hyperparameter dict to gin config and logs them. Args: - + hyperparams: Dictionary of hyperparameters. """ logging.info("Binding Hyperparameters:") for param, value in hyperparams.items(): gin.bind_parameter(param, value) logging.info(f"{param} = {value}") - wandb_log({param: value}) \ No newline at end of file + wandb_log({param: value}) From a2d33a80f59554ac8807e360ac0147614f9ccca7 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 19 Jun 2024 15:35:32 +0200 Subject: [PATCH 111/207] ci --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index da49ea2b..734f8789 100644 --- a/environment.yml +++ b/environment.yml @@ -1,4 +1,4 @@ -name: yaib_bench +name: yaib dependencies: - python=3.10 - pip=24.0 From 58f1ee599cbe5e164c78644319dbff13e007df7f Mon Sep 17 00:00:00 2001 From: Robin van de Water Date: Wed, 19 Jun 2024 15:36:17 +0200 Subject: [PATCH 112/207] Update ci.yml name --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 048e2275..32e3ca6d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: - uses: actions/checkout@v3 - uses: conda-incubator/setup-miniconda@v2 with: - activate-environment: yaib_updated + activate-environment: yaib environment-file: environment.yml auto-activate-base: false - name: Lint with flake8 @@ -40,4 +40,4 @@ jobs: # - name: Setup package # run: pip install -e . # - name: Test command line tool - # run: python -m icu_benchmarks.run --help \ No newline at end of file + # run: python -m icu_benchmarks.run --help From 0780499de86d8b6522b6af7332f8583e4bdcc35f Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 19 Jun 2024 15:38:04 +0200 Subject: [PATCH 113/207] conda forge --- environment.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/environment.yml b/environment.yml index 734f8789..abba2126 100644 --- a/environment.yml +++ b/environment.yml @@ -1,4 +1,6 @@ name: yaib +channels: + - conda-forge dependencies: - python=3.10 - pip=24.0 From 68920a9612b5056743cdc895cb23b48d38799d02 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 19 Jun 2024 15:51:09 +0200 Subject: [PATCH 114/207] formatting and update ci --- .github/workflows/ci.yml | 2 +- icu_benchmarks/run.py | 2 +- icu_benchmarks/tuning/gin_utils.py | 24 ++--- icu_benchmarks/tuning/hyperparameters.py | 115 ++++++++++++----------- icu_benchmarks/wandb_utils.py | 2 +- 5 files changed, 74 insertions(+), 71 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 32e3ca6d..acc5ed1f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # the GitHub editor is 127 chars wide - flake8 . --count --max-complexity=14 --max-line-length=127 --statistics + flake8 . --count --max-complexity=20 --max-line-length=127 --statistics # - name: Test with pytest # run: python -m pytest ./tests/recipes # If we want to test running the tool later on diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index c80d29d9..b0bd5a31 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -6,7 +6,7 @@ from pathlib import Path import torch.cuda from icu_benchmarks.wandb_utils import update_wandb_config, apply_wandb_sweep, set_wandb_experiment_name -from icu_benchmarks.tuning.hyperparameters import choose_and_bind_hyperparameters, choose_and_bind_hyperparameters_optuna +from icu_benchmarks.tuning.hyperparameters import choose_and_bind_hyperparameters_optuna from scripts.plotting.utils import plot_aggregated_results from icu_benchmarks.cross_validation import execute_repeated_cv from icu_benchmarks.run_utils import ( diff --git a/icu_benchmarks/tuning/gin_utils.py b/icu_benchmarks/tuning/gin_utils.py index 96437baf..0f9765a9 100644 --- a/icu_benchmarks/tuning/gin_utils.py +++ b/icu_benchmarks/tuning/gin_utils.py @@ -32,18 +32,18 @@ def get_gin_hyperparameters(class_to_tune: str = gin.REQUIRED, **hyperparams: di return hyperparams_to_tune -def bind_gin_params(hyperparams_names: list[str], hyperparams_values: list): - """Binds hyperparameters to gin config and logs them. - - Args: - hyperparams_names: List of hyperparameter names. - hyperparams_values: List of hyperparameter values. - """ - logging.info("Binding Hyperparameters:") - for param, value in zip(hyperparams_names, hyperparams_values): - gin.bind_parameter(param, value) - logging.info(f"{param} = {value}") - wandb_log({param: value}) +# def bind_gin_params(hyperparams_names: list[str], hyperparams_values: list): +# """Binds hyperparameters to gin config and logs them. +# +# Args: +# hyperparams_names: List of hyperparameter names. +# hyperparams_values: List of hyperparameter values. +# """ +# logging.info("Binding Hyperparameters:") +# for param, value in zip(hyperparams_names, hyperparams_values): +# gin.bind_parameter(param, value) +# logging.info(f"{param} = {value}") +# wandb_log({param: value}) def bind_gin_params(hyperparams: dict[str, any]): diff --git a/icu_benchmarks/tuning/hyperparameters.py b/icu_benchmarks/tuning/hyperparameters.py index fcb2293f..b1f5a73c 100644 --- a/icu_benchmarks/tuning/hyperparameters.py +++ b/icu_benchmarks/tuning/hyperparameters.py @@ -21,22 +21,22 @@ @gin.configurable("tune_hyperparameters_deprecated") def choose_and_bind_hyperparameters( - do_tune: bool, - data_dir: Path, - log_dir: Path, - seed: int, - run_mode: RunMode = RunMode.classification, - checkpoint: str = None, - scopes: list[str] = [], - n_initial_points: int = 3, - n_calls: int = 20, - folds_to_tune_on: int = None, - checkpoint_file: str = "hyperparameter_tuning_logs.json", - generate_cache: bool = False, - load_cache: bool = False, - debug: bool = False, - verbose: bool = False, - wandb: bool = False, + do_tune: bool, + data_dir: Path, + log_dir: Path, + seed: int, + run_mode: RunMode = RunMode.classification, + checkpoint: str = None, + scopes: list[str] = [], + n_initial_points: int = 3, + n_calls: int = 20, + folds_to_tune_on: int = None, + checkpoint_file: str = "hyperparameter_tuning_logs.json", + generate_cache: bool = False, + load_cache: bool = False, + debug: bool = False, + verbose: bool = False, + wandb: bool = False, ): """Choose hyperparameters to tune and bind them to gin. @@ -172,23 +172,23 @@ def tune_step_callback(res): @gin.configurable("tune_hyperparameters") def choose_and_bind_hyperparameters_optuna( - do_tune: bool, - data_dir: Path, - log_dir: Path, - seed: int, - run_mode: RunMode = RunMode.classification, - checkpoint: str = None, - scopes: list[str] = [], - n_initial_points: int = 3, - n_calls: int = 20, - sampler=optuna.samplers.GPSampler, - folds_to_tune_on: int = None, - checkpoint_file: str = "hyperparameter_tuning_logs.db", - generate_cache: bool = False, - load_cache: bool = False, - debug: bool = False, - verbose: bool = False, - wandb: bool = False, + do_tune: bool, + data_dir: Path, + log_dir: Path, + seed: int, + run_mode: RunMode = RunMode.classification, + checkpoint: str = None, + scopes: list[str] = [], + n_initial_points: int = 3, + n_calls: int = 20, + sampler=optuna.samplers.GPSampler, + folds_to_tune_on: int = None, + checkpoint_file: str = "hyperparameter_tuning_logs.db", + generate_cache: bool = False, + load_cache: bool = False, + debug: bool = False, + verbose: bool = False, + wandb: bool = False, ): """Choose hyperparameters to tune and bind them to gin. Uses Optuna for hyperparameter optimization. @@ -252,6 +252,21 @@ def choose_and_bind_hyperparameters_optuna( header = ["ITERATION"] + hyperparams_names + ["LOSS AT ITERATION"] + # Optuna objective function + def objective(trail, hyperparams_bounds, hyperparams_names): + # Optuna objective function + hyperparams = {} + logging.info(f"Bounds: {hyperparams_bounds}, Names: {hyperparams_names}") + for name, value in zip(hyperparams_names, hyperparams_bounds): + if isinstance(value, tuple): + if isinstance(value[0], int) and isinstance(value[1], int): + hyperparams[name] = trail.suggest_int(name, value[0], value[1], log=len(value) > 2 and value[2] == "log") + else: + hyperparams[name] = trail.suggest_float(name, value[0], value[1], log=len(value) > 2 and value[2] == "log") + else: + hyperparams[name] = trail.suggest_categorical(name, value) + return bind_params_and_train(hyperparams) + def tune_step_callback(study: optuna.study.Study, trial: optuna.trial.FrozenTrial): table_cells = [str(len(study.trials)), *list(study.trials[-1].params.values()), study.trials[-1].value] highlight = study.trials[-1] == study.best_trial # highlight if best so far @@ -263,8 +278,7 @@ def tune_step_callback(study: optuna.study.Study, trial: optuna.trial.FrozenTria log_full_line("STARTING TUNING", level=TUNE, char="=") logging.log( TUNE, - f"Applying {sampler} from {n_initial_points} points in {n_calls} " - f"iterations on {folds_to_tune_on} folds.", + f"Applying {sampler} from {n_initial_points} points in {n_calls} " f"iterations on {folds_to_tune_on} folds.", ) log_table_row(header, TUNE) else: @@ -278,7 +292,6 @@ def tune_step_callback(study: optuna.study.Study, trial: optuna.trial.FrozenTria n_initial_points = 1 n_calls = 1 - # Call gaussian process. To choose a random set of hyperparameters this functions is also called. def bind_params_and_train(hyperparams): with tempfile.TemporaryDirectory(dir=log_dir) as temp_dir: bind_gin_params(hyperparams) @@ -301,23 +314,6 @@ def bind_params_and_train(hyperparams): logging.info(f"Score: {score}") return score - # Optuna objective function - def objective(trail, hyperparams_bounds, hyperparams_names): - # Optuna objective function - hyperparams = {} - logging.info(f"Bounds: {hyperparams_bounds}, Names: {hyperparams_names}") - for name, value in zip(hyperparams_names, hyperparams_bounds): - if isinstance(value, tuple): - if isinstance(value[0], int) and isinstance(value[1], int): - hyperparams[name] = trail.suggest_int(name, value[0], value[1], - log=len(value) > 2 and value[2] == "log") - else: - hyperparams[name] = trail.suggest_float(name, value[0], value[1], - log=len(value) > 2 and value[2] == "log") - else: - hyperparams[name] = trail.suggest_categorical(name, value) - return bind_params_and_train(hyperparams) - if isinstance(sampler, optuna.samplers.GPSampler): sampler = sampler(seed=seed, n_startup_trials=n_initial_points, deterministic_objective=True) else: @@ -335,11 +331,15 @@ def objective(trail, hyperparams_bounds, hyperparams_names): wandb_kwargs = { "config": {"sampler": sampler}, } - wandbc = WeightsAndBiasesCallback(metric_name="loss",wandb_kwargs=wandb_kwargs) + wandbc = WeightsAndBiasesCallback(metric_name="loss", wandb_kwargs=wandb_kwargs) callbacks.append(wandbc) logging.info(f"Starting Optuna study with {n_calls} trials and callbacks: {callbacks}.") - study.optimize(lambda trail: objective(trail, hyperparams_bounds, hyperparams_names), n_trials=n_calls, - callbacks=callbacks, gc_after_trial = True) + study.optimize( + lambda trail: objective(trail, hyperparams_bounds, hyperparams_names), + n_trials=n_calls, + callbacks=callbacks, + gc_after_trial=True, + ) logging.disable(level=NOTSET) if do_tune: @@ -357,6 +357,7 @@ def collect_bound_hyperparameters(hyperparams, scopes): hyperparams_bounds = list(hyperparams.values()) return hyperparams_bounds, hyperparams_names + def load_optuna_checkpoint(checkpoint_path, n_calls): logging.info(f"Loading checkpoint at {checkpoint_path}") with open(checkpoint_path, "r") as f: @@ -366,6 +367,8 @@ def load_optuna_checkpoint(checkpoint_path, n_calls): n_calls -= len(x0) logging.log(TUNE, f"Checkpoint contains {len(x0)} points.") return n_calls, x0, y0 + + def load_checkpoint(checkpoint_path, n_calls): logging.info(f"Loading checkpoint at {checkpoint_path}") with open(checkpoint_path, "r") as f: diff --git a/icu_benchmarks/wandb_utils.py b/icu_benchmarks/wandb_utils.py index 40347dd4..6148af5d 100644 --- a/icu_benchmarks/wandb_utils.py +++ b/icu_benchmarks/wandb_utils.py @@ -74,4 +74,4 @@ def set_wandb_experiment_name(args, mode): if wandb_running(): wandb.config.update({"run-name": run_name}) - wandb.run.name = run_name \ No newline at end of file + wandb.run.name = run_name From ddd06abdaf3f4b6fbcf8bbdfbe14ebab63595650 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 19 Jun 2024 14:00:20 +0000 Subject: [PATCH 115/207] Bump pyarrow from 11.0.0 to 14.0.1 Bumps [pyarrow](https://github.com/apache/arrow) from 11.0.0 to 14.0.1. - [Commits](https://github.com/apache/arrow/compare/go/v11.0.0...go/v14.0.1) --- updated-dependencies: - dependency-name: pyarrow dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1cb55236..39509534 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ torchmetrics==1.0.3 lightgbm==3.3.5 numpy==1.24.3 pandas==2.0.0 -pyarrow==11.0.0 +pyarrow==14.0.1 pytest==7.3.1 scikit-learn==1.2.2 tensorboard==2.12.2 From f5ee65aec573a62fcd4fb3a552f04ed539bfbe29 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 19 Jun 2024 14:03:42 +0000 Subject: [PATCH 116/207] Bump scikit-learn from 1.2.2 to 1.5.0 Bumps [scikit-learn](https://github.com/scikit-learn/scikit-learn) from 1.2.2 to 1.5.0. - [Release notes](https://github.com/scikit-learn/scikit-learn/releases) - [Commits](https://github.com/scikit-learn/scikit-learn/compare/1.2.2...1.5.0) --- updated-dependencies: - dependency-name: scikit-learn dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 39509534..7b3035ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ numpy==1.24.3 pandas==2.0.0 pyarrow==14.0.1 pytest==7.3.1 -scikit-learn==1.2.2 +scikit-learn==1.5.0 tensorboard==2.12.2 tqdm==4.64.1 einops==0.6.1 From 89a835ac98d3d7aad918b43d9c221491b0f805ea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 19 Jun 2024 14:05:21 +0000 Subject: [PATCH 117/207] Bump black from 24.2.0 to 24.3.0 Bumps [black](https://github.com/psf/black) from 24.2.0 to 24.3.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/compare/24.2.0...24.3.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 39509534..4ef70492 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ --extra-index-url https://download.pytorch.org/whl/cu118 -black==24.2.0 +black==24.3.0 coverage==7.2.3 flake8==5.0.4 matplotlib==3.7.1 From 7b3fc4a1e707b91ab87b3600d286d068738a2c09 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 19 Jun 2024 14:09:12 +0000 Subject: [PATCH 118/207] Bump tqdm from 4.64.1 to 4.66.3 Bumps [tqdm](https://github.com/tqdm/tqdm) from 4.64.1 to 4.66.3. - [Release notes](https://github.com/tqdm/tqdm/releases) - [Commits](https://github.com/tqdm/tqdm/compare/v4.64.1...v4.66.3) --- updated-dependencies: - dependency-name: tqdm dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7b3035ff..8d682025 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,7 @@ pyarrow==14.0.1 pytest==7.3.1 scikit-learn==1.5.0 tensorboard==2.12.2 -tqdm==4.64.1 +tqdm==4.66.3 einops==0.6.1 hydra-core==1.3 optuna==3.6.1 From 96baca6ceb3acba62751c0bd57b6ed0085e24cba Mon Sep 17 00:00:00 2001 From: rvandewater Date: Mon, 24 Jun 2024 16:21:43 +0200 Subject: [PATCH 119/207] testing timesnet --- experiments/benchmark_cass.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/experiments/benchmark_cass.yml b/experiments/benchmark_cass.yml index e828ba3b..2309d286 100644 --- a/experiments/benchmark_cass.yml +++ b/experiments/benchmark_cass.yml @@ -5,13 +5,15 @@ command: - -d - ../data/ - -t - - CassClassification + - BinaryClassification +# - CassClassification - --log-dir - /sc-projects/sc-proj-cc08-cassandra/RW_Prospective/yaib_logs - --tune - --wandb-sweep - -tn - - SSI +# - SSI + - Mortality - -n - cass # - -gc @@ -21,11 +23,12 @@ name: yaib_classification_benchmark parameters: data_dir: values: - - /sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/notes + - /home/vandewrp/projects/YAIB/data/YAIB_Datasets/data/mortality24/miiv + #- /sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/notes model: values: # - LogisticRegression - - LGBMClassifier + - TimesNet # - GRU # - LSTM # - TCN From 4e2223a4d336d6761811072cc6484c356d6b9647 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 26 Jun 2024 12:31:44 +0200 Subject: [PATCH 120/207] Check for task/model --- icu_benchmarks/run.py | 9 ++++++++- icu_benchmarks/run_utils.py | 15 ++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index b0bd5a31..ae114e45 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -18,6 +18,7 @@ setup_logging, import_preprocessor, name_datasets, + get_config_files, ) from icu_benchmarks.contants import RunMode @@ -31,6 +32,7 @@ def get_mode(mode: gin.REQUIRED): def main(my_args=tuple(sys.argv[1:])): args, _ = build_parser().parse_known_args(my_args) + # Set arguments for wandb sweep if args.wandb_sweep: args = apply_wandb_sweep(args) set_wandb_experiment_name(args, "run") @@ -48,10 +50,15 @@ def main(my_args=tuple(sys.argv[1:])): evaluate = args.eval experiment = args.experiment source_dir = args.source_dir + tasks, models = get_config_files(Path("configs")) + + if task not in tasks: + raise ValueError(f"Task {task} not found in tasks.") + if model not in models: + raise ValueError(f"Model {model} not found in models.") # Load task config gin.parse_config_file(f"configs/tasks/{task}.gin") mode = get_mode() - # Set arguments for wandb sweep # Set experiment name if name is None: diff --git a/icu_benchmarks/run_utils.py b/icu_benchmarks/run_utils.py index 85b676ac..c73ca720 100644 --- a/icu_benchmarks/run_utils.py +++ b/icu_benchmarks/run_utils.py @@ -15,7 +15,8 @@ from statistics import mean, pstdev from icu_benchmarks.models.utils import JsonResultLoggingEncoder from icu_benchmarks.wandb_utils import wandb_log - +import os +import glob def build_parser() -> ArgumentParser: """Builds an ArgumentParser for the command line. @@ -235,3 +236,15 @@ def setup_logging(date_format, log_format, verbose): for logger in loggers: logging.getLogger(logger).setLevel(logging.DEBUG) warnings.filterwarnings("default") + + +def get_config_files(config_dir: Path): + tasks = glob.glob(os.path.join(config_dir / "tasks", '*')) + models = glob.glob(os.path.join(config_dir / "prediction_models", '*')) + tasks = [os.path.splitext(os.path.basename(task))[0] for task in tasks] + models = [os.path.splitext(os.path.basename(model))[0] for model in models] + tasks.remove("common") + models.remove("common") + logging.info(f"Found tasks: {tasks}") + logging.info(f"Found models: {models}") + return tasks, models \ No newline at end of file From e1e1549404ae85b3363dff78afd86d95dc872883 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 26 Jun 2024 12:45:38 +0200 Subject: [PATCH 121/207] refactor models: split into file per architecture for DL and per library for ML models --- .../BalancedRFClassifier.gin | 17 ++ configs/prediction_models/CBClassifier.gin | 16 ++ configs/prediction_models/RFClassifier.gin | 8 +- configs/prediction_models/XGBClassifier.gin | 16 ++ configs/prediction_models/common/DLCommon.gin | 4 +- configs/prediction_models/common/MLCommon.gin | 6 +- configs/prediction_models/common/MLTuning.gin | 4 +- icu_benchmarks/models/dl_models/__init__.py | 0 .../models/{ => dl_models}/layers.py | 0 icu_benchmarks/models/dl_models/rnn.py | 85 ++++++++++ icu_benchmarks/models/dl_models/tcn.py | 62 ++++++++ .../models/dl_models/transformer.py | 148 ++++++++++++++++++ icu_benchmarks/models/ml_models/__init__.py | 0 icu_benchmarks/models/ml_models/catboost.py | 18 +++ icu_benchmarks/models/ml_models/imblearn.py | 12 ++ icu_benchmarks/models/ml_models/lgbm.py | 49 ++++++ .../{ml_models.py => ml_models/sklearn.py} | 54 +------ icu_benchmarks/models/ml_models/xgboost.py | 15 ++ icu_benchmarks/wandb_utils.py | 2 +- requirements.txt | 11 +- 20 files changed, 462 insertions(+), 65 deletions(-) create mode 100644 configs/prediction_models/BalancedRFClassifier.gin create mode 100644 configs/prediction_models/CBClassifier.gin create mode 100644 configs/prediction_models/XGBClassifier.gin create mode 100644 icu_benchmarks/models/dl_models/__init__.py rename icu_benchmarks/models/{ => dl_models}/layers.py (100%) create mode 100644 icu_benchmarks/models/dl_models/rnn.py create mode 100644 icu_benchmarks/models/dl_models/tcn.py create mode 100644 icu_benchmarks/models/dl_models/transformer.py create mode 100644 icu_benchmarks/models/ml_models/__init__.py create mode 100644 icu_benchmarks/models/ml_models/catboost.py create mode 100644 icu_benchmarks/models/ml_models/imblearn.py create mode 100644 icu_benchmarks/models/ml_models/lgbm.py rename icu_benchmarks/models/{ml_models.py => ml_models/sklearn.py} (60%) create mode 100644 icu_benchmarks/models/ml_models/xgboost.py diff --git a/configs/prediction_models/BalancedRFClassifier.gin b/configs/prediction_models/BalancedRFClassifier.gin new file mode 100644 index 00000000..f0f73d84 --- /dev/null +++ b/configs/prediction_models/BalancedRFClassifier.gin @@ -0,0 +1,17 @@ +# Settings for ImbLearn Balanced Random Forest Classifier. + +# Common settings for ML models +include "configs/prediction_models/common/MLCommon.gin" + +# Train params +train_common.model = @BalancedRFClassifier + +model/hyperparameter.class_to_tune = @BalancedRFClassifier +model/hyperparameter.n_estimators = (10, 50, 100, 200, 500) +model/hyperparameter.max_depth = [3, 5, 10, 15] +model/hyperparameter.max_features = ['sqrt', 'log2', None] +model/hyperparameter.scale_pos_weight = [1, 5, 10, 25, 50, 75, 99, 100, 1000] +model/hyperparameter.learning_rate = (0.005, 1, "log-uniform") + + + diff --git a/configs/prediction_models/CBClassifier.gin b/configs/prediction_models/CBClassifier.gin new file mode 100644 index 00000000..a33cc611 --- /dev/null +++ b/configs/prediction_models/CBClassifier.gin @@ -0,0 +1,16 @@ +# Settings for XGBoost classifier. + +# Common settings for ML models +include "configs/prediction_models/common/MLCommon.gin" + +# Train params +train_common.model = @CBClassifier + +model/hyperparameter.class_to_tune = @CBClassifier +model/hyperparameter.learning_rate = (0.01, 0.1, "log", 2) +model/hyperparameter.num_trees = [50, 100, 250, 500, 750] +model/hyperparameter.depth = [3, 5, 10, 15] +model/hyperparameter.scale_pos_weight = [1, 5, 10, 25, 50, 75, 99, 100, 1000] +model/hyperparameter.min_child_weight = [1, 0.5] +model/hyperparameter.max_delta_step = [0, 1, 2, 3, 4, 5, 10] +model/hyperparameter.border_count = [5, 10, 20, 50, 100, 200] diff --git a/configs/prediction_models/RFClassifier.gin b/configs/prediction_models/RFClassifier.gin index 72d03e66..61d627d6 100644 --- a/configs/prediction_models/RFClassifier.gin +++ b/configs/prediction_models/RFClassifier.gin @@ -8,11 +8,11 @@ train_common.model = @RFClassifier model/hyperparameter.class_to_tune = @RFClassifier model/hyperparameter.n_estimators = (10, 50, 100, 200, 500) -model/hyperparameter.max_depth = (None, 5, 10, 20) +model/hyperparameter.max_depth = (5, 10, 20) model/hyperparameter.min_samples_split = (2, 5, 10) model/hyperparameter.min_samples_leaf = (1, 2, 4) -model/hyperparameter.max_features = ('sqrt', 'log2', None) -model/hyperparameter.bootstrap = (True, False) -model/hyperparameter.class_weight = (None, 'balanced') +model/hyperparameter.max_features = ['sqrt', 'log2', None] +model/hyperparameter.bootstrap = [True, False] +model/hyperparameter.class_weight = [None, 'balanced'] diff --git a/configs/prediction_models/XGBClassifier.gin b/configs/prediction_models/XGBClassifier.gin new file mode 100644 index 00000000..5f19866f --- /dev/null +++ b/configs/prediction_models/XGBClassifier.gin @@ -0,0 +1,16 @@ +# Settings for XGBoost classifier. + +# Common settings for ML models +include "configs/prediction_models/common/MLCommon.gin" + +# Train params +train_common.model = @XGBClassifier + +model/hyperparameter.class_to_tune = @XGBClassifier +model/hyperparameter.learning_rate = (0.01, 0.1, "log", 2) +model/hyperparameter.n_estimators = [50, 100, 250, 500, 750] +model/hyperparameter.max_depth = [3, 5, 10, 15] +model/hyperparameter.scale_pos_weight = [1, 5, 10, 25, 50, 75, 99, 100, 1000] +model/hyperparameter.min_child_weight = [1, 0.5] +model/hyperparameter.max_delta_step = [0, 1, 2, 3, 4, 5, 10] +model/hyperparameter.colsample_bytree = [0.1, 0.25, 0.5, 0.75, 1.0] diff --git a/configs/prediction_models/common/DLCommon.gin b/configs/prediction_models/common/DLCommon.gin index c220e6ab..fbec8ad1 100644 --- a/configs/prediction_models/common/DLCommon.gin +++ b/configs/prediction_models/common/DLCommon.gin @@ -3,7 +3,9 @@ # Imports to register the models import gin.torch.external_configurables import icu_benchmarks.models.wrappers -import icu_benchmarks.models.dl_models +import icu_benchmarks.models.dl_models.rnn +import icu_benchmarks.models.dl_models.transformer +import icu_benchmarks.models.dl_models.tcn import icu_benchmarks.models.utils # Do not generate features from dynamic data diff --git a/configs/prediction_models/common/MLCommon.gin b/configs/prediction_models/common/MLCommon.gin index 460bceba..4d26b8c7 100644 --- a/configs/prediction_models/common/MLCommon.gin +++ b/configs/prediction_models/common/MLCommon.gin @@ -3,7 +3,11 @@ # Imports to register the models import gin.torch.external_configurables import icu_benchmarks.models.wrappers -import icu_benchmarks.models.ml_models +import icu_benchmarks.models.ml_models.sklearn +import icu_benchmarks.models.ml_models.lgbm +import icu_benchmarks.models.ml_models.xgboost +import icu_benchmarks.models.ml_models.imblearn +import icu_benchmarks.models.ml_models.catboost import icu_benchmarks.models.utils # Patience for early stopping diff --git a/configs/prediction_models/common/MLTuning.gin b/configs/prediction_models/common/MLTuning.gin index 92fa0f0c..34bc070a 100644 --- a/configs/prediction_models/common/MLTuning.gin +++ b/configs/prediction_models/common/MLTuning.gin @@ -1,5 +1,5 @@ # Hyperparameter tuner settings for classical Machine Learning. tune_hyperparameters.scopes = ["model"] tune_hyperparameters.n_initial_points = 10 -tune_hyperparameters.n_calls = 3 -tune_hyperparameters.folds_to_tune_on = 1 \ No newline at end of file +tune_hyperparameters.n_calls = 50 +tune_hyperparameters.folds_to_tune_on = 5 \ No newline at end of file diff --git a/icu_benchmarks/models/dl_models/__init__.py b/icu_benchmarks/models/dl_models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/icu_benchmarks/models/layers.py b/icu_benchmarks/models/dl_models/layers.py similarity index 100% rename from icu_benchmarks/models/layers.py rename to icu_benchmarks/models/dl_models/layers.py diff --git a/icu_benchmarks/models/dl_models/rnn.py b/icu_benchmarks/models/dl_models/rnn.py new file mode 100644 index 00000000..327526c2 --- /dev/null +++ b/icu_benchmarks/models/dl_models/rnn.py @@ -0,0 +1,85 @@ +import gin +from torch import nn as nn + +from icu_benchmarks.contants import RunMode +from icu_benchmarks.models.wrappers import DLPredictionWrapper + + +@gin.configurable +class RNNet(DLPredictionWrapper): + """Torch standard RNN model""" + + _supported_run_modes = [RunMode.classification, RunMode.regression] + + def __init__(self, input_size, hidden_dim, layer_dim, num_classes, *args, **kwargs): + super().__init__( + input_size=input_size, hidden_dim=hidden_dim, layer_dim=layer_dim, num_classes=num_classes, *args, **kwargs + ) + self.hidden_dim = hidden_dim + self.layer_dim = layer_dim + self.rnn = nn.RNN(input_size[2], hidden_dim, layer_dim, batch_first=True) + self.logit = nn.Linear(hidden_dim, num_classes) + + def init_hidden(self, x): + h0 = x.new_zeros(self.layer_dim, x.size(0), self.hidden_dim) + return h0 + + def forward(self, x): + h0 = self.init_hidden(x) + out, hn = self.rnn(x, h0) + pred = self.logit(out) + return pred + + +@gin.configurable +class LSTMNet(DLPredictionWrapper): + """Torch standard LSTM model.""" + + _supported_run_modes = [RunMode.classification, RunMode.regression] + + def __init__(self, input_size, hidden_dim, layer_dim, num_classes, *args, **kwargs): + super().__init__( + input_size=input_size, hidden_dim=hidden_dim, layer_dim=layer_dim, num_classes=num_classes, *args, **kwargs + ) + self.hidden_dim = hidden_dim + self.layer_dim = layer_dim + self.rnn = nn.LSTM(input_size[2], hidden_dim, layer_dim, batch_first=True) + self.logit = nn.Linear(hidden_dim, num_classes) + + def init_hidden(self, x): + h0 = x.new_zeros(self.layer_dim, x.size(0), self.hidden_dim) + c0 = x.new_zeros(self.layer_dim, x.size(0), self.hidden_dim) + return [t for t in (h0, c0)] + + def forward(self, x): + h0, c0 = self.init_hidden(x) + out, h = self.rnn(x, (h0, c0)) + pred = self.logit(out) + return pred + + +@gin.configurable +class GRUNet(DLPredictionWrapper): + """Torch standard GRU model.""" + + _supported_run_modes = [RunMode.classification, RunMode.regression] + + def __init__(self, input_size, hidden_dim, layer_dim, num_classes, *args, **kwargs): + super().__init__( + input_size=input_size, hidden_dim=hidden_dim, layer_dim=layer_dim, num_classes=num_classes, *args, **kwargs + ) + self.hidden_dim = hidden_dim + self.layer_dim = layer_dim + self.rnn = nn.GRU(input_size[2], hidden_dim, layer_dim, batch_first=True) + self.logit = nn.Linear(hidden_dim, num_classes) + + def init_hidden(self, x): + h0 = x.new_zeros(self.layer_dim, x.size(0), self.hidden_dim) + return h0 + + def forward(self, x): + h0 = self.init_hidden(x) + out, hn = self.rnn(x, h0) + pred = self.logit(out) + + return pred diff --git a/icu_benchmarks/models/dl_models/tcn.py b/icu_benchmarks/models/dl_models/tcn.py new file mode 100644 index 00000000..950310e1 --- /dev/null +++ b/icu_benchmarks/models/dl_models/tcn.py @@ -0,0 +1,62 @@ +from numbers import Integral + +import gin +import numpy as np +from torch import nn as nn + +from icu_benchmarks.contants import RunMode +from icu_benchmarks.models.dl_models.layers import TemporalBlock +from icu_benchmarks.models.wrappers import DLPredictionWrapper + + +@gin.configurable +class TemporalConvNet(DLPredictionWrapper): + """Temporal Convolutional Network. Adapted from TCN original paper https://github.com/locuslab/TCN""" + + _supported_run_modes = [RunMode.classification, RunMode.regression] + + def __init__(self, input_size, num_channels, num_classes, *args, max_seq_length=0, kernel_size=2, dropout=0.0, **kwargs): + super().__init__( + input_size=input_size, + num_channels=num_channels, + num_classes=num_classes, + *args, + max_seq_length=max_seq_length, + kernel_size=kernel_size, + dropout=dropout, + **kwargs, + ) + layers = [] + + # We compute automatically the depth based on the desired seq_length. + if isinstance(num_channels, Integral) and max_seq_length: + num_channels = [num_channels] * int(np.ceil(np.log(max_seq_length / 2) / np.log(kernel_size))) + elif isinstance(num_channels, Integral) and not max_seq_length: + raise Exception("a maximum sequence length needs to be provided if num_channels is int") + + num_levels = len(num_channels) + for i in range(num_levels): + dilation_size = 2**i + in_channels = input_size[2] if i == 0 else num_channels[i - 1] + out_channels = num_channels[i] + layers += [ + TemporalBlock( + in_channels, + out_channels, + kernel_size, + stride=1, + dilation=dilation_size, + padding=(kernel_size - 1) * dilation_size, + dropout=dropout, + ) + ] + + self.network = nn.Sequential(*layers) + self.logit = nn.Linear(num_channels[-1], num_classes) + + def forward(self, x): + x = x.permute(0, 2, 1) # Permute to channel first + o = self.network(x) + o = o.permute(0, 2, 1) # Permute to channel last + pred = self.logit(o) + return pred diff --git a/icu_benchmarks/models/dl_models/transformer.py b/icu_benchmarks/models/dl_models/transformer.py new file mode 100644 index 00000000..a9273884 --- /dev/null +++ b/icu_benchmarks/models/dl_models/transformer.py @@ -0,0 +1,148 @@ +import gin +from torch import nn as nn + +from icu_benchmarks.contants import RunMode +from icu_benchmarks.models.dl_models.layers import PositionalEncoding, TransformerBlock, LocalBlock +from icu_benchmarks.models.wrappers import DLPredictionWrapper + + +@gin.configurable +class Transformer(DLPredictionWrapper): + """Transformer model as defined by the HiRID-Benchmark (https://github.com/ratschlab/HIRID-ICU-Benchmark).""" + + _supported_run_modes = [RunMode.classification, RunMode.regression] + + def __init__( + self, + input_size, + hidden, + heads, + ff_hidden_mult, + depth, + num_classes, + *args, + dropout=0.0, + l1_reg=0, + pos_encoding=True, + dropout_att=0.0, + **kwargs, + ): + super().__init__( + input_size=input_size, + hidden=hidden, + heads=heads, + ff_hidden_mult=ff_hidden_mult, + depth=depth, + num_classes=num_classes, + *args, + dropout=dropout, + l1_reg=l1_reg, + pos_encoding=pos_encoding, + dropout_att=dropout_att, + **kwargs, + ) + hidden = hidden if hidden % 2 == 0 else hidden + 1 # Make sure hidden is even + self.input_embedding = nn.Linear(input_size[2], hidden) # This acts as a time-distributed layer by defaults + if pos_encoding: + self.pos_encoder = PositionalEncoding(hidden) + else: + self.pos_encoder = None + + tblocks = [] + for i in range(depth): + tblocks.append( + TransformerBlock( + emb=hidden, + hidden=hidden, + heads=heads, + mask=True, + ff_hidden_mult=ff_hidden_mult, + dropout=dropout, + dropout_att=dropout_att, + ) + ) + + self.tblocks = nn.Sequential(*tblocks) + self.logit = nn.Linear(hidden, num_classes) + self.l1_reg = l1_reg + + def forward(self, x): + x = self.input_embedding(x) + if self.pos_encoder is not None: + x = self.pos_encoder(x) + x = self.tblocks(x) + pred = self.logit(x) + + return pred + + +@gin.configurable +class LocalTransformer(DLPredictionWrapper): + _supported_run_modes = [RunMode.classification, RunMode.regression] + + def __init__( + self, + input_size, + hidden, + heads, + ff_hidden_mult, + depth, + num_classes, + *args, + dropout=0.0, + l1_reg=0, + pos_encoding=True, + local_context=1, + dropout_att=0.0, + **kwargs, + ): + super().__init__( + input_size=input_size, + hidden=hidden, + heads=heads, + ff_hidden_mult=ff_hidden_mult, + depth=depth, + num_classes=num_classes, + *args, + dropout=dropout, + l1_reg=l1_reg, + pos_encoding=pos_encoding, + local_context=local_context, + dropout_att=dropout_att, + **kwargs, + ) + + hidden = hidden if hidden % 2 == 0 else hidden + 1 # Make sure hidden is even + self.input_embedding = nn.Linear(input_size[2], hidden) # This acts as a time-distributed layer by defaults + if pos_encoding: + self.pos_encoder = PositionalEncoding(hidden) + else: + self.pos_encoder = None + + tblocks = [] + for i in range(depth): + tblocks.append( + LocalBlock( + emb=hidden, + hidden=hidden, + heads=heads, + mask=True, + ff_hidden_mult=ff_hidden_mult, + local_context=local_context, + dropout=dropout, + dropout_att=dropout_att, + ) + ) + + self.tblocks = nn.Sequential(*tblocks) + self.logit = nn.Linear(hidden, num_classes) + self.l1_reg = l1_reg + + def forward(self, x): + x = self.input_embedding(x) + if self.pos_encoder is not None: + x = self.pos_encoder(x) + x = self.tblocks(x) + pred = self.logit(x) + + return pred diff --git a/icu_benchmarks/models/ml_models/__init__.py b/icu_benchmarks/models/ml_models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/icu_benchmarks/models/ml_models/catboost.py b/icu_benchmarks/models/ml_models/catboost.py new file mode 100644 index 00000000..857c0d68 --- /dev/null +++ b/icu_benchmarks/models/ml_models/catboost.py @@ -0,0 +1,18 @@ +import gin +import catboost as cb +import numpy as np +import wandb + +from icu_benchmarks.contants import RunMode +from icu_benchmarks.models.wrappers import MLWrapper +@gin.configurable +class CBClassifier(MLWrapper): + _supported_run_modes = [RunMode.classification] + + def __init__(self, *args, **kwargs): + self.model = self.set_model_args(cb.CatBoostClassifier, *args, **kwargs) + super().__init__(*args, **kwargs) + + def predict(self, features): + """Predicts labels for the given features.""" + return self.model.predict_proba(features) diff --git a/icu_benchmarks/models/ml_models/imblearn.py b/icu_benchmarks/models/ml_models/imblearn.py new file mode 100644 index 00000000..0e84d1f4 --- /dev/null +++ b/icu_benchmarks/models/ml_models/imblearn.py @@ -0,0 +1,12 @@ +from imblearn.ensemble import BalancedRandomForestClassifier +from icu_benchmarks.contants import RunMode +from icu_benchmarks.models.wrappers import MLWrapper +import gin + +@gin.configurable +class BalancedRFClassifier(MLWrapper): + _supported_run_modes = [RunMode.classification] + + def __init__(self, *args, **kwargs): + self.model = self.set_model_args(BalancedRandomForestClassifier, *args, **kwargs) + super().__init__(*args, **kwargs) \ No newline at end of file diff --git a/icu_benchmarks/models/ml_models/lgbm.py b/icu_benchmarks/models/ml_models/lgbm.py new file mode 100644 index 00000000..102da32d --- /dev/null +++ b/icu_benchmarks/models/ml_models/lgbm.py @@ -0,0 +1,49 @@ +import gin +import lightgbm as lgbm +import numpy as np +import wandb +from wandb.integration.lightgbm import wandb_callback as wandb_lgbm + +from icu_benchmarks.contants import RunMode +from icu_benchmarks.models.wrappers import MLWrapper + + +class LGBMWrapper(MLWrapper): + def fit_model(self, train_data, train_labels, val_data, val_labels): + """Fitting function for LGBM models.""" + self.model.set_params(random_state=np.random.get_state()[1][0]) + callbacks = [lgbm.early_stopping(self.hparams.patience, verbose=True), lgbm.log_evaluation(period=-1)] + + if wandb.run is not None: + callbacks.append(wandb_lgbm()) + + self.model = self.model.fit( + train_data, + train_labels, + eval_set=(val_data, val_labels), + callbacks=callbacks, + ) + val_loss = list(self.model.best_score_["valid_0"].values())[0] + return val_loss + + +@gin.configurable +class LGBMClassifier(LGBMWrapper): + _supported_run_modes = [RunMode.classification] + + def __init__(self, *args, **kwargs): + self.model = self.set_model_args(lgbm.LGBMClassifier, *args, **kwargs) + super().__init__(*args, **kwargs) + + def predict(self, features): + """Predicts labels for the given features.""" + return self.model.predict_proba(features) + + +@gin.configurable +class LGBMRegressor(LGBMWrapper): + _supported_run_modes = [RunMode.regression] + + def __init__(self, *args, **kwargs): + self.model = self.set_model_args(lgbm.LGBMRegressor, *args, **kwargs) + super().__init__(*args, **kwargs) diff --git a/icu_benchmarks/models/ml_models.py b/icu_benchmarks/models/ml_models/sklearn.py similarity index 60% rename from icu_benchmarks/models/ml_models.py rename to icu_benchmarks/models/ml_models/sklearn.py index 5e52921b..30312c28 100644 --- a/icu_benchmarks/models/ml_models.py +++ b/icu_benchmarks/models/ml_models/sklearn.py @@ -1,59 +1,9 @@ import gin -import lightgbm as lgbm -import numpy as np -import wandb -from sklearn import linear_model -from sklearn import ensemble -from sklearn import neural_network -from sklearn import svm -from icu_benchmarks.models.wrappers import MLWrapper +from sklearn import linear_model, ensemble, svm, neural_network from icu_benchmarks.contants import RunMode -from wandb.integration.lightgbm import wandb_callback - - -class LGBMWrapper(MLWrapper): - def fit_model(self, train_data, train_labels, val_data, val_labels): - """Fitting function for LGBM models.""" - self.model.set_params(random_state=np.random.get_state()[1][0]) - callbacks = [lgbm.early_stopping(self.hparams.patience, verbose=True), lgbm.log_evaluation(period=-1)] - - if wandb.run is not None: - callbacks.append(wandb_callback()) - - self.model = self.model.fit( - train_data, - train_labels, - eval_set=(val_data, val_labels), - verbose=True, - callbacks=callbacks, - ) - val_loss = list(self.model.best_score_["valid_0"].values())[0] - return val_loss - - -@gin.configurable -class LGBMClassifier(LGBMWrapper): - _supported_run_modes = [RunMode.classification] - - def __init__(self, *args, **kwargs): - self.model = self.set_model_args(lgbm.LGBMClassifier, *args, **kwargs) - super().__init__(*args, **kwargs) - - def predict(self, features): - """Predicts labels for the given features.""" - return self.model.predict_proba(features) - - -@gin.configurable -class LGBMRegressor(LGBMWrapper): - _supported_run_modes = [RunMode.regression] - - def __init__(self, *args, **kwargs): - self.model = self.set_model_args(lgbm.LGBMRegressor, *args, **kwargs) - super().__init__(*args, **kwargs) +from icu_benchmarks.models.wrappers import MLWrapper -# Scikit-learn models @gin.configurable class LogisticRegression(MLWrapper): __supported_run_modes = [RunMode.classification] diff --git a/icu_benchmarks/models/ml_models/xgboost.py b/icu_benchmarks/models/ml_models/xgboost.py new file mode 100644 index 00000000..22eda4d2 --- /dev/null +++ b/icu_benchmarks/models/ml_models/xgboost.py @@ -0,0 +1,15 @@ +import gin +from icu_benchmarks.contants import RunMode +from icu_benchmarks.models.wrappers import MLWrapper +import xgboost as xgb +@gin.configurable +class XGBClassifier(MLWrapper): + _supported_run_modes = [RunMode.classification] + + def __init__(self, *args, **kwargs): + self.model = self.set_model_args(xgb.XGBClassifier, *args, **kwargs) + super().__init__(*args, **kwargs) + + def predict(self, features): + """Predicts labels for the given features.""" + return self.model.predict_proba(features) \ No newline at end of file diff --git a/icu_benchmarks/wandb_utils.py b/icu_benchmarks/wandb_utils.py index 6148af5d..225693a4 100644 --- a/icu_benchmarks/wandb_utils.py +++ b/icu_benchmarks/wandb_utils.py @@ -30,7 +30,7 @@ def apply_wandb_sweep(args: Namespace) -> Namespace: Returns: Namespace: arguments with sweep configuration applied (some are applied via hyperparams) """ - wandb.init(allow_val_change=True) + wandb.init(allow_val_change=True, dir=args.log_dir) sweep_config = wandb.config args.__dict__.update(sweep_config) if args.hyperparams is None: diff --git a/requirements.txt b/requirements.txt index b24bd19f..ce4865c2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,9 +11,12 @@ torch==2.3.1 lightning==2.3.0 torchmetrics==1.0.3 #pytorch-cuda==11.8 -lightgbm==3.3.5 +lightgbm==4.4.0 +xgboost==2.1.0 +imbalanced-learn==0.12.3 +catboost==1.2.5 numpy==1.24.3 -pandas==2.0.0 +pandas==2.2.2 pyarrow==14.0.1 pytest==7.3.1 scikit-learn==1.5.0 @@ -23,9 +26,9 @@ einops==0.6.1 hydra-core==1.3 optuna==3.6.1 optuna-integration==3.6.0 -wandb==0.17.2 +wandb==0.17.3 recipies==0.1.3 #Fixed version because of NumPy incompatibility and stale development status. scikit-optimize-fix==0.9.1 hydra-submitit-launcher==1.2.0 -pytest-runner==6.0.1 \ No newline at end of file +pytest-runner==6.0.1 From 29f1abe82df3ffde6e4c8b9f893e0465f0b2a423 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 26 Jun 2024 12:46:15 +0200 Subject: [PATCH 122/207] Improve hyperparameter tuning parsing --- icu_benchmarks/tuning/hyperparameters.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/icu_benchmarks/tuning/hyperparameters.py b/icu_benchmarks/tuning/hyperparameters.py index b1f5a73c..852bf041 100644 --- a/icu_benchmarks/tuning/hyperparameters.py +++ b/icu_benchmarks/tuning/hyperparameters.py @@ -259,10 +259,22 @@ def objective(trail, hyperparams_bounds, hyperparams_names): logging.info(f"Bounds: {hyperparams_bounds}, Names: {hyperparams_names}") for name, value in zip(hyperparams_names, hyperparams_bounds): if isinstance(value, tuple): - if isinstance(value[0], int) and isinstance(value[1], int): - hyperparams[name] = trail.suggest_int(name, value[0], value[1], log=len(value) > 2 and value[2] == "log") + if isinstance(value[0], (int, float)) and isinstance(value[1], (int, float)): + if len(value)>2 and isinstance(value[2], str): + if isinstance(value[0], int) and isinstance(value[1], int): + hyperparams[name] = trail.suggest_int(name, value[0], value[1], log= value[2] == "log") + elif isinstance(value[0], float) and isinstance(value[1], float): + hyperparams[name] = trail.suggest_float(name, value[0], value[1], log=value[2] == "log") + else: + hyperparams[name] = trail.suggest_categorical(name, value) + # elif len(value)>2: + # hyperparams[name] = trail.suggest_categorical(name, value) + elif isinstance(value[0], int) and isinstance(value[1], int): + hyperparams[name] = trail.suggest_int(name, value[0], value[1]) + else: + hyperparams[name] = trail.suggest_float(name, value[0], value[1]) else: - hyperparams[name] = trail.suggest_float(name, value[0], value[1], log=len(value) > 2 and value[2] == "log") + hyperparams[name]=trail.suggest_categorical(name, value) else: hyperparams[name] = trail.suggest_categorical(name, value) return bind_params_and_train(hyperparams) @@ -272,7 +284,7 @@ def tune_step_callback(study: optuna.study.Study, trial: optuna.trial.FrozenTria highlight = study.trials[-1] == study.best_trial # highlight if best so far log_table_row(header, TUNE) log_table_row(table_cells, TUNE, align=Align.RIGHT, header=header, highlight=highlight) - wandb_log({"hp-iteration": len(study.trials)}) + # wandb_log({"hp-iteration": len(study.trials)}) if do_tune: log_full_line("STARTING TUNING", level=TUNE, char="=") @@ -330,6 +342,7 @@ def bind_params_and_train(hyperparams): if wandb: wandb_kwargs = { "config": {"sampler": sampler}, + "allow_val_change": True, } wandbc = WeightsAndBiasesCallback(metric_name="loss", wandb_kwargs=wandb_kwargs) callbacks.append(wandbc) From f1841d861b9c94d623a5fdca7b776b278470f34b Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 26 Jun 2024 12:46:39 +0200 Subject: [PATCH 123/207] deleted dl_models.py --- icu_benchmarks/models/dl_models.py | 282 ----------------------------- 1 file changed, 282 deletions(-) delete mode 100644 icu_benchmarks/models/dl_models.py diff --git a/icu_benchmarks/models/dl_models.py b/icu_benchmarks/models/dl_models.py deleted file mode 100644 index 0fb1b0d2..00000000 --- a/icu_benchmarks/models/dl_models.py +++ /dev/null @@ -1,282 +0,0 @@ -import gin -from numbers import Integral -import numpy as np -import torch.nn as nn -from icu_benchmarks.contants import RunMode -from icu_benchmarks.models.layers import TransformerBlock, LocalBlock, TemporalBlock, PositionalEncoding -from icu_benchmarks.models.wrappers import DLPredictionWrapper - - -@gin.configurable -class RNNet(DLPredictionWrapper): - """Torch standard RNN model""" - - _supported_run_modes = [RunMode.classification, RunMode.regression] - - def __init__(self, input_size, hidden_dim, layer_dim, num_classes, *args, **kwargs): - super().__init__( - input_size=input_size, hidden_dim=hidden_dim, layer_dim=layer_dim, num_classes=num_classes, *args, **kwargs - ) - self.hidden_dim = hidden_dim - self.layer_dim = layer_dim - self.rnn = nn.RNN(input_size[2], hidden_dim, layer_dim, batch_first=True) - self.logit = nn.Linear(hidden_dim, num_classes) - - def init_hidden(self, x): - h0 = x.new_zeros(self.layer_dim, x.size(0), self.hidden_dim) - return h0 - - def forward(self, x): - h0 = self.init_hidden(x) - out, hn = self.rnn(x, h0) - pred = self.logit(out) - return pred - - -@gin.configurable -class LSTMNet(DLPredictionWrapper): - """Torch standard LSTM model.""" - - _supported_run_modes = [RunMode.classification, RunMode.regression] - - def __init__(self, input_size, hidden_dim, layer_dim, num_classes, *args, **kwargs): - super().__init__( - input_size=input_size, hidden_dim=hidden_dim, layer_dim=layer_dim, num_classes=num_classes, *args, **kwargs - ) - self.hidden_dim = hidden_dim - self.layer_dim = layer_dim - self.rnn = nn.LSTM(input_size[2], hidden_dim, layer_dim, batch_first=True) - self.logit = nn.Linear(hidden_dim, num_classes) - - def init_hidden(self, x): - h0 = x.new_zeros(self.layer_dim, x.size(0), self.hidden_dim) - c0 = x.new_zeros(self.layer_dim, x.size(0), self.hidden_dim) - return [t for t in (h0, c0)] - - def forward(self, x): - h0, c0 = self.init_hidden(x) - out, h = self.rnn(x, (h0, c0)) - pred = self.logit(out) - return pred - - -@gin.configurable -class GRUNet(DLPredictionWrapper): - """Torch standard GRU model.""" - - _supported_run_modes = [RunMode.classification, RunMode.regression] - - def __init__(self, input_size, hidden_dim, layer_dim, num_classes, *args, **kwargs): - super().__init__( - input_size=input_size, hidden_dim=hidden_dim, layer_dim=layer_dim, num_classes=num_classes, *args, **kwargs - ) - self.hidden_dim = hidden_dim - self.layer_dim = layer_dim - self.rnn = nn.GRU(input_size[2], hidden_dim, layer_dim, batch_first=True) - self.logit = nn.Linear(hidden_dim, num_classes) - - def init_hidden(self, x): - h0 = x.new_zeros(self.layer_dim, x.size(0), self.hidden_dim) - return h0 - - def forward(self, x): - h0 = self.init_hidden(x) - out, hn = self.rnn(x, h0) - pred = self.logit(out) - - return pred - - -@gin.configurable -class Transformer(DLPredictionWrapper): - """Transformer model as defined by the HiRID-Benchmark (https://github.com/ratschlab/HIRID-ICU-Benchmark).""" - - _supported_run_modes = [RunMode.classification, RunMode.regression] - - def __init__( - self, - input_size, - hidden, - heads, - ff_hidden_mult, - depth, - num_classes, - *args, - dropout=0.0, - l1_reg=0, - pos_encoding=True, - dropout_att=0.0, - **kwargs, - ): - super().__init__( - input_size=input_size, - hidden=hidden, - heads=heads, - ff_hidden_mult=ff_hidden_mult, - depth=depth, - num_classes=num_classes, - *args, - dropout=dropout, - l1_reg=l1_reg, - pos_encoding=pos_encoding, - dropout_att=dropout_att, - **kwargs, - ) - hidden = hidden if hidden % 2 == 0 else hidden + 1 # Make sure hidden is even - self.input_embedding = nn.Linear(input_size[2], hidden) # This acts as a time-distributed layer by defaults - if pos_encoding: - self.pos_encoder = PositionalEncoding(hidden) - else: - self.pos_encoder = None - - tblocks = [] - for i in range(depth): - tblocks.append( - TransformerBlock( - emb=hidden, - hidden=hidden, - heads=heads, - mask=True, - ff_hidden_mult=ff_hidden_mult, - dropout=dropout, - dropout_att=dropout_att, - ) - ) - - self.tblocks = nn.Sequential(*tblocks) - self.logit = nn.Linear(hidden, num_classes) - self.l1_reg = l1_reg - - def forward(self, x): - x = self.input_embedding(x) - if self.pos_encoder is not None: - x = self.pos_encoder(x) - x = self.tblocks(x) - pred = self.logit(x) - - return pred - - -@gin.configurable -class LocalTransformer(DLPredictionWrapper): - _supported_run_modes = [RunMode.classification, RunMode.regression] - - def __init__( - self, - input_size, - hidden, - heads, - ff_hidden_mult, - depth, - num_classes, - *args, - dropout=0.0, - l1_reg=0, - pos_encoding=True, - local_context=1, - dropout_att=0.0, - **kwargs, - ): - super().__init__( - input_size=input_size, - hidden=hidden, - heads=heads, - ff_hidden_mult=ff_hidden_mult, - depth=depth, - num_classes=num_classes, - *args, - dropout=dropout, - l1_reg=l1_reg, - pos_encoding=pos_encoding, - local_context=local_context, - dropout_att=dropout_att, - **kwargs, - ) - - hidden = hidden if hidden % 2 == 0 else hidden + 1 # Make sure hidden is even - self.input_embedding = nn.Linear(input_size[2], hidden) # This acts as a time-distributed layer by defaults - if pos_encoding: - self.pos_encoder = PositionalEncoding(hidden) - else: - self.pos_encoder = None - - tblocks = [] - for i in range(depth): - tblocks.append( - LocalBlock( - emb=hidden, - hidden=hidden, - heads=heads, - mask=True, - ff_hidden_mult=ff_hidden_mult, - local_context=local_context, - dropout=dropout, - dropout_att=dropout_att, - ) - ) - - self.tblocks = nn.Sequential(*tblocks) - self.logit = nn.Linear(hidden, num_classes) - self.l1_reg = l1_reg - - def forward(self, x): - x = self.input_embedding(x) - if self.pos_encoder is not None: - x = self.pos_encoder(x) - x = self.tblocks(x) - pred = self.logit(x) - - return pred - - -@gin.configurable -class TemporalConvNet(DLPredictionWrapper): - """Temporal Convolutional Network. Adapted from TCN original paper https://github.com/locuslab/TCN""" - - _supported_run_modes = [RunMode.classification, RunMode.regression] - - def __init__(self, input_size, num_channels, num_classes, *args, max_seq_length=0, kernel_size=2, dropout=0.0, **kwargs): - super().__init__( - input_size=input_size, - num_channels=num_channels, - num_classes=num_classes, - *args, - max_seq_length=max_seq_length, - kernel_size=kernel_size, - dropout=dropout, - **kwargs, - ) - layers = [] - - # We compute automatically the depth based on the desired seq_length. - if isinstance(num_channels, Integral) and max_seq_length: - num_channels = [num_channels] * int(np.ceil(np.log(max_seq_length / 2) / np.log(kernel_size))) - elif isinstance(num_channels, Integral) and not max_seq_length: - raise Exception("a maximum sequence length needs to be provided if num_channels is int") - - num_levels = len(num_channels) - for i in range(num_levels): - dilation_size = 2**i - in_channels = input_size[2] if i == 0 else num_channels[i - 1] - out_channels = num_channels[i] - layers += [ - TemporalBlock( - in_channels, - out_channels, - kernel_size, - stride=1, - dilation=dilation_size, - padding=(kernel_size - 1) * dilation_size, - dropout=dropout, - ) - ] - - self.network = nn.Sequential(*layers) - self.logit = nn.Linear(num_channels[-1], num_classes) - - def forward(self, x): - x = x.permute(0, 2, 1) # Permute to channel first - o = self.network(x) - o = o.permute(0, 2, 1) # Permute to channel last - pred = self.logit(o) - return pred From 51b9ba8ff0fcb15b13ecc4bab34dab12fa438090 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 26 Jun 2024 13:06:17 +0200 Subject: [PATCH 124/207] gpu support for ml models --- icu_benchmarks/models/ml_models/catboost.py | 4 +++- icu_benchmarks/models/ml_models/xgboost.py | 2 +- icu_benchmarks/models/train.py | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/icu_benchmarks/models/ml_models/catboost.py b/icu_benchmarks/models/ml_models/catboost.py index 857c0d68..7ecf7f51 100644 --- a/icu_benchmarks/models/ml_models/catboost.py +++ b/icu_benchmarks/models/ml_models/catboost.py @@ -1,3 +1,5 @@ +import logging + import gin import catboost as cb import numpy as np @@ -10,7 +12,7 @@ class CBClassifier(MLWrapper): _supported_run_modes = [RunMode.classification] def __init__(self, *args, **kwargs): - self.model = self.set_model_args(cb.CatBoostClassifier, *args, **kwargs) + self.model = self.set_model_args(cb.CatBoostClassifier, task_type="GPU" if not kwargs['cpu'] else "CPU", *args, **kwargs) super().__init__(*args, **kwargs) def predict(self, features): diff --git a/icu_benchmarks/models/ml_models/xgboost.py b/icu_benchmarks/models/ml_models/xgboost.py index 22eda4d2..e8467e64 100644 --- a/icu_benchmarks/models/ml_models/xgboost.py +++ b/icu_benchmarks/models/ml_models/xgboost.py @@ -7,7 +7,7 @@ class XGBClassifier(MLWrapper): _supported_run_modes = [RunMode.classification] def __init__(self, *args, **kwargs): - self.model = self.set_model_args(xgb.XGBClassifier, *args, **kwargs) + self.model = self.set_model_args(xgb.XGBClassifier, device="cpu" if kwargs["cpu"] else "cuda", *args, **kwargs) super().__init__(*args, **kwargs) def predict(self, features): diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 321c30cb..b9f06e6d 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -116,9 +116,9 @@ def train_common( data_shape = next(iter(train_loader))[0].shape if load_weights: - model = load_model(model, source_dir, pl_model=pl_model) + model = load_model(model, source_dir, pl_model=pl_model, cpu=cpu) else: - model = model(optimizer=optimizer, input_size=data_shape, epochs=epochs, run_mode=mode) + model = model(optimizer=optimizer, input_size=data_shape, epochs=epochs, run_mode=mode, cpu=cpu) model.set_weight(weight, train_dataset) model.set_trained_columns(train_dataset.get_feature_names()) From 1722d95de0b46d548f11fc8c032570b28ba46046 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 26 Jun 2024 15:27:14 +0200 Subject: [PATCH 125/207] new model --- configs/prediction_models/RUSBClassifier.gin | 14 ++++++++++++++ icu_benchmarks/models/ml_models/imblearn.py | 11 +++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 configs/prediction_models/RUSBClassifier.gin diff --git a/configs/prediction_models/RUSBClassifier.gin b/configs/prediction_models/RUSBClassifier.gin new file mode 100644 index 00000000..836187c9 --- /dev/null +++ b/configs/prediction_models/RUSBClassifier.gin @@ -0,0 +1,14 @@ +# Settings for ImbLearn Balanced Random Forest Classifier. + +# Common settings for ML models +include "configs/prediction_models/common/MLCommon.gin" + +# Train params +train_common.model = @RUSBClassifier + +model/hyperparameter.class_to_tune = @RUSBClassifier +model/hyperparameter.n_estimators = (10, 50, 100, 200, 500) +model/hyperparameter.learning_rate = (0.005, 1, "log-uniform") +model/hyperparameter.sampling_strategy = "auto" + + diff --git a/icu_benchmarks/models/ml_models/imblearn.py b/icu_benchmarks/models/ml_models/imblearn.py index 0e84d1f4..3a83205a 100644 --- a/icu_benchmarks/models/ml_models/imblearn.py +++ b/icu_benchmarks/models/ml_models/imblearn.py @@ -1,12 +1,19 @@ -from imblearn.ensemble import BalancedRandomForestClassifier +from imblearn.ensemble import BalancedRandomForestClassifier, RUSBoostClassifier from icu_benchmarks.contants import RunMode from icu_benchmarks.models.wrappers import MLWrapper import gin @gin.configurable -class BalancedRFClassifier(MLWrapper): +class BRFClassifier(MLWrapper): _supported_run_modes = [RunMode.classification] def __init__(self, *args, **kwargs): self.model = self.set_model_args(BalancedRandomForestClassifier, *args, **kwargs) + super().__init__(*args, **kwargs) +@gin.configurable +class RUSBClassifier(MLWrapper): + _supported_run_modes = [RunMode.classification] + + def __init__(self, *args, **kwargs): + self.model = self.set_model_args(RUSBoostClassifier, *args, **kwargs) super().__init__(*args, **kwargs) \ No newline at end of file From c753039fbac5b6c23f65841b4d48736bbe12f119 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 26 Jun 2024 15:27:21 +0200 Subject: [PATCH 126/207] new model --- .../{BalancedRFClassifier.gin => BRFClassifier.gin} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename configs/prediction_models/{BalancedRFClassifier.gin => BRFClassifier.gin} (82%) diff --git a/configs/prediction_models/BalancedRFClassifier.gin b/configs/prediction_models/BRFClassifier.gin similarity index 82% rename from configs/prediction_models/BalancedRFClassifier.gin rename to configs/prediction_models/BRFClassifier.gin index f0f73d84..85e6ef98 100644 --- a/configs/prediction_models/BalancedRFClassifier.gin +++ b/configs/prediction_models/BRFClassifier.gin @@ -4,9 +4,9 @@ include "configs/prediction_models/common/MLCommon.gin" # Train params -train_common.model = @BalancedRFClassifier +train_common.model = @BRFClassifier -model/hyperparameter.class_to_tune = @BalancedRFClassifier +model/hyperparameter.class_to_tune = @BRFClassifier model/hyperparameter.n_estimators = (10, 50, 100, 200, 500) model/hyperparameter.max_depth = [3, 5, 10, 15] model/hyperparameter.max_features = ['sqrt', 'log2', None] From 8a7fbdf6a3424d625146c6051227fe0870af91b4 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 27 Jun 2024 11:27:02 +0200 Subject: [PATCH 127/207] hyperparameter tuning fixes: loading seems to work now --- icu_benchmarks/run.py | 8 +- icu_benchmarks/tuning/hyperparameters.py | 118 ++++++++++++----------- 2 files changed, 65 insertions(+), 61 deletions(-) diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index ae114e45..80fd05ad 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -136,10 +136,10 @@ def main(my_args=tuple(sys.argv[1:])): log_full_line(f"Data directory: {data_dir.resolve()}", level=logging.INFO) run_dir = create_run_dir(log_dir) choose_and_bind_hyperparameters_optuna( - args.tune, - data_dir, - run_dir, - args.seed, + do_tune=args.tune, + data_dir=data_dir, + log_dir=run_dir, + seed=args.seed, run_mode=mode, checkpoint=hp_checkpoint, debug=args.debug, diff --git a/icu_benchmarks/tuning/hyperparameters.py b/icu_benchmarks/tuning/hyperparameters.py index 852bf041..015fd918 100644 --- a/icu_benchmarks/tuning/hyperparameters.py +++ b/icu_benchmarks/tuning/hyperparameters.py @@ -2,6 +2,7 @@ import gin import logging from logging import NOTSET +import matplotlib.pyplot as plt import numpy as np from pathlib import Path from skopt import gp_minimize @@ -14,7 +15,7 @@ from icu_benchmarks.tuning.gin_utils import get_gin_hyperparameters, bind_gin_params from icu_benchmarks.contants import RunMode from icu_benchmarks.wandb_utils import wandb_log - +from optuna.visualization import plot_param_importances, plot_optimization_history TUNE = 25 logging.addLevelName(25, "TUNE") @@ -189,6 +190,7 @@ def choose_and_bind_hyperparameters_optuna( debug: bool = False, verbose: bool = False, wandb: bool = False, + plot: bool = True, ): """Choose hyperparameters to tune and bind them to gin. Uses Optuna for hyperparameter optimization. @@ -228,25 +230,16 @@ def choose_and_bind_hyperparameters_optuna( return # Attempt checkpoint loading - configuration, evaluation = None, None - if checkpoint: - return NotImplementedError("Checkpoint loading is not implemented for Optuna yet.") - # checkpoint_path = checkpoint / checkpoint_file - # if not checkpoint_path.exists(): - # logging.warning(f"Hyperparameter checkpoint {checkpoint_path} does not exist.") - # logging.info("Attempting to find latest checkpoint file.") - # checkpoint_path = find_checkpoint(log_dir.parent, checkpoint_file) - # # Check if we found a checkpoint file - # if checkpoint_path: - # n_calls, configuration, evaluation = load_checkpoint(checkpoint_path, n_calls) - # # # Check if we surpassed maximum tuning iterations - # # if n_calls <= 0: - # # logging.log(TUNE, "No more hyperparameter tuning iterations left, skipping tuning.") - # # logging.info("Training with these hyperparameters:") - # # bind_gin_params(hyperparams_names, configuration[np.argmin(evaluation)]) # bind best hyperparameters - # # return - # else: - # logging.warning("No checkpoint file found, starting from scratch.") + + if not checkpoint.exists(): + logging.warning(f"Hyperparameter checkpoint {checkpoint} does not exist.") + checkpoint = None + # logging.info("Attempting to find latest checkpoint file.") + # checkpoint_path = find_checkpoint(log_dir.parent, checkpoint_file) + # Check if we found a checkpoint file + # # Check if we surpassed maximum tuning iterations + else: + logging.warning("No checkpoint file found, starting from scratch.") # Function that trains the model with the given hyperparameters. @@ -259,22 +252,24 @@ def objective(trail, hyperparams_bounds, hyperparams_names): logging.info(f"Bounds: {hyperparams_bounds}, Names: {hyperparams_names}") for name, value in zip(hyperparams_names, hyperparams_bounds): if isinstance(value, tuple): + # Check for range or "list-type" hyperparameter bounds if isinstance(value[0], (int, float)) and isinstance(value[1], (int, float)): - if len(value)>2 and isinstance(value[2], str): + if len(value) == 3 and isinstance(value[2], str): if isinstance(value[0], int) and isinstance(value[1], int): - hyperparams[name] = trail.suggest_int(name, value[0], value[1], log= value[2] == "log") - elif isinstance(value[0], float) and isinstance(value[1], float): + hyperparams[name] = trail.suggest_int(name, value[0], value[1], log=value[2] == "log") + elif isinstance(value[0], (int, float)) and isinstance(value[1], (int, float)): hyperparams[name] = trail.suggest_float(name, value[0], value[1], log=value[2] == "log") else: hyperparams[name] = trail.suggest_categorical(name, value) - # elif len(value)>2: - # hyperparams[name] = trail.suggest_categorical(name, value) - elif isinstance(value[0], int) and isinstance(value[1], int): - hyperparams[name] = trail.suggest_int(name, value[0], value[1]) + elif len(value) == 2: + if isinstance(value[0], int) and isinstance(value[1], int): + hyperparams[name] = trail.suggest_int(name, value[0], value[1]) + elif isinstance(value[0], (int, float)) and isinstance(value[1], (int, float)): + hyperparams[name] = trail.suggest_float(name, value[0], value[1]) + else: + hyperparams[name] = trail.suggest_categorical(name, value) else: - hyperparams[name] = trail.suggest_float(name, value[0], value[1]) - else: - hyperparams[name]=trail.suggest_categorical(name, value) + hyperparams[name] = trail.suggest_categorical(name, value) else: hyperparams[name] = trail.suggest_categorical(name, value) return bind_params_and_train(hyperparams) @@ -295,12 +290,14 @@ def tune_step_callback(study: optuna.study.Study, trial: optuna.trial.FrozenTria log_table_row(header, TUNE) else: logging.log(TUNE, "Hyperparameter tuning disabled") - if configuration: + if checkpoint: + study = optuna.load_study(study_name="tuning", storage="sqlite:///" + str(checkpoint)) + configuration = study.best_params # We have loaded a checkpoint, use the best hyperparameters. logging.info("Training with the best hyperparameters from loaded checkpoint:") - bind_gin_params(hyperparams_names, configuration[np.argmin(evaluation)]) + bind_gin_params(configuration) else: - logging.log(TUNE, "Choosing hyperparameters randomly from bounds.") + logging.log(TUNE, "Choosing hyperparameters randomly from bounds using hp tuning.") n_initial_points = 1 n_calls = 1 @@ -332,12 +329,17 @@ def bind_params_and_train(hyperparams): sampler = sampler(seed=seed) # Optuna study - study = optuna.create_study( - sampler=sampler, - storage="sqlite:///" + str(log_dir / checkpoint_file), - study_name=str(data_dir) + str(seed), - pruner=optuna.pruners.HyperbandPruner(), - ) + if not checkpoint: + study = optuna.create_study( + sampler=sampler, + storage="sqlite:///" + str(log_dir / checkpoint_file), + study_name="tuning", + pruner=optuna.pruners.HyperbandPruner(), + load_if_exists=True, + ) + else: + logging.info(f"Loading checkpoint at {checkpoint}") + study = optuna.load_study(study_name="tuning", storage="sqlite:///" + str(checkpoint)) callbacks = [tune_step_callback] if wandb: wandb_kwargs = { @@ -346,13 +348,19 @@ def bind_params_and_train(hyperparams): } wandbc = WeightsAndBiasesCallback(metric_name="loss", wandb_kwargs=wandb_kwargs) callbacks.append(wandbc) - logging.info(f"Starting Optuna study with {n_calls} trials and callbacks: {callbacks}.") - study.optimize( - lambda trail: objective(trail, hyperparams_bounds, hyperparams_names), - n_trials=n_calls, - callbacks=callbacks, - gc_after_trial=True, - ) + if not checkpoint: + logging.info(f"Starting Optuna study with {n_calls} trials and callbacks: {callbacks}.") + study.optimize( + lambda trail: objective(trail, hyperparams_bounds, hyperparams_names), + n_trials=n_calls, + callbacks=callbacks, + gc_after_trial=True, + ) + else: + logging.info(f"Resuming Optuna study with {n_calls-len(study.trials)} trails and callbacks: {callbacks}.") + study.optimize( + lambda trail: objective(trail, hyperparams_bounds, hyperparams_names) + ) logging.disable(level=NOTSET) if do_tune: @@ -361,6 +369,13 @@ def bind_params_and_train(hyperparams): logging.info("Training with these hyperparameters:") bind_gin_params(study.best_params) + if plot: + logging.info("Plotting hyperparameter importances.") + plot_param_importances(study) + plt.savefig(log_dir / "param_importances.png") + plot_optimization_history(study) + plt.savefig(log_dir / "optimization_history.png") + def collect_bound_hyperparameters(hyperparams, scopes): for scope in scopes: @@ -371,17 +386,6 @@ def collect_bound_hyperparameters(hyperparams, scopes): return hyperparams_bounds, hyperparams_names -def load_optuna_checkpoint(checkpoint_path, n_calls): - logging.info(f"Loading checkpoint at {checkpoint_path}") - with open(checkpoint_path, "r") as f: - data = json.loads(f.read()) - x0 = data["x_iters"] - y0 = data["func_vals"] - n_calls -= len(x0) - logging.log(TUNE, f"Checkpoint contains {len(x0)} points.") - return n_calls, x0, y0 - - def load_checkpoint(checkpoint_path, n_calls): logging.info(f"Loading checkpoint at {checkpoint_path}") with open(checkpoint_path, "r") as f: From 7afe2bd907c91e758dd434e744a391919dbf7a1f Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 27 Jun 2024 11:27:23 +0200 Subject: [PATCH 128/207] revert to normal dataloader cores --- icu_benchmarks/models/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index b9f06e6d..14e10170 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -49,7 +49,7 @@ def train_common( ram_cache=False, pl_model=True, train_only=False, - num_workers: int = 1 #min(cpu_core_count, torch.cuda.device_count() * 8 * int(torch.cuda.is_available()), 32), + num_workers: int = min(cpu_core_count, torch.cuda.device_count() * 8 * int(torch.cuda.is_available()), 32), ): """Common wrapper to train all benchmarked models. From bce11c52f01afc2b2ed74820adcb2a8141fb849c Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 27 Jun 2024 11:27:53 +0200 Subject: [PATCH 129/207] fixes and extensions in hyperparameter tuning configs --- configs/prediction_models/RUSBClassifier.gin | 2 +- configs/prediction_models/Transformer.gin | 8 ++++---- configs/prediction_models/common/DLTuning.gin | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/configs/prediction_models/RUSBClassifier.gin b/configs/prediction_models/RUSBClassifier.gin index 836187c9..e8f17722 100644 --- a/configs/prediction_models/RUSBClassifier.gin +++ b/configs/prediction_models/RUSBClassifier.gin @@ -8,7 +8,7 @@ train_common.model = @RUSBClassifier model/hyperparameter.class_to_tune = @RUSBClassifier model/hyperparameter.n_estimators = (10, 50, 100, 200, 500) -model/hyperparameter.learning_rate = (0.005, 1, "log-uniform") +model/hyperparameter.learning_rate = (0.005, 1, "log") model/hyperparameter.sampling_strategy = "auto" diff --git a/configs/prediction_models/Transformer.gin b/configs/prediction_models/Transformer.gin index 2767fd37..6201f3e6 100644 --- a/configs/prediction_models/Transformer.gin +++ b/configs/prediction_models/Transformer.gin @@ -12,11 +12,11 @@ optimizer/hyperparameter.lr = (1e-5, 3e-4) # Encoder params model/hyperparameter.class_to_tune = @Transformer -model/hyperparameter.ff_hidden_mult = 2 -model/hyperparameter.l1_reg = 0.0 +model/hyperparameter.ff_hidden_mult = (2,4,6,8) +model/hyperparameter.l1_reg = (0.0,1.0) model/hyperparameter.num_classes = %NUM_CLASSES -model/hyperparameter.hidden = (32, 256, "log-uniform", 2) -model/hyperparameter.heads = (1, 8, "log-uniform", 2) +model/hyperparameter.hidden = (32, 256, "log") +model/hyperparameter.heads = (1, 8, "log") model/hyperparameter.depth = (1, 3) model/hyperparameter.dropout = (0.0, 0.4) model/hyperparameter.dropout_att = (0.0, 0.4) diff --git a/configs/prediction_models/common/DLTuning.gin b/configs/prediction_models/common/DLTuning.gin index b4d13e12..0d71c2f8 100644 --- a/configs/prediction_models/common/DLTuning.gin +++ b/configs/prediction_models/common/DLTuning.gin @@ -2,4 +2,4 @@ tune_hyperparameters.scopes = ["model", "optimizer"] tune_hyperparameters.n_initial_points = 5 tune_hyperparameters.n_calls = 30 -tune_hyperparameters.folds_to_tune_on = 2 \ No newline at end of file +tune_hyperparameters.folds_to_tune_on = 5 \ No newline at end of file From 7b1ad9d40cc76e78dbebb5bff3f3249992f30838 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 4 Jul 2024 12:45:16 +0200 Subject: [PATCH 130/207] hyperparameter bound changes to enable a better search space --- configs/prediction_models/BRFClassifier.gin | 11 ++++++----- configs/prediction_models/CBClassifier.gin | 9 ++++----- configs/prediction_models/GRU.gin | 6 +++--- configs/prediction_models/TCN.gin | 6 +++--- configs/prediction_models/Transformer.gin | 8 ++++---- configs/prediction_models/XGBClassifier.gin | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/configs/prediction_models/BRFClassifier.gin b/configs/prediction_models/BRFClassifier.gin index 85e6ef98..3682bb8c 100644 --- a/configs/prediction_models/BRFClassifier.gin +++ b/configs/prediction_models/BRFClassifier.gin @@ -7,11 +7,12 @@ include "configs/prediction_models/common/MLCommon.gin" train_common.model = @BRFClassifier model/hyperparameter.class_to_tune = @BRFClassifier -model/hyperparameter.n_estimators = (10, 50, 100, 200, 500) +model/hyperparameter.n_estimators = [50, 100, 250, 500, 750,1000,1500] model/hyperparameter.max_depth = [3, 5, 10, 15] -model/hyperparameter.max_features = ['sqrt', 'log2', None] -model/hyperparameter.scale_pos_weight = [1, 5, 10, 25, 50, 75, 99, 100, 1000] -model/hyperparameter.learning_rate = (0.005, 1, "log-uniform") - +model/hyperparameter.min_samples_split = (2, 5, 10) +model/hyperparameter.min_samples_leaf = (1, 2, 4) +model/hyperparameter.max_features = ['sqrt', 'log2', 1.0] +model/hyperparameter.bootstrap = [True, False] +model/hyperparameter.class_weight = [None, 'balanced'] diff --git a/configs/prediction_models/CBClassifier.gin b/configs/prediction_models/CBClassifier.gin index a33cc611..e9abbecd 100644 --- a/configs/prediction_models/CBClassifier.gin +++ b/configs/prediction_models/CBClassifier.gin @@ -1,4 +1,4 @@ -# Settings for XGBoost classifier. +# Settings for Catboost classifier. # Common settings for ML models include "configs/prediction_models/common/MLCommon.gin" @@ -7,10 +7,9 @@ include "configs/prediction_models/common/MLCommon.gin" train_common.model = @CBClassifier model/hyperparameter.class_to_tune = @CBClassifier -model/hyperparameter.learning_rate = (0.01, 0.1, "log", 2) -model/hyperparameter.num_trees = [50, 100, 250, 500, 750] +model/hyperparameter.learning_rate = (1e-4, 0.5, "log") +model/hyperparameter.num_trees = [50, 100, 250, 500, 750,1000,1500] model/hyperparameter.depth = [3, 5, 10, 15] model/hyperparameter.scale_pos_weight = [1, 5, 10, 25, 50, 75, 99, 100, 1000] -model/hyperparameter.min_child_weight = [1, 0.5] -model/hyperparameter.max_delta_step = [0, 1, 2, 3, 4, 5, 10] model/hyperparameter.border_count = [5, 10, 20, 50, 100, 200] +model/hyperparameter.l2_leaf_reg = [1, 3, 5, 7, 9] \ No newline at end of file diff --git a/configs/prediction_models/GRU.gin b/configs/prediction_models/GRU.gin index d2a28a79..43cb0218 100644 --- a/configs/prediction_models/GRU.gin +++ b/configs/prediction_models/GRU.gin @@ -9,11 +9,11 @@ train_common.model = @GRUNet # Optimizer params optimizer/hyperparameter.class_to_tune = @Adam optimizer/hyperparameter.weight_decay = 1e-6 -optimizer/hyperparameter.lr = (1e-5, 3e-4) +optimizer/hyperparameter.lr = (1e-6, 1e-4, "log") # Encoder params model/hyperparameter.class_to_tune = @GRUNet model/hyperparameter.num_classes = %NUM_CLASSES -model/hyperparameter.hidden_dim = (32, 256, "log-uniform", 2) -model/hyperparameter.layer_dim = (1, 3) +model/hyperparameter.hidden_dim = (32, 512, "log") +model/hyperparameter.layer_dim = (1, 10) diff --git a/configs/prediction_models/TCN.gin b/configs/prediction_models/TCN.gin index c6b314db..d1cb748a 100644 --- a/configs/prediction_models/TCN.gin +++ b/configs/prediction_models/TCN.gin @@ -9,12 +9,12 @@ train_common.model = @TemporalConvNet # Optimizer params optimizer/hyperparameter.class_to_tune = @Adam optimizer/hyperparameter.weight_decay = 1e-6 -optimizer/hyperparameter.lr = (1e-5, 3e-4) +optimizer/hyperparameter.lr = (1e-6, 3e-4) # Encoder params model/hyperparameter.class_to_tune = @TemporalConvNet model/hyperparameter.num_classes = %NUM_CLASSES model/hyperparameter.max_seq_length = %HORIZON -model/hyperparameter.num_channels = (32, 256, "log-uniform", 2) -model/hyperparameter.kernel_size = (2, 32, "log-uniform", 2) +model/hyperparameter.num_channels = (32, 256, "log") +model/hyperparameter.kernel_size = (2, 128, "log") model/hyperparameter.dropout = (0.0, 0.4) diff --git a/configs/prediction_models/Transformer.gin b/configs/prediction_models/Transformer.gin index 6201f3e6..69f31e51 100644 --- a/configs/prediction_models/Transformer.gin +++ b/configs/prediction_models/Transformer.gin @@ -8,17 +8,17 @@ train_common.model = @Transformer optimizer/hyperparameter.class_to_tune = @Adam optimizer/hyperparameter.weight_decay = 1e-6 -optimizer/hyperparameter.lr = (1e-5, 3e-4) +optimizer/hyperparameter.lr = (1e-6, 1e-4) # Encoder params model/hyperparameter.class_to_tune = @Transformer model/hyperparameter.ff_hidden_mult = (2,4,6,8) model/hyperparameter.l1_reg = (0.0,1.0) model/hyperparameter.num_classes = %NUM_CLASSES -model/hyperparameter.hidden = (32, 256, "log") +model/hyperparameter.hidden = (32, 512, "log") model/hyperparameter.heads = (1, 8, "log") model/hyperparameter.depth = (1, 3) -model/hyperparameter.dropout = (0.0, 0.4) -model/hyperparameter.dropout_att = (0.0, 0.4) +model/hyperparameter.dropout = 0 # no improvement (0.0, 0.4) +model/hyperparameter.dropout_att = (0.0, 1.0) diff --git a/configs/prediction_models/XGBClassifier.gin b/configs/prediction_models/XGBClassifier.gin index 5f19866f..beb22e66 100644 --- a/configs/prediction_models/XGBClassifier.gin +++ b/configs/prediction_models/XGBClassifier.gin @@ -7,8 +7,8 @@ include "configs/prediction_models/common/MLCommon.gin" train_common.model = @XGBClassifier model/hyperparameter.class_to_tune = @XGBClassifier -model/hyperparameter.learning_rate = (0.01, 0.1, "log", 2) -model/hyperparameter.n_estimators = [50, 100, 250, 500, 750] +model/hyperparameter.learning_rate = (0.01, 0.1, "log") +model/hyperparameter.n_estimators = [50, 100, 250, 500, 750, 1000,1500,2000] model/hyperparameter.max_depth = [3, 5, 10, 15] model/hyperparameter.scale_pos_weight = [1, 5, 10, 25, 50, 75, 99, 100, 1000] model/hyperparameter.min_child_weight = [1, 0.5] From ef7374d988e7d9899f23bbe6aefb61f0a78b0648 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 4 Jul 2024 12:45:40 +0200 Subject: [PATCH 131/207] catboost gpu leads to errors --- icu_benchmarks/models/ml_models/catboost.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/icu_benchmarks/models/ml_models/catboost.py b/icu_benchmarks/models/ml_models/catboost.py index 7ecf7f51..2da5ff75 100644 --- a/icu_benchmarks/models/ml_models/catboost.py +++ b/icu_benchmarks/models/ml_models/catboost.py @@ -12,7 +12,8 @@ class CBClassifier(MLWrapper): _supported_run_modes = [RunMode.classification] def __init__(self, *args, **kwargs): - self.model = self.set_model_args(cb.CatBoostClassifier, task_type="GPU" if not kwargs['cpu'] else "CPU", *args, **kwargs) + #self.model = self.set_model_args(cb.CatBoostClassifier, task_type="GPU" if not kwargs['cpu'] else "CPU", *args, **kwargs) + self.model = self.set_model_args(cb.CatBoostClassifier, task_type="CPU", *args, **kwargs) super().__init__(*args, **kwargs) def predict(self, features): From 17d2268b38e2cd189e4b84c5cc67981776e716f1 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 4 Jul 2024 12:46:28 +0200 Subject: [PATCH 132/207] logging --- icu_benchmarks/models/wrappers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index e8595a70..27c47c85 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -498,6 +498,7 @@ def set_model_args(self, model, *args, **kwargs): # Get passed keyword arguments arguments = locals()["kwargs"] # Get valid hyperparameters + logging.debug(f"Possible hps: {possible_hps}") hyperparams = {key: value for key, value in arguments.items() if key in possible_hps} logging.debug(f"Creating model with: {hyperparams}.") return model(**hyperparams) From 01e13f872b881fa5b494d34f8438446c55f1f11c Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 4 Jul 2024 12:46:39 +0200 Subject: [PATCH 133/207] logging --- icu_benchmarks/data/preprocessor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/icu_benchmarks/data/preprocessor.py b/icu_benchmarks/data/preprocessor.py index 0c3fb190..d7e7e8cb 100644 --- a/icu_benchmarks/data/preprocessor.py +++ b/icu_benchmarks/data/preprocessor.py @@ -110,7 +110,8 @@ def apply(self, data, vars) -> dict[dict[pd.DataFrame]]: data[Split.val][Segment.features] = data[Split.val].pop(Segment.dynamic) data[Split.test][Segment.features] = data[Split.test].pop(Segment.dynamic) - logging.info(data[Split.train][Segment.features].head()) + logging.debug("Data head") + logging.debug(data[Split.train][Segment.features].head()) logging.info(f"Generate features: {self.generate_features}") return data From b7ff8260a123cc85f57f52d1775eb3b4c525b6b8 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 4 Jul 2024 12:47:01 +0200 Subject: [PATCH 134/207] xgboost model configuration --- icu_benchmarks/models/ml_models/xgboost.py | 31 ++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/icu_benchmarks/models/ml_models/xgboost.py b/icu_benchmarks/models/ml_models/xgboost.py index e8467e64..26f5b211 100644 --- a/icu_benchmarks/models/ml_models/xgboost.py +++ b/icu_benchmarks/models/ml_models/xgboost.py @@ -1,15 +1,42 @@ +import inspect +import logging + import gin from icu_benchmarks.contants import RunMode from icu_benchmarks.models.wrappers import MLWrapper import xgboost as xgb +from xgboost.callback import EarlyStopping, LearningRateScheduler +from wandb.integration.xgboost import wandb_callback as wandb_xgb +import wandb +from optuna.integration import XGBoostPruningCallback @gin.configurable class XGBClassifier(MLWrapper): _supported_run_modes = [RunMode.classification] def __init__(self, *args, **kwargs): - self.model = self.set_model_args(xgb.XGBClassifier, device="cpu" if kwargs["cpu"] else "cuda", *args, **kwargs) + self.model = self.set_model_args(xgb.XGBClassifier, device="cpu",*args, **kwargs) super().__init__(*args, **kwargs) def predict(self, features): """Predicts labels for the given features.""" - return self.model.predict_proba(features) \ No newline at end of file + return self.model.predict_proba(features) + + def fit_model(self, train_data, train_labels, val_data, val_labels): + """Fit the model to the training data (default SKlearn syntax)""" + callbacks = [EarlyStopping(self.hparams.patience)] + + if wandb.run is not None: + callbacks.append(wandb_xgb()) + self.model.fit(train_data, train_labels, eval_set=[(val_data, val_labels)], callbacks=callbacks) + + + def set_model_args(self, model, *args, **kwargs): + """XGBoost signature does not include the hyperparams so we need to pass them manually.""" + signature = inspect.signature(model.__init__).parameters + possible_hps = list(signature.keys()) + # Get passed keyword arguments + arguments = locals()["kwargs"] + # Get valid hyperparameters + hyperparams = arguments + logging.debug(f"Creating model with: {hyperparams}.") + return model(**hyperparams) \ No newline at end of file From 42ed12867ba4be24ac971d2490ade85b6daadcdb Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 4 Jul 2024 12:47:26 +0200 Subject: [PATCH 135/207] cleaned up hp tuning --- icu_benchmarks/tuning/hyperparameters.py | 64 +++++++++++++----------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/icu_benchmarks/tuning/hyperparameters.py b/icu_benchmarks/tuning/hyperparameters.py index 015fd918..eed90e7c 100644 --- a/icu_benchmarks/tuning/hyperparameters.py +++ b/icu_benchmarks/tuning/hyperparameters.py @@ -229,18 +229,6 @@ def choose_and_bind_hyperparameters_optuna( logging.info("No hyperparameters to tune, skipping tuning.") return - # Attempt checkpoint loading - - if not checkpoint.exists(): - logging.warning(f"Hyperparameter checkpoint {checkpoint} does not exist.") - checkpoint = None - # logging.info("Attempting to find latest checkpoint file.") - # checkpoint_path = find_checkpoint(log_dir.parent, checkpoint_file) - # Check if we found a checkpoint file - # # Check if we surpassed maximum tuning iterations - else: - logging.warning("No checkpoint file found, starting from scratch.") - # Function that trains the model with the given hyperparameters. header = ["ITERATION"] + hyperparams_names + ["LOSS AT ITERATION"] @@ -296,8 +284,10 @@ def tune_step_callback(study: optuna.study.Study, trial: optuna.trial.FrozenTria # We have loaded a checkpoint, use the best hyperparameters. logging.info("Training with the best hyperparameters from loaded checkpoint:") bind_gin_params(configuration) + return else: - logging.log(TUNE, "Choosing hyperparameters randomly from bounds using hp tuning.") + logging.log(TUNE, "Choosing hyperparameters randomly from bounds using hp tuning as no earlier checkpoint " + "supplied.") n_initial_points = 1 n_calls = 1 @@ -327,19 +317,29 @@ def bind_params_and_train(hyperparams): sampler = sampler(seed=seed, n_startup_trials=n_initial_points, deterministic_objective=True) else: sampler = sampler(seed=seed) - + pruner = optuna.pruners.HyperbandPruner() # Optuna study - if not checkpoint: + # Attempt checkpoint loading + if checkpoint and checkpoint.exists(): + logging.warning(f"Hyperparameter checkpoint {checkpoint} does not exist.") + # logging.info("Attempting to find latest checkpoint file.") + # checkpoint_path = find_checkpoint(log_dir.parent, checkpoint_file) + # Check if we found a checkpoint file + logging.info(f"Loading checkpoint at {checkpoint}") + study = optuna.load_study(study_name="tuning", storage="sqlite:///" + str(checkpoint), sampler=sampler, + pruner=pruner) + n_calls = n_calls - len(study.trials) + else: + if checkpoint: + logging.warning("Checkpoint path given as flag but not found, starting from scratch.") study = optuna.create_study( sampler=sampler, storage="sqlite:///" + str(log_dir / checkpoint_file), study_name="tuning", - pruner=optuna.pruners.HyperbandPruner(), + pruner=pruner, load_if_exists=True, ) - else: - logging.info(f"Loading checkpoint at {checkpoint}") - study = optuna.load_study(study_name="tuning", storage="sqlite:///" + str(checkpoint)) + callbacks = [tune_step_callback] if wandb: wandb_kwargs = { @@ -348,8 +348,9 @@ def bind_params_and_train(hyperparams): } wandbc = WeightsAndBiasesCallback(metric_name="loss", wandb_kwargs=wandb_kwargs) callbacks.append(wandbc) - if not checkpoint: - logging.info(f"Starting Optuna study with {n_calls} trials and callbacks: {callbacks}.") + + logging.info(f"Starting or resuming Optuna study with {n_calls} trails and callbacks: {callbacks}.") + if n_calls > 0: study.optimize( lambda trail: objective(trail, hyperparams_bounds, hyperparams_names), n_trials=n_calls, @@ -357,10 +358,10 @@ def bind_params_and_train(hyperparams): gc_after_trial=True, ) else: - logging.info(f"Resuming Optuna study with {n_calls-len(study.trials)} trails and callbacks: {callbacks}.") - study.optimize( - lambda trail: objective(trail, hyperparams_bounds, hyperparams_names) - ) + logging.info("No more hyperparameter tuning iterations left, skipping tuning.") + logging.info("Training with these hyperparameters:") + bind_gin_params(study.best_params) + return logging.disable(level=NOTSET) if do_tune: @@ -370,11 +371,14 @@ def bind_params_and_train(hyperparams): bind_gin_params(study.best_params) if plot: - logging.info("Plotting hyperparameter importances.") - plot_param_importances(study) - plt.savefig(log_dir / "param_importances.png") - plot_optimization_history(study) - plt.savefig(log_dir / "optimization_history.png") + try: + logging.info("Plotting hyperparameter importances.") + plot_param_importances(study) + plt.savefig(log_dir / "param_importances.png") + plot_optimization_history(study) + plt.savefig(log_dir / "optimization_history.png") + except Exception as e: + logging.error(f"Failed to plot hyperparameter importances: {e}") def collect_bound_hyperparameters(hyperparams, scopes): From c4dffb0cf71f78c592023a41d52f7a5377a9dc8b Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 4 Jul 2024 12:47:46 +0200 Subject: [PATCH 136/207] cpu submission --- experiments/charhpc_wandb_sweep_cpu.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 experiments/charhpc_wandb_sweep_cpu.sh diff --git a/experiments/charhpc_wandb_sweep_cpu.sh b/experiments/charhpc_wandb_sweep_cpu.sh new file mode 100644 index 00000000..f9377a02 --- /dev/null +++ b/experiments/charhpc_wandb_sweep_cpu.sh @@ -0,0 +1,13 @@ +#!/bin/bash +#SBATCH --job-name=yaib_experiment +#SBATCH --partition=compute # -p +#SBATCH --cpus-per-task=10 # -c +#SBATCH --mem=200gb +#SBATCH --output=logs/classification_%a_%j.log # %j is job id +#SBATCH --time=48:00:00 + +eval "$(conda shell.bash hook)" +conda activate yaib_req +wandb agent --count 1 cassandra_hpi/cassandra/"$1" + + From 105d5aec84ca1ee88cf87cac4b168da58e1f9e1e Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 4 Jul 2024 12:48:18 +0200 Subject: [PATCH 137/207] increased mem, time for slurm job --- experiments/charhpc_wandb_sweep.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/experiments/charhpc_wandb_sweep.sh b/experiments/charhpc_wandb_sweep.sh index 8c29aa02..30975f4f 100644 --- a/experiments/charhpc_wandb_sweep.sh +++ b/experiments/charhpc_wandb_sweep.sh @@ -1,14 +1,14 @@ #!/bin/bash #SBATCH --job-name=yaib_experiment -#SBATCH --partition=gpu,pgpu,agpu # -p +#SBATCH --partition=gpu # -p #SBATCH --cpus-per-task=8 # -c -#SBATCH --mem=100gb +#SBATCH --mem=200gb #SBATCH --output=logs/classification_%a_%j.log # %j is job id #SBATCH --gpus=1 -#SBATCH --time=12:00:00 +#SBATCH --time=48:00:00 eval "$(conda shell.bash hook)" -conda activate yaib_benchmark +conda activate yaib_req wandb agent --count 1 cassandra_hpi/cassandra/"$1" From d7252c1eafe8d30b84fa5169fdb8c5feebc7a47a Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 4 Jul 2024 17:10:23 +0200 Subject: [PATCH 138/207] Polars version of the dataloader --- configs/tasks/BinaryClassification.gin | 2 + icu_benchmarks/data/loader.py | 192 ++++++++++++++++++--- icu_benchmarks/models/ml_models/xgboost.py | 4 +- icu_benchmarks/models/train.py | 21 ++- 4 files changed, 189 insertions(+), 30 deletions(-) diff --git a/configs/tasks/BinaryClassification.gin b/configs/tasks/BinaryClassification.gin index 492a12eb..d1ca94fd 100644 --- a/configs/tasks/BinaryClassification.gin +++ b/configs/tasks/BinaryClassification.gin @@ -24,6 +24,8 @@ preprocess.use_static = True # SELECTING DATASET PredictionDataset.vars = %vars +PredictionPolarsDataset.vars = %vars + PredictionDataset.ram_cache = True diff --git a/icu_benchmarks/data/loader.py b/icu_benchmarks/data/loader.py index 20fb4f27..18586cec 100644 --- a/icu_benchmarks/data/loader.py +++ b/icu_benchmarks/data/loader.py @@ -6,12 +6,161 @@ from torch.utils.data import Dataset import logging from typing import Dict, Tuple - +import polars as pl from icu_benchmarks.imputation.amputations import ampute_data from .constants import DataSegment as Segment from .constants import DataSplit as Split +@gin.configurable("CommonPolarsDataset") +class CommonPolarsDataset(Dataset): + def __init__( + self, + data: dict, + split: str = Split.train, + vars: Dict[str, str] = gin.REQUIRED, + grouping_segment: str = Segment.outcome, + mps: bool = False, + name: str = "", + ): + self.split = split + self.vars = vars + self.grouping_df = data[split][grouping_segment] #.set_index(self.vars["GROUP"]) + # logging.info(f"data split: {data[split]}") + # self.features_df = ( + # data[split][Segment.features].set_index(self.vars["GROUP"]).drop(labels=self.vars["SEQUENCE"], axis=1) + # ) + self.features_df = data[split][Segment.features].drop(self.vars["SEQUENCE"]) + # calculate basic info for the data + self.num_stays = self.grouping_df[self.vars["GROUP"]].unique().shape[0] + self.maxlen = self.features_df.group_by([self.vars["GROUP"]]).len().max().item(0, 1) + self.mps = mps + self.name = name + + def ram_cache(self, cache: bool = False): + self._cached_dataset = None + if cache: + logging.info(f"Caching {self.split} dataset in ram.") + self._cached_dataset = [self[i] for i in range(len(self))] + + def __len__(self) -> int: + """Returns number of stays in the data. + + Returns: + number of stays in the data + """ + return self.num_stays + + def get_feature_names(self): + return self.features_df.columns + + def to_tensor(self): + values = [] + for entry in self: + for i, value in enumerate(entry): + if len(values) <= i: + values.append([]) + values[i].append(value.unsqueeze(0)) + return [cat(value, dim=0) for value in values] + + +@gin.configurable("PredictionPolarsDataset") +class PredictionPolarsDataset(CommonPolarsDataset): + """Subclass of common dataset for prediction tasks. + + Args: + ram_cache (bool, optional): Whether the complete dataset should be stored in ram. Defaults to True. + """ + + def __init__(self, *args, ram_cache: bool = False, **kwargs): + super().__init__(*args, grouping_segment=Segment.outcome, **kwargs) + self.outcome_df = self.grouping_df + self.ram_cache(ram_cache) + + def __getitem__(self, idx: int) -> Tuple[Tensor, Tensor, Tensor]: + """Function to sample from the data split of choice. Used for deep learning implementations. + + Args: + idx: A specific row index to sample. + + Returns: + A sample from the data, consisting of data, labels and padding mask. + """ + if self._cached_dataset is not None: + return self._cached_dataset[idx] + + pad_value = 0.0 + # stay_id = self.outcome_df.index.unique()[idx] # [self.vars["GROUP"]] + stay_id = self.outcome_df[self.vars["GROUP"]].unique()[idx] # [self.vars["GROUP"]] + + # slice to make sure to always return a DF + # window = self.features_df.loc[stay_id:stay_id].to_numpy() + # labels = self.outcome_df.loc[stay_id:stay_id][self.vars["LABEL"]].to_numpy(dtype=float) + window = self.features_df.filter(pl.col(self.vars["GROUP"]) == stay_id).to_numpy() + labels = self.outcome_df.filter(pl.col(self.vars["GROUP"]) == stay_id)[self.vars["LABEL"]].to_numpy().astype(float) + + if len(labels) == 1: + # only one label per stay, align with window + labels = np.concatenate([np.empty(window.shape[0] - 1) * np.nan, labels], axis=0) + + length_diff = self.maxlen - window.shape[0] + pad_mask = np.ones(window.shape[0]) + + # Padding the array to fulfill size requirement + if length_diff > 0: + # window shorter than the longest window in dataset, pad to same length + window = np.concatenate([window, np.ones((length_diff, window.shape[1])) * pad_value], axis=0) + labels = np.concatenate([labels, np.ones(length_diff) * pad_value], axis=0) + pad_mask = np.concatenate([pad_mask, np.zeros(length_diff)], axis=0) + + not_labeled = np.argwhere(np.isnan(labels)) + if len(not_labeled) > 0: + labels[not_labeled] = -1 + pad_mask[not_labeled] = 0 + + pad_mask = pad_mask.astype(bool) + labels = labels.astype(np.float32) + data = window.astype(np.float32) + + return from_numpy(data), from_numpy(labels), from_numpy(pad_mask) + + def get_balance(self) -> list: + """Return the weight balance for the split of interest. + + Returns: + Weights for each label. + """ + counts = self.outcome_df[self.vars["LABEL"]].value_counts(parallel=True).get_columns()[1] + counts = counts.to_numpy() + weights = list((1 / counts) * np.sum(counts) / counts.shape[0]) + return weights + + def get_data_and_labels(self) -> Tuple[np.array, np.array]: + """Function to return all the data and labels aligned at once. + + We use this function for the ML methods which don't require an iterator. + + Returns: + A Tuple containing data points and label for the split. + """ + labels = self.outcome_df[self.vars["LABEL"]].to_numpy().astype(float) + rep = self.features_df + if len(labels) == self.num_stays: + # order of groups could be random, we make sure not to change it + # rep = rep.groupby(level=self.vars["GROUP"], sort=False).last() + rep = rep.group_by(self.vars["GROUP"]).last() + rep = rep.to_numpy().astype(float) + + return rep, labels + + def to_tensor(self): + data, labels = self.get_data_and_labels() + if self.mps: + return from_numpy(data).to(float32), from_numpy(labels).to(float32) + else: + return from_numpy(data), from_numpy(labels) + + class CommonDataset(Dataset): """Common dataset: subclass of Torch Dataset that represents the data to learn on. @@ -21,13 +170,13 @@ class CommonDataset(Dataset): """ def __init__( - self, - data: dict, - split: str = Split.train, - vars: Dict[str, str] = gin.REQUIRED, - grouping_segment: str = Segment.outcome, - mps: bool = False, - name: str = "", + self, + data: dict, + split: str = Split.train, + vars: Dict[str, str] = gin.REQUIRED, + grouping_segment: str = Segment.outcome, + mps: bool = False, + name: str = "", ): self.split = split self.vars = vars @@ -134,6 +283,7 @@ def get_balance(self) -> list: Weights for each label. """ counts = self.outcome_df[self.vars["LABEL"]].value_counts() + weights = list((1 / counts) * np.sum(counts) / counts.shape[0]) return list((1 / counts) * np.sum(counts) / counts.shape[0]) def get_data_and_labels(self) -> Tuple[np.array, np.array]: @@ -166,14 +316,14 @@ class ImputationDataset(CommonDataset): """Subclass of Common Dataset that contains data for imputation models.""" def __init__( - self, - data: Dict[str, DataFrame], - split: str = Split.train, - vars: Dict[str, str] = gin.REQUIRED, - mask_proportion=0.3, - mask_method="MCAR", - mask_observation_proportion=0.3, - ram_cache: bool = True, + self, + data: Dict[str, DataFrame], + split: str = Split.train, + vars: Dict[str, str] = gin.REQUIRED, + mask_proportion=0.3, + mask_method="MCAR", + mask_observation_proportion=0.3, + ram_cache: bool = True, ): """ Args: @@ -240,11 +390,11 @@ class ImputationPredictionDataset(Dataset): """ def __init__( - self, - data: DataFrame, - grouping_column: str = "stay_id", - select_columns: List[str] = None, - ram_cache: bool = True, + self, + data: DataFrame, + grouping_column: str = "stay_id", + select_columns: List[str] = None, + ram_cache: bool = True, ): self.dyn_df = data diff --git a/icu_benchmarks/models/ml_models/xgboost.py b/icu_benchmarks/models/ml_models/xgboost.py index 26f5b211..a13f8c75 100644 --- a/icu_benchmarks/models/ml_models/xgboost.py +++ b/icu_benchmarks/models/ml_models/xgboost.py @@ -8,6 +8,7 @@ from xgboost.callback import EarlyStopping, LearningRateScheduler from wandb.integration.xgboost import wandb_callback as wandb_xgb import wandb +from statistics import mean from optuna.integration import XGBoostPruningCallback @gin.configurable class XGBClassifier(MLWrapper): @@ -27,7 +28,8 @@ def fit_model(self, train_data, train_labels, val_data, val_labels): if wandb.run is not None: callbacks.append(wandb_xgb()) - self.model.fit(train_data, train_labels, eval_set=[(val_data, val_labels)], callbacks=callbacks) + self.model.fit(train_data, train_labels, eval_set=[(val_data, val_labels)]) + return mean(self.model.evals_result_["validation_0"]["logloss"])#, callbacks=callbacks) def set_model_args(self, model, *args, **kwargs): diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 14e10170..905d1245 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -3,6 +3,7 @@ import torch import logging import pandas as pd +import polars as pl from joblib import load from torch.optim import Adam from torch.utils.data import DataLoader @@ -10,7 +11,7 @@ from pytorch_lightning import Trainer from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint, TQDMProgressBar, LearningRateMonitor from pathlib import Path -from icu_benchmarks.data.loader import PredictionDataset, ImputationDataset +from icu_benchmarks.data.loader import PredictionDataset, ImputationDataset, PredictionPolarsDataset from icu_benchmarks.models.utils import save_config_file, JSONMetricsLogger from icu_benchmarks.contants import RunMode from icu_benchmarks.data.constants import DataSplit as Split @@ -79,13 +80,16 @@ def train_common( """ logging.info(f"Training model: {model.__name__}.") - dataset_class = ImputationDataset if mode == RunMode.imputation else PredictionDataset - + # dataset_class = ImputationDataset if mode == RunMode.imputation else PredictionDataset + for dict in data.values(): + for key,val in dict.items(): + dict[key] = pl.from_pandas(val) + dataset_class = PredictionPolarsDataset logging.info(f"Logging to directory: {log_dir}.") save_config_file(log_dir) # We save the operative config before and also after training - train_dataset = dataset_class(data, split=Split.train, ram_cache=ram_cache, name=dataset_names["train"]) - val_dataset = dataset_class(data, split=Split.val, ram_cache=ram_cache, name=dataset_names["val"]) + train_dataset = dataset_class(data, split=Split.train, ram_cache=False, name=dataset_names["train"]) + val_dataset = dataset_class(data, split=Split.val, ram_cache=False, name=dataset_names["val"]) train_dataset, val_dataset = assure_minimum_length(train_dataset), assure_minimum_length(val_dataset) batch_size = min(batch_size, len(train_dataset), len(val_dataset)) @@ -95,13 +99,14 @@ def train_common( f" {len(val_dataset)} samples." ) logging.info(f"Using {num_workers} workers for data loading.") - + cpu=True + batch_size=1 train_loader = DataLoader( train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers, - pin_memory=True, + pin_memory=not cpu, drop_last=True, ) val_loader = DataLoader( @@ -109,7 +114,7 @@ def train_common( batch_size=batch_size, shuffle=False, num_workers=num_workers, - pin_memory=True, + pin_memory=not cpu, drop_last=True, ) From a64e79ab4030bd1a6a0c528f1c7434bfc397df05 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Fri, 5 Jul 2024 10:02:34 +0200 Subject: [PATCH 139/207] polars version of split_process_data.py (backwards compatible with pandas) --- icu_benchmarks/data/split_process_data.py | 75 ++++++++++++++++------- 1 file changed, 54 insertions(+), 21 deletions(-) diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index 5831a457..e465ade2 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -4,6 +4,7 @@ import json import hashlib import pandas as pd +import polars as pl import pyarrow.parquet as pq from pathlib import Path import pickle @@ -91,7 +92,7 @@ def preprocess_data( # Read parquet files into pandas dataframes and remove the parquet file from memory logging.info(f"Loading data from directory {data_dir.absolute()}") - data = {f: pq.read_table(data_dir / file_names[f]).to_pandas(self_destruct=True) for f in file_names.keys()} + data = {f: pl.read_parquet(data_dir / file_names[f]) for f in file_names.keys()} # Generate the splits logging.info("Generating splits.") # complete_train = True @@ -187,6 +188,7 @@ def make_train_val( return data_split + def make_single_split( data: dict[pd.DataFrame], vars: dict[str], @@ -198,6 +200,7 @@ def make_single_split( seed: int = 42, debug: bool = False, runmode: RunMode = RunMode.classification, + polars: bool = True, ) -> dict[dict[pd.DataFrame]]: """Randomly split the data into training, validation, and test set. @@ -222,19 +225,34 @@ def make_single_split( if debug: # Only use 1% of the data logging.info("Using only 1% of the data for debugging. Note that this might lead to errors for small datasets.") - data[Segment.outcome] = data[Segment.outcome].sample(frac=0.01, random_state=seed) + if polars: + data[Segment.outcome] = data[Segment.outcome].sample(fraction=0.01, seed=seed) + else: + data[Segment.outcome] = data[Segment.outcome].sample(frac=0.01, random_state=seed) # Get stay IDs from outcome segment - stays = pd.Series(data[Segment.outcome][id].unique(), name=id) + if polars: + stays = pl.Series(name=id, values=data[Segment.outcome][id].unique()) + else: + stays = pd.Series(data[Segment.outcome][id].unique(), name=id) # If there are labels, and the task is classification, use stratified k-fold if Var.label in vars and runmode is RunMode.classification: # Get labels from outcome data (takes the highest value (or True) in case seq2seq classification) - labels = data[Segment.outcome].groupby(id).max()[vars[Var.label]].reset_index(drop=True) - if labels.value_counts().min() < cv_folds: - raise Exception( - f"The smallest amount of samples in a class is: {labels.value_counts().min()}, " - f"but {cv_folds} folds are requested. Reduce the number of folds or use more data." - ) + if polars: + labels = data[Segment.outcome].group_by(id).max()[vars[Var.label]] + if labels.value_counts().min().item(0, 1) < cv_folds: + raise Exception( + f"The smallest amount of samples in a class is: {labels.value_counts().min()}, " + f"but {cv_folds} folds are requested. Reduce the number of folds or use more data." + ) + else: + labels = data[Segment.outcome].groupby(id).max()[vars[Var.label]].reset_index(drop=True) + if labels.value_counts().min() < cv_folds: + raise Exception( + f"The smallest amount of samples in a class is: {labels.value_counts().min()}, " + f"but {cv_folds} folds are requested. Reduce the number of folds or use more data." + ) + if train_size: outer_cv = StratifiedShuffleSplit(cv_repetitions, train_size=train_size) else: @@ -242,8 +260,12 @@ def make_single_split( inner_cv = StratifiedKFold(cv_folds, shuffle=True, random_state=seed) dev, test = list(outer_cv.split(stays, labels))[repetition_index] - dev_stays = stays.iloc[dev] - train, val = list(inner_cv.split(dev_stays, labels.iloc[dev]))[fold_index] + if polars: + dev_stays = stays[dev] + train, val = list(inner_cv.split(dev_stays, labels[dev]))[fold_index] + else: + dev_stays = stays.iloc[dev] + train, val = list(inner_cv.split(dev_stays, labels.iloc[dev]))[fold_index] else: # If there are no labels, or the task is regression, use regular k-fold. if train_size: @@ -255,21 +277,32 @@ def make_single_split( dev, test = list(outer_cv.split(stays))[repetition_index] dev_stays = stays.iloc[dev] train, val = list(inner_cv.split(dev_stays))[fold_index] - - split = { - Split.train: dev_stays.iloc[train], - Split.val: dev_stays.iloc[val], - Split.test: stays.iloc[test], - } + if polars: + split = { + Split.train: dev_stays[train].to_frame(), + Split.val: dev_stays[val].to_frame(), + Split.test: stays[test].to_frame(), + } + else: + split = { + Split.train: dev_stays.iloc[train], + Split.val: dev_stays.iloc[val], + Split.test: stays.iloc[test], + } data_split = {} for fold in split.keys(): # Loop through splits (train / val / test) # Loop through segments (DYNAMIC / STATIC / OUTCOME) # set sort to true to make sure that IDs are reordered after scrambling earlier - data_split[fold] = { - data_type: data[data_type].merge(split[fold], on=id, how="right", sort=True) for data_type in data.keys() - } - # logging.info(f"Data split: {data_split}") + if polars: + data_split[fold] = { + data_type: split[fold].join(data[data_type], on=id, how="left").sort(by=id) for data_type in data.keys() + } + else: + data_split[fold] = { + data_type: data[data_type].merge(split[fold], on=id, how="right", sort=True) for data_type in data.keys() + } + logging.debug(f"Data split: {data_split}") return data_split From 3b2183804145d6d3ac2c0127f895627058e5d842 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Fri, 5 Jul 2024 12:29:16 +0200 Subject: [PATCH 140/207] requirements update --- environment.yml | 5 ++--- requirements.txt | 8 +++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/environment.yml b/environment.yml index abba2126..5616269a 100644 --- a/environment.yml +++ b/environment.yml @@ -1,10 +1,9 @@ -name: yaib +name: yaib_19 channels: - conda-forge dependencies: - python=3.10 - - pip=24.0 - - flake8=7.1.0 + - pip>=24.0 # - pip: # - -r requirements.txt diff --git a/requirements.txt b/requirements.txt index ce4865c2..1c4dd0e8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,14 @@ --extra-index-url https://download.pytorch.org/whl/cu118 black==24.3.0 coverage==7.2.3 -flake8==5.0.4 +flake8>=7.0.0 matplotlib==3.7.1 gin-config==0.5.0 pytorch-ignite==0.5.0.post2 # Note: versioning of Pytorch might be dependent on compatible CUDA version. # Please check yourself if your Pytorch installation supports cuda (for gpu acceleration) torch==2.3.1 -lightning==2.3.0 +pytorch-lightning==2.3.2 torchmetrics==1.0.3 #pytorch-cuda==11.8 lightgbm==4.4.0 @@ -27,8 +27,10 @@ hydra-core==1.3 optuna==3.6.1 optuna-integration==3.6.0 wandb==0.17.3 -recipies==0.1.3 +#recipies==0.1.3 #Fixed version because of NumPy incompatibility and stale development status. scikit-optimize-fix==0.9.1 hydra-submitit-launcher==1.2.0 pytest-runner==6.0.1 +recipys @ git+https://github.com/rvandewater/ReciPys@polars + From b7c80c4ab92b316177856e29055f88ba55d5bcb3 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Fri, 5 Jul 2024 12:29:50 +0200 Subject: [PATCH 141/207] preprocessing using polars partially working --- icu_benchmarks/data/preprocessor.py | 146 +++++++++++++++++++++- icu_benchmarks/data/split_process_data.py | 4 +- icu_benchmarks/models/train.py | 6 +- 3 files changed, 148 insertions(+), 8 deletions(-) diff --git a/icu_benchmarks/data/preprocessor.py b/icu_benchmarks/data/preprocessor.py index d7e7e8cb..77b6b4db 100644 --- a/icu_benchmarks/data/preprocessor.py +++ b/icu_benchmarks/data/preprocessor.py @@ -6,12 +6,15 @@ import gin import pandas as pd +import polars.selectors as cs +import polars as pl from recipys.recipe import Recipe from recipys.selector import all_numeric_predictors, all_outcomes, has_type, all_of from recipys.step import ( StepScale, - StepImputeFastForwardFill, - StepImputeFastZeroFill, + # StepImputeFastForwardFill, + # StepImputeFastZeroFill, + StepImputeFill, StepSklearn, StepHistorical, Accumulator, @@ -41,8 +44,145 @@ def set_imputation_model(self, imputation_model): if self.imputation_model is not None: update_wandb_config({"imputation_model": self.imputation_model.__class__.__name__}) - @gin.configurable("base_classification_preprocessor") +class PolarsClassificationPreprocessor(Preprocessor): + def __init__( + self, + generate_features: bool = True, + scaling: bool = True, + use_static_features: bool = True, + save_cache=None, + load_cache=None, + ): + """ + Args: + generate_features: Generate features for dynamic data. + scaling: Scaling of dynamic and static data. + use_static_features: Use static features. + save_cache: Save recipe cache from this path. + load_cache: Load recipe cache from this path. + Returns: + Preprocessed data. + """ + self.generate_features = generate_features + self.scaling = scaling + self.use_static_features = use_static_features + self.imputation_model = None + self.save_cache = save_cache + self.load_cache = load_cache + + def apply(self, data, vars) -> dict[dict[pd.DataFrame]]: + """ + Args: + data: Train, validation and test data dictionary. Further divided in static, dynamic, and outcome. + vars: Variables for static, dynamic, outcome. + Returns: + Preprocessed data. + """ + logging.info("Preprocessing dynamic features.") + + data = self._process_dynamic(data, vars) + if self.use_static_features: + logging.info("Preprocessing static features.") + data = self._process_static(data, vars) + + # Set index to grouping variable + data[Split.train][Segment.static] = data[Split.train][Segment.static]#.set_index(vars["GROUP"]) + data[Split.val][Segment.static] = data[Split.val][Segment.static]#.set_index(vars["GROUP"]) + data[Split.test][Segment.static] = data[Split.test][Segment.static]#.set_index(vars["GROUP"]) + + # Join static and dynamic data. + data[Split.train][Segment.dynamic] = data[Split.train][Segment.dynamic].join( + data[Split.train][Segment.static], on=vars["GROUP"] + ) + data[Split.val][Segment.dynamic] = data[Split.val][Segment.dynamic].join( + data[Split.val][Segment.static], on=vars["GROUP"] + ) + data[Split.test][Segment.dynamic] = data[Split.test][Segment.dynamic].join( + data[Split.test][Segment.static], on=vars["GROUP"] + ) + + # Remove static features from splits + data[Split.train][Segment.features] = data[Split.train].pop(Segment.static) + data[Split.val][Segment.features] = data[Split.val].pop(Segment.static) + data[Split.test][Segment.features] = data[Split.test].pop(Segment.static) + + # Create feature splits + data[Split.train][Segment.features] = data[Split.train].pop(Segment.dynamic) + data[Split.val][Segment.features] = data[Split.val].pop(Segment.dynamic) + data[Split.test][Segment.features] = data[Split.test].pop(Segment.dynamic) + + logging.debug("Data head") + logging.debug(data[Split.train][Segment.features].head()) + logging.info(f"Generate features: {self.generate_features}") + return data + + def _process_static(self, data, vars): + sta_rec = Recipe(data[Split.train][Segment.static], [], vars[Segment.static]) + if self.scaling: + sta_rec.add_step(StepScale()) + + # sta_rec.add_step(StepImputeFastZeroFill(sel=all_numeric_predictors())) + sta_rec.add_step(StepImputeFill(sel=all_numeric_predictors(),strategy="zero")) + # if len(data[Split.train][Segment.static].select_dtypes(include=["object"]).columns) > 0: + pl_dtypes = [pl.String, pl.Object, pl.Categorical] + types = ["String", "Object", "Categorical"] + sel = has_type(types) + if(len(sel(sta_rec.data))>0): + # if len(data[Split.train][Segment.static].select(cs.by_dtype(types)).columns) > 0: + sta_rec.add_step(StepSklearn(SimpleImputer(missing_values=None, strategy="most_frequent"), sel=has_type(types))) + sta_rec.add_step(StepSklearn(LabelEncoder(), sel=has_type(types), columnwise=True)) + + data = apply_recipe_to_splits(sta_rec, data, Segment.static, self.save_cache, self.load_cache) + + return data + + def _model_impute(self, data, group=None): + dataset = ImputationPredictionDataset(data, group, self.imputation_model.trained_columns) + input_data = torch.cat([data_point.unsqueeze(0) for data_point in dataset], dim=0) + self.imputation_model.eval() + with torch.no_grad(): + logging.info(f"Imputing with {self.imputation_model.__class__.__name__}.") + imputation = self.imputation_model.predict(input_data) + logging.info("Imputation done.") + assert imputation.isnan().sum() == 0 + data = data.copy() + data.loc[:, self.imputation_model.trained_columns] = imputation.flatten(end_dim=1).to("cpu") + if group is not None: + data.drop(columns=group, inplace=True) + return data + + def _process_dynamic(self, data, vars): + dyn_rec = Recipe(data[Split.train][Segment.dynamic], [], vars[Segment.dynamic], vars["GROUP"], vars["SEQUENCE"]) + if self.scaling: + dyn_rec.add_step(StepScale()) + if self.imputation_model is not None: + dyn_rec.add_step(StepImputeModel(model=self.model_impute, sel=all_of(vars[Segment.dynamic]))) + dyn_rec.add_step(StepSklearn(MissingIndicator(), sel=all_of(vars[Segment.dynamic]), in_place=False)) + # dyn_rec.add_step(StepImputeFastForwardFill()) + dyn_rec.add_step(StepImputeFill(strategy="forward")) + # dyn_rec.add_step(StepImputeFastZeroFill()) + dyn_rec.add_step(StepImputeFill(strategy="zero")) + if self.generate_features: + dyn_rec = self._dynamic_feature_generation(dyn_rec, all_of(vars[Segment.dynamic])) + data = apply_recipe_to_splits(dyn_rec, data, Segment.dynamic, self.save_cache, self.load_cache) + return data + + def _dynamic_feature_generation(self, data, dynamic_vars): + logging.debug("Adding dynamic feature generation.") + data.add_step(StepHistorical(sel=dynamic_vars, fun=Accumulator.MIN, suffix="min_hist")) + data.add_step(StepHistorical(sel=dynamic_vars, fun=Accumulator.MAX, suffix="max_hist")) + data.add_step(StepHistorical(sel=dynamic_vars, fun=Accumulator.COUNT, suffix="count_hist")) + data.add_step(StepHistorical(sel=dynamic_vars, fun=Accumulator.MEAN, suffix="mean_hist")) + return data + + def to_cache_string(self): + return ( + super().to_cache_string() + + f"_classification_{self.generate_features}_{self.scaling}_{self.imputation_model.__class__.__name__}" + ) + +# @gin.configurable("base_classification_preprocessor") class DefaultClassificationPreprocessor(Preprocessor): def __init__( self, diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index e465ade2..8d92a4f1 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -11,7 +11,7 @@ from sklearn.model_selection import StratifiedKFold, KFold, StratifiedShuffleSplit, ShuffleSplit -from icu_benchmarks.data.preprocessor import Preprocessor, DefaultClassificationPreprocessor +from icu_benchmarks.data.preprocessor import Preprocessor, DefaultClassificationPreprocessor, PolarsClassificationPreprocessor from icu_benchmarks.contants import RunMode from .constants import DataSplit as Split, DataSegment as Segment, VarType as Var @@ -20,7 +20,7 @@ def preprocess_data( data_dir: Path, file_names: dict[str] = gin.REQUIRED, - preprocessor: Preprocessor = DefaultClassificationPreprocessor, + preprocessor: Preprocessor = PolarsClassificationPreprocessor, use_static: bool = True, vars: dict[str] = gin.REQUIRED, seed: int = 42, diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 905d1245..e3464caa 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -81,9 +81,9 @@ def train_common( logging.info(f"Training model: {model.__name__}.") # dataset_class = ImputationDataset if mode == RunMode.imputation else PredictionDataset - for dict in data.values(): - for key,val in dict.items(): - dict[key] = pl.from_pandas(val) + # for dict in data.values(): + # for key,val in dict.items(): + # dict[key] = pl.from_pandas(val) dataset_class = PredictionPolarsDataset logging.info(f"Logging to directory: {log_dir}.") save_config_file(log_dir) # We save the operative config before and also after training From b93a923166d32e8fd20b066715805ba533169e65 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Fri, 5 Jul 2024 16:33:25 +0200 Subject: [PATCH 142/207] preprocessing working experimentally --- icu_benchmarks/data/preprocessor.py | 6 +++--- icu_benchmarks/data/split_process_data.py | 5 ++++- icu_benchmarks/models/train.py | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/icu_benchmarks/data/preprocessor.py b/icu_benchmarks/data/preprocessor.py index 77b6b4db..57e12648 100644 --- a/icu_benchmarks/data/preprocessor.py +++ b/icu_benchmarks/data/preprocessor.py @@ -119,11 +119,11 @@ def apply(self, data, vars) -> dict[dict[pd.DataFrame]]: def _process_static(self, data, vars): sta_rec = Recipe(data[Split.train][Segment.static], [], vars[Segment.static]) + sta_rec.add_step(StepImputeFill(sel=all_numeric_predictors(),strategy="zero")) if self.scaling: sta_rec.add_step(StepScale()) # sta_rec.add_step(StepImputeFastZeroFill(sel=all_numeric_predictors())) - sta_rec.add_step(StepImputeFill(sel=all_numeric_predictors(),strategy="zero")) # if len(data[Split.train][Segment.static].select_dtypes(include=["object"]).columns) > 0: pl_dtypes = [pl.String, pl.Object, pl.Categorical] types = ["String", "Object", "Categorical"] @@ -154,8 +154,6 @@ def _model_impute(self, data, group=None): def _process_dynamic(self, data, vars): dyn_rec = Recipe(data[Split.train][Segment.dynamic], [], vars[Segment.dynamic], vars["GROUP"], vars["SEQUENCE"]) - if self.scaling: - dyn_rec.add_step(StepScale()) if self.imputation_model is not None: dyn_rec.add_step(StepImputeModel(model=self.model_impute, sel=all_of(vars[Segment.dynamic]))) dyn_rec.add_step(StepSklearn(MissingIndicator(), sel=all_of(vars[Segment.dynamic]), in_place=False)) @@ -163,6 +161,8 @@ def _process_dynamic(self, data, vars): dyn_rec.add_step(StepImputeFill(strategy="forward")) # dyn_rec.add_step(StepImputeFastZeroFill()) dyn_rec.add_step(StepImputeFill(strategy="zero")) + if self.scaling: + dyn_rec.add_step(StepScale()) if self.generate_features: dyn_rec = self._dynamic_feature_generation(dyn_rec, all_of(vars[Segment.dynamic])) data = apply_recipe_to_splits(dyn_rec, data, Segment.dynamic, self.save_cache, self.load_cache) diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index 8d92a4f1..d8fe3247 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -8,7 +8,7 @@ import pyarrow.parquet as pq from pathlib import Path import pickle - +from timeit import default_timer as timer from sklearn.model_selection import StratifiedKFold, KFold, StratifiedShuffleSplit, ShuffleSplit from icu_benchmarks.data.preprocessor import Preprocessor, DefaultClassificationPreprocessor, PolarsClassificationPreprocessor @@ -116,7 +116,10 @@ def preprocess_data( # Apply preprocessing # data = {Split.train: data, Split.val: data, Split.test: data} + start = timer() data = preprocessor.apply(data, vars) + end = timer() + logging.info(f"Preprocessing took: {end - start}") # data[Split.train][Segment.dynamic].to_parquet(data_dir / "preprocessed.parquet") # data[Split.train][Segment.outcome].to_parquet(data_dir / "outcome.parquet") diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index e3464caa..b9ffd70e 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -100,7 +100,7 @@ def train_common( ) logging.info(f"Using {num_workers} workers for data loading.") cpu=True - batch_size=1 + # batch_size=1 train_loader = DataLoader( train_dataset, batch_size=batch_size, From f074b8ca1ce8c1c3b88354959a6e5bffa9126e78 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Fri, 5 Jul 2024 18:02:40 +0200 Subject: [PATCH 143/207] ordering missing indicator back because of nan/none fix --- icu_benchmarks/data/preprocessor.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/icu_benchmarks/data/preprocessor.py b/icu_benchmarks/data/preprocessor.py index 57e12648..57197f59 100644 --- a/icu_benchmarks/data/preprocessor.py +++ b/icu_benchmarks/data/preprocessor.py @@ -119,13 +119,12 @@ def apply(self, data, vars) -> dict[dict[pd.DataFrame]]: def _process_static(self, data, vars): sta_rec = Recipe(data[Split.train][Segment.static], [], vars[Segment.static]) - sta_rec.add_step(StepImputeFill(sel=all_numeric_predictors(),strategy="zero")) + sta_rec.add_step(StepSklearn(MissingIndicator(features="all"), sel=all_of(vars[Segment.static]), in_place=False)) if self.scaling: sta_rec.add_step(StepScale()) - + sta_rec.add_step(StepImputeFill(sel=all_numeric_predictors(),strategy="zero")) # sta_rec.add_step(StepImputeFastZeroFill(sel=all_numeric_predictors())) # if len(data[Split.train][Segment.static].select_dtypes(include=["object"]).columns) > 0: - pl_dtypes = [pl.String, pl.Object, pl.Categorical] types = ["String", "Object", "Categorical"] sel = has_type(types) if(len(sel(sta_rec.data))>0): @@ -154,15 +153,15 @@ def _model_impute(self, data, group=None): def _process_dynamic(self, data, vars): dyn_rec = Recipe(data[Split.train][Segment.dynamic], [], vars[Segment.dynamic], vars["GROUP"], vars["SEQUENCE"]) + if self.scaling: + dyn_rec.add_step(StepScale()) if self.imputation_model is not None: dyn_rec.add_step(StepImputeModel(model=self.model_impute, sel=all_of(vars[Segment.dynamic]))) - dyn_rec.add_step(StepSklearn(MissingIndicator(), sel=all_of(vars[Segment.dynamic]), in_place=False)) + dyn_rec.add_step(StepSklearn(MissingIndicator(features="all"), sel=all_of(vars[Segment.dynamic]), in_place=False)) # dyn_rec.add_step(StepImputeFastForwardFill()) dyn_rec.add_step(StepImputeFill(strategy="forward")) # dyn_rec.add_step(StepImputeFastZeroFill()) dyn_rec.add_step(StepImputeFill(strategy="zero")) - if self.scaling: - dyn_rec.add_step(StepScale()) if self.generate_features: dyn_rec = self._dynamic_feature_generation(dyn_rec, all_of(vars[Segment.dynamic])) data = apply_recipe_to_splits(dyn_rec, data, Segment.dynamic, self.save_cache, self.load_cache) From 3aac4f19f2e6a8f8dbce8411fd965adeefd47a70 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Mon, 8 Jul 2024 09:23:11 +0200 Subject: [PATCH 144/207] preproc timing --- icu_benchmarks/data/split_process_data.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index 5831a457..5bacc8f1 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -7,7 +7,7 @@ import pyarrow.parquet as pq from pathlib import Path import pickle - +from timeit import default_timer as timer from sklearn.model_selection import StratifiedKFold, KFold, StratifiedShuffleSplit, ShuffleSplit from icu_benchmarks.data.preprocessor import Preprocessor, DefaultClassificationPreprocessor @@ -115,7 +115,11 @@ def preprocess_data( # Apply preprocessing # data = {Split.train: data, Split.val: data, Split.test: data} + + start = timer() data = preprocessor.apply(data, vars) + end = timer() + logging.info(f"Preprocessing took {end - start:.2f} seconds.") # data[Split.train][Segment.dynamic].to_parquet(data_dir / "preprocessed.parquet") # data[Split.train][Segment.outcome].to_parquet(data_dir / "outcome.parquet") From 5b37115b66af92014fe32138f76193bfa4eeaa4b Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 9 Jul 2024 11:53:09 +0200 Subject: [PATCH 145/207] catching errors --- icu_benchmarks/cross_validation.py | 5 ++++- icu_benchmarks/run.py | 5 ++++- icu_benchmarks/tuning/hyperparameters.py | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/icu_benchmarks/cross_validation.py b/icu_benchmarks/cross_validation.py index 95e44e1a..730f2a63 100644 --- a/icu_benchmarks/cross_validation.py +++ b/icu_benchmarks/cross_validation.py @@ -133,7 +133,10 @@ def execute_repeated_cv( if wandb: wandb_log({"Iteration": repetition * cv_folds_to_train + fold_index}) if repetition * cv_folds_to_train + fold_index > 1: - aggregate_results(log_dir) + try: + aggregate_results(log_dir) + except Exception as e: + logging.error(f"Failed to aggregate results: {e}") log_full_line(f"FINISHED CV REPETITION {repetition}", level=logging.INFO, char="=", num_newlines=3) return agg_loss / (cv_repetitions_to_train * cv_folds_to_train) diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index 80fd05ad..1dc5921e 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -182,7 +182,10 @@ def main(my_args=tuple(sys.argv[1:])): log_full_line("FINISHED TRAINING", level=logging.INFO, char="=", num_newlines=3) execution_time = datetime.now() - start_time log_full_line(f"DURATION: {execution_time}", level=logging.INFO, char="") - aggregate_results(run_dir, execution_time) + try: + aggregate_results(run_dir, execution_time) + except Exception as e: + logging.error(f"Failed to aggregate results: {e}") if args.plot: plot_aggregated_results(run_dir, "aggregated_test_metrics.json") diff --git a/icu_benchmarks/tuning/hyperparameters.py b/icu_benchmarks/tuning/hyperparameters.py index eed90e7c..68eaa047 100644 --- a/icu_benchmarks/tuning/hyperparameters.py +++ b/icu_benchmarks/tuning/hyperparameters.py @@ -267,7 +267,7 @@ def tune_step_callback(study: optuna.study.Study, trial: optuna.trial.FrozenTria highlight = study.trials[-1] == study.best_trial # highlight if best so far log_table_row(header, TUNE) log_table_row(table_cells, TUNE, align=Align.RIGHT, header=header, highlight=highlight) - # wandb_log({"hp-iteration": len(study.trials)}) + wandb_log({"HP-optimization-iteration": len(study.trials)}) if do_tune: log_full_line("STARTING TUNING", level=TUNE, char="=") From 06cf6c0ff3d1f58c87c781594e8fcace0ee7ac95 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 9 Jul 2024 14:20:27 +0200 Subject: [PATCH 146/207] Cast to get consistent join key --- icu_benchmarks/data/split_process_data.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index e7d0c40e..a2fc5ad8 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -185,7 +185,8 @@ def make_train_val( # Loop through segments (DYNAMIC / STATIC / OUTCOME) # set sort to true to make sure that IDs are reordered after scrambling earlier data_split[fold] = { - data_type: data[data_type].merge(split[fold], on=id, how="right", sort=True) for data_type in data.keys() + data_type: split[fold].join(data[data_type].with_columns(pl.col(id).cast(pl.datatypes.Int64)), on=id, + how="left").sort(by=id) for data_type in data.keys() } # Maintain compatibility with test split data_split[Split.test] = copy.deepcopy(data_split[Split.val]) @@ -283,9 +284,9 @@ def make_single_split( train, val = list(inner_cv.split(dev_stays))[fold_index] if polars: split = { - Split.train: dev_stays[train].to_frame(), - Split.val: dev_stays[val].to_frame(), - Split.test: stays[test].to_frame(), + Split.train: dev_stays[train].cast(pl.datatypes.Int64).to_frame(), + Split.val: dev_stays[val].cast(pl.datatypes.Int64).to_frame(), + Split.test: stays[test].cast(pl.datatypes.Int64).to_frame(), } else: split = { @@ -300,7 +301,7 @@ def make_single_split( # set sort to true to make sure that IDs are reordered after scrambling earlier if polars: data_split[fold] = { - data_type: split[fold].join(data[data_type], on=id, how="left").sort(by=id) for data_type in data.keys() + data_type: split[fold].join(data[data_type].with_columns(pl.col(id).cast(pl.datatypes.Int64)), on=id, how="left").sort(by=id) for data_type in data.keys() } else: data_split[fold] = { From 957c49964c0ec3f59d001d0dc310b497a370c2ea Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 9 Jul 2024 14:41:30 +0200 Subject: [PATCH 147/207] refactor and cleanup. Added support for regression --- configs/tasks/DatasetImputation.gin | 4 +- configs/tasks/Regression.gin | 3 +- configs/tasks/common/Dataloader.gin | 7 +++ environment.yml | 2 +- icu_benchmarks/data/loader.py | 12 ++-- icu_benchmarks/data/preprocessor.py | 73 +++++++++++++++++++++-- icu_benchmarks/data/split_process_data.py | 9 ++- icu_benchmarks/models/train.py | 9 ++- 8 files changed, 95 insertions(+), 24 deletions(-) create mode 100644 configs/tasks/common/Dataloader.gin diff --git a/configs/tasks/DatasetImputation.gin b/configs/tasks/DatasetImputation.gin index ddbd56a2..55914adc 100644 --- a/configs/tasks/DatasetImputation.gin +++ b/configs/tasks/DatasetImputation.gin @@ -22,6 +22,6 @@ preprocess.file_names = { preprocess.preprocessor = @base_imputation_preprocessor preprocess.vars = %vars -ImputationDataset.vars = %vars -ImputationDataset.ram_cache = True + +include "configs/tasks/common/Dataloader.gin" diff --git a/configs/tasks/Regression.gin b/configs/tasks/Regression.gin index 5cf3f8d9..c2c54174 100644 --- a/configs/tasks/Regression.gin +++ b/configs/tasks/Regression.gin @@ -28,6 +28,5 @@ base_regression_preprocessor.outcome_min = 0 base_regression_preprocessor.outcome_max = 15 # SELECTING DATASET -PredictionDataset.vars = %vars -PredictionDataset.ram_cache = True +include "configs/tasks/common/Dataloader.gin" diff --git a/configs/tasks/common/Dataloader.gin b/configs/tasks/common/Dataloader.gin new file mode 100644 index 00000000..eee40a76 --- /dev/null +++ b/configs/tasks/common/Dataloader.gin @@ -0,0 +1,7 @@ +# Prediction +PredictionPandasDataset.vars = %vars +PredictionPolarsDataset.vars = %vars +PredictionPandasDataset.ram_cache = True +# Imputation +ImputationPandasDataset.vars = %vars +ImputationPandasDataset.ram_cache = True \ No newline at end of file diff --git a/environment.yml b/environment.yml index 5616269a..65e048d2 100644 --- a/environment.yml +++ b/environment.yml @@ -1,4 +1,4 @@ -name: yaib_19 +name: yaib channels: - conda-forge dependencies: diff --git a/icu_benchmarks/data/loader.py b/icu_benchmarks/data/loader.py index 18586cec..b95b8cd4 100644 --- a/icu_benchmarks/data/loader.py +++ b/icu_benchmarks/data/loader.py @@ -160,8 +160,8 @@ def to_tensor(self): else: return from_numpy(data), from_numpy(labels) - -class CommonDataset(Dataset): +@gin.configurable("CommonPandasDataset") +class CommonPandasDataset(Dataset): """Common dataset: subclass of Torch Dataset that represents the data to learn on. Args: data: Dict of the different splits of the data. split: Either 'train','val' or 'test'. vars: Contains the names of @@ -219,8 +219,8 @@ def to_tensor(self): return [cat(value, dim=0) for value in values] -@gin.configurable("PredictionDataset") -class PredictionDataset(CommonDataset): +@gin.configurable("PredictionPandasDataset") +class PredictionPandasDataset(CommonPandasDataset): """Subclass of common dataset for prediction tasks. Args: @@ -311,8 +311,8 @@ def to_tensor(self): return from_numpy(data), from_numpy(labels) -@gin.configurable("ImputationDataset") -class ImputationDataset(CommonDataset): +@gin.configurable("ImputationPandasDataset") +class ImputationPandasDataset(CommonPandasDataset): """Subclass of Common Dataset that contains data for imputation models.""" def __init__( diff --git a/icu_benchmarks/data/preprocessor.py b/icu_benchmarks/data/preprocessor.py index 57197f59..76865a49 100644 --- a/icu_benchmarks/data/preprocessor.py +++ b/icu_benchmarks/data/preprocessor.py @@ -180,9 +180,71 @@ def to_cache_string(self): super().to_cache_string() + f"_classification_{self.generate_features}_{self.scaling}_{self.imputation_model.__class__.__name__}" ) +@gin.configurable("base_regression_preprocessor") +class PolarsRegressionPreprocessor(PolarsClassificationPreprocessor): + # Override base classification preprocessor + def __init__( + self, + generate_features: bool = True, + scaling: bool = True, + use_static_features: bool = True, + outcome_max=None, + outcome_min=None, + save_cache=None, + load_cache=None, + ): + """ + Args: + generate_features: Generate features for dynamic data. + scaling: Scaling of dynamic and static data. + use_static_features: Use static features. + max_range: Maximum value in outcome. + min_range: Minimum value in outcome. + save_cache: Save recipe cache. + load_cache: Load recipe cache. + Returns: + Preprocessed data. + """ + super().__init__(generate_features, scaling, use_static_features, save_cache, load_cache) + self.outcome_max = outcome_max + self.outcome_min = outcome_min + + def apply(self, data, vars): + """ + Args: + data: Train, validation and test data dictionary. Further divided in static, dynamic, and outcome. + vars: Variables for static, dynamic, outcome. + Returns: + Preprocessed data. + """ + for split in [Split.train, Split.val, Split.test]: + data = self._process_outcome(data, vars, split) + + data = super().apply(data, vars) + return data -# @gin.configurable("base_classification_preprocessor") -class DefaultClassificationPreprocessor(Preprocessor): + def _process_outcome(self, data, vars, split): + logging.debug(f"Processing {split} outcome values.") + outcome_rec = Recipe(data[split][Segment.outcome], vars["LABEL"], [], vars["GROUP"]) + # If the range is predefined, use predefined transformation function + if self.outcome_max is not None and self.outcome_min is not None: + outcome_rec.add_step( + StepSklearn( + sklearn_transformer=FunctionTransformer( + func=lambda x: ((x - self.outcome_min) / (self.outcome_max - self.outcome_min)) + ), + sel=all_outcomes(), + ) + ) + else: + # If the range is not predefined, use MinMaxScaler + outcome_rec.add_step(StepSklearn(MinMaxScaler(), sel=all_outcomes())) + outcome_rec.prep() + data[split][Segment.outcome] = outcome_rec.bake() + return data + +@gin.configurable("pandas_classification_preprocessor") +class PandasClassificationPreprocessor(Preprocessor): def __init__( self, generate_features: bool = True, @@ -312,8 +374,8 @@ def to_cache_string(self): ) -@gin.configurable("base_regression_preprocessor") -class DefaultRegressionPreprocessor(DefaultClassificationPreprocessor): +@gin.configurable("pandas_regression_preprocessor") +class PandasRegressionPreprocessor(PandasClassificationPreprocessor): # Override base classification preprocessor def __init__( self, @@ -377,7 +439,7 @@ def _process_outcome(self, data, vars, split): @gin.configurable("base_imputation_preprocessor") -class DefaultImputationPreprocessor(Preprocessor): +class PandasImputationPreprocessor(Preprocessor): def __init__( self, scaling: bool = True, @@ -438,6 +500,7 @@ def apply_recipe_to_splits( recipe: Recipe, data: dict[dict[pd.DataFrame]], type: str, save_cache=None, load_cache=None ) -> dict[dict[pd.DataFrame]]: """Fits and transforms the training features, then transforms the validation and test features with the recipe. + Works with both Polars and Pandas versions of recipys. Args: load_cache: Load recipe from cache, for e.g. transfer learning. diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index a2fc5ad8..d00a9225 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -11,7 +11,7 @@ from timeit import default_timer as timer from sklearn.model_selection import StratifiedKFold, KFold, StratifiedShuffleSplit, ShuffleSplit -from icu_benchmarks.data.preprocessor import Preprocessor, DefaultClassificationPreprocessor, PolarsClassificationPreprocessor +from icu_benchmarks.data.preprocessor import Preprocessor, PandasClassificationPreprocessor, PolarsClassificationPreprocessor from icu_benchmarks.contants import RunMode from .constants import DataSplit as Split, DataSegment as Segment, VarType as Var @@ -75,7 +75,7 @@ def preprocess_data( logging.log(logging.INFO, f"Using preprocessor: {preprocessor.__name__}") preprocessor = preprocessor(use_static_features=use_static, save_cache=data_dir / "preproc" / (cache_filename + "_recipe")) - if isinstance(preprocessor, DefaultClassificationPreprocessor): + if isinstance(preprocessor, PandasClassificationPreprocessor): preprocessor.set_imputation_model(pretrained_imputation_model) hash_config = hashlib.md5(f"{preprocessor.to_cache_string()}{dumped_file_names}{dumped_vars}".encode("utf-8")) @@ -280,7 +280,10 @@ def make_single_split( inner_cv = KFold(cv_folds, shuffle=True, random_state=seed) dev, test = list(outer_cv.split(stays))[repetition_index] - dev_stays = stays.iloc[dev] + if polars: + dev_stays = stays[dev] + else: + dev_stays = stays.iloc[dev] train, val = list(inner_cv.split(dev_stays))[fold_index] if polars: split = { diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index b9ffd70e..dba2b70f 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -3,7 +3,6 @@ import torch import logging import pandas as pd -import polars as pl from joblib import load from torch.optim import Adam from torch.utils.data import DataLoader @@ -11,7 +10,7 @@ from pytorch_lightning import Trainer from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint, TQDMProgressBar, LearningRateMonitor from pathlib import Path -from icu_benchmarks.data.loader import PredictionDataset, ImputationDataset, PredictionPolarsDataset +from icu_benchmarks.data.loader import PredictionPandasDataset, ImputationPandasDataset, PredictionPolarsDataset from icu_benchmarks.models.utils import save_config_file, JSONMetricsLogger from icu_benchmarks.contants import RunMode from icu_benchmarks.data.constants import DataSplit as Split @@ -51,6 +50,7 @@ def train_common( pl_model=True, train_only=False, num_workers: int = min(cpu_core_count, torch.cuda.device_count() * 8 * int(torch.cuda.is_available()), 32), + polars=True, ): """Common wrapper to train all benchmarked models. @@ -80,11 +80,10 @@ def train_common( """ logging.info(f"Training model: {model.__name__}.") - # dataset_class = ImputationDataset if mode == RunMode.imputation else PredictionDataset + dataset_class = ImputationPandasDataset if mode == RunMode.imputation else PredictionPolarsDataset if polars else PredictionPandasDataset # for dict in data.values(): # for key,val in dict.items(): # dict[key] = pl.from_pandas(val) - dataset_class = PredictionPolarsDataset logging.info(f"Logging to directory: {log_dir}.") save_config_file(log_dir) # We save the operative config before and also after training @@ -99,8 +98,8 @@ def train_common( f" {len(val_dataset)} samples." ) logging.info(f"Using {num_workers} workers for data loading.") + # todo: compare memory pinning cpu=True - # batch_size=1 train_loader = DataLoader( train_dataset, batch_size=batch_size, From 6d15904c74cb8461fa17d6df786d0d9bd8b37c9f Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 9 Jul 2024 14:41:51 +0200 Subject: [PATCH 148/207] binary classification --- configs/tasks/BinaryClassification.gin | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/configs/tasks/BinaryClassification.gin b/configs/tasks/BinaryClassification.gin index d1ca94fd..70eed783 100644 --- a/configs/tasks/BinaryClassification.gin +++ b/configs/tasks/BinaryClassification.gin @@ -23,9 +23,5 @@ preprocess.vars = %vars preprocess.use_static = True # SELECTING DATASET -PredictionDataset.vars = %vars -PredictionPolarsDataset.vars = %vars - -PredictionDataset.ram_cache = True - +include "configs/tasks/common/Dataloader.gin" From c226a12f4a107ae94ef7aa6ea6ceca8e33089567 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 9 Jul 2024 15:05:59 +0200 Subject: [PATCH 149/207] cleanup --- icu_benchmarks/models/train.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index dba2b70f..f0a5a43d 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -80,15 +80,14 @@ def train_common( """ logging.info(f"Training model: {model.__name__}.") + # todo: add support for polars versions of datasets dataset_class = ImputationPandasDataset if mode == RunMode.imputation else PredictionPolarsDataset if polars else PredictionPandasDataset - # for dict in data.values(): - # for key,val in dict.items(): - # dict[key] = pl.from_pandas(val) + logging.info(f"Using dataset class: {dataset_class.__name__}.") logging.info(f"Logging to directory: {log_dir}.") save_config_file(log_dir) # We save the operative config before and also after training - train_dataset = dataset_class(data, split=Split.train, ram_cache=False, name=dataset_names["train"]) - val_dataset = dataset_class(data, split=Split.val, ram_cache=False, name=dataset_names["val"]) + train_dataset = dataset_class(data, split=Split.train, ram_cache=ram_cache, name=dataset_names["train"]) + val_dataset = dataset_class(data, split=Split.val, ram_cache=ram_cache, name=dataset_names["val"]) train_dataset, val_dataset = assure_minimum_length(train_dataset), assure_minimum_length(val_dataset) batch_size = min(batch_size, len(train_dataset), len(val_dataset)) From bdbbff5dd197f6f0986de8cf6cd165c529e4ba03 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 9 Jul 2024 15:06:21 +0200 Subject: [PATCH 150/207] cass related changes --- experiments/benchmark_cass.yml | 25 +++++++++++++++++-------- experiments/charhpc_wandb_sweep.sh | 2 +- experiments/charhpc_wandb_sweep_cpu.sh | 2 +- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/experiments/benchmark_cass.yml b/experiments/benchmark_cass.yml index 2309d286..936b06c9 100644 --- a/experiments/benchmark_cass.yml +++ b/experiments/benchmark_cass.yml @@ -5,17 +5,20 @@ command: - -d - ../data/ - -t - - BinaryClassification -# - CassClassification +# - BinaryClassification + - CassClassification - --log-dir - /sc-projects/sc-proj-cc08-cassandra/RW_Prospective/yaib_logs - --tune - --wandb-sweep - -tn -# - SSI - - Mortality + - SSI +# - Mortality - -n - cass +# - --hp-checkpoint +# - /sc-projects/sc-proj-cc08-cassandra/RW_Prospective/yaib_logs/cass/SSI/XGBClassifier/2024-06-28T17-28-04/hyperparameter_tuning_logs.db +# - /sc-projects/sc-proj-cc08-cassandra/RW_Prospective/yaib_logs/cass/SSI/Transformer/2024-06-26T15-30-53/hyperparameter_tuning_logs.db # - -gc # - -lc method: grid @@ -23,16 +26,22 @@ name: yaib_classification_benchmark parameters: data_dir: values: - - /home/vandewrp/projects/YAIB/data/YAIB_Datasets/data/mortality24/miiv - #- /sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/notes +# - /home/vandewrp/projects/YAIB/data/YAIB_Datasets/data/mortality24/miiv + - /sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/notes_full model: values: # - LogisticRegression - - TimesNet +# - TimesNet +# - LGBMClassifier +# - XGBClassifier +# - CBClassifier +# - BRFClassifier +# - RUSBClassifier +# - RFClassifier # - GRU # - LSTM # - TCN -# - Transformer + - Transformer seed: values: - 1111 diff --git a/experiments/charhpc_wandb_sweep.sh b/experiments/charhpc_wandb_sweep.sh index 30975f4f..4c713d16 100644 --- a/experiments/charhpc_wandb_sweep.sh +++ b/experiments/charhpc_wandb_sweep.sh @@ -8,7 +8,7 @@ #SBATCH --time=48:00:00 eval "$(conda shell.bash hook)" -conda activate yaib_req +conda activate yaib_req_pl wandb agent --count 1 cassandra_hpi/cassandra/"$1" diff --git a/experiments/charhpc_wandb_sweep_cpu.sh b/experiments/charhpc_wandb_sweep_cpu.sh index f9377a02..1662d6c9 100644 --- a/experiments/charhpc_wandb_sweep_cpu.sh +++ b/experiments/charhpc_wandb_sweep_cpu.sh @@ -7,7 +7,7 @@ #SBATCH --time=48:00:00 eval "$(conda shell.bash hook)" -conda activate yaib_req +conda activate yaib_req_pl wandb agent --count 1 cassandra_hpi/cassandra/"$1" From b1ffc318bf02cc9bb8d9008b524701f706e46788 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 9 Jul 2024 15:33:09 +0200 Subject: [PATCH 151/207] cleaning up requirements with release candidate recipies --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1c4dd0e8..b03e3df5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,10 +27,9 @@ hydra-core==1.3 optuna==3.6.1 optuna-integration==3.6.0 wandb==0.17.3 -#recipies==0.1.3 +recipies==1.0rc1 #Fixed version because of NumPy incompatibility and stale development status. scikit-optimize-fix==0.9.1 hydra-submitit-launcher==1.2.0 pytest-runner==6.0.1 -recipys @ git+https://github.com/rvandewater/ReciPys@polars From 7eab87d4526d322a16a355f51b5b723a312ea01c Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 9 Jul 2024 16:52:20 +0200 Subject: [PATCH 152/207] complete train for polars --- icu_benchmarks/data/split_process_data.py | 53 +++++++++++++++-------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index d00a9225..78a6d7a0 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -110,20 +110,16 @@ def preprocess_data( runmode=runmode, ) else: - data = data # If full train is set, we use all data for training/validation - # data = make_train_val(data, vars, train_size=None, seed=seed, debug=debug, runmode=runmode) + data = make_train_val(data, vars, train_size=None, seed=seed, debug=debug, runmode=runmode) # Apply preprocessing - # data = {Split.train: data, Split.val: data, Split.test: data} start = timer() data = preprocessor.apply(data, vars) end = timer() logging.info(f"Preprocessing took {end - start:.2f} seconds.") - # data[Split.train][Segment.dynamic].to_parquet(data_dir / "preprocessed.parquet") - # data[Split.train][Segment.outcome].to_parquet(data_dir / "outcome.parquet") # Generate cache if generate_cache: @@ -143,7 +139,8 @@ def make_train_val( seed: int = 42, debug: bool = False, runmode: RunMode = RunMode.classification, -) -> dict[dict[pd.DataFrame]]: + polars: bool = True, +) -> dict[dict[pl.DataFrame]]: """Randomly split the data into training and validation sets for fitting a full model. Args: @@ -163,12 +160,19 @@ def make_train_val( if debug: # Only use 1% of the data - stays = stays.sample(frac=0.01, random_state=seed) + logging.info("Using only 1% of the data for debugging. Note that this might lead to errors for small datasets.") + if polars: + data[Segment.outcome] = data[Segment.outcome].sample(fraction=0.01, seed=seed) + else: + data[Segment.outcome] = data[Segment.outcome].sample(frac=0.01, random_state=seed) # If there are labels, and the task is classification, use stratified k-fold - if Var.label in vars and runmode is RunMode.classification : - # Get labels from outcome data (takes the highest value (or True) in case seq2seq classification) - labels = data[Segment.outcome].groupby(id).max()[vars[Var.label]].reset_index(drop=True) + if Var.label in vars and runmode is RunMode.classification: + if polars: + labels = data[Segment.outcome].group_by(id).max()[vars[Var.label]] + else: + # Get labels from outcome data (takes the highest value (or True) in case seq2seq classification) + labels = data[Segment.outcome].groupby(id).max()[vars[Var.label]].reset_index(drop=True) if train_size: train_val = StratifiedShuffleSplit(train_size=train_size, random_state=seed, n_splits=1) train, val = list(train_val.split(stays, labels))[0] @@ -177,23 +181,33 @@ def make_train_val( train_val = ShuffleSplit(train_size=train_size, random_state=seed) train, val = list(train_val.split(stays))[0] - split = {Split.train: stays.iloc[train], Split.val: stays.iloc[val]} + if polars: + split = {Split.train: stays.loc[train], Split.val: stays.loc[val]} + else: + split = {Split.train: stays.iloc[train], Split.val: stays.iloc[val]} data_split = {} for fold in split.keys(): # Loop through splits (train / val / test) # Loop through segments (DYNAMIC / STATIC / OUTCOME) # set sort to true to make sure that IDs are reordered after scrambling earlier - data_split[fold] = { - data_type: split[fold].join(data[data_type].with_columns(pl.col(id).cast(pl.datatypes.Int64)), on=id, - how="left").sort(by=id) for data_type in data.keys() - } + if polars: + data_split[fold] = { + data_type: split[fold] + .join(data[data_type].with_columns(pl.col(id).cast(pl.datatypes.Int64)), on=id, how="left") + .sort(by=id) + for data_type in data.keys() + } + else: + data_split[fold] = { + data_type: data[data_type].merge(split[fold], on=id, how="right", sort=True) for data_type in data.keys() + } + # Maintain compatibility with test split data_split[Split.test] = copy.deepcopy(data_split[Split.val]) return data_split - def make_single_split( data: dict[pd.DataFrame], vars: dict[str], @@ -206,7 +220,7 @@ def make_single_split( debug: bool = False, runmode: RunMode = RunMode.classification, polars: bool = True, -) -> dict[dict[pd.DataFrame]]: +) -> dict[dict[pl.DataFrame]]: """Randomly split the data into training, validation, and test set. Args: @@ -304,7 +318,10 @@ def make_single_split( # set sort to true to make sure that IDs are reordered after scrambling earlier if polars: data_split[fold] = { - data_type: split[fold].join(data[data_type].with_columns(pl.col(id).cast(pl.datatypes.Int64)), on=id, how="left").sort(by=id) for data_type in data.keys() + data_type: split[fold] + .join(data[data_type].with_columns(pl.col(id).cast(pl.datatypes.Int64)), on=id, how="left") + .sort(by=id) + for data_type in data.keys() } else: data_split[fold] = { From d966fc92f9a33496bdc68ee6ed0aef1bc0757c91 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 9 Jul 2024 17:05:23 +0200 Subject: [PATCH 153/207] Complete train extended --- configs/tasks/common/Dataloader.gin | 2 +- icu_benchmarks/data/split_process_data.py | 17 ++++++++++------- icu_benchmarks/models/train.py | 5 ++--- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/configs/tasks/common/Dataloader.gin b/configs/tasks/common/Dataloader.gin index eee40a76..424bbaf6 100644 --- a/configs/tasks/common/Dataloader.gin +++ b/configs/tasks/common/Dataloader.gin @@ -1,7 +1,7 @@ # Prediction PredictionPandasDataset.vars = %vars PredictionPolarsDataset.vars = %vars -PredictionPandasDataset.ram_cache = True +PredictionPandasDataset.ram_cache = False # Imputation ImputationPandasDataset.vars = %vars ImputationPandasDataset.ram_cache = True \ No newline at end of file diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index 78a6d7a0..0260562e 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -155,9 +155,6 @@ def make_train_val( # ID variable id = vars[Var.group] - # Get stay IDs from outcome segment - stays = pd.Series(data[Segment.outcome][id].unique(), name=id) - if debug: # Only use 1% of the data logging.info("Using only 1% of the data for debugging. Note that this might lead to errors for small datasets.") @@ -166,6 +163,12 @@ def make_train_val( else: data[Segment.outcome] = data[Segment.outcome].sample(frac=0.01, random_state=seed) + # Get stay IDs from outcome segment + if polars: + stays = pl.Series(name=id, values=data[Segment.outcome][id].unique()) + else: + stays = pd.Series(data[Segment.outcome][id].unique(), name=id) + # If there are labels, and the task is classification, use stratified k-fold if Var.label in vars and runmode is RunMode.classification: if polars: @@ -173,16 +176,16 @@ def make_train_val( else: # Get labels from outcome data (takes the highest value (or True) in case seq2seq classification) labels = data[Segment.outcome].groupby(id).max()[vars[Var.label]].reset_index(drop=True) - if train_size: - train_val = StratifiedShuffleSplit(train_size=train_size, random_state=seed, n_splits=1) - train, val = list(train_val.split(stays, labels))[0] + train_val = StratifiedShuffleSplit(train_size=train_size, random_state=seed, n_splits=1) + train, val = list(train_val.split(stays, labels))[0] + else: # If there are no labels, use random split train_val = ShuffleSplit(train_size=train_size, random_state=seed) train, val = list(train_val.split(stays))[0] if polars: - split = {Split.train: stays.loc[train], Split.val: stays.loc[val]} + split = {Split.train: stays[train].cast(pl.datatypes.Int64).to_frame(), Split.val: stays[val].cast(pl.datatypes.Int64).to_frame()} else: split = {Split.train: stays.iloc[train], Split.val: stays.iloc[val]} diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index f0a5a43d..e57e0a68 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -2,7 +2,7 @@ import gin import torch import logging -import pandas as pd +import polars as pl from joblib import load from torch.optim import Adam from torch.utils.data import DataLoader @@ -26,7 +26,7 @@ def assure_minimum_length(dataset): @gin.configurable("train_common") def train_common( - data: dict[str, pd.DataFrame], + data: dict[str, pl.DataFrame], log_dir: Path, eval_only: bool = False, load_weights: bool = False, @@ -85,7 +85,6 @@ def train_common( logging.info(f"Using dataset class: {dataset_class.__name__}.") logging.info(f"Logging to directory: {log_dir}.") save_config_file(log_dir) # We save the operative config before and also after training - train_dataset = dataset_class(data, split=Split.train, ram_cache=ram_cache, name=dataset_names["train"]) val_dataset = dataset_class(data, split=Split.val, ram_cache=ram_cache, name=dataset_names["val"]) train_dataset, val_dataset = assure_minimum_length(train_dataset), assure_minimum_length(val_dataset) From 0f7b366634729f105205ec10d68826abdd7c6dd1 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 10 Jul 2024 16:12:15 +0200 Subject: [PATCH 154/207] debugging looped dl training --- icu_benchmarks/data/loader.py | 3 +++ icu_benchmarks/data/split_process_data.py | 17 +++++++++++++++++ icu_benchmarks/models/train.py | 6 +++++- 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/icu_benchmarks/data/loader.py b/icu_benchmarks/data/loader.py index b95b8cd4..961f8998 100644 --- a/icu_benchmarks/data/loader.py +++ b/icu_benchmarks/data/loader.py @@ -22,7 +22,10 @@ def __init__( grouping_segment: str = Segment.outcome, mps: bool = False, name: str = "", + *args, + **kwargs ): + # super().__init__(*args, **kwargs) self.split = split self.vars = vars self.grouping_df = data[split][grouping_segment] #.set_index(self.vars["GROUP"]) diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index 0260562e..e0490b6f 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -119,6 +119,23 @@ def preprocess_data( data = preprocessor.apply(data, vars) end = timer() logging.info(f"Preprocessing took {end - start:.2f} seconds.") + logging.info(f"Checking for NaNs and nulls in {data.keys()}.") + for dict in data.values(): + for key,val in dict.items(): + logging.debug(f"Data type: {key}") + logging.debug(f"Is NaN:") + sel = dict[key].select(pl.selectors.numeric().is_nan().max()) + logging.info(sel.select(col.name for col in sel if col.item(0))) + #logging.info(dict[key].select(pl.all().has_nulls()).sum_horizontal()) + logging.debug(f"Has nulls:") + sel=dict[key].select(pl.all().has_nulls()) + logging.info(sel.select(col.name for col in sel if col.item(0))) + # dict[key] = val[:, [not (s.null_count() > 0) for s in val]] + dict[key] = val.fill_null(strategy="zero") + dict[key] = val.fill_nan(0) + logging.debug(f"Dropping columns with nulls") + sel=dict[key].select(pl.all().has_nulls()) + logging.info(sel.select(col.name for col in sel if col.item(0))) # Generate cache diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index e57e0a68..3db52c31 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -85,6 +85,7 @@ def train_common( logging.info(f"Using dataset class: {dataset_class.__name__}.") logging.info(f"Logging to directory: {log_dir}.") save_config_file(log_dir) # We save the operative config before and also after training + persistent_workers = False train_dataset = dataset_class(data, split=Split.train, ram_cache=ram_cache, name=dataset_names["train"]) val_dataset = dataset_class(data, split=Split.val, ram_cache=ram_cache, name=dataset_names["val"]) train_dataset, val_dataset = assure_minimum_length(train_dataset), assure_minimum_length(val_dataset) @@ -97,7 +98,7 @@ def train_common( ) logging.info(f"Using {num_workers} workers for data loading.") # todo: compare memory pinning - cpu=True + # cpu=True train_loader = DataLoader( train_dataset, batch_size=batch_size, @@ -105,6 +106,7 @@ def train_common( num_workers=num_workers, pin_memory=not cpu, drop_last=True, + persistent_workers=persistent_workers ) val_loader = DataLoader( val_dataset, @@ -113,6 +115,7 @@ def train_common( num_workers=num_workers, pin_memory=not cpu, drop_last=True, + persistent_workers=persistent_workers ) data_shape = next(iter(train_loader))[0].shape @@ -175,6 +178,7 @@ def train_common( num_workers=num_workers, pin_memory=True, drop_last=True, + persistent_workers=persistent_workers ) if model.requires_backprop else DataLoader([test_dataset.to_tensor()], batch_size=1) From 2e921ea422fbde7a65823daf270902838d894ff8 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 10 Jul 2024 17:16:20 +0200 Subject: [PATCH 155/207] classification task --- configs/tasks/CassClassification.gin | 36 ++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 configs/tasks/CassClassification.gin diff --git a/configs/tasks/CassClassification.gin b/configs/tasks/CassClassification.gin new file mode 100644 index 00000000..3e8e8700 --- /dev/null +++ b/configs/tasks/CassClassification.gin @@ -0,0 +1,36 @@ +# COMMON IMPORTS +include "configs/tasks/common/Imports.gin" + + +# CROSS-VALIDATION +include "configs/tasks/common/CrossValidation.gin" + +# MODE SETTINGS +Run.mode = "Classification" +NUM_CLASSES = 2 # Binary classification +HORIZON = 12 +train_common.weight = "balanced" +train_common.ram_cache = False + +# DEEP LEARNING +DLPredictionWrapper.loss = @cross_entropy + +# SELECTING PREPROCESSOR +preprocess.preprocessor = @base_classification_preprocessor +preprocess.vars = %vars +preprocess.use_static = True + +# SELECTING DATASET +# include "configs/tasks/common/Dataloader.gin" +# SELECTING DATASET +PredictionDataset.vars = %vars +PredictionDataset.ram_cache = True + +# DATASET CONFIGURATION +preprocess.file_names = { + "DYNAMIC": "dyn.parquet", + "OUTCOME": "outc.parquet", + "STATIC": "sta.parquet", +} + +include "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/notes_full/vars.gin" \ No newline at end of file From 2ad630f6554b0c5422790bba102749b3b2632f8d Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 11 Jul 2024 12:52:47 +0200 Subject: [PATCH 156/207] fix dataloader hanging due to not caching --- configs/prediction_models/common/DLCommon.gin | 2 +- configs/tasks/CassClassification.gin | 8 ++++---- configs/tasks/common/Dataloader.gin | 3 ++- experiments/charhpc_wandb_sweep.sh | 4 +--- icu_benchmarks/cross_validation.py | 7 +++++-- icu_benchmarks/data/loader.py | 6 +++--- icu_benchmarks/models/train.py | 18 ++++++++++++------ 7 files changed, 28 insertions(+), 20 deletions(-) diff --git a/configs/prediction_models/common/DLCommon.gin b/configs/prediction_models/common/DLCommon.gin index fbec8ad1..9d790775 100644 --- a/configs/prediction_models/common/DLCommon.gin +++ b/configs/prediction_models/common/DLCommon.gin @@ -14,7 +14,7 @@ base_regression_preprocessor.generate_features = False # Train params train_common.optimizer = @Adam -train_common.epochs = 1000 +train_common.epochs = 50 train_common.batch_size = 64 train_common.patience = 10 train_common.min_delta = 1e-4 diff --git a/configs/tasks/CassClassification.gin b/configs/tasks/CassClassification.gin index 3e8e8700..a243db25 100644 --- a/configs/tasks/CassClassification.gin +++ b/configs/tasks/CassClassification.gin @@ -10,7 +10,7 @@ Run.mode = "Classification" NUM_CLASSES = 2 # Binary classification HORIZON = 12 train_common.weight = "balanced" -train_common.ram_cache = False +train_common.ram_cache = True # DEEP LEARNING DLPredictionWrapper.loss = @cross_entropy @@ -21,10 +21,10 @@ preprocess.vars = %vars preprocess.use_static = True # SELECTING DATASET -# include "configs/tasks/common/Dataloader.gin" +include "configs/tasks/common/Dataloader.gin" # SELECTING DATASET -PredictionDataset.vars = %vars -PredictionDataset.ram_cache = True +# PredictionDataset.vars = %vars +# PredictionDataset.ram_cache = True # DATASET CONFIGURATION preprocess.file_names = { diff --git a/configs/tasks/common/Dataloader.gin b/configs/tasks/common/Dataloader.gin index 424bbaf6..6bed1b7e 100644 --- a/configs/tasks/common/Dataloader.gin +++ b/configs/tasks/common/Dataloader.gin @@ -1,7 +1,8 @@ # Prediction PredictionPandasDataset.vars = %vars +PredictionPandasDataset.ram_cache = True PredictionPolarsDataset.vars = %vars -PredictionPandasDataset.ram_cache = False +PredictionPolarsDataset.ram_cache = True # Imputation ImputationPandasDataset.vars = %vars ImputationPandasDataset.ram_cache = True \ No newline at end of file diff --git a/experiments/charhpc_wandb_sweep.sh b/experiments/charhpc_wandb_sweep.sh index 4c713d16..bec499f1 100644 --- a/experiments/charhpc_wandb_sweep.sh +++ b/experiments/charhpc_wandb_sweep.sh @@ -9,6 +9,4 @@ eval "$(conda shell.bash hook)" conda activate yaib_req_pl -wandb agent --count 1 cassandra_hpi/cassandra/"$1" - - +wandb agent --count 1 cassandra_hpi/cassandra/"$1" \ No newline at end of file diff --git a/icu_benchmarks/cross_validation.py b/icu_benchmarks/cross_validation.py index 730f2a63..3137821a 100644 --- a/icu_benchmarks/cross_validation.py +++ b/icu_benchmarks/cross_validation.py @@ -81,8 +81,9 @@ def execute_repeated_cv( else: logging.info(f"Starting nested CV with {cv_repetitions_to_train} repetitions of {cv_folds_to_train} folds.") - + # Train model for each repetition (a manner of splitting the folds) for repetition in range(cv_repetitions_to_train): + # Train model for each fold configuration (i.e, one fold is test fold and the rest are train/val folds) for fold_index in range(cv_folds_to_train): repetition_fold_dir = log_dir / f"repetition_{repetition}" / f"fold_{fold_index}" repetition_fold_dir.mkdir(parents=True, exist_ok=True) @@ -118,7 +119,9 @@ def execute_repeated_cv( cpu=cpu, verbose=verbose, use_wandb=wandb, - train_only=complete_train + train_only=complete_train, + epochs=20, + patience=5 ) train_time = datetime.now() - start_time diff --git a/icu_benchmarks/data/loader.py b/icu_benchmarks/data/loader.py index 961f8998..95ac63ba 100644 --- a/icu_benchmarks/data/loader.py +++ b/icu_benchmarks/data/loader.py @@ -40,7 +40,7 @@ def __init__( self.mps = mps self.name = name - def ram_cache(self, cache: bool = False): + def ram_cache(self, cache: bool = True): self._cached_dataset = None if cache: logging.info(f"Caching {self.split} dataset in ram.") @@ -75,8 +75,8 @@ class PredictionPolarsDataset(CommonPolarsDataset): ram_cache (bool, optional): Whether the complete dataset should be stored in ram. Defaults to True. """ - def __init__(self, *args, ram_cache: bool = False, **kwargs): - super().__init__(*args, grouping_segment=Segment.outcome, **kwargs) + def __init__(self, *args, ram_cache: bool = True, **kwargs): + super().__init__(*args, **kwargs) self.outcome_df = self.grouping_df self.ram_cache(ram_cache) diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 3db52c31..5a9f2df3 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -38,7 +38,7 @@ def train_common( optimizer: type = Adam, precision=32, batch_size=1, - epochs=1000, + epochs=100, patience=20, min_delta=1e-5, test_on: str = Split.test, @@ -82,10 +82,15 @@ def train_common( logging.info(f"Training model: {model.__name__}.") # todo: add support for polars versions of datasets dataset_class = ImputationPandasDataset if mode == RunMode.imputation else PredictionPolarsDataset if polars else PredictionPandasDataset + # dataset_class = ImputationPandasDataset if mode == RunMode.imputation else PredictionPandasDataset + logging.info(f"Using dataset class: {dataset_class.__name__}.") logging.info(f"Logging to directory: {log_dir}.") save_config_file(log_dir) # We save the operative config before and also after training - persistent_workers = False + persistent_workers = None + # for dataset in data.values(): + # for key, value in dataset.items(): + # dataset[key] = value.to_pandas() train_dataset = dataset_class(data, split=Split.train, ram_cache=ram_cache, name=dataset_names["train"]) val_dataset = dataset_class(data, split=Split.val, ram_cache=ram_cache, name=dataset_names["val"]) train_dataset, val_dataset = assure_minimum_length(train_dataset), assure_minimum_length(val_dataset) @@ -104,7 +109,7 @@ def train_common( batch_size=batch_size, shuffle=True, num_workers=num_workers, - pin_memory=not cpu, + # pin_memory=not cpu, drop_last=True, persistent_workers=persistent_workers ) @@ -113,7 +118,7 @@ def train_common( batch_size=batch_size, shuffle=False, num_workers=num_workers, - pin_memory=not cpu, + # pin_memory=not cpu, drop_last=True, persistent_workers=persistent_workers ) @@ -142,6 +147,7 @@ def train_common( trainer = Trainer( max_epochs=epochs if model.requires_backprop else 1, + min_epochs=1, callbacks=callbacks, precision=precision, accelerator="auto" if not cpu else "cpu", @@ -150,7 +156,7 @@ def train_common( benchmark=not reproducible, enable_progress_bar=verbose, logger=loggers, - num_sanity_val_steps=-1, + num_sanity_val_steps=2, log_every_n_steps=5, ) if not eval_only: @@ -167,7 +173,7 @@ def train_common( logging.info("Finished training full model.") save_config_file(log_dir) return 0 - test_dataset = dataset_class(data, split=test_on, name=dataset_names["test"]) + test_dataset = dataset_class(data, split=test_on, name=dataset_names["test"], ram_cache=ram_cache) test_dataset = assure_minimum_length(test_dataset) logging.info(f"Testing on {test_dataset.name} with {len(test_dataset)} samples.") test_loader = ( From af74e29caef3f532d8a372ea7e70357f6fe36d23 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Fri, 12 Jul 2024 15:57:30 +0200 Subject: [PATCH 157/207] Modality selection --- .../tasks/common/PredictionTaskVariables.gin | 8 +++++++ icu_benchmarks/data/split_process_data.py | 21 ++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/configs/tasks/common/PredictionTaskVariables.gin b/configs/tasks/common/PredictionTaskVariables.gin index 6e38638e..d5006041 100644 --- a/configs/tasks/common/PredictionTaskVariables.gin +++ b/configs/tasks/common/PredictionTaskVariables.gin @@ -15,4 +15,12 @@ vars = { "methb", "mg", "na", "neut", "o2sat", "pco2", "ph", "phos", "plt", "po2", "ptt", "resp", "sbp", "temp", "tnt", "urine", "wbc"], "STATIC": ["age", "sex", "height", "weight"], +} + +modality_mapping = { + "DYNAMIC": ["alb", "alp", "alt", "ast", "be", "bicar", "bili", "bili_dir", "bnd", "bun", "ca", "cai", "ck", "ckmb", "cl", + "crea", "crp", "dbp", "fgn", "fio2", "glu", "hgb", "hr", "inr_pt", "k", "lact", "lymph", "map", "mch", "mchc", "mcv", + "methb", "mg", "na", "neut", "o2sat", "pco2", "ph", "phos", "plt", "po2", "ptt", "resp", "sbp", "temp", "tnt", "urine", + "wbc"], + "STATIC": ["age", "sex", "height", "weight"], } \ No newline at end of file diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index e0490b6f..747767b8 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -23,6 +23,8 @@ def preprocess_data( preprocessor: Preprocessor = PolarsClassificationPreprocessor, use_static: bool = True, vars: dict[str] = gin.REQUIRED, + modality_mapping: dict[str] = [], + selected_modalities: list[str] = ["DYNAMIC", "STATIC"], seed: int = 42, debug: bool = False, cv_repetitions: int = 5, @@ -35,7 +37,7 @@ def preprocess_data( pretrained_imputation_model: str = None, complete_train: bool = False, runmode: RunMode = RunMode.classification, -) -> dict[dict[pd.DataFrame]]: +) -> dict[dict[pl.DataFrame]] or dict[dict[pd.DataFrame]]: """Perform loading, splitting, imputing and normalising of task data. Args: @@ -93,6 +95,12 @@ def preprocess_data( # Read parquet files into pandas dataframes and remove the parquet file from memory logging.info(f"Loading data from directory {data_dir.absolute()}") data = {f: pl.read_parquet(data_dir / file_names[f]) for f in file_names.keys()} + + if len(modality_mapping) > 0: + # Optional modality selection + data = modality_selection(data, modality_mapping, selected_modalities, vars) + + # Generate the splits logging.info("Generating splits.") # complete_train = True @@ -148,6 +156,17 @@ def preprocess_data( return data +def modality_selection(data: dict[pl.DataFrame], modality_mapping: dict[str], selected_modalities: list[str], vars) -> dict[pl.DataFrame]: + logging.info(f"Selected modalities: {selected_modalities}") + selected_columns =[modality_mapping[cols] for cols in selected_modalities if cols in modality_mapping.keys()] + selected_columns = sum(selected_columns, []) + selected_columns.extend([vars[Var.group], vars[Var.label], vars[Var.sequence]]) + logging.info(f"Selected columns: {selected_columns}") + for key in data.keys(): + sel_col = [col for col in data[key].columns if col in selected_columns] + data[key] = data[key].select(sel_col) + logging.debug(f"Selected columns in {key}: {data[key].columns}") + return data def make_train_val( data: dict[pd.DataFrame], From 148843dbdb988c24a10bb2809a586fbf2a8ce19e Mon Sep 17 00:00:00 2001 From: rvandewater Date: Fri, 12 Jul 2024 16:10:41 +0200 Subject: [PATCH 158/207] flake for ci --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index 65e048d2..405d9b47 100644 --- a/environment.yml +++ b/environment.yml @@ -4,6 +4,7 @@ channels: dependencies: - python=3.10 - pip>=24.0 + - flake8=7.1.0 # - pip: # - -r requirements.txt From 2d146476e1caa91d51b6969d3d4d1f672cb1d511 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 24 Jul 2024 16:42:27 +0200 Subject: [PATCH 159/207] modality mapping change to include all by default --- icu_benchmarks/data/split_process_data.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index 747767b8..76656bc4 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -24,7 +24,7 @@ def preprocess_data( use_static: bool = True, vars: dict[str] = gin.REQUIRED, modality_mapping: dict[str] = [], - selected_modalities: list[str] = ["DYNAMIC", "STATIC"], + selected_modalities: list[str] = "all", seed: int = 42, debug: bool = False, cv_repetitions: int = 5, @@ -98,7 +98,10 @@ def preprocess_data( if len(modality_mapping) > 0: # Optional modality selection - data = modality_selection(data, modality_mapping, selected_modalities, vars) + if not selected_modalities == "all": + data = modality_selection(data, modality_mapping, selected_modalities, vars) + else: + logging.info(f"Selecting all modalities.") # Generate the splits From b81346dc95b25f0493a08d954b3971fcfedaf6f5 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 24 Jul 2024 17:22:16 +0200 Subject: [PATCH 160/207] modality mapping bug fixes --- configs/tasks/BinaryClassification.gin | 1 + icu_benchmarks/data/preprocessor.py | 4 ++-- icu_benchmarks/data/split_process_data.py | 7 +++++-- icu_benchmarks/run.py | 4 ++++ icu_benchmarks/run_utils.py | 1 + 5 files changed, 13 insertions(+), 4 deletions(-) diff --git a/configs/tasks/BinaryClassification.gin b/configs/tasks/BinaryClassification.gin index 70eed783..f86436a4 100644 --- a/configs/tasks/BinaryClassification.gin +++ b/configs/tasks/BinaryClassification.gin @@ -19,6 +19,7 @@ DLPredictionWrapper.loss = @cross_entropy # SELECTING PREPROCESSOR preprocess.preprocessor = @base_classification_preprocessor +preprocess.modality_mapping = %modality_mapping preprocess.vars = %vars preprocess.use_static = True diff --git a/icu_benchmarks/data/preprocessor.py b/icu_benchmarks/data/preprocessor.py index 76865a49..6c2e1397 100644 --- a/icu_benchmarks/data/preprocessor.py +++ b/icu_benchmarks/data/preprocessor.py @@ -71,7 +71,7 @@ def __init__( self.save_cache = save_cache self.load_cache = load_cache - def apply(self, data, vars) -> dict[dict[pd.DataFrame]]: + def apply(self, data, vars) -> dict[dict[pl.DataFrame]]: """ Args: data: Train, validation and test data dictionary. Further divided in static, dynamic, and outcome. @@ -82,7 +82,7 @@ def apply(self, data, vars) -> dict[dict[pd.DataFrame]]: logging.info("Preprocessing dynamic features.") data = self._process_dynamic(data, vars) - if self.use_static_features: + if self.use_static_features and len(vars[Segment.static]) > 0: logging.info("Preprocessing static features.") data = self._process_static(data, vars) diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index 76656bc4..a9ae6ed7 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -99,7 +99,7 @@ def preprocess_data( if len(modality_mapping) > 0: # Optional modality selection if not selected_modalities == "all": - data = modality_selection(data, modality_mapping, selected_modalities, vars) + data, vars = modality_selection(data, modality_mapping, selected_modalities, vars) else: logging.info(f"Selecting all modalities.") @@ -164,12 +164,15 @@ def modality_selection(data: dict[pl.DataFrame], modality_mapping: dict[str], se selected_columns =[modality_mapping[cols] for cols in selected_modalities if cols in modality_mapping.keys()] selected_columns = sum(selected_columns, []) selected_columns.extend([vars[Var.group], vars[Var.label], vars[Var.sequence]]) + for key, value in vars.items(): + if key not in [Var.group, Var.label, Var.sequence]: + vars[key] = [col for col in value if col in selected_columns] logging.info(f"Selected columns: {selected_columns}") for key in data.keys(): sel_col = [col for col in data[key].columns if col in selected_columns] data[key] = data[key].select(sel_col) logging.debug(f"Selected columns in {key}: {data[key].columns}") - return data + return data, vars def make_train_val( data: dict[pd.DataFrame], diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index 1dc5921e..08987765 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -50,6 +50,10 @@ def main(my_args=tuple(sys.argv[1:])): evaluate = args.eval experiment = args.experiment source_dir = args.source_dir + modalities = args.modalities + if modalities: + logging.debug(f"Binding modalities: {modalities}") + gin.bind_parameter("preprocess.selected_modalities", modalities) tasks, models = get_config_files(Path("configs")) if task not in tasks: diff --git a/icu_benchmarks/run_utils.py b/icu_benchmarks/run_utils.py index c73ca720..f29419c4 100644 --- a/icu_benchmarks/run_utils.py +++ b/icu_benchmarks/run_utils.py @@ -53,6 +53,7 @@ def build_parser() -> ArgumentParser: parser.add_argument("-sn", "--source-name", type=Path, help="Name of the source dataset.") parser.add_argument("--source-dir", type=Path, help="Directory containing gin and model weights.") parser.add_argument("-sa", "--samples", type=int, default=None, help="Number of samples to use for evaluation.") + parser.add_argument("-mo", "--modalities", nargs="+", help="Modalities to use for evaluation.") return parser From a15db4fab57dd134dcea335654d889660deb77ef Mon Sep 17 00:00:00 2001 From: rvandewater Date: Mon, 29 Jul 2024 11:01:26 +0200 Subject: [PATCH 161/207] modality mapping enhancements and naming clash --- icu_benchmarks/data/split_process_data.py | 16 +++++++++++++--- icu_benchmarks/run_utils.py | 13 ++++++++++--- icu_benchmarks/wandb_utils.py | 3 ++- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index a9ae6ed7..5c0effbb 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -96,9 +96,10 @@ def preprocess_data( logging.info(f"Loading data from directory {data_dir.absolute()}") data = {f: pl.read_parquet(data_dir / file_names[f]) for f in file_names.keys()} + logging.debug(f"Modality mapping: {modality_mapping}") if len(modality_mapping) > 0: # Optional modality selection - if not selected_modalities == "all": + if not (selected_modalities == "all" or selected_modalities == ["all"] or selected_modalities == None): data, vars = modality_selection(data, modality_mapping, selected_modalities, vars) else: logging.info(f"Selecting all modalities.") @@ -162,16 +163,25 @@ def preprocess_data( def modality_selection(data: dict[pl.DataFrame], modality_mapping: dict[str], selected_modalities: list[str], vars) -> dict[pl.DataFrame]: logging.info(f"Selected modalities: {selected_modalities}") selected_columns =[modality_mapping[cols] for cols in selected_modalities if cols in modality_mapping.keys()] + if selected_columns == []: + logging.info(f"No columns selected. Using all columns.") + return data, vars selected_columns = sum(selected_columns, []) selected_columns.extend([vars[Var.group], vars[Var.label], vars[Var.sequence]]) + old_columns =[] + # Update vars dict for key, value in vars.items(): if key not in [Var.group, Var.label, Var.sequence]: + old_columns.extend(value) vars[key] = [col for col in value if col in selected_columns] - logging.info(f"Selected columns: {selected_columns}") + # -3 becaus of standard columns + logging.info(f"Selected columns: {len(selected_columns)-3}, old columns: {len(old_columns)}") + logging.debug(f"Difference: {set(old_columns) - set(selected_columns)}") + # Update data dict for key in data.keys(): sel_col = [col for col in data[key].columns if col in selected_columns] data[key] = data[key].select(sel_col) - logging.debug(f"Selected columns in {key}: {data[key].columns}") + logging.debug(f"Selected columns in {key}: {len(data[key].columns)}") return data, vars def make_train_val( diff --git a/icu_benchmarks/run_utils.py b/icu_benchmarks/run_utils.py index f29419c4..a38a4a1b 100644 --- a/icu_benchmarks/run_utils.py +++ b/icu_benchmarks/run_utils.py @@ -74,7 +74,12 @@ def create_run_dir(log_dir: Path, randomly_searched_params: str = None) -> Path: log_dir_run = log_dir / str(datetime.now().strftime("%Y-%m-%dT%H-%M-%S")) else: log_dir_run = log_dir / str(datetime.now().strftime("%Y-%m-%dT%H-%M-%S.%f")) - log_dir_run.mkdir(parents=True) + if not log_dir_run.exists(): + log_dir_run.mkdir(parents=True) + else: + # Directory clash at last moment + log_dir_run = log_dir / str(datetime.now().strftime("%Y-%m-%dT%H-%M-%S.%f")) + log_dir_run.mkdir(parents=True) if randomly_searched_params: (log_dir_run / randomly_searched_params).touch() return log_dir_run @@ -244,8 +249,10 @@ def get_config_files(config_dir: Path): models = glob.glob(os.path.join(config_dir / "prediction_models", '*')) tasks = [os.path.splitext(os.path.basename(task))[0] for task in tasks] models = [os.path.splitext(os.path.basename(model))[0] for model in models] - tasks.remove("common") - models.remove("common") + if "common" in tasks: + tasks.remove("common") + if "common" in models: + models.remove("common") logging.info(f"Found tasks: {tasks}") logging.info(f"Found models: {models}") return tasks, models \ No newline at end of file diff --git a/icu_benchmarks/wandb_utils.py b/icu_benchmarks/wandb_utils.py index 225693a4..2ea06b57 100644 --- a/icu_benchmarks/wandb_utils.py +++ b/icu_benchmarks/wandb_utils.py @@ -62,7 +62,8 @@ def set_wandb_experiment_name(args, mode): data_dir = Path(args.data_dir) args.name = data_dir.name run_name = f"{mode}_{args.model}_{args.name}" - + if args.modalities: + run_name += f"_mods_{args.modalities}" if args.fine_tune: run_name += f"_source_{args.source_name}_fine-tune_{args.fine_tune}_samples" elif args.eval: From d00f26f1811bd40d5d8d25758472f9f0644cb740 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 30 Jul 2024 17:24:42 +0200 Subject: [PATCH 162/207] Reduce logging "spam" --- icu_benchmarks/data/split_process_data.py | 6 +++--- icu_benchmarks/models/ml_models/xgboost.py | 10 ++++++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index 5c0effbb..df369d2c 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -137,17 +137,17 @@ def preprocess_data( logging.debug(f"Data type: {key}") logging.debug(f"Is NaN:") sel = dict[key].select(pl.selectors.numeric().is_nan().max()) - logging.info(sel.select(col.name for col in sel if col.item(0))) + logging.debug(sel.select(col.name for col in sel if col.item(0))) #logging.info(dict[key].select(pl.all().has_nulls()).sum_horizontal()) logging.debug(f"Has nulls:") sel=dict[key].select(pl.all().has_nulls()) - logging.info(sel.select(col.name for col in sel if col.item(0))) + logging.debug(sel.select(col.name for col in sel if col.item(0))) # dict[key] = val[:, [not (s.null_count() > 0) for s in val]] dict[key] = val.fill_null(strategy="zero") dict[key] = val.fill_nan(0) logging.debug(f"Dropping columns with nulls") sel=dict[key].select(pl.all().has_nulls()) - logging.info(sel.select(col.name for col in sel if col.item(0))) + logging.debug(sel.select(col.name for col in sel if col.item(0))) # Generate cache diff --git a/icu_benchmarks/models/ml_models/xgboost.py b/icu_benchmarks/models/ml_models/xgboost.py index a13f8c75..0d1358c9 100644 --- a/icu_benchmarks/models/ml_models/xgboost.py +++ b/icu_benchmarks/models/ml_models/xgboost.py @@ -28,7 +28,10 @@ def fit_model(self, train_data, train_labels, val_data, val_labels): if wandb.run is not None: callbacks.append(wandb_xgb()) - self.model.fit(train_data, train_labels, eval_set=[(val_data, val_labels)]) + self.model.fit(train_data, train_labels, eval_set=[(val_data, val_labels)], verbose=False) + logging.info(self.model.get_booster().get_score(importance_type='weight')) + + self.log_dict(self.model.get_booster().get_score(importance_type='weight')) return mean(self.model.evals_result_["validation_0"]["logloss"])#, callbacks=callbacks) @@ -41,4 +44,7 @@ def set_model_args(self, model, *args, **kwargs): # Get valid hyperparameters hyperparams = arguments logging.debug(f"Creating model with: {hyperparams}.") - return model(**hyperparams) \ No newline at end of file + return model(**hyperparams) + + def get_feature_importance(self): + return self.model.feature_importances_ \ No newline at end of file From db07daf31170adb6ccf913e77b30657bfa9b923f Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 14 Aug 2024 10:20:20 +0200 Subject: [PATCH 163/207] Multiple label support --- icu_benchmarks/data/split_process_data.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index df369d2c..a3944c44 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -37,6 +37,7 @@ def preprocess_data( pretrained_imputation_model: str = None, complete_train: bool = False, runmode: RunMode = RunMode.classification, + label: str = None, ) -> dict[dict[pl.DataFrame]] or dict[dict[pd.DataFrame]]: """Perform loading, splitting, imputing and normalising of task data. @@ -69,7 +70,13 @@ def preprocess_data( if not use_static: file_names.pop(Segment.static) vars.pop(Segment.static) - + if vars[Var.label]>1: + if label is not None: + vars[Var.label] = [label] + else: + logging.debug(f"Multiple labels found and no value provided. Using first label: {vars[Var.label]}") + vars[Var.label] = vars[Var.label][0] + logging.info(f"Using label: {vars[Var.label]}") dumped_file_names = json.dumps(file_names, sort_keys=True) dumped_vars = json.dumps(vars, sort_keys=True) From 56eca4fc192064d9c262f3ef07ca66d9c0f559ab Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 14 Aug 2024 10:22:26 +0200 Subject: [PATCH 164/207] Multiple label support --- icu_benchmarks/run.py | 2 ++ icu_benchmarks/run_utils.py | 1 + 2 files changed, 3 insertions(+) diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index 08987765..b8c007ed 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -54,6 +54,8 @@ def main(my_args=tuple(sys.argv[1:])): if modalities: logging.debug(f"Binding modalities: {modalities}") gin.bind_parameter("preprocess.selected_modalities", modalities) + if args.label: + gin.bind_parameter("preprocess.label", args.label) tasks, models = get_config_files(Path("configs")) if task not in tasks: diff --git a/icu_benchmarks/run_utils.py b/icu_benchmarks/run_utils.py index a38a4a1b..d2917056 100644 --- a/icu_benchmarks/run_utils.py +++ b/icu_benchmarks/run_utils.py @@ -54,6 +54,7 @@ def build_parser() -> ArgumentParser: parser.add_argument("--source-dir", type=Path, help="Directory containing gin and model weights.") parser.add_argument("-sa", "--samples", type=int, default=None, help="Number of samples to use for evaluation.") parser.add_argument("-mo", "--modalities", nargs="+", help="Modalities to use for evaluation.") + parser.add_argument("--label", type=str, help="Label to use for evaluation in case of multiple labels.", default=None) return parser From d0ea1a2254e0c05689cd9dae8d7b03fab6a1220b Mon Sep 17 00:00:00 2001 From: rvandewater Date: Fri, 16 Aug 2024 13:17:02 +0200 Subject: [PATCH 165/207] Fix for endpoint checking --- icu_benchmarks/data/split_process_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index a3944c44..5f82356e 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -70,7 +70,7 @@ def preprocess_data( if not use_static: file_names.pop(Segment.static) vars.pop(Segment.static) - if vars[Var.label]>1: + if isinstance(vars[Var.label], list) and len(vars[Var.label])>1: if label is not None: vars[Var.label] = [label] else: From 358654c7ee4479b9785d597925064bb4c18f9257 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Fri, 16 Aug 2024 14:37:17 +0200 Subject: [PATCH 166/207] Introduced shap value logging --- icu_benchmarks/models/ml_models/xgboost.py | 28 +++++++++++++++++++--- icu_benchmarks/models/train.py | 18 ++++++++++++++ icu_benchmarks/models/wrappers.py | 4 +++- 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/icu_benchmarks/models/ml_models/xgboost.py b/icu_benchmarks/models/ml_models/xgboost.py index 0d1358c9..0a7ecc0a 100644 --- a/icu_benchmarks/models/ml_models/xgboost.py +++ b/icu_benchmarks/models/ml_models/xgboost.py @@ -2,6 +2,8 @@ import logging import gin +import numpy as np + from icu_benchmarks.contants import RunMode from icu_benchmarks.models.wrappers import MLWrapper import xgboost as xgb @@ -10,6 +12,8 @@ import wandb from statistics import mean from optuna.integration import XGBoostPruningCallback +import shap + @gin.configurable class XGBClassifier(MLWrapper): _supported_run_modes = [RunMode.classification] @@ -29,11 +33,29 @@ def fit_model(self, train_data, train_labels, val_data, val_labels): if wandb.run is not None: callbacks.append(wandb_xgb()) self.model.fit(train_data, train_labels, eval_set=[(val_data, val_labels)], verbose=False) - logging.info(self.model.get_booster().get_score(importance_type='weight')) - - self.log_dict(self.model.get_booster().get_score(importance_type='weight')) + self.explainer = shap.TreeExplainer(self.model) + self.train_shap_values = self.explainer(train_data) + # shap.summary_plot(shap_values, X_test, feature_names=features) + # logging.info(self.model.get_booster().get_score(importance_type='weight')) + # self.log_dict(self.model.get_booster().get_score(importance_type='weight')) return mean(self.model.evals_result_["validation_0"]["logloss"])#, callbacks=callbacks) + def test_step(self, dataset, _): + test_rep, test_label = dataset + test_rep, test_label = test_rep.squeeze().cpu().numpy(), test_label.squeeze().cpu().numpy() + self.set_metrics(test_label) + test_pred = self.predict(test_rep) + if self.explainer is not None: + self.test_shap_values = self.explainer(test_rep) + logging.info(f"Shap values: {self.test_shap_values}") + # self.log("test/shap_values", self.test_shap_values, sync_dist=True) + if self.mps: + self.log("test/loss", np.float32(self.loss(test_label, test_pred)), sync_dist=True) + self.log_metrics(np.float32(test_label), np.float32(test_pred), "test") + else: + self.log("test/loss", self.loss(test_label, test_pred), sync_dist=True) + self.log_metrics(test_label, test_pred, "test") + logging.debug(f"Test loss: {self.loss(test_label, test_pred)}") def set_model_args(self, model, *args, **kwargs): """XGBoost signature does not include the hyperparams so we need to pass them manually.""" diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 5a9f2df3..12baafb3 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -1,5 +1,6 @@ import os import gin +import numpy as np import torch import logging import polars as pl @@ -192,9 +193,26 @@ def train_common( model.set_weight("balanced", train_dataset) test_loss = trainer.test(model, dataloaders=test_loader, verbose=verbose)[0]["test/loss"] + persist_data(trainer, log_dir) save_config_file(log_dir) return test_loss +def persist_data(trainer, log_dir): + if trainer.lightning_module.test_shap_values is not None: + shap_values = trainer.lightning_module.test_shap_values + # sdf_test = pl.DataFrame({ + # 'features': trainer.lightning_module.trained_columns, + # 'feature_value': np.transpose(shap_values.values.mean(axis=0)), + # }) + shaps_test = pl.DataFrame(schema = trainer.lightning_module.trained_columns, + data = np.transpose(shap_values.values)) + shaps_test.write_parquet(log_dir / "shap_values_test.parquet") + logging.info(f"Saved shap values to {log_dir / 'test_shap_values.parquet'}") + if trainer.lightning_module.train_shap_values is not None: + shap_values = trainer.lightning_module.train_shap_values + shaps_train = pl.DataFrame(schema = trainer.lightning_module.trained_columns, + data = np.transpose(shap_values.values)) + shaps_train.write_parquet(log_dir / "shap_values_train.parquet") def load_model(model, source_dir, pl_model=True): if source_dir.exists(): diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index 27c47c85..6770bc97 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -442,7 +442,9 @@ def test_step(self, dataset, _): test_rep, test_label = test_rep.squeeze().cpu().numpy(), test_label.squeeze().cpu().numpy() self.set_metrics(test_label) test_pred = self.predict(test_rep) - + # if self.explainer is not None: + # self.test_shap_values = self.explainer(test_rep) + # logging.info(f"Shap values: {self.test_shap_values}") if self.mps: self.log("test/loss", np.float32(self.loss(test_label, test_pred)), sync_dist=True) self.log_metrics(np.float32(test_label), np.float32(test_pred), "test") From a8a06ca2bffed5990b7182b5e0ef8d3f2682f782 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Mon, 19 Aug 2024 16:38:29 +0200 Subject: [PATCH 167/207] xgboost adjustments --- configs/prediction_models/XGBClassifier.gin | 1 + icu_benchmarks/models/ml_models/xgboost.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/configs/prediction_models/XGBClassifier.gin b/configs/prediction_models/XGBClassifier.gin index beb22e66..fcca6b22 100644 --- a/configs/prediction_models/XGBClassifier.gin +++ b/configs/prediction_models/XGBClassifier.gin @@ -14,3 +14,4 @@ model/hyperparameter.scale_pos_weight = [1, 5, 10, 25, 50, 75, 99, 100, 1000] model/hyperparameter.min_child_weight = [1, 0.5] model/hyperparameter.max_delta_step = [0, 1, 2, 3, 4, 5, 10] model/hyperparameter.colsample_bytree = [0.1, 0.25, 0.5, 0.75, 1.0] +model/hyperparameter.eval_metric = "aucpr" \ No newline at end of file diff --git a/icu_benchmarks/models/ml_models/xgboost.py b/icu_benchmarks/models/ml_models/xgboost.py index 0a7ecc0a..9aec17f9 100644 --- a/icu_benchmarks/models/ml_models/xgboost.py +++ b/icu_benchmarks/models/ml_models/xgboost.py @@ -38,7 +38,9 @@ def fit_model(self, train_data, train_labels, val_data, val_labels): # shap.summary_plot(shap_values, X_test, feature_names=features) # logging.info(self.model.get_booster().get_score(importance_type='weight')) # self.log_dict(self.model.get_booster().get_score(importance_type='weight')) - return mean(self.model.evals_result_["validation_0"]["logloss"])#, callbacks=callbacks) + # Return the first metric we use for validation + eval_score = mean(next(iter(self.model.evals_result_["validation_0"].values()))) + return eval_score #, callbacks=callbacks) def test_step(self, dataset, _): test_rep, test_label = dataset @@ -47,7 +49,7 @@ def test_step(self, dataset, _): test_pred = self.predict(test_rep) if self.explainer is not None: self.test_shap_values = self.explainer(test_rep) - logging.info(f"Shap values: {self.test_shap_values}") + # logging.debug(f"Shap values: {self.test_shap_values}") # self.log("test/shap_values", self.test_shap_values, sync_dist=True) if self.mps: self.log("test/loss", np.float32(self.loss(test_label, test_pred)), sync_dist=True) From d7ed831f20210e6a6504c94a04371fa431820ba6 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Mon, 19 Aug 2024 16:38:59 +0200 Subject: [PATCH 168/207] added option to use more sklearn metrics --- icu_benchmarks/models/utils.py | 20 ++++++++++++++++++++ icu_benchmarks/models/wrappers.py | 10 ++++++---- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/icu_benchmarks/models/utils.py b/icu_benchmarks/models/utils.py index 6c944ae7..e75c94b1 100644 --- a/icu_benchmarks/models/utils.py +++ b/icu_benchmarks/models/utils.py @@ -11,6 +11,7 @@ from pytorch_lightning.loggers.logger import Logger from pytorch_lightning.utilities import rank_zero_only +from sklearn.metrics import average_precision_score from torch.nn import Module from torch.optim import Optimizer, Adam, SGD, RAdam from typing import Optional, Union @@ -188,3 +189,22 @@ def version(self): @rank_zero_only def log_hyperparams(self, params): pass + +class scorer_wrapper(): + """ + Wrapper that flattens the binary classification input such that we can use a broader range of sklearn metrics. + """ + def __init__(self, scorer=average_precision_score): + self.scorer = scorer + + def __call__(self, y_true, y_pred): + if len(np.unique(y_true))<=2 and y_pred.ndim > 1: + return self.scorer(y_true, np.argmax(y_pred,axis=1)) + else: + return self.scorer(y_true, y_pred) + + def __name__(self): + return "scorer_wrapper" + + + diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index 6770bc97..2210d883 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -3,7 +3,7 @@ from typing import Dict, Any, List, Optional, Union import torchmetrics -from sklearn.metrics import log_loss, mean_squared_error +from sklearn.metrics import log_loss, mean_squared_error, average_precision_score, make_scorer import torch from torch.nn import MSELoss, CrossEntropyLoss @@ -16,7 +16,7 @@ import numpy as np from ignite.exceptions import NotComputableError from icu_benchmarks.models.constants import ImputationInit -from icu_benchmarks.models.utils import create_optimizer, create_scheduler +from icu_benchmarks.models.utils import create_optimizer, create_scheduler, scorer_wrapper from joblib import dump from pytorch_lightning import LightningModule @@ -29,7 +29,7 @@ gin.config.external_configurable(mean_squared_error, module="sklearn.metrics") gin.config.external_configurable(log_loss, module="sklearn.metrics") - +gin.config.external_configurable(scorer_wrapper, module="icu_benchmarks.models.utils") @gin.configurable("BaseModule") class BaseModule(LightningModule): @@ -363,7 +363,7 @@ class MLWrapper(BaseModule, ABC): requires_backprop = False _supported_run_modes = [RunMode.classification, RunMode.regression] - def __init__(self, *args, run_mode=RunMode.classification, loss=log_loss, patience=10, mps=False, **kwargs): + def __init__(self, *args, run_mode=RunMode.classification, loss=scorer_wrapper(average_precision_score), patience=10, mps=False, **kwargs): super().__init__() self.save_hyperparameters() self.scaler = None @@ -372,6 +372,7 @@ def __init__(self, *args, run_mode=RunMode.classification, loss=log_loss, patien self.loss = loss self.patience = patience self.mps = mps + self.loss_weight = None def set_metrics(self, labels): if self.run_mode == RunMode.classification: @@ -414,6 +415,7 @@ def fit(self, train_dataset, val_dataset): train_pred = self.predict(train_rep) logging.debug(f"Model:{self.model}") + self.log("train/loss", self.loss(train_label, train_pred), sync_dist=True) logging.debug(f"Train loss: {self.loss(train_label, train_pred)}") self.log("val/loss", val_loss, sync_dist=True) From 61f5f288f03e392d1cd570438a11718aeb221146 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Mon, 19 Aug 2024 16:39:22 +0200 Subject: [PATCH 169/207] failsave if not using shap --- icu_benchmarks/models/train.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 12baafb3..2266664e 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -198,21 +198,25 @@ def train_common( return test_loss def persist_data(trainer, log_dir): - if trainer.lightning_module.test_shap_values is not None: - shap_values = trainer.lightning_module.test_shap_values - # sdf_test = pl.DataFrame({ - # 'features': trainer.lightning_module.trained_columns, - # 'feature_value': np.transpose(shap_values.values.mean(axis=0)), - # }) - shaps_test = pl.DataFrame(schema = trainer.lightning_module.trained_columns, - data = np.transpose(shap_values.values)) - shaps_test.write_parquet(log_dir / "shap_values_test.parquet") - logging.info(f"Saved shap values to {log_dir / 'test_shap_values.parquet'}") - if trainer.lightning_module.train_shap_values is not None: - shap_values = trainer.lightning_module.train_shap_values - shaps_train = pl.DataFrame(schema = trainer.lightning_module.trained_columns, - data = np.transpose(shap_values.values)) - shaps_train.write_parquet(log_dir / "shap_values_train.parquet") + try: + if trainer.lightning_module.test_shap_values is not None: + shap_values = trainer.lightning_module.test_shap_values + # sdf_test = pl.DataFrame({ + # 'features': trainer.lightning_module.trained_columns, + # 'feature_value': np.transpose(shap_values.values.mean(axis=0)), + # }) + shaps_test = pl.DataFrame(schema = trainer.lightning_module.trained_columns, + data = np.transpose(shap_values.values)) + shaps_test.write_parquet(log_dir / "shap_values_test.parquet") + logging.info(f"Saved shap values to {log_dir / 'test_shap_values.parquet'}") + if trainer.lightning_module.train_shap_values is not None: + shap_values = trainer.lightning_module.train_shap_values + shaps_train = pl.DataFrame(schema = trainer.lightning_module.trained_columns, + data = np.transpose(shap_values.values)) + shaps_train.write_parquet(log_dir / "shap_values_train.parquet") + + except Exception as e: + logging.error(f"Failed to save shap values: {e}") def load_model(model, source_dir, pl_model=True): if source_dir.exists(): From 8243dca771e2fe43c7049b0a07e404069fab7636 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Mon, 19 Aug 2024 16:39:38 +0200 Subject: [PATCH 170/207] concatenating shap --- icu_benchmarks/run_utils.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/icu_benchmarks/run_utils.py b/icu_benchmarks/run_utils.py index d2917056..ab38a901 100644 --- a/icu_benchmarks/run_utils.py +++ b/icu_benchmarks/run_utils.py @@ -17,6 +17,7 @@ from icu_benchmarks.wandb_utils import wandb_log import os import glob +import polars as pl def build_parser() -> ArgumentParser: """Builds an ArgumentParser for the command line. @@ -107,6 +108,8 @@ def aggregate_results(log_dir: Path, execution_time: timedelta = None): execution_time: Overall execution time. """ aggregated = {} + shap_values_test = [] + shap_values_train = [] for repetition in log_dir.iterdir(): if repetition.is_dir(): aggregated[repetition.name] = {} @@ -125,6 +128,12 @@ def aggregate_results(log_dir: Path, execution_time: timedelta = None): with open(fold_iter / "durations.json", "r") as f: result = json.load(f) aggregated[repetition.name][fold_iter.name].update(result) + if (fold_iter / "test_shap_values.parquet").is_file(): + shap_values_test.append(pl.read_parquet(fold_iter / "test_shap_values.parquet")) + + if shap_values_test: + shap_values = pl.concat(shap_values_test) + shap_values.write_parquet(log_dir / "aggregated_shap_values.parquet") # Aggregate results per metric list_scores = {} From 93f680b0fc5c4a2a4c734bf4f9ef6e6f77c851b4 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Wed, 21 Aug 2024 15:50:05 +0200 Subject: [PATCH 171/207] add support for excluding variables from feature generation --- icu_benchmarks/data/preprocessor.py | 18 +++++++--- icu_benchmarks/data/split_process_data.py | 40 +++++++++++++++-------- 2 files changed, 40 insertions(+), 18 deletions(-) diff --git a/icu_benchmarks/data/preprocessor.py b/icu_benchmarks/data/preprocessor.py index 6c2e1397..27f847cb 100644 --- a/icu_benchmarks/data/preprocessor.py +++ b/icu_benchmarks/data/preprocessor.py @@ -32,7 +32,7 @@ class Preprocessor: @abc.abstractmethod - def apply(self, data, vars, save_cache=False, load_cache=None): + def apply(self, data, vars, save_cache=False, load_cache=None, vars_to_exclude=None): return data @abc.abstractmethod @@ -48,11 +48,12 @@ def set_imputation_model(self, imputation_model): class PolarsClassificationPreprocessor(Preprocessor): def __init__( self, - generate_features: bool = True, + generate_features: bool = False, scaling: bool = True, use_static_features: bool = True, save_cache=None, load_cache=None, + vars_to_exclude=None, ): """ Args: @@ -61,6 +62,7 @@ def __init__( use_static_features: Use static features. save_cache: Save recipe cache from this path. load_cache: Load recipe cache from this path. + vars_to_exclude: Variables to exclude from missing indicator/ feature generation. Returns: Preprocessed data. """ @@ -70,6 +72,7 @@ def __init__( self.imputation_model = None self.save_cache = save_cache self.load_cache = load_cache + self.vars_to_exclude = vars_to_exclude def apply(self, data, vars) -> dict[dict[pl.DataFrame]]: """ @@ -157,13 +160,18 @@ def _process_dynamic(self, data, vars): dyn_rec.add_step(StepScale()) if self.imputation_model is not None: dyn_rec.add_step(StepImputeModel(model=self.model_impute, sel=all_of(vars[Segment.dynamic]))) - dyn_rec.add_step(StepSklearn(MissingIndicator(features="all"), sel=all_of(vars[Segment.dynamic]), in_place=False)) + if self.vars_to_exclude is not None: + # Exclude vars_to_exclude from missing indicator/ feature generation + vars_to_apply = list(set(vars[Segment.dynamic]) - set(self.vars_to_exclude)) + else: + vars_to_apply = vars[Segment.dynamic] + dyn_rec.add_step(StepSklearn(MissingIndicator(features="all"), sel=all_of(vars_to_apply), in_place=False)) # dyn_rec.add_step(StepImputeFastForwardFill()) dyn_rec.add_step(StepImputeFill(strategy="forward")) # dyn_rec.add_step(StepImputeFastZeroFill()) dyn_rec.add_step(StepImputeFill(strategy="zero")) if self.generate_features: - dyn_rec = self._dynamic_feature_generation(dyn_rec, all_of(vars[Segment.dynamic])) + dyn_rec = self._dynamic_feature_generation(dyn_rec, all_of(vars_to_apply)) data = apply_recipe_to_splits(dyn_rec, data, Segment.dynamic, self.save_cache, self.load_cache) return data @@ -185,7 +193,7 @@ class PolarsRegressionPreprocessor(PolarsClassificationPreprocessor): # Override base classification preprocessor def __init__( self, - generate_features: bool = True, + generate_features: bool = False, scaling: bool = True, use_static_features: bool = True, outcome_max=None, diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index 5f82356e..3caed968 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -70,7 +70,7 @@ def preprocess_data( if not use_static: file_names.pop(Segment.static) vars.pop(Segment.static) - if isinstance(vars[Var.label], list) and len(vars[Var.label])>1: + if isinstance(vars[Var.label], list) and len(vars[Var.label]) > 1: if label is not None: vars[Var.label] = [label] else: @@ -83,7 +83,16 @@ def preprocess_data( cache_filename = f"s_{seed}_r_{repetition_index}_f_{fold_index}_t_{train_size}_d_{debug}" logging.log(logging.INFO, f"Using preprocessor: {preprocessor.__name__}") - preprocessor = preprocessor(use_static_features=use_static, save_cache=data_dir / "preproc" / (cache_filename + "_recipe")) + preprocessor = preprocessor( + use_static_features=use_static, + save_cache=data_dir / "preproc" / (cache_filename + "_recipe"), + vars_to_exclude=modality_mapping.get("cat_clinical_notes") + modality_mapping.get("cat_med_embeddings_map") + if ( + modality_mapping.get("cat_clinical_notes") is not None + and modality_mapping.get("cat_med_embeddings_map") is not None + ) + else None, + ) if isinstance(preprocessor, PandasClassificationPreprocessor): preprocessor.set_imputation_model(pretrained_imputation_model) @@ -111,7 +120,6 @@ def preprocess_data( else: logging.info(f"Selecting all modalities.") - # Generate the splits logging.info("Generating splits.") # complete_train = True @@ -140,23 +148,22 @@ def preprocess_data( logging.info(f"Preprocessing took {end - start:.2f} seconds.") logging.info(f"Checking for NaNs and nulls in {data.keys()}.") for dict in data.values(): - for key,val in dict.items(): + for key, val in dict.items(): logging.debug(f"Data type: {key}") logging.debug(f"Is NaN:") sel = dict[key].select(pl.selectors.numeric().is_nan().max()) logging.debug(sel.select(col.name for col in sel if col.item(0))) - #logging.info(dict[key].select(pl.all().has_nulls()).sum_horizontal()) + # logging.info(dict[key].select(pl.all().has_nulls()).sum_horizontal()) logging.debug(f"Has nulls:") - sel=dict[key].select(pl.all().has_nulls()) + sel = dict[key].select(pl.all().has_nulls()) logging.debug(sel.select(col.name for col in sel if col.item(0))) # dict[key] = val[:, [not (s.null_count() > 0) for s in val]] dict[key] = val.fill_null(strategy="zero") dict[key] = val.fill_nan(0) logging.debug(f"Dropping columns with nulls") - sel=dict[key].select(pl.all().has_nulls()) + sel = dict[key].select(pl.all().has_nulls()) logging.debug(sel.select(col.name for col in sel if col.item(0))) - # Generate cache if generate_cache: caching(cache_dir, cache_file, data, load_cache) @@ -167,22 +174,25 @@ def preprocess_data( return data -def modality_selection(data: dict[pl.DataFrame], modality_mapping: dict[str], selected_modalities: list[str], vars) -> dict[pl.DataFrame]: + +def modality_selection( + data: dict[pl.DataFrame], modality_mapping: dict[str], selected_modalities: list[str], vars +) -> dict[pl.DataFrame]: logging.info(f"Selected modalities: {selected_modalities}") - selected_columns =[modality_mapping[cols] for cols in selected_modalities if cols in modality_mapping.keys()] + selected_columns = [modality_mapping[cols] for cols in selected_modalities if cols in modality_mapping.keys()] if selected_columns == []: logging.info(f"No columns selected. Using all columns.") return data, vars selected_columns = sum(selected_columns, []) selected_columns.extend([vars[Var.group], vars[Var.label], vars[Var.sequence]]) - old_columns =[] + old_columns = [] # Update vars dict for key, value in vars.items(): if key not in [Var.group, Var.label, Var.sequence]: old_columns.extend(value) vars[key] = [col for col in value if col in selected_columns] # -3 becaus of standard columns - logging.info(f"Selected columns: {len(selected_columns)-3}, old columns: {len(old_columns)}") + logging.info(f"Selected columns: {len(selected_columns) - 3}, old columns: {len(old_columns)}") logging.debug(f"Difference: {set(old_columns) - set(selected_columns)}") # Update data dict for key in data.keys(): @@ -191,6 +201,7 @@ def modality_selection(data: dict[pl.DataFrame], modality_mapping: dict[str], se logging.debug(f"Selected columns in {key}: {len(data[key].columns)}") return data, vars + def make_train_val( data: dict[pd.DataFrame], vars: dict[str], @@ -244,7 +255,10 @@ def make_train_val( train, val = list(train_val.split(stays))[0] if polars: - split = {Split.train: stays[train].cast(pl.datatypes.Int64).to_frame(), Split.val: stays[val].cast(pl.datatypes.Int64).to_frame()} + split = { + Split.train: stays[train].cast(pl.datatypes.Int64).to_frame(), + Split.val: stays[val].cast(pl.datatypes.Int64).to_frame(), + } else: split = {Split.train: stays.iloc[train], Split.val: stays.iloc[val]} From 90b0b05ad04fd2acc5cee1c7a6d199ebd4927196 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 17 Sep 2024 12:59:24 +0200 Subject: [PATCH 172/207] Added auprc loss and prediction indicators to inspect model predictions --- icu_benchmarks/models/ml_models/xgboost.py | 21 ++++++++++++++--- icu_benchmarks/models/wrappers.py | 27 ++++++++++++++-------- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/icu_benchmarks/models/ml_models/xgboost.py b/icu_benchmarks/models/ml_models/xgboost.py index 9aec17f9..381e0276 100644 --- a/icu_benchmarks/models/ml_models/xgboost.py +++ b/icu_benchmarks/models/ml_models/xgboost.py @@ -3,7 +3,8 @@ import gin import numpy as np - +import torch +import os from icu_benchmarks.contants import RunMode from icu_benchmarks.models.wrappers import MLWrapper import xgboost as xgb @@ -42,11 +43,23 @@ def fit_model(self, train_data, train_labels, val_data, val_labels): eval_score = mean(next(iter(self.model.evals_result_["validation_0"].values()))) return eval_score #, callbacks=callbacks) + def test_step(self, dataset, _): - test_rep, test_label = dataset - test_rep, test_label = test_rep.squeeze().cpu().numpy(), test_label.squeeze().cpu().numpy() + # Pred indicators indicates the row id, and time in hours + test_rep, test_label, pred_indicators = dataset + test_rep, test_label, pred_indicators = test_rep.squeeze().cpu().numpy(), test_label.squeeze().cpu().numpy(), pred_indicators.squeeze().cpu().numpy() self.set_metrics(test_label) test_pred = self.predict(test_rep) + if pred_indicators.shape[1] == test_pred.shape[1]: + pred_indicators = np.hstack((pred_indicators, test_label.reshape(-1, 1))) + # test_reshaped = test_pred.reshape(-1, 1) + pred_indicators = np.hstack((pred_indicators, test_pred)) + sav = self.logger.save_dir + # Save as: id, time (hours), ground truth, prediction 0, prediction 1 + np.save(os.path.join(self.logger.save_dir,f'pred_indicators.npy'), pred_indicators) + logging.debug(f"Saved row indicators to {os.path.join(self.logger.save_dir,f'row_indicators.npy')}") + # self.log_numpy_array(row_indicators, "test/pred_labels_windows", self.global_step) + # self.log("test/pred_labels_windows", torch.from_numpy(row_indicators), sync_dist=True) if self.explainer is not None: self.test_shap_values = self.explainer(test_rep) # logging.debug(f"Shap values: {self.test_shap_values}") @@ -59,6 +72,8 @@ def test_step(self, dataset, _): self.log_metrics(test_label, test_pred, "test") logging.debug(f"Test loss: {self.loss(test_label, test_pred)}") + + def set_model_args(self, model, *args, **kwargs): """XGBoost signature does not include the hyperparams so we need to pass them manually.""" signature = inspect.signature(model.__init__).parameters diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index 2210d883..cdfa7f7c 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -3,7 +3,7 @@ from typing import Dict, Any, List, Optional, Union import torchmetrics -from sklearn.metrics import log_loss, mean_squared_error, average_precision_score, make_scorer +from sklearn.metrics import log_loss, mean_squared_error, average_precision_score, roc_auc_score import torch from torch.nn import MSELoss, CrossEntropyLoss @@ -16,6 +16,7 @@ import numpy as np from ignite.exceptions import NotComputableError from icu_benchmarks.models.constants import ImputationInit +from icu_benchmarks.models.custom_metrics import confusion_matrix from icu_benchmarks.models.utils import create_optimizer, create_scheduler, scorer_wrapper from joblib import dump from pytorch_lightning import LightningModule @@ -29,7 +30,9 @@ gin.config.external_configurable(mean_squared_error, module="sklearn.metrics") gin.config.external_configurable(log_loss, module="sklearn.metrics") -gin.config.external_configurable(scorer_wrapper, module="icu_benchmarks.models.utils") +gin.config.external_configurable(average_precision_score, module="sklearn.metrics") +gin.config.external_configurable(roc_auc_score, module="sklearn.metrics") +# gin.config.external_configurable(scorer_wrapper, module="icu_benchmarks.models.utils") @gin.configurable("BaseModule") class BaseModule(LightningModule): @@ -363,13 +366,16 @@ class MLWrapper(BaseModule, ABC): requires_backprop = False _supported_run_modes = [RunMode.classification, RunMode.regression] - def __init__(self, *args, run_mode=RunMode.classification, loss=scorer_wrapper(average_precision_score), patience=10, mps=False, **kwargs): + def __init__(self, *args, run_mode=RunMode.classification, loss=average_precision_score, patience=10, mps=False, **kwargs): super().__init__() self.save_hyperparameters() self.scaler = None self.check_supported_runmode(run_mode) self.run_mode = run_mode - self.loss = loss + if loss.__name__ in ["average_precision_score", "roc_auc_score"]: + self.loss = scorer_wrapper(average_precision_score) + else: + self.loss = self.loss self.patience = patience self.mps = mps self.loss_weight = None @@ -402,8 +408,8 @@ def set_metrics(self, labels): def fit(self, train_dataset, val_dataset): """Fit the model to the training data.""" - train_rep, train_label = train_dataset.get_data_and_labels() - val_rep, val_label = val_dataset.get_data_and_labels() + train_rep, train_label, row_indicators = train_dataset.get_data_and_labels() + val_rep, val_label, row_indicators = val_dataset.get_data_and_labels() self.set_metrics(train_label) @@ -429,7 +435,7 @@ def fit_model(self, train_data, train_labels, val_data, val_labels): return val_loss def validation_step(self, val_dataset, _): - val_rep, val_label = val_dataset.get_data_and_labels() + val_rep, val_label, row_indicators = val_dataset.get_data_and_labels() val_rep, val_label = torch.from_numpy(val_rep).to(self.device), torch.from_numpy(val_label).to(self.device) self.set_metrics(val_label) @@ -463,17 +469,18 @@ def predict(self, features): def log_metrics(self, label, pred, metric_type): """Log metrics to the PL logs.""" - + if "Confusion_Matrix" in self.metrics.keys(): + self.log_dict(confusion_matrix(self.label_transform(label), self.output_transform(pred)), sync_dist=True) self.log_dict( { # MPS dependent type casting f"{metric_type}/{name}": metric(self.label_transform(label), self.output_transform(pred)) if not self.mps else metric(self.label_transform(label), self.output_transform(pred)) - # Fore very metric + # For every metric for name, metric in self.metrics.items() # Filter out metrics that return a tuple (e.g. precision_recall_curve) - if not isinstance(metric(self.label_transform(label), self.output_transform(pred)), tuple) + if not isinstance(metric(self.label_transform(label), self.output_transform(pred)), tuple) and name != "Confusion_Matrix" }, sync_dist=True, ) From b9aad2ba8999002fbc6d12c2f626f1ea48f6468e Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 17 Sep 2024 12:59:39 +0200 Subject: [PATCH 173/207] Optuna update --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b03e3df5..a1539483 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,7 +24,7 @@ tensorboard==2.12.2 tqdm==4.66.3 einops==0.6.1 hydra-core==1.3 -optuna==3.6.1 +optuna==4.0.0 optuna-integration==3.6.0 wandb==0.17.3 recipies==1.0rc1 From 32d1707deda5368d8a12716b773f92aa2023709e Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 17 Sep 2024 13:00:01 +0200 Subject: [PATCH 174/207] Confusion matrix --- icu_benchmarks/models/custom_metrics.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/icu_benchmarks/models/custom_metrics.py b/icu_benchmarks/models/custom_metrics.py index ddb5d37e..75648466 100644 --- a/icu_benchmarks/models/custom_metrics.py +++ b/icu_benchmarks/models/custom_metrics.py @@ -1,8 +1,11 @@ +import logging + import torch from typing import Callable import numpy as np from ignite.metrics import EpochMetric -from sklearn.metrics import balanced_accuracy_score, mean_absolute_error +from numpy import ndarray +from sklearn.metrics import balanced_accuracy_score, mean_absolute_error, confusion_matrix as sk_confusion_matrix from sklearn.calibration import calibration_curve from scipy.spatial.distance import jensenshannon from torchmetrics.classification import BinaryFairness @@ -130,3 +133,18 @@ def feature_helper(self, trainer, step_prefix): else: feature_names = trainer.test_dataloaders.dataset.features return feature_names + +def confusion_matrix(y_true: ndarray, y_pred: ndarray, normalize=False) -> torch.tensor: + y_pred = np.rint(y_pred).astype(int) + confusion = sk_confusion_matrix(y_true, y_pred) + if normalize: + confusion = confusion / confusion.sum() + confusion_tensor = torch.tensor(confusion) + # confusion = confusion.tolist() + confusion_dict = {} + for i in range(confusion.shape[0]): + for j in range(confusion.shape[1]): + confusion_dict[f"class_{i}_pred_{j}"] = confusion[i][j] + # logging.info(f"Confusion matrix: {confusion_dict}") + # dict = {"TP": confusion[0][0], "FP": confusion[0][1], "FN": confusion[1][0], "TN": confusion[1][1]} + return confusion_dict From babaaedbf04d226b48e9868d0796e33dd12ab3fd Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 17 Sep 2024 13:00:18 +0200 Subject: [PATCH 175/207] Added to constants --- icu_benchmarks/models/constants.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/icu_benchmarks/models/constants.py b/icu_benchmarks/models/constants.py index 45af8271..41eee851 100644 --- a/icu_benchmarks/models/constants.py +++ b/icu_benchmarks/models/constants.py @@ -26,6 +26,7 @@ MAE, JSD, BinaryFairnessWrapper, + confusion_matrix ) @@ -36,13 +37,17 @@ class MLMetrics: "PR": average_precision_score, "PR_Curve": precision_recall_curve, "RO_Curve": roc_curve, + "Confusion_Matrix": confusion_matrix, + } MULTICLASS_CLASSIFICATION = { "Accuracy": accuracy_score, "AUC": roc_auc_score, "Balanced_Accuracy": balanced_accuracy_score, - "PR": average_precision_score, + # "PR": average_precision_score, + "Confusion_Matrix": confusion_matrix, + } REGRESSION = { From 561f3e7d450115c369f16edf44d25a06f2029ccd Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 17 Sep 2024 13:00:41 +0200 Subject: [PATCH 176/207] prediction indicators in dataloader --- icu_benchmarks/data/loader.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/icu_benchmarks/data/loader.py b/icu_benchmarks/data/loader.py index 95ac63ba..a303a478 100644 --- a/icu_benchmarks/data/loader.py +++ b/icu_benchmarks/data/loader.py @@ -33,6 +33,9 @@ def __init__( # self.features_df = ( # data[split][Segment.features].set_index(self.vars["GROUP"]).drop(labels=self.vars["SEQUENCE"], axis=1) # ) + # Get the row indicators for the data to be able to match predicted labels + self.row_indicators = data[split][Segment.features][self.vars["GROUP"], self.vars["SEQUENCE"]] + self.row_indicators = self.row_indicators.with_columns(pl.col(self.vars["SEQUENCE"]).dt.total_hours()) self.features_df = data[split][Segment.features].drop(self.vars["SEQUENCE"]) # calculate basic info for the data self.num_stays = self.grouping_df[self.vars["GROUP"]].unique().shape[0] @@ -138,7 +141,7 @@ def get_balance(self) -> list: weights = list((1 / counts) * np.sum(counts) / counts.shape[0]) return weights - def get_data_and_labels(self) -> Tuple[np.array, np.array]: + def get_data_and_labels(self) -> Tuple[np.array, np.array, np.array]: """Function to return all the data and labels aligned at once. We use this function for the ML methods which don't require an iterator. @@ -154,14 +157,14 @@ def get_data_and_labels(self) -> Tuple[np.array, np.array]: rep = rep.group_by(self.vars["GROUP"]).last() rep = rep.to_numpy().astype(float) - return rep, labels + return rep, labels, self.row_indicators.to_numpy() def to_tensor(self): - data, labels = self.get_data_and_labels() + data, labels, row_indicators = self.get_data_and_labels() if self.mps: return from_numpy(data).to(float32), from_numpy(labels).to(float32) else: - return from_numpy(data), from_numpy(labels) + return from_numpy(data), from_numpy(labels), row_indicators @gin.configurable("CommonPandasDataset") class CommonPandasDataset(Dataset): From 487ed819b36dcce73eb0750421874276771a1150 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 17 Sep 2024 13:01:20 +0200 Subject: [PATCH 177/207] todo --- icu_benchmarks/data/split_process_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index 3caed968..37768b92 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -91,7 +91,7 @@ def preprocess_data( modality_mapping.get("cat_clinical_notes") is not None and modality_mapping.get("cat_med_embeddings_map") is not None ) - else None, + else None, # Todo: Exclude clinical notes and med embeddings from missing indicator and feature generation ) if isinstance(preprocessor, PandasClassificationPreprocessor): preprocessor.set_imputation_model(pretrained_imputation_model) From 0592cdbc11230aabaeeb44aab4d008bcae822697 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 17 Sep 2024 13:01:47 +0200 Subject: [PATCH 178/207] hyperparameter ranges with scale_pos_weight --- configs/prediction_models/XGBClassifier.gin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/prediction_models/XGBClassifier.gin b/configs/prediction_models/XGBClassifier.gin index fcca6b22..f1070672 100644 --- a/configs/prediction_models/XGBClassifier.gin +++ b/configs/prediction_models/XGBClassifier.gin @@ -10,7 +10,7 @@ model/hyperparameter.class_to_tune = @XGBClassifier model/hyperparameter.learning_rate = (0.01, 0.1, "log") model/hyperparameter.n_estimators = [50, 100, 250, 500, 750, 1000,1500,2000] model/hyperparameter.max_depth = [3, 5, 10, 15] -model/hyperparameter.scale_pos_weight = [1, 5, 10, 25, 50, 75, 99, 100, 1000] +model/hyperparameter.scale_pos_weight = [1, 5, 10, 15, 20, 25, 30, 35, 40, 50, 75, 99, 100, 1000] model/hyperparameter.min_child_weight = [1, 0.5] model/hyperparameter.max_delta_step = [0, 1, 2, 3, 4, 5, 10] model/hyperparameter.colsample_bytree = [0.1, 0.25, 0.5, 0.75, 1.0] From 6365a09113142e08bb0dc8459097d94840fa3a16 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 17 Sep 2024 13:02:55 +0200 Subject: [PATCH 179/207] Added smoothed labels code (not implemented yet) --- icu_benchmarks/models/utils.py | 63 +++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/icu_benchmarks/models/utils.py b/icu_benchmarks/models/utils.py index e75c94b1..c6e128eb 100644 --- a/icu_benchmarks/models/utils.py +++ b/icu_benchmarks/models/utils.py @@ -199,12 +199,73 @@ def __init__(self, scorer=average_precision_score): def __call__(self, y_true, y_pred): if len(np.unique(y_true))<=2 and y_pred.ndim > 1: - return self.scorer(y_true, np.argmax(y_pred,axis=1)) + y_pred_argmax = np.argmax(y_pred,axis=1) + return self.scorer(y_true, y_pred_argmax) else: return self.scorer(y_true, y_pred) def __name__(self): return "scorer_wrapper" +@gin.configurable('get_smoothed_labels') +def get_smoothed_labels(label, event, smoothing_fn=gin.REQUIRED, h_true=gin.REQUIRED, h_min=gin.REQUIRED, + h_max=gin.REQUIRED, delta_h=12, gamma=0.1): + diffs = np.concatenate([np.zeros(1), event[1:] - event[:-1]], axis=-1) + pos_event_change_full = np.where((diffs == 1) & (event == 1))[0] + + multihorizon = isinstance(h_true, list) + if multihorizon: + label_for_event = label[0] + h_for_event = h_true[0] + else: + label_for_event = label + h_for_event = h_true + diffs_label = np.concatenate([np.zeros(1), label_for_event[1:] - label_for_event[:-1]], axis=-1) + + # Event that occurred after the end of the stay for M3B. + # In that case event are equal to the number of hours after the end of stay when the event occured. + pos_event_change_delayed = np.where((diffs >= 1) & (event > 1))[0] + if len(pos_event_change_delayed) > 0: + delays = event[pos_event_change_delayed] - 1 + pos_event_change_delayed += delays.astype(int) + pos_event_change_full = np.sort(np.concatenate([pos_event_change_full, pos_event_change_delayed])) + + last_know_label = label_for_event[np.where(label_for_event != -1)][-1] + last_know_idx = np.where(label_for_event == last_know_label)[0][-1] + + # Need to handle the case where the ts was truncatenated at 2016 for HiB + if ((last_know_label == 1) and (len(pos_event_change_full) == 0)) or ( + (last_know_label == 1) and (last_know_idx >= pos_event_change_full[-1])): + last_know_event = 0 + if len(pos_event_change_full) > 0: + last_know_event = pos_event_change_full[-1] + + last_known_stable = 0 + known_stable = np.where(label_for_event == 0)[0] + if len(known_stable) > 0: + last_known_stable = known_stable[-1] + + pos_change = np.where((diffs_label >= 1) & (label_for_event == 1))[0] + last_pos_change = pos_change[np.where(pos_change > max(last_know_event, last_known_stable))][0] + pos_event_change_full = np.concatenate([pos_event_change_full, [last_pos_change + h_for_event]]) + + # No event case + if len(pos_event_change_full) == 0: + pos_event_change_full = np.array([np.inf]) + + time_array = np.arange(len(label)) + dist = pos_event_change_full.reshape(-1, 1) - time_array + dte = np.where(dist > 0, dist, np.inf).min(axis=0) + if multihorizon: + smoothed_labels = [] + for k in range(label.shape[-1]): + smoothed_labels.append(np.array(list( + map(lambda x: smoothing_fn(x, h_true=h_true[k], h_min=h_min[k], h_max=h_max[k], delta_h=delta_h, + gamma=gamma), dte)))) + return np.stack(smoothed_labels, axis=-1) + else: + return np.array( + list(map(lambda x: smoothing_fn(x, h_true=h_true, h_min=h_min, + h_max=h_max, delta_h=delta_h, gamma=gamma), dte))) From b9456df07c7cde0384fd69c8302bcd470f34b1c8 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 17 Sep 2024 13:05:11 +0200 Subject: [PATCH 180/207] Added smoothed labels code (not implemented yet) --- icu_benchmarks/models/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icu_benchmarks/models/utils.py b/icu_benchmarks/models/utils.py index c6e128eb..65e87bce 100644 --- a/icu_benchmarks/models/utils.py +++ b/icu_benchmarks/models/utils.py @@ -206,7 +206,7 @@ def __call__(self, y_true, y_pred): def __name__(self): return "scorer_wrapper" - +# Source: https://github.com/ratschlab/tls @gin.configurable('get_smoothed_labels') def get_smoothed_labels(label, event, smoothing_fn=gin.REQUIRED, h_true=gin.REQUIRED, h_min=gin.REQUIRED, h_max=gin.REQUIRED, delta_h=12, gamma=0.1): From 640c8da1435fcf83c30a28643a55c948e4dd8cd1 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 15 Oct 2024 09:49:06 +0200 Subject: [PATCH 181/207] experiments --- experiments/benchmark_cass.yml | 35 +++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/experiments/benchmark_cass.yml b/experiments/benchmark_cass.yml index 936b06c9..4389deb8 100644 --- a/experiments/benchmark_cass.yml +++ b/experiments/benchmark_cass.yml @@ -14,9 +14,12 @@ command: - -tn - SSI # - Mortality - - -n - - cass +# - --verbose # - --hp-checkpoint +# - /sc-projects/sc-proj-cc08-cassandra/RW_Prospective/yaib_logs/activity_notes/SSI/XGBClassifier/2024-07-25T14-32-55/hyperparameter_tuning_logs.db + - --modalities + - "all" +# - --verbose # - /sc-projects/sc-proj-cc08-cassandra/RW_Prospective/yaib_logs/cass/SSI/XGBClassifier/2024-06-28T17-28-04/hyperparameter_tuning_logs.db # - /sc-projects/sc-proj-cc08-cassandra/RW_Prospective/yaib_logs/cass/SSI/Transformer/2024-06-26T15-30-53/hyperparameter_tuning_logs.db # - -gc @@ -26,22 +29,44 @@ name: yaib_classification_benchmark parameters: data_dir: values: + - "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/dataset_segment_1.0_horizon_6:00:00_transfer_3_2024-10-07T23:26:22" +# - "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/dataset_segment_1.0_horizon_6:00:00_transfer_full_2024-09-27T16:14:05_complication" +# - "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/dataset_segment_1.0_horizon_6:00:00_transfer_full_2024-09-26T16:26:24" +# - "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/dataset_x_segment_duration_x_transfer_2024-09-19T13:57:36" +# - "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/dataset_baseline_dynamic_6h_2024-09-18T11:24:23" +# - "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/dataset_baseline_flat_2024-09-17T17:25:20" # - /home/vandewrp/projects/YAIB/data/YAIB_Datasets/data/mortality24/miiv - - /sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/notes_full +# - /sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/activity_notes_fixed +# - /sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/19-08-2024_all_wards +# - /sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/dataset_x_segment_duration_x_transfer_2024-08-26 13:43:20 +# - "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/dataset_x_segment_duration_x_transfer_2024-09-12T16:22:01" model: values: # - LogisticRegression # - TimesNet # - LGBMClassifier -# - XGBClassifier + - XGBClassifier # - CBClassifier # - BRFClassifier # - RUSBClassifier # - RFClassifier -# - GRU + - GRU # - LSTM # - TCN - Transformer + modalities: + values: + - "all" +# - [copra_observations, copra_scores, copra_fluid_balance, ishmed_lab_numeric, cat_med_embeddings_map, cat_clinical_notes, wearable_activity, wearable_core, static] +# - [copra_observations, copra_scores, copra_fluid_balance, ishmed_lab_numeric, cat_med_embeddings_map, wearable_activity, wearable_core, static] +# - [copra_observations, copra_scores, copra_fluid_balance, ishmed_lab_numeric, static] # without wearable data +# - [copra_observations, copra_scores, copra_fluid_balance, cat_med_embeddings_map, wearable_activity, wearable_core, static] # without ishmed lab data and clinical notes +# - [copra_observations, copra_scores, copra_fluid_balance, cat_med_embeddings_map, static ] # without wearable data and ishmed lab data, clinical notes +## - [copra_observations, copra_scores, copra_fluid_balance, ishmed_lab_numeric, cat_med_embeddings_map, wearable_activity, wearable_core, static] # without clinical notes +# - [copra_observations, copra_scores, copra_fluid_balance, ishmed_lab_numeric, cat_clinical_notes, wearable_activity, wearable_core, static] # without med embeddings +# - [wearable_activity, wearable_core, static] +# - [ishmed_lab_numeric, static] +# - [static] seed: values: - 1111 From f93d9fb32cad40ca89b4275bd7c07b8f9622eedb Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 15 Oct 2024 09:49:34 +0200 Subject: [PATCH 182/207] More changes in experiments --- configs/tasks/CassClassification.gin | 9 ++++++++- experiments/charhpc_wandb_sweep.sh | 6 ++++-- experiments/charhpc_wandb_sweep_cpu.sh | 12 +++++++----- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/configs/tasks/CassClassification.gin b/configs/tasks/CassClassification.gin index a243db25..b9576f7d 100644 --- a/configs/tasks/CassClassification.gin +++ b/configs/tasks/CassClassification.gin @@ -33,4 +33,11 @@ preprocess.file_names = { "STATIC": "sta.parquet", } -include "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/notes_full/vars.gin" \ No newline at end of file +#include "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/14-08-2024_normal_ward/vars.gin" +#include "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/19-08-2024_all_wards/vars.gin" +# include "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/dataset_x_segment_duration_x_transfer_2024-08-26 13:43:20/vars.gin" +# include "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/dataset_x_segment_duration_x_transfer_2024-09-12T16:22:01/vars.gin" +# include "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/dataset_baseline_flat_2024-09-17T17:25:20/vars.gin" +# include "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/dataset_segment_1.0_horizon_6:00:00_transfer_full_2024-09-27T16:14:05/vars.gin" +include "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/dataset_segment_1.0_horizon_6:00:00_transfer_3_2024-10-07T23:26:22/vars.gin" +# preprocess.modality_mapping = %modality_mapping diff --git a/experiments/charhpc_wandb_sweep.sh b/experiments/charhpc_wandb_sweep.sh index bec499f1..3426d5fd 100644 --- a/experiments/charhpc_wandb_sweep.sh +++ b/experiments/charhpc_wandb_sweep.sh @@ -1,11 +1,13 @@ #!/bin/bash #SBATCH --job-name=yaib_experiment -#SBATCH --partition=gpu # -p +#SBATCH --partition=pgpu # -p #SBATCH --cpus-per-task=8 # -c #SBATCH --mem=200gb #SBATCH --output=logs/classification_%a_%j.log # %j is job id #SBATCH --gpus=1 -#SBATCH --time=48:00:00 +#SBATCH --time=24:00:00 + +source /etc/profile.d/conda.sh eval "$(conda shell.bash hook)" conda activate yaib_req_pl diff --git a/experiments/charhpc_wandb_sweep_cpu.sh b/experiments/charhpc_wandb_sweep_cpu.sh index 1662d6c9..de81d07e 100644 --- a/experiments/charhpc_wandb_sweep_cpu.sh +++ b/experiments/charhpc_wandb_sweep_cpu.sh @@ -1,13 +1,15 @@ #!/bin/bash #SBATCH --job-name=yaib_experiment -#SBATCH --partition=compute # -p -#SBATCH --cpus-per-task=10 # -c -#SBATCH --mem=200gb +#SBATCH --partition=pgpu # -p +#SBATCH --cpus-per-task=16 # -c +#SBATCH --mem=100gb #SBATCH --output=logs/classification_%a_%j.log # %j is job id -#SBATCH --time=48:00:00 +#SBATCH --time=24:00:00 + +source /etc/profile.d/conda.sh eval "$(conda shell.bash hook)" conda activate yaib_req_pl wandb agent --count 1 cassandra_hpi/cassandra/"$1" - +# Debug instance: srun -p gpu --pty -t 5:00:00 --gres=gpu:1 --cpus-per-task=16 --mem=100GB bash \ No newline at end of file From 786e256d90a4e9e4c3ff45f2c8fe4f974f3bfba9 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 15 Oct 2024 09:50:12 +0200 Subject: [PATCH 183/207] debugging --- icu_benchmarks/cross_validation.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/icu_benchmarks/cross_validation.py b/icu_benchmarks/cross_validation.py index 3137821a..3a22f627 100644 --- a/icu_benchmarks/cross_validation.py +++ b/icu_benchmarks/cross_validation.py @@ -1,4 +1,5 @@ import json +import pickle from datetime import datetime import logging import gin @@ -104,7 +105,10 @@ def execute_repeated_cv( runmode=mode, complete_train=complete_train ) - + # logging.debug(f"{data}") + # data_pickle_path = log_dir / "data.pkl" + # with open(data_pickle_path, "wb") as f: + # pickle.dump(data, f, pickle.HIGHEST_PROTOCOL) preprocess_time = datetime.now() - start_time start_time = datetime.now() agg_loss += train_common( From 125e4f003b34b5c1719a3b26af9fd30a8cb8e22e Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 15 Oct 2024 09:52:07 +0200 Subject: [PATCH 184/207] Compatibility with static (non temporal) datasets --- icu_benchmarks/data/loader.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/icu_benchmarks/data/loader.py b/icu_benchmarks/data/loader.py index a303a478..eb544d64 100644 --- a/icu_benchmarks/data/loader.py +++ b/icu_benchmarks/data/loader.py @@ -34,9 +34,19 @@ def __init__( # data[split][Segment.features].set_index(self.vars["GROUP"]).drop(labels=self.vars["SEQUENCE"], axis=1) # ) # Get the row indicators for the data to be able to match predicted labels - self.row_indicators = data[split][Segment.features][self.vars["GROUP"], self.vars["SEQUENCE"]] - self.row_indicators = self.row_indicators.with_columns(pl.col(self.vars["SEQUENCE"]).dt.total_hours()) - self.features_df = data[split][Segment.features].drop(self.vars["SEQUENCE"]) + if "SEQUENCE" in self.vars and self.vars["SEQUENCE"] in data[split][Segment.features].columns: + # We have a time series dataset + self.row_indicators = data[split][Segment.features][self.vars["GROUP"], self.vars["SEQUENCE"]] + self.row_indicators = self.row_indicators.with_columns(pl.col(self.vars["SEQUENCE"]).dt.total_hours()) + self.features_df = data[split][Segment.features] + self.features_df = self.features_df.sort([self.vars["GROUP"], self.vars["SEQUENCE"]]) + self.features_df = self.features_df.drop(self.vars["SEQUENCE"]) + self.grouping_df = self.grouping_df.sort([self.vars["GROUP"], self.vars["SEQUENCE"]]) + else: + # We have a static dataset + logging.info("Using static dataset") + self.row_indicators = data[split][Segment.features][self.vars["GROUP"]] + self.features_df = data[split][Segment.features] # calculate basic info for the data self.num_stays = self.grouping_df[self.vars["GROUP"]].unique().shape[0] self.maxlen = self.features_df.group_by([self.vars["GROUP"]]).len().max().item(0, 1) @@ -151,12 +161,19 @@ def get_data_and_labels(self) -> Tuple[np.array, np.array, np.array]: """ labels = self.outcome_df[self.vars["LABEL"]].to_numpy().astype(float) rep = self.features_df + if len(labels) == self.num_stays: # order of groups could be random, we make sure not to change it # rep = rep.groupby(level=self.vars["GROUP"], sort=False).last() rep = rep.group_by(self.vars["GROUP"]).last() + else: + # Adding segment count for each stay id and timestep. + rep = rep.with_columns( + pl.col(self.vars["GROUP"]).cum_count().over(self.vars["GROUP"]).alias("counter") + ) rep = rep.to_numpy().astype(float) - + logging.debug(f"rep shape: {rep.shape}") + logging.debug(f"labels shape: {labels.shape}") return rep, labels, self.row_indicators.to_numpy() def to_tensor(self): From b3074742bb45dd9370c5bb3e15b2e130767077b3 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 15 Oct 2024 09:52:25 +0200 Subject: [PATCH 185/207] Experiment related tuning --- configs/prediction_models/common/MLTuning.gin | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/prediction_models/common/MLTuning.gin b/configs/prediction_models/common/MLTuning.gin index 34bc070a..9df38c47 100644 --- a/configs/prediction_models/common/MLTuning.gin +++ b/configs/prediction_models/common/MLTuning.gin @@ -1,5 +1,5 @@ # Hyperparameter tuner settings for classical Machine Learning. tune_hyperparameters.scopes = ["model"] -tune_hyperparameters.n_initial_points = 10 -tune_hyperparameters.n_calls = 50 +tune_hyperparameters.n_initial_points = 5 +tune_hyperparameters.n_calls = 30 tune_hyperparameters.folds_to_tune_on = 5 \ No newline at end of file From 977741807bf1e87d69b2300ce4cd641ecabc1a39 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 15 Oct 2024 09:52:56 +0200 Subject: [PATCH 186/207] revert to log loss --- icu_benchmarks/models/wrappers.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index cdfa7f7c..a9dc9caa 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -366,16 +366,17 @@ class MLWrapper(BaseModule, ABC): requires_backprop = False _supported_run_modes = [RunMode.classification, RunMode.regression] - def __init__(self, *args, run_mode=RunMode.classification, loss=average_precision_score, patience=10, mps=False, **kwargs): + def __init__(self, *args, run_mode=RunMode.classification, loss=log_loss, patience=10, mps=False, **kwargs): super().__init__() self.save_hyperparameters() self.scaler = None self.check_supported_runmode(run_mode) self.run_mode = run_mode - if loss.__name__ in ["average_precision_score", "roc_auc_score"]: - self.loss = scorer_wrapper(average_precision_score) - else: - self.loss = self.loss + # if loss.__name__ in ["average_precision_score", "roc_auc_score"]: + # self.loss = scorer_wrapper(average_precision_score) + # else: + # self.loss = self.loss + self.loss = loss self.patience = patience self.mps = mps self.loss_weight = None From 808eb9f5107e71288af629c600f40fd702c63f4a Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 15 Oct 2024 09:53:59 +0200 Subject: [PATCH 187/207] Saving pred indicators to evaluate predictions --- icu_benchmarks/models/ml_models/xgboost.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/icu_benchmarks/models/ml_models/xgboost.py b/icu_benchmarks/models/ml_models/xgboost.py index 381e0276..14b8c365 100644 --- a/icu_benchmarks/models/ml_models/xgboost.py +++ b/icu_benchmarks/models/ml_models/xgboost.py @@ -33,6 +33,8 @@ def fit_model(self, train_data, train_labels, val_data, val_labels): if wandb.run is not None: callbacks.append(wandb_xgb()) + logging.info(f"train_data: {train_data.shape}, train_labels: {train_labels.shape}") + logging.info(train_labels) self.model.fit(train_data, train_labels, eval_set=[(val_data, val_labels)], verbose=False) self.explainer = shap.TreeExplainer(self.model) self.train_shap_values = self.explainer(train_data) @@ -50,20 +52,14 @@ def test_step(self, dataset, _): test_rep, test_label, pred_indicators = test_rep.squeeze().cpu().numpy(), test_label.squeeze().cpu().numpy(), pred_indicators.squeeze().cpu().numpy() self.set_metrics(test_label) test_pred = self.predict(test_rep) - if pred_indicators.shape[1] == test_pred.shape[1]: + if len(pred_indicators.shape)>1 and len(test_pred.shape)>1 and pred_indicators.shape[1] == test_pred.shape[1]: pred_indicators = np.hstack((pred_indicators, test_label.reshape(-1, 1))) - # test_reshaped = test_pred.reshape(-1, 1) pred_indicators = np.hstack((pred_indicators, test_pred)) - sav = self.logger.save_dir # Save as: id, time (hours), ground truth, prediction 0, prediction 1 - np.save(os.path.join(self.logger.save_dir,f'pred_indicators.npy'), pred_indicators) - logging.debug(f"Saved row indicators to {os.path.join(self.logger.save_dir,f'row_indicators.npy')}") - # self.log_numpy_array(row_indicators, "test/pred_labels_windows", self.global_step) - # self.log("test/pred_labels_windows", torch.from_numpy(row_indicators), sync_dist=True) + np.savetxt(os.path.join(self.logger.save_dir,f'pred_indicators.csv'), pred_indicators, delimiter=",") + logging.debug(f"Saved row indicators to {os.path.join(self.logger.save_dir,f'row_indicators.csv')}") if self.explainer is not None: self.test_shap_values = self.explainer(test_rep) - # logging.debug(f"Shap values: {self.test_shap_values}") - # self.log("test/shap_values", self.test_shap_values, sync_dist=True) if self.mps: self.log("test/loss", np.float32(self.loss(test_label, test_pred)), sync_dist=True) self.log_metrics(np.float32(test_label), np.float32(test_pred), "test") From 34e057367805ef194f322e6972aeda3a1f6d83e2 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 15 Oct 2024 09:56:30 +0200 Subject: [PATCH 188/207] Data sanitization --- icu_benchmarks/data/split_process_data.py | 55 ++++++++++++++++++----- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index 37768b92..f40e08c9 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -1,5 +1,7 @@ import copy import logging +import os + import gin import json import hashlib @@ -9,7 +11,10 @@ from pathlib import Path import pickle from timeit import default_timer as timer + +from setuptools.dist import sequence from sklearn.model_selection import StratifiedKFold, KFold, StratifiedShuffleSplit, ShuffleSplit +from sqlalchemy import false from icu_benchmarks.data.preprocessor import Preprocessor, PandasClassificationPreprocessor, PolarsClassificationPreprocessor from icu_benchmarks.contants import RunMode @@ -23,7 +28,7 @@ def preprocess_data( preprocessor: Preprocessor = PolarsClassificationPreprocessor, use_static: bool = True, vars: dict[str] = gin.REQUIRED, - modality_mapping: dict[str] = [], + modality_mapping: dict[str] = {}, selected_modalities: list[str] = "all", seed: int = 42, debug: bool = False, @@ -38,6 +43,8 @@ def preprocess_data( complete_train: bool = False, runmode: RunMode = RunMode.classification, label: str = None, + vars_to_exclude: list[str] = [], + ) -> dict[dict[pl.DataFrame]] or dict[dict[pd.DataFrame]]: """Perform loading, splitting, imputing and normalising of task data. @@ -83,15 +90,14 @@ def preprocess_data( cache_filename = f"s_{seed}_r_{repetition_index}_f_{fold_index}_t_{train_size}_d_{debug}" logging.log(logging.INFO, f"Using preprocessor: {preprocessor.__name__}") + vars_to_exclude = modality_mapping.get("cat_clinical_notes") + modality_mapping.get("cat_med_embeddings_map") if ( + modality_mapping.get("cat_clinical_notes") is not None + and modality_mapping.get("cat_med_embeddings_map") is not None) else None + preprocessor = preprocessor( use_static_features=use_static, save_cache=data_dir / "preproc" / (cache_filename + "_recipe"), - vars_to_exclude=modality_mapping.get("cat_clinical_notes") + modality_mapping.get("cat_med_embeddings_map") - if ( - modality_mapping.get("cat_clinical_notes") is not None - and modality_mapping.get("cat_med_embeddings_map") is not None - ) - else None, # Todo: Exclude clinical notes and med embeddings from missing indicator and feature generation + vars_to_exclude=vars_to_exclude, ) if isinstance(preprocessor, PandasClassificationPreprocessor): preprocessor.set_imputation_model(pretrained_imputation_model) @@ -110,7 +116,12 @@ def preprocess_data( # Read parquet files into pandas dataframes and remove the parquet file from memory logging.info(f"Loading data from directory {data_dir.absolute()}") - data = {f: pl.read_parquet(data_dir / file_names[f]) for f in file_names.keys()} + data = {f: pl.read_parquet(data_dir / file_names[f]) for f in file_names.keys() if os.path.exists(data_dir / file_names[f])} + logging.info(f"Loaded data: {list(data.keys())}") + data = check_sanitize_data(data, vars) + + if not (Segment.dynamic in data.keys()): + logging.warning("No dynamic data found, using only static data.") logging.debug(f"Modality mapping: {modality_mapping}") if len(modality_mapping) > 0: @@ -175,6 +186,30 @@ def preprocess_data( return data +def check_sanitize_data(data, vars): + """Check for duplicates in the loaded data and remove them.""" + group = vars[Var.group] if Var.group in vars.keys() else None + sequence = vars[Var.sequence] if Var.sequence in vars.keys() else None + keep = "last" + if Segment.static in data.keys(): + old_len = len(data[Segment.static]) + data[Segment.static] = data[Segment.static].unique(subset=group, keep=keep, maintain_order=True) + logging.warning(f"Removed {old_len - len(data[Segment.static])} duplicates from static data.") + if Segment.dynamic in data.keys(): + old_len = len(data[Segment.dynamic]) + data[Segment.dynamic] = data[Segment.dynamic].unique(subset=[group,sequence], keep=keep, maintain_order=True) + logging.warning(f"Removed {old_len - len(data[Segment.dynamic])} duplicates from dynamic data.") + if Segment.outcome in data.keys(): + old_len = len(data[Segment.outcome]) + if sequence in data[Segment.outcome].columns: + # We have a dynamic outcome with group and sequence + data[Segment.outcome] = data[Segment.outcome].unique(subset=[group,sequence], keep=keep, maintain_order=True) + else: + data[Segment.outcome] = data[Segment.outcome].unique(subset=[group], keep=keep, maintain_order=True) + logging.warning(f"Removed {old_len - len(data[Segment.outcome])} duplicates from outcome data.") + return data + + def modality_selection( data: dict[pl.DataFrame], modality_mapping: dict[str], selected_modalities: list[str], vars ) -> dict[pl.DataFrame]: @@ -191,7 +226,7 @@ def modality_selection( if key not in [Var.group, Var.label, Var.sequence]: old_columns.extend(value) vars[key] = [col for col in value if col in selected_columns] - # -3 becaus of standard columns + # -3 because of standard columns logging.info(f"Selected columns: {len(selected_columns) - 3}, old columns: {len(old_columns)}") logging.debug(f"Difference: {set(old_columns) - set(selected_columns)}") # Update data dict @@ -403,7 +438,7 @@ def make_single_split( data_split[fold] = { data_type: data[data_type].merge(split[fold], on=id, how="right", sort=True) for data_type in data.keys() } - logging.debug(f"Data split: {data_split}") + logging.info(f"Data split: {data_split}") return data_split From ab3ab4ba0f42df25b1eff0cf47c91f5cfd5e1958 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Tue, 15 Oct 2024 09:58:29 +0200 Subject: [PATCH 189/207] Preprocessing improvements and failsafe when using just static features --- icu_benchmarks/data/preprocessor.py | 67 ++++++++++++++++++----------- 1 file changed, 41 insertions(+), 26 deletions(-) diff --git a/icu_benchmarks/data/preprocessor.py b/icu_benchmarks/data/preprocessor.py index 27f847cb..a00303b5 100644 --- a/icu_benchmarks/data/preprocessor.py +++ b/icu_benchmarks/data/preprocessor.py @@ -82,41 +82,55 @@ def apply(self, data, vars) -> dict[dict[pl.DataFrame]]: Returns: Preprocessed data. """ - logging.info("Preprocessing dynamic features.") - - data = self._process_dynamic(data, vars) - if self.use_static_features and len(vars[Segment.static]) > 0: + # Check if dynamic features are present + if self.use_static_features and all(Segment.static in value for value in data.values()) and len(vars[Segment.static]) > 0: logging.info("Preprocessing static features.") data = self._process_static(data, vars) + else: + self.use_static_features = False + + if all(Segment.dynamic in value for value in data.values()): + logging.info("Preprocessing dynamic features.") + logging.info(data.keys()) + data = self._process_dynamic(data, vars) + if self.use_static_features: + # Join static and dynamic data. + data[Split.train][Segment.dynamic] = data[Split.train][Segment.dynamic].join( + data[Split.train][Segment.static], on=vars["GROUP"] + ) + data[Split.val][Segment.dynamic] = data[Split.val][Segment.dynamic].join( + data[Split.val][Segment.static], on=vars["GROUP"] + ) + data[Split.test][Segment.dynamic] = data[Split.test][Segment.dynamic].join( + data[Split.test][Segment.static], on=vars["GROUP"] + ) - # Set index to grouping variable - data[Split.train][Segment.static] = data[Split.train][Segment.static]#.set_index(vars["GROUP"]) - data[Split.val][Segment.static] = data[Split.val][Segment.static]#.set_index(vars["GROUP"]) - data[Split.test][Segment.static] = data[Split.test][Segment.static]#.set_index(vars["GROUP"]) + # Remove static features from splits + data[Split.train][Segment.features] = data[Split.train].pop(Segment.static) + data[Split.val][Segment.features] = data[Split.val].pop(Segment.static) + data[Split.test][Segment.features] = data[Split.test].pop(Segment.static) - # Join static and dynamic data. - data[Split.train][Segment.dynamic] = data[Split.train][Segment.dynamic].join( - data[Split.train][Segment.static], on=vars["GROUP"] - ) - data[Split.val][Segment.dynamic] = data[Split.val][Segment.dynamic].join( - data[Split.val][Segment.static], on=vars["GROUP"] - ) - data[Split.test][Segment.dynamic] = data[Split.test][Segment.dynamic].join( - data[Split.test][Segment.static], on=vars["GROUP"] - ) - - # Remove static features from splits + # Create feature splits + data[Split.train][Segment.features] = data[Split.train].pop(Segment.dynamic) + data[Split.val][Segment.features] = data[Split.val].pop(Segment.dynamic) + data[Split.test][Segment.features] = data[Split.test].pop(Segment.dynamic) + elif self.use_static_features: data[Split.train][Segment.features] = data[Split.train].pop(Segment.static) data[Split.val][Segment.features] = data[Split.val].pop(Segment.static) data[Split.test][Segment.features] = data[Split.test].pop(Segment.static) - - # Create feature splits - data[Split.train][Segment.features] = data[Split.train].pop(Segment.dynamic) - data[Split.val][Segment.features] = data[Split.val].pop(Segment.dynamic) - data[Split.test][Segment.features] = data[Split.test].pop(Segment.dynamic) - + else: + raise Exception(f"No recognized data segments data to preprocess. Available: {data.keys()}") logging.debug("Data head") logging.debug(data[Split.train][Segment.features].head()) + logging.debug(data[Split.train][Segment.outcome]) + for split in [Split.train, Split.val, Split.test]: + if vars["SEQUENCE"] in data[split][Segment.outcome] and len(data[split][Segment.features]) != len(data[split][Segment.outcome]): + raise Exception(f"Data and outcome length mismatch in {split} split: " + f"features: {len(data[split][Segment.features])}, outcome: {len(data[split][Segment.outcome])}") + data[Split.train][Segment.features] = data[Split.train][Segment.features].unique() + data[Split.val][Segment.features] = data[Split.val][Segment.features].unique() + data[Split.test][Segment.features] = data[Split.test][Segment.features].unique() + logging.info(f"Generate features: {self.generate_features}") return data @@ -321,6 +335,7 @@ def apply(self, data, vars) -> dict[dict[pd.DataFrame]]: logging.debug("Data head") logging.debug(data[Split.train][Segment.features].head()) + logging.debug(data[Split.train][Segment.outcome].head()) logging.info(f"Generate features: {self.generate_features}") return data From a194cf61a467c71c39f049079c1dac8b874ae9d1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 17 Oct 2024 08:07:52 +0000 Subject: [PATCH 190/207] Bump lightning from 2.3.0 to 2.3.3 Bumps [lightning](https://github.com/Lightning-AI/lightning) from 2.3.0 to 2.3.3. - [Release notes](https://github.com/Lightning-AI/lightning/releases) - [Commits](https://github.com/Lightning-AI/lightning/compare/2.3.0...2.3.3) --- updated-dependencies: - dependency-name: lightning dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index b24bd19f..96d9ea12 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ pytorch-ignite==0.5.0.post2 # Note: versioning of Pytorch might be dependent on compatible CUDA version. # Please check yourself if your Pytorch installation supports cuda (for gpu acceleration) torch==2.3.1 -lightning==2.3.0 +lightning==2.3.3 torchmetrics==1.0.3 #pytorch-cuda==11.8 lightgbm==3.3.5 From 9905abcb7c9c06572f240730c228e1481c2a8fa4 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 17 Oct 2024 10:15:51 +0200 Subject: [PATCH 191/207] Remove cass specific files --- experiments/benchmark_cass.yml | 76 -------------------------- experiments/charhpc_wandb_sweep.sh | 14 ----- experiments/charhpc_wandb_sweep_cpu.sh | 15 ----- experiments/slurm_base_char_sc.sh | 44 --------------- 4 files changed, 149 deletions(-) delete mode 100644 experiments/benchmark_cass.yml delete mode 100644 experiments/charhpc_wandb_sweep.sh delete mode 100644 experiments/charhpc_wandb_sweep_cpu.sh delete mode 100644 experiments/slurm_base_char_sc.sh diff --git a/experiments/benchmark_cass.yml b/experiments/benchmark_cass.yml deleted file mode 100644 index 4389deb8..00000000 --- a/experiments/benchmark_cass.yml +++ /dev/null @@ -1,76 +0,0 @@ -command: - - ${env} - - ${program} - - train - - -d - - ../data/ - - -t -# - BinaryClassification - - CassClassification - - --log-dir - - /sc-projects/sc-proj-cc08-cassandra/RW_Prospective/yaib_logs - - --tune - - --wandb-sweep - - -tn - - SSI -# - Mortality -# - --verbose -# - --hp-checkpoint -# - /sc-projects/sc-proj-cc08-cassandra/RW_Prospective/yaib_logs/activity_notes/SSI/XGBClassifier/2024-07-25T14-32-55/hyperparameter_tuning_logs.db - - --modalities - - "all" -# - --verbose -# - /sc-projects/sc-proj-cc08-cassandra/RW_Prospective/yaib_logs/cass/SSI/XGBClassifier/2024-06-28T17-28-04/hyperparameter_tuning_logs.db -# - /sc-projects/sc-proj-cc08-cassandra/RW_Prospective/yaib_logs/cass/SSI/Transformer/2024-06-26T15-30-53/hyperparameter_tuning_logs.db -# - -gc -# - -lc -method: grid -name: yaib_classification_benchmark -parameters: - data_dir: - values: - - "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/dataset_segment_1.0_horizon_6:00:00_transfer_3_2024-10-07T23:26:22" -# - "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/dataset_segment_1.0_horizon_6:00:00_transfer_full_2024-09-27T16:14:05_complication" -# - "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/dataset_segment_1.0_horizon_6:00:00_transfer_full_2024-09-26T16:26:24" -# - "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/dataset_x_segment_duration_x_transfer_2024-09-19T13:57:36" -# - "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/dataset_baseline_dynamic_6h_2024-09-18T11:24:23" -# - "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/dataset_baseline_flat_2024-09-17T17:25:20" -# - /home/vandewrp/projects/YAIB/data/YAIB_Datasets/data/mortality24/miiv -# - /sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/activity_notes_fixed -# - /sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/19-08-2024_all_wards -# - /sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/dataset_x_segment_duration_x_transfer_2024-08-26 13:43:20 -# - "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/dataset_x_segment_duration_x_transfer_2024-09-12T16:22:01" - model: - values: -# - LogisticRegression -# - TimesNet -# - LGBMClassifier - - XGBClassifier -# - CBClassifier -# - BRFClassifier -# - RUSBClassifier -# - RFClassifier - - GRU -# - LSTM -# - TCN - - Transformer - modalities: - values: - - "all" -# - [copra_observations, copra_scores, copra_fluid_balance, ishmed_lab_numeric, cat_med_embeddings_map, cat_clinical_notes, wearable_activity, wearable_core, static] -# - [copra_observations, copra_scores, copra_fluid_balance, ishmed_lab_numeric, cat_med_embeddings_map, wearable_activity, wearable_core, static] -# - [copra_observations, copra_scores, copra_fluid_balance, ishmed_lab_numeric, static] # without wearable data -# - [copra_observations, copra_scores, copra_fluid_balance, cat_med_embeddings_map, wearable_activity, wearable_core, static] # without ishmed lab data and clinical notes -# - [copra_observations, copra_scores, copra_fluid_balance, cat_med_embeddings_map, static ] # without wearable data and ishmed lab data, clinical notes -## - [copra_observations, copra_scores, copra_fluid_balance, ishmed_lab_numeric, cat_med_embeddings_map, wearable_activity, wearable_core, static] # without clinical notes -# - [copra_observations, copra_scores, copra_fluid_balance, ishmed_lab_numeric, cat_clinical_notes, wearable_activity, wearable_core, static] # without med embeddings -# - [wearable_activity, wearable_core, static] -# - [ishmed_lab_numeric, static] -# - [static] - seed: - values: - - 1111 - use_pretrained_imputation: - values: - - None -program: icu-benchmarks \ No newline at end of file diff --git a/experiments/charhpc_wandb_sweep.sh b/experiments/charhpc_wandb_sweep.sh deleted file mode 100644 index 3426d5fd..00000000 --- a/experiments/charhpc_wandb_sweep.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=yaib_experiment -#SBATCH --partition=pgpu # -p -#SBATCH --cpus-per-task=8 # -c -#SBATCH --mem=200gb -#SBATCH --output=logs/classification_%a_%j.log # %j is job id -#SBATCH --gpus=1 -#SBATCH --time=24:00:00 - -source /etc/profile.d/conda.sh - -eval "$(conda shell.bash hook)" -conda activate yaib_req_pl -wandb agent --count 1 cassandra_hpi/cassandra/"$1" \ No newline at end of file diff --git a/experiments/charhpc_wandb_sweep_cpu.sh b/experiments/charhpc_wandb_sweep_cpu.sh deleted file mode 100644 index de81d07e..00000000 --- a/experiments/charhpc_wandb_sweep_cpu.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=yaib_experiment -#SBATCH --partition=pgpu # -p -#SBATCH --cpus-per-task=16 # -c -#SBATCH --mem=100gb -#SBATCH --output=logs/classification_%a_%j.log # %j is job id -#SBATCH --time=24:00:00 - -source /etc/profile.d/conda.sh - -eval "$(conda shell.bash hook)" -conda activate yaib_req_pl -wandb agent --count 1 cassandra_hpi/cassandra/"$1" - -# Debug instance: srun -p gpu --pty -t 5:00:00 --gres=gpu:1 --cpus-per-task=16 --mem=100GB bash \ No newline at end of file diff --git a/experiments/slurm_base_char_sc.sh b/experiments/slurm_base_char_sc.sh deleted file mode 100644 index 6693b339..00000000 --- a/experiments/slurm_base_char_sc.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=default -#SBATCH --mail-type=ALL -#SBATCH --mail-user=[INSERT:EMAIL] -#SBATCH --partition=gpu # -p -#SBATCH --cpus-per-task=4 # -c -#SBATCH --mem=48gb -#SBATCH --gpus=1 -#SBATCH --output=%x_%a_%j.log # %x is job-name, %j is job id, %a is array id -#SBATCH --array=0-3 - -# Submit with e.g. --export=TASK_NAME=mortality24,MODEL_NAME=LGBMClassifier -# Basic experiment variables, please exchange [INSERT] for your experiment parameters - -TASK=BinaryClassification # BinaryClassification -YAIB_PATH=/home/vandewrp/projects/YAIB #/dhc/home/robin.vandewater/projects/yaib -EXPERIMENT_PATH=../yaib_logs/${TASK_NAME}_experiment -DATASET_ROOT_PATH= /sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format #data/YAIB_Datasets/data - -echo "This is a SLURM job named" $SLURM_JOB_NAME "with array id" $SLURM_ARRAY_TASK_ID "and job id" $SLURM_JOB_ID -echo "Resources allocated: " $SLURM_CPUS_PER_TASK "CPUs, " $SLURM_MEM_PER_NODE "GB RAM, " $SLURM_GPUS_PER_NODE "GPUs" -echi "Task type:" ${TASK} -echo "Task: " ${TASK_NAME} -echo "Model: "${MODEL_NAME} -echo "Dataset: "${DATASETS[$SLURM_ARRAY_TASK_ID]} -echo "Experiment path: "${EXPERIMENT_PATH} - -cd ${YAIB_PATH} - -eval "$(conda shell.bash hook)" -conda activate yaib - - - -icu-benchmarks train \ - -d ${DATASET_ROOT_PATH} \ - -n ${DATASETS} \ - -t ${TASK} \ - -tn ${TASK_NAME} \ - -m ${MODEL_NAME} \ - -c \ - -s 1111 \ - -l ${EXPERIMENT_PATH} \ - --tune \ No newline at end of file From 860bf3a1166fd17534224782c516b94fb08e1402 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 17 Oct 2024 10:16:03 +0200 Subject: [PATCH 192/207] update recipies to 1.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a1539483..de6f04e0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,7 +27,7 @@ hydra-core==1.3 optuna==4.0.0 optuna-integration==3.6.0 wandb==0.17.3 -recipies==1.0rc1 +recipies==1.0 #Fixed version because of NumPy incompatibility and stale development status. scikit-optimize-fix==0.9.1 hydra-submitit-launcher==1.2.0 From 4a4ab0c3103675184a8c0477de3bf134468c40fc Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 17 Oct 2024 10:27:57 +0200 Subject: [PATCH 193/207] Major refactor --- icu_benchmarks/cross_validation.py | 7 +- icu_benchmarks/data/loader.py | 73 ++++++++++----------- icu_benchmarks/data/pooling.py | 56 ++++++++-------- icu_benchmarks/data/preprocessor.py | 31 ++++++--- icu_benchmarks/data/split_process_data.py | 35 +++++----- icu_benchmarks/models/constants.py | 4 +- icu_benchmarks/models/custom_metrics.py | 5 +- icu_benchmarks/models/ml_models/catboost.py | 10 ++- icu_benchmarks/models/ml_models/imblearn.py | 5 +- icu_benchmarks/models/ml_models/xgboost.py | 25 +++---- icu_benchmarks/models/train.py | 20 +++--- icu_benchmarks/models/utils.py | 43 +++++++----- icu_benchmarks/models/wrappers.py | 14 ++-- icu_benchmarks/run.py | 8 +-- icu_benchmarks/run_utils.py | 9 +-- icu_benchmarks/tuning/hyperparameters.py | 19 +++--- 16 files changed, 199 insertions(+), 165 deletions(-) diff --git a/icu_benchmarks/cross_validation.py b/icu_benchmarks/cross_validation.py index 3a22f627..b14e0e87 100644 --- a/icu_benchmarks/cross_validation.py +++ b/icu_benchmarks/cross_validation.py @@ -1,5 +1,4 @@ import json -import pickle from datetime import datetime import logging import gin @@ -38,7 +37,7 @@ def execute_repeated_cv( cpu: bool = False, verbose: bool = False, wandb: bool = False, - complete_train: bool = False + complete_train: bool = False, ) -> float: """Preprocesses data and trains a model for each fold. @@ -103,7 +102,7 @@ def execute_repeated_cv( fold_index=fold_index, pretrained_imputation_model=pretrained_imputation_model, runmode=mode, - complete_train=complete_train + complete_train=complete_train, ) # logging.debug(f"{data}") # data_pickle_path = log_dir / "data.pkl" @@ -125,7 +124,7 @@ def execute_repeated_cv( use_wandb=wandb, train_only=complete_train, epochs=20, - patience=5 + patience=5, ) train_time = datetime.now() - start_time diff --git a/icu_benchmarks/data/loader.py b/icu_benchmarks/data/loader.py index eb544d64..ffb00c65 100644 --- a/icu_benchmarks/data/loader.py +++ b/icu_benchmarks/data/loader.py @@ -15,20 +15,20 @@ @gin.configurable("CommonPolarsDataset") class CommonPolarsDataset(Dataset): def __init__( - self, - data: dict, - split: str = Split.train, - vars: Dict[str, str] = gin.REQUIRED, - grouping_segment: str = Segment.outcome, - mps: bool = False, - name: str = "", - *args, - **kwargs + self, + data: dict, + split: str = Split.train, + vars: Dict[str, str] = gin.REQUIRED, + grouping_segment: str = Segment.outcome, + mps: bool = False, + name: str = "", + *args, + **kwargs, ): # super().__init__(*args, **kwargs) self.split = split self.vars = vars - self.grouping_df = data[split][grouping_segment] #.set_index(self.vars["GROUP"]) + self.grouping_df = data[split][grouping_segment] # .set_index(self.vars["GROUP"]) # logging.info(f"data split: {data[split]}") # self.features_df = ( # data[split][Segment.features].set_index(self.vars["GROUP"]).drop(labels=self.vars["SEQUENCE"], axis=1) @@ -84,9 +84,9 @@ def to_tensor(self): class PredictionPolarsDataset(CommonPolarsDataset): """Subclass of common dataset for prediction tasks. - Args: - ram_cache (bool, optional): Whether the complete dataset should be stored in ram. Defaults to True. - """ + Args: + ram_cache (bool, optional): Whether the complete dataset should be stored in ram. Defaults to True. + """ def __init__(self, *args, ram_cache: bool = True, **kwargs): super().__init__(*args, **kwargs) @@ -168,9 +168,7 @@ def get_data_and_labels(self) -> Tuple[np.array, np.array, np.array]: rep = rep.group_by(self.vars["GROUP"]).last() else: # Adding segment count for each stay id and timestep. - rep = rep.with_columns( - pl.col(self.vars["GROUP"]).cum_count().over(self.vars["GROUP"]).alias("counter") - ) + rep = rep.with_columns(pl.col(self.vars["GROUP"]).cum_count().over(self.vars["GROUP"]).alias("counter")) rep = rep.to_numpy().astype(float) logging.debug(f"rep shape: {rep.shape}") logging.debug(f"labels shape: {labels.shape}") @@ -183,6 +181,7 @@ def to_tensor(self): else: return from_numpy(data), from_numpy(labels), row_indicators + @gin.configurable("CommonPandasDataset") class CommonPandasDataset(Dataset): """Common dataset: subclass of Torch Dataset that represents the data to learn on. @@ -193,13 +192,13 @@ class CommonPandasDataset(Dataset): """ def __init__( - self, - data: dict, - split: str = Split.train, - vars: Dict[str, str] = gin.REQUIRED, - grouping_segment: str = Segment.outcome, - mps: bool = False, - name: str = "", + self, + data: dict, + split: str = Split.train, + vars: Dict[str, str] = gin.REQUIRED, + grouping_segment: str = Segment.outcome, + mps: bool = False, + name: str = "", ): self.split = split self.vars = vars @@ -306,7 +305,7 @@ def get_balance(self) -> list: Weights for each label. """ counts = self.outcome_df[self.vars["LABEL"]].value_counts() - weights = list((1 / counts) * np.sum(counts) / counts.shape[0]) + # weights = list((1 / counts) * np.sum(counts) / counts.shape[0]) return list((1 / counts) * np.sum(counts) / counts.shape[0]) def get_data_and_labels(self) -> Tuple[np.array, np.array]: @@ -339,14 +338,14 @@ class ImputationPandasDataset(CommonPandasDataset): """Subclass of Common Dataset that contains data for imputation models.""" def __init__( - self, - data: Dict[str, DataFrame], - split: str = Split.train, - vars: Dict[str, str] = gin.REQUIRED, - mask_proportion=0.3, - mask_method="MCAR", - mask_observation_proportion=0.3, - ram_cache: bool = True, + self, + data: Dict[str, DataFrame], + split: str = Split.train, + vars: Dict[str, str] = gin.REQUIRED, + mask_proportion=0.3, + mask_method="MCAR", + mask_observation_proportion=0.3, + ram_cache: bool = True, ): """ Args: @@ -413,11 +412,11 @@ class ImputationPredictionDataset(Dataset): """ def __init__( - self, - data: DataFrame, - grouping_column: str = "stay_id", - select_columns: List[str] = None, - ram_cache: bool = True, + self, + data: DataFrame, + grouping_column: str = "stay_id", + select_columns: List[str] = None, + ram_cache: bool = True, ): self.dyn_df = data diff --git a/icu_benchmarks/data/pooling.py b/icu_benchmarks/data/pooling.py index 7eeb2949..48689d35 100644 --- a/icu_benchmarks/data/pooling.py +++ b/icu_benchmarks/data/pooling.py @@ -16,16 +16,17 @@ class PooledDataset: class PooledData: - def __init__(self, - data_dir, - vars, - datasets, - file_names, - shuffle=False, - stratify=None, - runmode=RunMode.classification, - save_test=True, - ): + def __init__( + self, + data_dir, + vars, + datasets, + file_names, + shuffle=False, + stratify=None, + runmode=RunMode.classification, + save_test=True, + ): """ Generate pooled data from existing datasets. Args: @@ -48,10 +49,10 @@ def __init__(self, self.save_test = save_test def generate( - self, - datasets, - samples=10000, - seed=42, + self, + datasets, + samples=10000, + seed=42, ): """ Generate pooled data from existing datasets. @@ -65,8 +66,8 @@ def generate( if folder.is_dir(): if folder.name in datasets: data[folder.name] = { - f: pq.read_table(folder / self.file_names[f]).to_pandas(self_destruct=True) for f in - self.file_names.keys() + f: pq.read_table(folder / self.file_names[f]).to_pandas(self_destruct=True) + for f in self.file_names.keys() } data = self._pool_datasets( datasets=data, @@ -101,15 +102,15 @@ def _save_pooled_data(self, data_dir, data, datasets, file_names, samples=10000) logging.info(f"Saved pooled data at {save_dir}") def _pool_datasets( - self, - datasets={}, - samples=10000, - vars=[], - seed=42, - shuffle=True, - runmode=RunMode.classification, - data_dir=Path("data"), - save_test=True, + self, + datasets={}, + samples=10000, + vars=[], + seed=42, + shuffle=True, + runmode=RunMode.classification, + data_dir=Path("data"), + save_test=True, ): """ Pool datasets into a single dataset. @@ -144,8 +145,9 @@ def _pool_datasets( # If we have more outcomes than stays, check max label value per stay id labels = outcome.groupby(id).max()[vars[Var.label]].reset_index(drop=True) # if pd.Series(outcome[id].unique()) is outcome[id]): - selected_stays = train_test_split(stays, stratify=labels, shuffle=shuffle, random_state=seed, - train_size=samples) + selected_stays = train_test_split( + stays, stratify=labels, shuffle=shuffle, random_state=seed, train_size=samples + ) else: selected_stays = train_test_split(stays, shuffle=shuffle, random_state=seed, train_size=samples) # Select only stays that are in the selected_stays diff --git a/icu_benchmarks/data/preprocessor.py b/icu_benchmarks/data/preprocessor.py index a00303b5..6a288b54 100644 --- a/icu_benchmarks/data/preprocessor.py +++ b/icu_benchmarks/data/preprocessor.py @@ -6,14 +6,13 @@ import gin import pandas as pd -import polars.selectors as cs import polars as pl from recipys.recipe import Recipe from recipys.selector import all_numeric_predictors, all_outcomes, has_type, all_of from recipys.step import ( StepScale, - # StepImputeFastForwardFill, - # StepImputeFastZeroFill, + StepImputeFastForwardFill, + StepImputeFastZeroFill, StepImputeFill, StepSklearn, StepHistorical, @@ -44,6 +43,7 @@ def set_imputation_model(self, imputation_model): if self.imputation_model is not None: update_wandb_config({"imputation_model": self.imputation_model.__class__.__name__}) + @gin.configurable("base_classification_preprocessor") class PolarsClassificationPreprocessor(Preprocessor): def __init__( @@ -83,7 +83,11 @@ def apply(self, data, vars) -> dict[dict[pl.DataFrame]]: Preprocessed data. """ # Check if dynamic features are present - if self.use_static_features and all(Segment.static in value for value in data.values()) and len(vars[Segment.static]) > 0: + if ( + self.use_static_features + and all(Segment.static in value for value in data.values()) + and len(vars[Segment.static]) > 0 + ): logging.info("Preprocessing static features.") data = self._process_static(data, vars) else: @@ -124,9 +128,13 @@ def apply(self, data, vars) -> dict[dict[pl.DataFrame]]: logging.debug(data[Split.train][Segment.features].head()) logging.debug(data[Split.train][Segment.outcome]) for split in [Split.train, Split.val, Split.test]: - if vars["SEQUENCE"] in data[split][Segment.outcome] and len(data[split][Segment.features]) != len(data[split][Segment.outcome]): - raise Exception(f"Data and outcome length mismatch in {split} split: " - f"features: {len(data[split][Segment.features])}, outcome: {len(data[split][Segment.outcome])}") + if vars["SEQUENCE"] in data[split][Segment.outcome] and len(data[split][Segment.features]) != len( + data[split][Segment.outcome] + ): + raise Exception( + f"Data and outcome length mismatch in {split} split: " + f"features: {len(data[split][Segment.features])}, outcome: {len(data[split][Segment.outcome])}" + ) data[Split.train][Segment.features] = data[Split.train][Segment.features].unique() data[Split.val][Segment.features] = data[Split.val][Segment.features].unique() data[Split.test][Segment.features] = data[Split.test][Segment.features].unique() @@ -139,13 +147,13 @@ def _process_static(self, data, vars): sta_rec.add_step(StepSklearn(MissingIndicator(features="all"), sel=all_of(vars[Segment.static]), in_place=False)) if self.scaling: sta_rec.add_step(StepScale()) - sta_rec.add_step(StepImputeFill(sel=all_numeric_predictors(),strategy="zero")) + sta_rec.add_step(StepImputeFill(sel=all_numeric_predictors(), strategy="zero")) # sta_rec.add_step(StepImputeFastZeroFill(sel=all_numeric_predictors())) # if len(data[Split.train][Segment.static].select_dtypes(include=["object"]).columns) > 0: types = ["String", "Object", "Categorical"] sel = has_type(types) - if(len(sel(sta_rec.data))>0): - # if len(data[Split.train][Segment.static].select(cs.by_dtype(types)).columns) > 0: + if len(sel(sta_rec.data)) > 0: + # if len(data[Split.train][Segment.static].select(cs.by_dtype(types)).columns) > 0: sta_rec.add_step(StepSklearn(SimpleImputer(missing_values=None, strategy="most_frequent"), sel=has_type(types))) sta_rec.add_step(StepSklearn(LabelEncoder(), sel=has_type(types), columnwise=True)) @@ -202,6 +210,8 @@ def to_cache_string(self): super().to_cache_string() + f"_classification_{self.generate_features}_{self.scaling}_{self.imputation_model.__class__.__name__}" ) + + @gin.configurable("base_regression_preprocessor") class PolarsRegressionPreprocessor(PolarsClassificationPreprocessor): # Override base classification preprocessor @@ -265,6 +275,7 @@ def _process_outcome(self, data, vars, split): data[split][Segment.outcome] = outcome_rec.bake() return data + @gin.configurable("pandas_classification_preprocessor") class PandasClassificationPreprocessor(Preprocessor): def __init__( diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index f40e08c9..1600af2e 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -7,15 +7,10 @@ import hashlib import pandas as pd import polars as pl -import pyarrow.parquet as pq from pathlib import Path import pickle from timeit import default_timer as timer - -from setuptools.dist import sequence from sklearn.model_selection import StratifiedKFold, KFold, StratifiedShuffleSplit, ShuffleSplit -from sqlalchemy import false - from icu_benchmarks.data.preprocessor import Preprocessor, PandasClassificationPreprocessor, PolarsClassificationPreprocessor from icu_benchmarks.contants import RunMode from .constants import DataSplit as Split, DataSegment as Segment, VarType as Var @@ -44,7 +39,6 @@ def preprocess_data( runmode: RunMode = RunMode.classification, label: str = None, vars_to_exclude: list[str] = [], - ) -> dict[dict[pl.DataFrame]] or dict[dict[pd.DataFrame]]: """Perform loading, splitting, imputing and normalising of task data. @@ -90,9 +84,14 @@ def preprocess_data( cache_filename = f"s_{seed}_r_{repetition_index}_f_{fold_index}_t_{train_size}_d_{debug}" logging.log(logging.INFO, f"Using preprocessor: {preprocessor.__name__}") - vars_to_exclude = modality_mapping.get("cat_clinical_notes") + modality_mapping.get("cat_med_embeddings_map") if ( + vars_to_exclude = ( + modality_mapping.get("cat_clinical_notes") + modality_mapping.get("cat_med_embeddings_map") + if ( modality_mapping.get("cat_clinical_notes") is not None - and modality_mapping.get("cat_med_embeddings_map") is not None) else None + and modality_mapping.get("cat_med_embeddings_map") is not None + ) + else None + ) preprocessor = preprocessor( use_static_features=use_static, @@ -116,7 +115,9 @@ def preprocess_data( # Read parquet files into pandas dataframes and remove the parquet file from memory logging.info(f"Loading data from directory {data_dir.absolute()}") - data = {f: pl.read_parquet(data_dir / file_names[f]) for f in file_names.keys() if os.path.exists(data_dir / file_names[f])} + data = { + f: pl.read_parquet(data_dir / file_names[f]) for f in file_names.keys() if os.path.exists(data_dir / file_names[f]) + } logging.info(f"Loaded data: {list(data.keys())}") data = check_sanitize_data(data, vars) @@ -126,10 +127,10 @@ def preprocess_data( logging.debug(f"Modality mapping: {modality_mapping}") if len(modality_mapping) > 0: # Optional modality selection - if not (selected_modalities == "all" or selected_modalities == ["all"] or selected_modalities == None): + if selected_modalities not in [None, "all", ["all"]]: data, vars = modality_selection(data, modality_mapping, selected_modalities, vars) else: - logging.info(f"Selecting all modalities.") + logging.info("Selecting all modalities.") # Generate the splits logging.info("Generating splits.") @@ -161,17 +162,17 @@ def preprocess_data( for dict in data.values(): for key, val in dict.items(): logging.debug(f"Data type: {key}") - logging.debug(f"Is NaN:") + logging.debug("Is NaN:") sel = dict[key].select(pl.selectors.numeric().is_nan().max()) logging.debug(sel.select(col.name for col in sel if col.item(0))) # logging.info(dict[key].select(pl.all().has_nulls()).sum_horizontal()) - logging.debug(f"Has nulls:") + logging.debug("Has nulls:") sel = dict[key].select(pl.all().has_nulls()) logging.debug(sel.select(col.name for col in sel if col.item(0))) # dict[key] = val[:, [not (s.null_count() > 0) for s in val]] dict[key] = val.fill_null(strategy="zero") dict[key] = val.fill_nan(0) - logging.debug(f"Dropping columns with nulls") + logging.debug("Dropping columns with nulls") sel = dict[key].select(pl.all().has_nulls()) logging.debug(sel.select(col.name for col in sel if col.item(0))) @@ -197,13 +198,13 @@ def check_sanitize_data(data, vars): logging.warning(f"Removed {old_len - len(data[Segment.static])} duplicates from static data.") if Segment.dynamic in data.keys(): old_len = len(data[Segment.dynamic]) - data[Segment.dynamic] = data[Segment.dynamic].unique(subset=[group,sequence], keep=keep, maintain_order=True) + data[Segment.dynamic] = data[Segment.dynamic].unique(subset=[group, sequence], keep=keep, maintain_order=True) logging.warning(f"Removed {old_len - len(data[Segment.dynamic])} duplicates from dynamic data.") if Segment.outcome in data.keys(): old_len = len(data[Segment.outcome]) if sequence in data[Segment.outcome].columns: # We have a dynamic outcome with group and sequence - data[Segment.outcome] = data[Segment.outcome].unique(subset=[group,sequence], keep=keep, maintain_order=True) + data[Segment.outcome] = data[Segment.outcome].unique(subset=[group, sequence], keep=keep, maintain_order=True) else: data[Segment.outcome] = data[Segment.outcome].unique(subset=[group], keep=keep, maintain_order=True) logging.warning(f"Removed {old_len - len(data[Segment.outcome])} duplicates from outcome data.") @@ -216,7 +217,7 @@ def modality_selection( logging.info(f"Selected modalities: {selected_modalities}") selected_columns = [modality_mapping[cols] for cols in selected_modalities if cols in modality_mapping.keys()] if selected_columns == []: - logging.info(f"No columns selected. Using all columns.") + logging.info("No columns selected. Using all columns.") return data, vars selected_columns = sum(selected_columns, []) selected_columns.extend([vars[Var.group], vars[Var.label], vars[Var.sequence]]) diff --git a/icu_benchmarks/models/constants.py b/icu_benchmarks/models/constants.py index 41eee851..43843db8 100644 --- a/icu_benchmarks/models/constants.py +++ b/icu_benchmarks/models/constants.py @@ -26,7 +26,7 @@ MAE, JSD, BinaryFairnessWrapper, - confusion_matrix + confusion_matrix, ) @@ -38,7 +38,6 @@ class MLMetrics: "PR_Curve": precision_recall_curve, "RO_Curve": roc_curve, "Confusion_Matrix": confusion_matrix, - } MULTICLASS_CLASSIFICATION = { @@ -47,7 +46,6 @@ class MLMetrics: "Balanced_Accuracy": balanced_accuracy_score, # "PR": average_precision_score, "Confusion_Matrix": confusion_matrix, - } REGRESSION = { diff --git a/icu_benchmarks/models/custom_metrics.py b/icu_benchmarks/models/custom_metrics.py index 75648466..271aa55c 100644 --- a/icu_benchmarks/models/custom_metrics.py +++ b/icu_benchmarks/models/custom_metrics.py @@ -1,5 +1,3 @@ -import logging - import torch from typing import Callable import numpy as np @@ -134,12 +132,13 @@ def feature_helper(self, trainer, step_prefix): feature_names = trainer.test_dataloaders.dataset.features return feature_names + def confusion_matrix(y_true: ndarray, y_pred: ndarray, normalize=False) -> torch.tensor: y_pred = np.rint(y_pred).astype(int) confusion = sk_confusion_matrix(y_true, y_pred) if normalize: confusion = confusion / confusion.sum() - confusion_tensor = torch.tensor(confusion) + # confusion_tensor = torch.tensor(confusion) # confusion = confusion.tolist() confusion_dict = {} for i in range(confusion.shape[0]): diff --git a/icu_benchmarks/models/ml_models/catboost.py b/icu_benchmarks/models/ml_models/catboost.py index 2da5ff75..7821ed26 100644 --- a/icu_benchmarks/models/ml_models/catboost.py +++ b/icu_benchmarks/models/ml_models/catboost.py @@ -1,18 +1,16 @@ -import logging - import gin import catboost as cb -import numpy as np -import wandb - from icu_benchmarks.contants import RunMode from icu_benchmarks.models.wrappers import MLWrapper + + @gin.configurable class CBClassifier(MLWrapper): _supported_run_modes = [RunMode.classification] def __init__(self, *args, **kwargs): - #self.model = self.set_model_args(cb.CatBoostClassifier, task_type="GPU" if not kwargs['cpu'] else "CPU", *args, **kwargs) + # self.model = self.set_model_args(cb.CatBoostClassifier, task_type="GPU" + # if not kwargs['cpu'] else "CPU", *args, **kwargs) self.model = self.set_model_args(cb.CatBoostClassifier, task_type="CPU", *args, **kwargs) super().__init__(*args, **kwargs) diff --git a/icu_benchmarks/models/ml_models/imblearn.py b/icu_benchmarks/models/ml_models/imblearn.py index 3a83205a..1e5fdbf3 100644 --- a/icu_benchmarks/models/ml_models/imblearn.py +++ b/icu_benchmarks/models/ml_models/imblearn.py @@ -3,6 +3,7 @@ from icu_benchmarks.models.wrappers import MLWrapper import gin + @gin.configurable class BRFClassifier(MLWrapper): _supported_run_modes = [RunMode.classification] @@ -10,10 +11,12 @@ class BRFClassifier(MLWrapper): def __init__(self, *args, **kwargs): self.model = self.set_model_args(BalancedRandomForestClassifier, *args, **kwargs) super().__init__(*args, **kwargs) + + @gin.configurable class RUSBClassifier(MLWrapper): _supported_run_modes = [RunMode.classification] def __init__(self, *args, **kwargs): self.model = self.set_model_args(RUSBoostClassifier, *args, **kwargs) - super().__init__(*args, **kwargs) \ No newline at end of file + super().__init__(*args, **kwargs) diff --git a/icu_benchmarks/models/ml_models/xgboost.py b/icu_benchmarks/models/ml_models/xgboost.py index 14b8c365..81f0f0d8 100644 --- a/icu_benchmarks/models/ml_models/xgboost.py +++ b/icu_benchmarks/models/ml_models/xgboost.py @@ -3,24 +3,24 @@ import gin import numpy as np -import torch import os from icu_benchmarks.contants import RunMode from icu_benchmarks.models.wrappers import MLWrapper import xgboost as xgb -from xgboost.callback import EarlyStopping, LearningRateScheduler +from xgboost.callback import EarlyStopping from wandb.integration.xgboost import wandb_callback as wandb_xgb import wandb from statistics import mean -from optuna.integration import XGBoostPruningCallback +# from optuna.integration import XGBoostPruningCallback import shap + @gin.configurable class XGBClassifier(MLWrapper): _supported_run_modes = [RunMode.classification] def __init__(self, *args, **kwargs): - self.model = self.set_model_args(xgb.XGBClassifier, device="cpu",*args, **kwargs) + self.model = self.set_model_args(xgb.XGBClassifier, device="cpu", *args, **kwargs) super().__init__(*args, **kwargs) def predict(self, features): @@ -43,20 +43,23 @@ def fit_model(self, train_data, train_labels, val_data, val_labels): # self.log_dict(self.model.get_booster().get_score(importance_type='weight')) # Return the first metric we use for validation eval_score = mean(next(iter(self.model.evals_result_["validation_0"].values()))) - return eval_score #, callbacks=callbacks) - + return eval_score # , callbacks=callbacks) def test_step(self, dataset, _): # Pred indicators indicates the row id, and time in hours test_rep, test_label, pred_indicators = dataset - test_rep, test_label, pred_indicators = test_rep.squeeze().cpu().numpy(), test_label.squeeze().cpu().numpy(), pred_indicators.squeeze().cpu().numpy() + test_rep, test_label, pred_indicators = ( + test_rep.squeeze().cpu().numpy(), + test_label.squeeze().cpu().numpy(), + pred_indicators.squeeze().cpu().numpy(), + ) self.set_metrics(test_label) test_pred = self.predict(test_rep) - if len(pred_indicators.shape)>1 and len(test_pred.shape)>1 and pred_indicators.shape[1] == test_pred.shape[1]: + if len(pred_indicators.shape) > 1 and len(test_pred.shape) > 1 and pred_indicators.shape[1] == test_pred.shape[1]: pred_indicators = np.hstack((pred_indicators, test_label.reshape(-1, 1))) pred_indicators = np.hstack((pred_indicators, test_pred)) # Save as: id, time (hours), ground truth, prediction 0, prediction 1 - np.savetxt(os.path.join(self.logger.save_dir,f'pred_indicators.csv'), pred_indicators, delimiter=",") + np.savetxt(os.path.join(self.logger.save_dir, "pred_indicators.csv"), pred_indicators, delimiter=",") logging.debug(f"Saved row indicators to {os.path.join(self.logger.save_dir,f'row_indicators.csv')}") if self.explainer is not None: self.test_shap_values = self.explainer(test_rep) @@ -68,8 +71,6 @@ def test_step(self, dataset, _): self.log_metrics(test_label, test_pred, "test") logging.debug(f"Test loss: {self.loss(test_label, test_pred)}") - - def set_model_args(self, model, *args, **kwargs): """XGBoost signature does not include the hyperparams so we need to pass them manually.""" signature = inspect.signature(model.__init__).parameters @@ -82,4 +83,4 @@ def set_model_args(self, model, *args, **kwargs): return model(**hyperparams) def get_feature_importance(self): - return self.model.feature_importances_ \ No newline at end of file + return self.model.feature_importances_ diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index 2266664e..f2eddc02 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -82,7 +82,11 @@ def train_common( logging.info(f"Training model: {model.__name__}.") # todo: add support for polars versions of datasets - dataset_class = ImputationPandasDataset if mode == RunMode.imputation else PredictionPolarsDataset if polars else PredictionPandasDataset + dataset_class = ( + ImputationPandasDataset + if mode == RunMode.imputation + else PredictionPolarsDataset if polars else PredictionPandasDataset + ) # dataset_class = ImputationPandasDataset if mode == RunMode.imputation else PredictionPandasDataset logging.info(f"Using dataset class: {dataset_class.__name__}.") @@ -112,7 +116,7 @@ def train_common( num_workers=num_workers, # pin_memory=not cpu, drop_last=True, - persistent_workers=persistent_workers + persistent_workers=persistent_workers, ) val_loader = DataLoader( val_dataset, @@ -121,7 +125,7 @@ def train_common( num_workers=num_workers, # pin_memory=not cpu, drop_last=True, - persistent_workers=persistent_workers + persistent_workers=persistent_workers, ) data_shape = next(iter(train_loader))[0].shape @@ -185,7 +189,7 @@ def train_common( num_workers=num_workers, pin_memory=True, drop_last=True, - persistent_workers=persistent_workers + persistent_workers=persistent_workers, ) if model.requires_backprop else DataLoader([test_dataset.to_tensor()], batch_size=1) @@ -197,6 +201,7 @@ def train_common( save_config_file(log_dir) return test_loss + def persist_data(trainer, log_dir): try: if trainer.lightning_module.test_shap_values is not None: @@ -205,19 +210,18 @@ def persist_data(trainer, log_dir): # 'features': trainer.lightning_module.trained_columns, # 'feature_value': np.transpose(shap_values.values.mean(axis=0)), # }) - shaps_test = pl.DataFrame(schema = trainer.lightning_module.trained_columns, - data = np.transpose(shap_values.values)) + shaps_test = pl.DataFrame(schema=trainer.lightning_module.trained_columns, data=np.transpose(shap_values.values)) shaps_test.write_parquet(log_dir / "shap_values_test.parquet") logging.info(f"Saved shap values to {log_dir / 'test_shap_values.parquet'}") if trainer.lightning_module.train_shap_values is not None: shap_values = trainer.lightning_module.train_shap_values - shaps_train = pl.DataFrame(schema = trainer.lightning_module.trained_columns, - data = np.transpose(shap_values.values)) + shaps_train = pl.DataFrame(schema=trainer.lightning_module.trained_columns, data=np.transpose(shap_values.values)) shaps_train.write_parquet(log_dir / "shap_values_train.parquet") except Exception as e: logging.error(f"Failed to save shap values: {e}") + def load_model(model, source_dir, pl_model=True): if source_dir.exists(): if model.requires_backprop: diff --git a/icu_benchmarks/models/utils.py b/icu_benchmarks/models/utils.py index 65e87bce..fc5b5506 100644 --- a/icu_benchmarks/models/utils.py +++ b/icu_benchmarks/models/utils.py @@ -190,26 +190,31 @@ def version(self): def log_hyperparams(self, params): pass -class scorer_wrapper(): + +class scorer_wrapper: """ - Wrapper that flattens the binary classification input such that we can use a broader range of sklearn metrics. + Wrapper that flattens the binary classification input such that we can use a broader range of sklearn metrics. """ + def __init__(self, scorer=average_precision_score): self.scorer = scorer def __call__(self, y_true, y_pred): - if len(np.unique(y_true))<=2 and y_pred.ndim > 1: - y_pred_argmax = np.argmax(y_pred,axis=1) + if len(np.unique(y_true)) <= 2 and y_pred.ndim > 1: + y_pred_argmax = np.argmax(y_pred, axis=1) return self.scorer(y_true, y_pred_argmax) else: return self.scorer(y_true, y_pred) def __name__(self): return "scorer_wrapper" + + # Source: https://github.com/ratschlab/tls -@gin.configurable('get_smoothed_labels') -def get_smoothed_labels(label, event, smoothing_fn=gin.REQUIRED, h_true=gin.REQUIRED, h_min=gin.REQUIRED, - h_max=gin.REQUIRED, delta_h=12, gamma=0.1): +@gin.configurable("get_smoothed_labels") +def get_smoothed_labels( + label, event, smoothing_fn=gin.REQUIRED, h_true=gin.REQUIRED, h_min=gin.REQUIRED, h_max=gin.REQUIRED, delta_h=12, gamma=0.1 +): diffs = np.concatenate([np.zeros(1), event[1:] - event[:-1]], axis=-1) pos_event_change_full = np.where((diffs == 1) & (event == 1))[0] @@ -235,7 +240,8 @@ def get_smoothed_labels(label, event, smoothing_fn=gin.REQUIRED, h_true=gin.REQU # Need to handle the case where the ts was truncatenated at 2016 for HiB if ((last_know_label == 1) and (len(pos_event_change_full) == 0)) or ( - (last_know_label == 1) and (last_know_idx >= pos_event_change_full[-1])): + (last_know_label == 1) and (last_know_idx >= pos_event_change_full[-1]) + ): last_know_event = 0 if len(pos_event_change_full) > 0: last_know_event = pos_event_change_full[-1] @@ -259,13 +265,20 @@ def get_smoothed_labels(label, event, smoothing_fn=gin.REQUIRED, h_true=gin.REQU if multihorizon: smoothed_labels = [] for k in range(label.shape[-1]): - smoothed_labels.append(np.array(list( - map(lambda x: smoothing_fn(x, h_true=h_true[k], h_min=h_min[k], h_max=h_max[k], delta_h=delta_h, - gamma=gamma), dte)))) + smoothed_labels.append( + np.array( + list( + map( + lambda x: smoothing_fn( + x, h_true=h_true[k], h_min=h_min[k], h_max=h_max[k], delta_h=delta_h, gamma=gamma + ), + dte, + ) + ) + ) + ) return np.stack(smoothed_labels, axis=-1) else: return np.array( - list(map(lambda x: smoothing_fn(x, h_true=h_true, h_min=h_min, - h_max=h_max, delta_h=delta_h, gamma=gamma), dte))) - - + list(map(lambda x: smoothing_fn(x, h_true=h_true, h_min=h_min, h_max=h_max, delta_h=delta_h, gamma=gamma), dte)) + ) diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index a9dc9caa..022c0e90 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -17,7 +17,7 @@ from ignite.exceptions import NotComputableError from icu_benchmarks.models.constants import ImputationInit from icu_benchmarks.models.custom_metrics import confusion_matrix -from icu_benchmarks.models.utils import create_optimizer, create_scheduler, scorer_wrapper +from icu_benchmarks.models.utils import create_optimizer, create_scheduler from joblib import dump from pytorch_lightning import LightningModule @@ -34,6 +34,7 @@ gin.config.external_configurable(roc_auc_score, module="sklearn.metrics") # gin.config.external_configurable(scorer_wrapper, module="icu_benchmarks.models.utils") + @gin.configurable("BaseModule") class BaseModule(LightningModule): # DL type models, requires backpropagation @@ -475,13 +476,16 @@ def log_metrics(self, label, pred, metric_type): self.log_dict( { # MPS dependent type casting - f"{metric_type}/{name}": metric(self.label_transform(label), self.output_transform(pred)) - if not self.mps - else metric(self.label_transform(label), self.output_transform(pred)) + f"{metric_type}/{name}": ( + metric(self.label_transform(label), self.output_transform(pred)) + if not self.mps + else metric(self.label_transform(label), self.output_transform(pred)) + ) # For every metric for name, metric in self.metrics.items() # Filter out metrics that return a tuple (e.g. precision_recall_curve) - if not isinstance(metric(self.label_transform(label), self.output_transform(pred)), tuple) and name != "Confusion_Matrix" + if not isinstance(metric(self.label_transform(label), self.output_transform(pred)), tuple) + and name != "Confusion_Matrix" }, sync_dist=True, ) diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index b8c007ed..7e3ecf79 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -81,9 +81,9 @@ def main(my_args=tuple(sys.argv[1:])): # Log imputation model to wandb update_wandb_config( { - "pretrained_imputation_model": pretrained_imputation_model.__class__.__name__ - if pretrained_imputation_model is not None - else "None" + "pretrained_imputation_model": ( + pretrained_imputation_model.__class__.__name__ if pretrained_imputation_model is not None else "None" + ) } ) @@ -131,7 +131,7 @@ def main(my_args=tuple(sys.argv[1:])): name_datasets(args.name, args.name, args.name) hp_checkpoint = log_dir / args.hp_checkpoint if args.hp_checkpoint else None model_path = ( - Path("configs") / ("imputation_models" if mode == RunMode.imputation else "prediction_models") / f"{model}.gin" + Path("configs") / ("imputation_models" if mode == RunMode.imputation else "prediction_models") / f"{model}.gin" ) gin_config_files = ( [Path(f"configs/experiments/{args.experiment}.gin")] diff --git a/icu_benchmarks/run_utils.py b/icu_benchmarks/run_utils.py index ab38a901..2c5da67b 100644 --- a/icu_benchmarks/run_utils.py +++ b/icu_benchmarks/run_utils.py @@ -19,6 +19,7 @@ import glob import polars as pl + def build_parser() -> ArgumentParser: """Builds an ArgumentParser for the command line. @@ -109,7 +110,7 @@ def aggregate_results(log_dir: Path, execution_time: timedelta = None): """ aggregated = {} shap_values_test = [] - shap_values_train = [] + # shap_values_train = [] for repetition in log_dir.iterdir(): if repetition.is_dir(): aggregated[repetition.name] = {} @@ -255,8 +256,8 @@ def setup_logging(date_format, log_format, verbose): def get_config_files(config_dir: Path): - tasks = glob.glob(os.path.join(config_dir / "tasks", '*')) - models = glob.glob(os.path.join(config_dir / "prediction_models", '*')) + tasks = glob.glob(os.path.join(config_dir / "tasks", "*")) + models = glob.glob(os.path.join(config_dir / "prediction_models", "*")) tasks = [os.path.splitext(os.path.basename(task))[0] for task in tasks] models = [os.path.splitext(os.path.basename(model))[0] for model in models] if "common" in tasks: @@ -265,4 +266,4 @@ def get_config_files(config_dir: Path): models.remove("common") logging.info(f"Found tasks: {tasks}") logging.info(f"Found models: {models}") - return tasks, models \ No newline at end of file + return tasks, models diff --git a/icu_benchmarks/tuning/hyperparameters.py b/icu_benchmarks/tuning/hyperparameters.py index 68eaa047..c3a93370 100644 --- a/icu_benchmarks/tuning/hyperparameters.py +++ b/icu_benchmarks/tuning/hyperparameters.py @@ -16,6 +16,7 @@ from icu_benchmarks.contants import RunMode from icu_benchmarks.wandb_utils import wandb_log from optuna.visualization import plot_param_importances, plot_optimization_history + TUNE = 25 logging.addLevelName(25, "TUNE") @@ -286,8 +287,9 @@ def tune_step_callback(study: optuna.study.Study, trial: optuna.trial.FrozenTria bind_gin_params(configuration) return else: - logging.log(TUNE, "Choosing hyperparameters randomly from bounds using hp tuning as no earlier checkpoint " - "supplied.") + logging.log( + TUNE, "Choosing hyperparameters randomly from bounds using hp tuning as no earlier checkpoint " "supplied." + ) n_initial_points = 1 n_calls = 1 @@ -321,14 +323,13 @@ def bind_params_and_train(hyperparams): # Optuna study # Attempt checkpoint loading if checkpoint and checkpoint.exists(): - logging.warning(f"Hyperparameter checkpoint {checkpoint} does not exist.") - # logging.info("Attempting to find latest checkpoint file.") - # checkpoint_path = find_checkpoint(log_dir.parent, checkpoint_file) + logging.warning(f"Hyperparameter checkpoint {checkpoint} does not exist.") + # logging.info("Attempting to find latest checkpoint file.") + # checkpoint_path = find_checkpoint(log_dir.parent, checkpoint_file) # Check if we found a checkpoint file - logging.info(f"Loading checkpoint at {checkpoint}") - study = optuna.load_study(study_name="tuning", storage="sqlite:///" + str(checkpoint), sampler=sampler, - pruner=pruner) - n_calls = n_calls - len(study.trials) + logging.info(f"Loading checkpoint at {checkpoint}") + study = optuna.load_study(study_name="tuning", storage="sqlite:///" + str(checkpoint), sampler=sampler, pruner=pruner) + n_calls = n_calls - len(study.trials) else: if checkpoint: logging.warning("Checkpoint path given as flag but not found, starting from scratch.") From 9547bfaf9767408a980b455acca37ff499a82f5a Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 17 Oct 2024 13:15:55 +0200 Subject: [PATCH 194/207] Major refactoring round 2 --- docs/adding_model/rnn.py | 2 +- icu_benchmarks/{contants.py => constants.py} | 0 icu_benchmarks/cross_validation.py | 2 +- icu_benchmarks/data/loader.py | 12 +++-- icu_benchmarks/data/pooling.py | 4 +- icu_benchmarks/data/preprocessor.py | 2 +- icu_benchmarks/data/split_process_data.py | 30 ++++++++---- icu_benchmarks/models/dl_models/rnn.py | 2 +- icu_benchmarks/models/dl_models/tcn.py | 2 +- .../models/dl_models/transformer.py | 2 +- icu_benchmarks/models/ml_models/catboost.py | 12 ++++- icu_benchmarks/models/ml_models/imblearn.py | 2 +- icu_benchmarks/models/ml_models/lgbm.py | 12 ++++- icu_benchmarks/models/ml_models/sklearn.py | 2 +- icu_benchmarks/models/ml_models/xgboost.py | 35 ++++++++++---- icu_benchmarks/models/train.py | 47 +++++++++---------- icu_benchmarks/models/wrappers.py | 2 +- icu_benchmarks/run.py | 13 ++--- icu_benchmarks/run_utils.py | 45 ++++++++++++------ icu_benchmarks/tuning/hyperparameters.py | 40 ++++++++-------- 20 files changed, 162 insertions(+), 106 deletions(-) rename icu_benchmarks/{contants.py => constants.py} (100%) diff --git a/docs/adding_model/rnn.py b/docs/adding_model/rnn.py index 5500e672..d2215627 100644 --- a/docs/adding_model/rnn.py +++ b/docs/adding_model/rnn.py @@ -1,6 +1,6 @@ import gin import torch.nn as nn -from icu_benchmarks.contants import RunMode +from icu_benchmarks.constants import RunMode from icu_benchmarks.models.wrappers import DLPredictionWrapper diff --git a/icu_benchmarks/contants.py b/icu_benchmarks/constants.py similarity index 100% rename from icu_benchmarks/contants.py rename to icu_benchmarks/constants.py diff --git a/icu_benchmarks/cross_validation.py b/icu_benchmarks/cross_validation.py index b14e0e87..bcad6bde 100644 --- a/icu_benchmarks/cross_validation.py +++ b/icu_benchmarks/cross_validation.py @@ -11,7 +11,7 @@ from icu_benchmarks.models.train import train_common from icu_benchmarks.models.utils import JsonResultLoggingEncoder from icu_benchmarks.run_utils import log_full_line -from icu_benchmarks.contants import RunMode +from icu_benchmarks.constants import RunMode @gin.configurable diff --git a/icu_benchmarks/data/loader.py b/icu_benchmarks/data/loader.py index ffb00c65..4cf67757 100644 --- a/icu_benchmarks/data/loader.py +++ b/icu_benchmarks/data/loader.py @@ -1,3 +1,4 @@ +import warnings from typing import List from pandas import DataFrame import gin @@ -67,10 +68,10 @@ def __len__(self) -> int: """ return self.num_stays - def get_feature_names(self): + def get_feature_names(self) -> List[str]: return self.features_df.columns - def to_tensor(self): + def to_tensor(self) -> List[Tensor]: values = [] for entry in self: for i, value in enumerate(entry): @@ -174,7 +175,7 @@ def get_data_and_labels(self) -> Tuple[np.array, np.array, np.array]: logging.debug(f"labels shape: {labels.shape}") return rep, labels, self.row_indicators.to_numpy() - def to_tensor(self): + def to_tensor(self) -> Tuple[Tensor, Tensor, Tensor]: data, labels, row_indicators = self.get_data_and_labels() if self.mps: return from_numpy(data).to(float32), from_numpy(labels).to(float32) @@ -200,6 +201,7 @@ def __init__( mps: bool = False, name: str = "", ): + warnings.warn("CommonPandasDataset is deprecated. Use CommonPolarsDataset instead.", DeprecationWarning, stacklevel=2) self.split = split self.vars = vars self.grouping_df = data[split][grouping_segment].set_index(self.vars["GROUP"]) @@ -228,10 +230,10 @@ def __len__(self) -> int: """ return self.num_stays - def get_feature_names(self): + def get_feature_names(self) -> List[str]: return self.features_df.columns - def to_tensor(self): + def to_tensor(self) -> List[Tensor]: values = [] for entry in self: for i, value in enumerate(entry): diff --git a/icu_benchmarks/data/pooling.py b/icu_benchmarks/data/pooling.py index 48689d35..57ca84d2 100644 --- a/icu_benchmarks/data/pooling.py +++ b/icu_benchmarks/data/pooling.py @@ -3,7 +3,7 @@ import pandas as pd from sklearn.model_selection import train_test_split from .constants import DataSegment as Segment, VarType as Var -from icu_benchmarks.contants import RunMode +from icu_benchmarks.constants import RunMode import pyarrow.parquet as pq @@ -67,7 +67,7 @@ def generate( if folder.name in datasets: data[folder.name] = { f: pq.read_table(folder / self.file_names[f]).to_pandas(self_destruct=True) - for f in self.file_names.keys() + for f in self.file_names } data = self._pool_datasets( datasets=data, diff --git a/icu_benchmarks/data/preprocessor.py b/icu_benchmarks/data/preprocessor.py index 6a288b54..610e0e33 100644 --- a/icu_benchmarks/data/preprocessor.py +++ b/icu_benchmarks/data/preprocessor.py @@ -29,7 +29,7 @@ import abc -class Preprocessor: +class Preprocessor(abc.ABC): @abc.abstractmethod def apply(self, data, vars, save_cache=False, load_cache=None, vars_to_exclude=None): return data diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index 1600af2e..3ae63ce2 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -12,7 +12,7 @@ from timeit import default_timer as timer from sklearn.model_selection import StratifiedKFold, KFold, StratifiedShuffleSplit, ShuffleSplit from icu_benchmarks.data.preprocessor import Preprocessor, PandasClassificationPreprocessor, PolarsClassificationPreprocessor -from icu_benchmarks.contants import RunMode +from icu_benchmarks.constants import RunMode from .constants import DataSplit as Split, DataSegment as Segment, VarType as Var @@ -78,6 +78,8 @@ def preprocess_data( logging.debug(f"Multiple labels found and no value provided. Using first label: {vars[Var.label]}") vars[Var.label] = vars[Var.label][0] logging.info(f"Using label: {vars[Var.label]}") + if not vars[Var.label]: + raise ValueError("No label selected after filtering.") dumped_file_names = json.dumps(file_names, sort_keys=True) dumped_vars = json.dumps(vars, sort_keys=True) @@ -216,6 +218,8 @@ def modality_selection( ) -> dict[pl.DataFrame]: logging.info(f"Selected modalities: {selected_modalities}") selected_columns = [modality_mapping[cols] for cols in selected_modalities if cols in modality_mapping.keys()] + if not any(col in modality_mapping.keys() for col in selected_modalities): + raise ValueError("None of the selected modalities found in modality mapping.") if selected_columns == []: logging.info("No columns selected. Using all columns.") return data, vars @@ -270,18 +274,12 @@ def make_train_val( data[Segment.outcome] = data[Segment.outcome].sample(frac=0.01, random_state=seed) # Get stay IDs from outcome segment - if polars: - stays = pl.Series(name=id, values=data[Segment.outcome][id].unique()) - else: - stays = pd.Series(data[Segment.outcome][id].unique(), name=id) + stays = _get_stays(data, id, polars) # If there are labels, and the task is classification, use stratified k-fold if Var.label in vars and runmode is RunMode.classification: - if polars: - labels = data[Segment.outcome].group_by(id).max()[vars[Var.label]] - else: - # Get labels from outcome data (takes the highest value (or True) in case seq2seq classification) - labels = data[Segment.outcome].groupby(id).max()[vars[Var.label]].reset_index(drop=True) + # Get labels from outcome data (takes the highest value (or True) in case seq2seq classification) + labels = _get_labels(data, id, vars, polars) train_val = StratifiedShuffleSplit(train_size=train_size, random_state=seed, n_splits=1) train, val = list(train_val.split(stays, labels))[0] @@ -319,6 +317,18 @@ def make_train_val( data_split[Split.test] = copy.deepcopy(data_split[Split.val]) return data_split +def _get_stays(data, id, polars): + return pl.Series(name=id, values=data[Segment.outcome][id].unique()) if polars else pd.Series(data[Segment.outcome][id].unique(), name=id) + +def _get_labels(data, id, vars, polars): + # Get labels from outcome data (takes the highest value (or True) in case seq2seq classification) + if polars: + return data[Segment.outcome].group_by(id).max()[vars[Var.label]] + else: + return data[Segment.outcome].groupby(id).max()[vars[Var.label]].reset_index(drop=True) + +# Use these helper functions in both make_train_val and make_single_split + def make_single_split( data: dict[pd.DataFrame], diff --git a/icu_benchmarks/models/dl_models/rnn.py b/icu_benchmarks/models/dl_models/rnn.py index 327526c2..7174e090 100644 --- a/icu_benchmarks/models/dl_models/rnn.py +++ b/icu_benchmarks/models/dl_models/rnn.py @@ -1,7 +1,7 @@ import gin from torch import nn as nn -from icu_benchmarks.contants import RunMode +from icu_benchmarks.constants import RunMode from icu_benchmarks.models.wrappers import DLPredictionWrapper diff --git a/icu_benchmarks/models/dl_models/tcn.py b/icu_benchmarks/models/dl_models/tcn.py index 950310e1..28547a57 100644 --- a/icu_benchmarks/models/dl_models/tcn.py +++ b/icu_benchmarks/models/dl_models/tcn.py @@ -4,7 +4,7 @@ import numpy as np from torch import nn as nn -from icu_benchmarks.contants import RunMode +from icu_benchmarks.constants import RunMode from icu_benchmarks.models.dl_models.layers import TemporalBlock from icu_benchmarks.models.wrappers import DLPredictionWrapper diff --git a/icu_benchmarks/models/dl_models/transformer.py b/icu_benchmarks/models/dl_models/transformer.py index a9273884..95021379 100644 --- a/icu_benchmarks/models/dl_models/transformer.py +++ b/icu_benchmarks/models/dl_models/transformer.py @@ -1,7 +1,7 @@ import gin from torch import nn as nn -from icu_benchmarks.contants import RunMode +from icu_benchmarks.constants import RunMode from icu_benchmarks.models.dl_models.layers import PositionalEncoding, TransformerBlock, LocalBlock from icu_benchmarks.models.wrappers import DLPredictionWrapper diff --git a/icu_benchmarks/models/ml_models/catboost.py b/icu_benchmarks/models/ml_models/catboost.py index 7821ed26..2e7d63ed 100644 --- a/icu_benchmarks/models/ml_models/catboost.py +++ b/icu_benchmarks/models/ml_models/catboost.py @@ -1,6 +1,6 @@ import gin import catboost as cb -from icu_benchmarks.contants import RunMode +from icu_benchmarks.constants import RunMode from icu_benchmarks.models.wrappers import MLWrapper @@ -15,5 +15,13 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) def predict(self, features): - """Predicts labels for the given features.""" + """ + Predicts class probabilities for the given features. + + Args: + features: Input features for prediction. + + Returns: + numpy.ndarray: Predicted probabilities for each class. + """ return self.model.predict_proba(features) diff --git a/icu_benchmarks/models/ml_models/imblearn.py b/icu_benchmarks/models/ml_models/imblearn.py index 1e5fdbf3..d1db0703 100644 --- a/icu_benchmarks/models/ml_models/imblearn.py +++ b/icu_benchmarks/models/ml_models/imblearn.py @@ -1,5 +1,5 @@ from imblearn.ensemble import BalancedRandomForestClassifier, RUSBoostClassifier -from icu_benchmarks.contants import RunMode +from icu_benchmarks.constants import RunMode from icu_benchmarks.models.wrappers import MLWrapper import gin diff --git a/icu_benchmarks/models/ml_models/lgbm.py b/icu_benchmarks/models/ml_models/lgbm.py index 102da32d..c2207555 100644 --- a/icu_benchmarks/models/ml_models/lgbm.py +++ b/icu_benchmarks/models/ml_models/lgbm.py @@ -4,7 +4,7 @@ import wandb from wandb.integration.lightgbm import wandb_callback as wandb_lgbm -from icu_benchmarks.contants import RunMode +from icu_benchmarks.constants import RunMode from icu_benchmarks.models.wrappers import MLWrapper @@ -36,7 +36,15 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) def predict(self, features): - """Predicts labels for the given features.""" + """ + Predicts class probabilities for the given features. + + Args: + features: Input features for prediction. + + Returns: + numpy.ndarray: Predicted probabilities for each class. + """ return self.model.predict_proba(features) diff --git a/icu_benchmarks/models/ml_models/sklearn.py b/icu_benchmarks/models/ml_models/sklearn.py index 30312c28..1fe7a87b 100644 --- a/icu_benchmarks/models/ml_models/sklearn.py +++ b/icu_benchmarks/models/ml_models/sklearn.py @@ -1,6 +1,6 @@ import gin from sklearn import linear_model, ensemble, svm, neural_network -from icu_benchmarks.contants import RunMode +from icu_benchmarks.constants import RunMode from icu_benchmarks.models.wrappers import MLWrapper diff --git a/icu_benchmarks/models/ml_models/xgboost.py b/icu_benchmarks/models/ml_models/xgboost.py index 81f0f0d8..dfdaa841 100644 --- a/icu_benchmarks/models/ml_models/xgboost.py +++ b/icu_benchmarks/models/ml_models/xgboost.py @@ -1,30 +1,42 @@ import inspect import logging +import os +from statistics import mean import gin import numpy as np -import os -from icu_benchmarks.contants import RunMode -from icu_benchmarks.models.wrappers import MLWrapper +import shap +import wandb import xgboost as xgb from xgboost.callback import EarlyStopping from wandb.integration.xgboost import wandb_callback as wandb_xgb -import wandb -from statistics import mean + +from icu_benchmarks.constants import RunMode +from icu_benchmarks.models.wrappers import MLWrapper + +# Uncomment if needed in the future # from optuna.integration import XGBoostPruningCallback -import shap @gin.configurable class XGBClassifier(MLWrapper): _supported_run_modes = [RunMode.classification] + _explain_values = False def __init__(self, *args, **kwargs): self.model = self.set_model_args(xgb.XGBClassifier, device="cpu", *args, **kwargs) super().__init__(*args, **kwargs) def predict(self, features): - """Predicts labels for the given features.""" + """ + Predicts class probabilities for the given features. + + Args: + features: Input features for prediction. + + Returns: + numpy.ndarray: Predicted probabilities for each class. + """ return self.model.predict_proba(features) def fit_model(self, train_data, train_labels, val_data, val_labels): @@ -36,8 +48,9 @@ def fit_model(self, train_data, train_labels, val_data, val_labels): logging.info(f"train_data: {train_data.shape}, train_labels: {train_labels.shape}") logging.info(train_labels) self.model.fit(train_data, train_labels, eval_set=[(val_data, val_labels)], verbose=False) - self.explainer = shap.TreeExplainer(self.model) - self.train_shap_values = self.explainer(train_data) + if self._explain_values: + self.explainer = shap.TreeExplainer(self.model) + self.train_shap_values = self.explainer(train_data) # shap.summary_plot(shap_values, X_test, feature_names=features) # logging.info(self.model.get_booster().get_score(importance_type='weight')) # self.log_dict(self.model.get_booster().get_score(importance_type='weight')) @@ -61,7 +74,7 @@ def test_step(self, dataset, _): # Save as: id, time (hours), ground truth, prediction 0, prediction 1 np.savetxt(os.path.join(self.logger.save_dir, "pred_indicators.csv"), pred_indicators, delimiter=",") logging.debug(f"Saved row indicators to {os.path.join(self.logger.save_dir,f'row_indicators.csv')}") - if self.explainer is not None: + if self._explain_values and self.explainer is not None: self.test_shap_values = self.explainer(test_rep) if self.mps: self.log("test/loss", np.float32(self.loss(test_label, test_pred)), sync_dist=True) @@ -83,4 +96,6 @@ def set_model_args(self, model, *args, **kwargs): return model(**hyperparams) def get_feature_importance(self): + if not hasattr(self.model, 'feature_importances_'): + raise ValueError("Model has not been fit yet. Call fit_model() before getting feature importances.") return self.model.feature_importances_ diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index f2eddc02..e53d7700 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -13,7 +13,7 @@ from pathlib import Path from icu_benchmarks.data.loader import PredictionPandasDataset, ImputationPandasDataset, PredictionPolarsDataset from icu_benchmarks.models.utils import save_config_file, JSONMetricsLogger -from icu_benchmarks.contants import RunMode +from icu_benchmarks.constants import RunMode from icu_benchmarks.data.constants import DataSplit as Split cpu_core_count = len(os.sched_getaffinity(0)) if hasattr(os, "sched_getaffinity") else os.cpu_count() @@ -52,6 +52,7 @@ def train_common( train_only=False, num_workers: int = min(cpu_core_count, torch.cuda.device_count() * 8 * int(torch.cuda.is_available()), 32), polars=True, + persistent_workers=None, ): """Common wrapper to train all benchmarked models. @@ -82,20 +83,16 @@ def train_common( logging.info(f"Training model: {model.__name__}.") # todo: add support for polars versions of datasets - dataset_class = ( - ImputationPandasDataset - if mode == RunMode.imputation - else PredictionPolarsDataset if polars else PredictionPandasDataset - ) - # dataset_class = ImputationPandasDataset if mode == RunMode.imputation else PredictionPandasDataset + dataset_classes = { + RunMode.imputation: ImputationPandasDataset, + RunMode.classification: PredictionPolarsDataset if polars else PredictionPandasDataset, + RunMode.regression: PredictionPolarsDataset if polars else PredictionPandasDataset, + } + dataset_class = dataset_classes[mode] logging.info(f"Using dataset class: {dataset_class.__name__}.") logging.info(f"Logging to directory: {log_dir}.") save_config_file(log_dir) # We save the operative config before and also after training - persistent_workers = None - # for dataset in data.values(): - # for key, value in dataset.items(): - # dataset[key] = value.to_pandas() train_dataset = dataset_class(data, split=Split.train, ram_cache=ram_cache, name=dataset_names["train"]) val_dataset = dataset_class(data, split=Split.val, ram_cache=ram_cache, name=dataset_names["val"]) train_dataset, val_dataset = assure_minimum_length(train_dataset), assure_minimum_length(val_dataset) @@ -107,14 +104,11 @@ def train_common( f" {len(val_dataset)} samples." ) logging.info(f"Using {num_workers} workers for data loading.") - # todo: compare memory pinning - # cpu=True train_loader = DataLoader( train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers, - # pin_memory=not cpu, drop_last=True, persistent_workers=persistent_workers, ) @@ -123,7 +117,6 @@ def train_common( batch_size=batch_size, shuffle=False, num_workers=num_workers, - # pin_memory=not cpu, drop_last=True, persistent_workers=persistent_workers, ) @@ -152,7 +145,7 @@ def train_common( trainer = Trainer( max_epochs=epochs if model.requires_backprop else 1, - min_epochs=1, + min_epochs=1, # We need at least one epoch to get results. callbacks=callbacks, precision=precision, accelerator="auto" if not cpu else "cpu", @@ -161,7 +154,7 @@ def train_common( benchmark=not reproducible, enable_progress_bar=verbose, logger=loggers, - num_sanity_val_steps=2, + num_sanity_val_steps=2, # Helps catch errors in the validation loop before training begins. log_every_n_steps=5, ) if not eval_only: @@ -197,26 +190,30 @@ def train_common( model.set_weight("balanced", train_dataset) test_loss = trainer.test(model, dataloaders=test_loader, verbose=verbose)[0]["test/loss"] - persist_data(trainer, log_dir) + persist_shap_data(trainer, log_dir) save_config_file(log_dir) return test_loss -def persist_data(trainer, log_dir): +def persist_shap_data(trainer: Trainer, log_dir: Path): + """ + Persist shap values to disk. + Args: + trainer: Pytorch lightning trainer object + log_dir: Log directory + """ try: if trainer.lightning_module.test_shap_values is not None: shap_values = trainer.lightning_module.test_shap_values - # sdf_test = pl.DataFrame({ - # 'features': trainer.lightning_module.trained_columns, - # 'feature_value': np.transpose(shap_values.values.mean(axis=0)), - # }) shaps_test = pl.DataFrame(schema=trainer.lightning_module.trained_columns, data=np.transpose(shap_values.values)) - shaps_test.write_parquet(log_dir / "shap_values_test.parquet") + with (log_dir / "shap_values_test.parquet").open("wb") as f: + shaps_test.write_parquet(f) logging.info(f"Saved shap values to {log_dir / 'test_shap_values.parquet'}") if trainer.lightning_module.train_shap_values is not None: shap_values = trainer.lightning_module.train_shap_values shaps_train = pl.DataFrame(schema=trainer.lightning_module.trained_columns, data=np.transpose(shap_values.values)) - shaps_train.write_parquet(log_dir / "shap_values_train.parquet") + with (log_dir / "shap_values_train.parquet").open("wb") as f: + shaps_train.write_parquet(f) except Exception as e: logging.error(f"Failed to save shap values: {e}") diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index 022c0e90..b5e96313 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -22,7 +22,7 @@ from pytorch_lightning import LightningModule from icu_benchmarks.models.constants import MLMetrics, DLMetrics -from icu_benchmarks.contants import RunMode +from icu_benchmarks.constants import RunMode gin.config.external_configurable(nn.functional.nll_loss, module="torch.nn.functional") gin.config.external_configurable(nn.functional.cross_entropy, module="torch.nn.functional") diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index 7e3ecf79..187de3c1 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -20,7 +20,7 @@ name_datasets, get_config_files, ) -from icu_benchmarks.contants import RunMode +from icu_benchmarks.constants import RunMode @gin.configurable("Run") @@ -55,13 +55,13 @@ def main(my_args=tuple(sys.argv[1:])): logging.debug(f"Binding modalities: {modalities}") gin.bind_parameter("preprocess.selected_modalities", modalities) if args.label: + logging.debug(f"Binding label: {args.label}") gin.bind_parameter("preprocess.label", args.label) tasks, models = get_config_files(Path("configs")) - - if task not in tasks: - raise ValueError(f"Task {task} not found in tasks.") - if model not in models: - raise ValueError(f"Model {model} not found in models.") + if task not in tasks or model not in models: + raise ValueError( + f"Invalid task or model. Task: {task} {'not ' if task not in tasks else ''} found. " + f"Model: {model} {'not ' if model not in models else ''}found.") # Load task config gin.parse_config_file(f"configs/tasks/{task}.gin") mode = get_mode() @@ -192,6 +192,7 @@ def main(my_args=tuple(sys.argv[1:])): aggregate_results(run_dir, execution_time) except Exception as e: logging.error(f"Failed to aggregate results: {e}") + logging.debug("Error details:", exc_info=True) if args.plot: plot_aggregated_results(run_dir, "aggregated_test_metrics.json") diff --git a/icu_benchmarks/run_utils.py b/icu_benchmarks/run_utils.py index 2c5da67b..bf790caf 100644 --- a/icu_benchmarks/run_utils.py +++ b/icu_benchmarks/run_utils.py @@ -55,7 +55,12 @@ def build_parser() -> ArgumentParser: parser.add_argument("-sn", "--source-name", type=Path, help="Name of the source dataset.") parser.add_argument("--source-dir", type=Path, help="Directory containing gin and model weights.") parser.add_argument("-sa", "--samples", type=int, default=None, help="Number of samples to use for evaluation.") - parser.add_argument("-mo", "--modalities", nargs="+", help="Modalities to use for evaluation.") + parser.add_argument( + "-mo", + "--modalities", + nargs="+", + help="Optional modality selection to use. Specify multiple modalities separated by spaces.", + ) parser.add_argument("--label", type=str, help="Label to use for evaluation in case of multiple labels.", default=None) return parser @@ -73,16 +78,10 @@ def create_run_dir(log_dir: Path, randomly_searched_params: str = None) -> Path: Returns: Path to the created run log directory. """ - if not (log_dir / str(datetime.now().strftime("%Y-%m-%dT%H-%M-%S"))).exists(): - log_dir_run = log_dir / str(datetime.now().strftime("%Y-%m-%dT%H-%M-%S")) - else: - log_dir_run = log_dir / str(datetime.now().strftime("%Y-%m-%dT%H-%M-%S.%f")) - if not log_dir_run.exists(): - log_dir_run.mkdir(parents=True) - else: - # Directory clash at last moment + log_dir_run = log_dir / str(datetime.now().strftime("%Y-%m-%dT%H-%M-%S")) + while log_dir_run.exists(): log_dir_run = log_dir / str(datetime.now().strftime("%Y-%m-%dT%H-%M-%S.%f")) - log_dir_run.mkdir(parents=True) + log_dir_run.mkdir(parents=True) if randomly_searched_params: (log_dir_run / randomly_searched_params).touch() return log_dir_run @@ -136,6 +135,11 @@ def aggregate_results(log_dir: Path, execution_time: timedelta = None): shap_values = pl.concat(shap_values_test) shap_values.write_parquet(log_dir / "aggregated_shap_values.parquet") + try: + shap_values = pl.concat(shap_values_test) + shap_values.write_parquet(log_dir / "aggregated_shap_values.parquet") + except Exception as e: + logging.error(f"Error aggregating or writing SHAP values: {e}") # Aggregate results per metric list_scores = {} for repetition, folds in aggregated.items(): @@ -256,10 +260,23 @@ def setup_logging(date_format, log_format, verbose): def get_config_files(config_dir: Path): - tasks = glob.glob(os.path.join(config_dir / "tasks", "*")) - models = glob.glob(os.path.join(config_dir / "prediction_models", "*")) - tasks = [os.path.splitext(os.path.basename(task))[0] for task in tasks] - models = [os.path.splitext(os.path.basename(model))[0] for model in models] + """ + Get all task and model config files in the specified directory. + Args: + config_dir: Name of the directory containing the config gin files. + + Returns: + tasks: List of task names + models: List of model names + """ + try: + tasks = list((config_dir / "tasks").glob("*")) + models = list((config_dir / "prediction_models").glob("*")) + tasks = [task.stem for task in tasks if task.is_file()] + models = [model.stem for model in models if model.is_file()] + except Exception as e: + logging.error(f"Error retrieving config files: {e}") + return [], [] if "common" in tasks: tasks.remove("common") if "common" in models: diff --git a/icu_benchmarks/tuning/hyperparameters.py b/icu_benchmarks/tuning/hyperparameters.py index c3a93370..83af80aa 100644 --- a/icu_benchmarks/tuning/hyperparameters.py +++ b/icu_benchmarks/tuning/hyperparameters.py @@ -13,7 +13,7 @@ from icu_benchmarks.cross_validation import execute_repeated_cv from icu_benchmarks.run_utils import log_full_line from icu_benchmarks.tuning.gin_utils import get_gin_hyperparameters, bind_gin_params -from icu_benchmarks.contants import RunMode +from icu_benchmarks.constants import RunMode from icu_benchmarks.wandb_utils import wandb_log from optuna.visualization import plot_param_importances, plot_optimization_history @@ -235,32 +235,30 @@ def choose_and_bind_hyperparameters_optuna( header = ["ITERATION"] + hyperparams_names + ["LOSS AT ITERATION"] # Optuna objective function - def objective(trail, hyperparams_bounds, hyperparams_names): + def objective(trial, hyperparams_bounds, hyperparams_names): # Optuna objective function hyperparams = {} logging.info(f"Bounds: {hyperparams_bounds}, Names: {hyperparams_names}") for name, value in zip(hyperparams_names, hyperparams_bounds): if isinstance(value, tuple): - # Check for range or "list-type" hyperparameter bounds - if isinstance(value[0], (int, float)) and isinstance(value[1], (int, float)): - if len(value) == 3 and isinstance(value[2], str): - if isinstance(value[0], int) and isinstance(value[1], int): - hyperparams[name] = trail.suggest_int(name, value[0], value[1], log=value[2] == "log") - elif isinstance(value[0], (int, float)) and isinstance(value[1], (int, float)): - hyperparams[name] = trail.suggest_float(name, value[0], value[1], log=value[2] == "log") - else: - hyperparams[name] = trail.suggest_categorical(name, value) - elif len(value) == 2: - if isinstance(value[0], int) and isinstance(value[1], int): - hyperparams[name] = trail.suggest_int(name, value[0], value[1]) - elif isinstance(value[0], (int, float)) and isinstance(value[1], (int, float)): - hyperparams[name] = trail.suggest_float(name, value[0], value[1]) - else: - hyperparams[name] = trail.suggest_categorical(name, value) - else: - hyperparams[name] = trail.suggest_categorical(name, value) + def suggest_int_param(trial, name, value): + return trial.suggest_int(name, value[0], value[1], log=value[2] == "log" if len(value) == 3 else False) + + def suggest_float_param(trial, name, value): + return trial.suggest_float(name, value[0], value[1], log=value[2] == "log" if len(value) == 3 else False) + + def suggest_categorical_param(trial, name, value): + return trial.suggest_categorical(name, value) + + # Then in the objective function: + if isinstance(value[0], int) and isinstance(value[1], int): + hyperparams[name] = suggest_int_param(trial, name, value) + elif isinstance(value[0], (int, float)) and isinstance(value[1], (int, float)): + hyperparams[name] = suggest_float_param(trial, name, value) + else: + hyperparams[name] = suggest_categorical_param(trial, name, value) else: - hyperparams[name] = trail.suggest_categorical(name, value) + hyperparams[name] = trial.suggest_categorical(name, value) return bind_params_and_train(hyperparams) def tune_step_callback(study: optuna.study.Study, trial: optuna.trial.FrozenTrial): From 5d3ee6901e0c4cc3995575a0ac4c9aae48346ea3 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 17 Oct 2024 15:47:25 +0200 Subject: [PATCH 195/207] remove cass files --- configs/tasks/CassClassification.gin | 1 - 1 file changed, 1 deletion(-) diff --git a/configs/tasks/CassClassification.gin b/configs/tasks/CassClassification.gin index b9576f7d..50a6d013 100644 --- a/configs/tasks/CassClassification.gin +++ b/configs/tasks/CassClassification.gin @@ -25,7 +25,6 @@ include "configs/tasks/common/Dataloader.gin" # SELECTING DATASET # PredictionDataset.vars = %vars # PredictionDataset.ram_cache = True - # DATASET CONFIGURATION preprocess.file_names = { "DYNAMIC": "dyn.parquet", From 99ce855529b0505597fb2581f1f8b0e84bb2b9af Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 17 Oct 2024 15:47:56 +0200 Subject: [PATCH 196/207] remove cass files --- configs/tasks/CassClassification.gin | 42 ---------------------------- 1 file changed, 42 deletions(-) delete mode 100644 configs/tasks/CassClassification.gin diff --git a/configs/tasks/CassClassification.gin b/configs/tasks/CassClassification.gin deleted file mode 100644 index 50a6d013..00000000 --- a/configs/tasks/CassClassification.gin +++ /dev/null @@ -1,42 +0,0 @@ -# COMMON IMPORTS -include "configs/tasks/common/Imports.gin" - - -# CROSS-VALIDATION -include "configs/tasks/common/CrossValidation.gin" - -# MODE SETTINGS -Run.mode = "Classification" -NUM_CLASSES = 2 # Binary classification -HORIZON = 12 -train_common.weight = "balanced" -train_common.ram_cache = True - -# DEEP LEARNING -DLPredictionWrapper.loss = @cross_entropy - -# SELECTING PREPROCESSOR -preprocess.preprocessor = @base_classification_preprocessor -preprocess.vars = %vars -preprocess.use_static = True - -# SELECTING DATASET -include "configs/tasks/common/Dataloader.gin" -# SELECTING DATASET -# PredictionDataset.vars = %vars -# PredictionDataset.ram_cache = True -# DATASET CONFIGURATION -preprocess.file_names = { - "DYNAMIC": "dyn.parquet", - "OUTCOME": "outc.parquet", - "STATIC": "sta.parquet", -} - -#include "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/14-08-2024_normal_ward/vars.gin" -#include "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/19-08-2024_all_wards/vars.gin" -# include "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/dataset_x_segment_duration_x_transfer_2024-08-26 13:43:20/vars.gin" -# include "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/dataset_x_segment_duration_x_transfer_2024-09-12T16:22:01/vars.gin" -# include "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/dataset_baseline_flat_2024-09-17T17:25:20/vars.gin" -# include "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/dataset_segment_1.0_horizon_6:00:00_transfer_full_2024-09-27T16:14:05/vars.gin" -include "/sc-projects/sc-proj-cc08-cassandra/Prospective_Preprocessed/yaib_format/dataset_segment_1.0_horizon_6:00:00_transfer_3_2024-10-07T23:26:22/vars.gin" -# preprocess.modality_mapping = %modality_mapping From 151d68f4c6086dc579b8711ec1bbfc8a749f747c Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 17 Oct 2024 16:47:41 +0200 Subject: [PATCH 197/207] Added polars --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index b068c8e3..69814bd3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,16 +7,16 @@ gin-config==0.5.0 pytorch-ignite==0.5.0.post2 # Note: versioning of Pytorch might be dependent on compatible CUDA version. # Please check yourself if your Pytorch installation supports cuda (for gpu acceleration) -torch==2.3.1 +torch==2.4 lightning==2.4.0 torchmetrics==1.0.3 -#pytorch-cuda==11.8 lightgbm==4.4.0 xgboost==2.1.0 imbalanced-learn==0.12.3 catboost==1.2.5 numpy==1.24.3 pandas==2.2.2 +polars==1.9.0 pyarrow==14.0.1 pytest==7.3.1 scikit-learn==1.5.0 From 47c594a7dd5c32737a1de0827736a64413f591fa Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 17 Oct 2024 16:50:27 +0200 Subject: [PATCH 198/207] More major refactoring --- icu_benchmarks/cross_validation.py | 8 +- icu_benchmarks/data/loader.py | 1 - icu_benchmarks/data/pooling.py | 11 ++- icu_benchmarks/data/preprocessor.py | 19 ++-- icu_benchmarks/data/split_process_data.py | 5 +- icu_benchmarks/models/custom_metrics.py | 4 - icu_benchmarks/models/dl_models/rnn.py | 7 +- icu_benchmarks/models/dl_models/tcn.py | 2 +- .../models/dl_models/transformer.py | 90 +++++++++---------- icu_benchmarks/models/ml_models/catboost.py | 7 +- icu_benchmarks/models/ml_models/xgboost.py | 2 - icu_benchmarks/models/wrappers.py | 61 ++++++++----- icu_benchmarks/run_utils.py | 15 ++++ icu_benchmarks/tuning/hyperparameters.py | 4 +- 14 files changed, 126 insertions(+), 110 deletions(-) diff --git a/icu_benchmarks/cross_validation.py b/icu_benchmarks/cross_validation.py index bcad6bde..0d6c64ea 100644 --- a/icu_benchmarks/cross_validation.py +++ b/icu_benchmarks/cross_validation.py @@ -13,7 +13,6 @@ from icu_benchmarks.run_utils import log_full_line from icu_benchmarks.constants import RunMode - @gin.configurable def execute_repeated_cv( data_dir: Path, @@ -104,10 +103,6 @@ def execute_repeated_cv( runmode=mode, complete_train=complete_train, ) - # logging.debug(f"{data}") - # data_pickle_path = log_dir / "data.pkl" - # with open(data_pickle_path, "wb") as f: - # pickle.dump(data, f, pickle.HIGHEST_PROTOCOL) preprocess_time = datetime.now() - start_time start_time = datetime.now() agg_loss += train_common( @@ -123,8 +118,6 @@ def execute_repeated_cv( verbose=verbose, use_wandb=wandb, train_only=complete_train, - epochs=20, - patience=5, ) train_time = datetime.now() - start_time @@ -146,3 +139,4 @@ def execute_repeated_cv( log_full_line(f"FINISHED CV REPETITION {repetition}", level=logging.INFO, char="=", num_newlines=3) return agg_loss / (cv_repetitions_to_train * cv_folds_to_train) + diff --git a/icu_benchmarks/data/loader.py b/icu_benchmarks/data/loader.py index 4cf67757..b227dbf2 100644 --- a/icu_benchmarks/data/loader.py +++ b/icu_benchmarks/data/loader.py @@ -42,7 +42,6 @@ def __init__( self.features_df = data[split][Segment.features] self.features_df = self.features_df.sort([self.vars["GROUP"], self.vars["SEQUENCE"]]) self.features_df = self.features_df.drop(self.vars["SEQUENCE"]) - self.grouping_df = self.grouping_df.sort([self.vars["GROUP"], self.vars["SEQUENCE"]]) else: # We have a static dataset logging.info("Using static dataset") diff --git a/icu_benchmarks/data/pooling.py b/icu_benchmarks/data/pooling.py index 57ca84d2..e8ee8ba6 100644 --- a/icu_benchmarks/data/pooling.py +++ b/icu_benchmarks/data/pooling.py @@ -66,8 +66,7 @@ def generate( if folder.is_dir(): if folder.name in datasets: data[folder.name] = { - f: pq.read_table(folder / self.file_names[f]).to_pandas(self_destruct=True) - for f in self.file_names + f: pq.read_table(folder / self.file_names[f]).to_pandas(self_destruct=True) for f in self.file_names } data = self._pool_datasets( datasets=data, @@ -103,9 +102,9 @@ def _save_pooled_data(self, data_dir, data, datasets, file_names, samples=10000) def _pool_datasets( self, - datasets={}, + datasets=None, samples=10000, - vars=[], + vars=None, seed=42, shuffle=True, runmode=RunMode.classification, @@ -126,6 +125,10 @@ def _pool_datasets( Returns: pooled dataset """ + if datasets is None: + datasets = {} + if vars is None: + vars = [] if len(datasets) == 0: raise ValueError("No datasets supplied.") pooled_data = {Segment.static: [], Segment.dynamic: [], Segment.outcome: []} diff --git a/icu_benchmarks/data/preprocessor.py b/icu_benchmarks/data/preprocessor.py index 610e0e33..5366b868 100644 --- a/icu_benchmarks/data/preprocessor.py +++ b/icu_benchmarks/data/preprocessor.py @@ -44,6 +44,8 @@ def set_imputation_model(self, imputation_model): update_wandb_config({"imputation_model": self.imputation_model.__class__.__name__}) + + @gin.configurable("base_classification_preprocessor") class PolarsClassificationPreprocessor(Preprocessor): def __init__( @@ -260,14 +262,17 @@ def _process_outcome(self, data, vars, split): outcome_rec = Recipe(data[split][Segment.outcome], vars["LABEL"], [], vars["GROUP"]) # If the range is predefined, use predefined transformation function if self.outcome_max is not None and self.outcome_min is not None: - outcome_rec.add_step( - StepSklearn( - sklearn_transformer=FunctionTransformer( - func=lambda x: ((x - self.outcome_min) / (self.outcome_max - self.outcome_min)) - ), - sel=all_outcomes(), + if self.outcome_max == self.outcome_min: + logging.warning("outcome_max equals outcome_min. Skipping outcome scaling.") + else: + outcome_rec.add_step( + StepSklearn( + sklearn_transformer=FunctionTransformer( + func=lambda x: ((x - self.outcome_min) / (self.outcome_max - self.outcome_min)) + ), + sel=all_outcomes(), + ) ) - ) else: # If the range is not predefined, use MinMaxScaler outcome_rec.add_step(StepSklearn(MinMaxScaler(), sel=all_outcomes())) diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index 3ae63ce2..6d059428 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -13,6 +13,7 @@ from sklearn.model_selection import StratifiedKFold, KFold, StratifiedShuffleSplit, ShuffleSplit from icu_benchmarks.data.preprocessor import Preprocessor, PandasClassificationPreprocessor, PolarsClassificationPreprocessor from icu_benchmarks.constants import RunMode +from icu_benchmarks.run_utils import check_required_keys from .constants import DataSplit as Split, DataSegment as Segment, VarType as Var @@ -38,7 +39,6 @@ def preprocess_data( complete_train: bool = False, runmode: RunMode = RunMode.classification, label: str = None, - vars_to_exclude: list[str] = [], ) -> dict[dict[pl.DataFrame]] or dict[dict[pd.DataFrame]]: """Perform loading, splitting, imputing and normalising of task data. @@ -67,7 +67,8 @@ def preprocess_data( """ cache_dir = data_dir / "cache" - + required_keys = ["GROUP", "SEQUENCE", "LABEL"] + check_required_keys(vars, required_keys) if not use_static: file_names.pop(Segment.static) vars.pop(Segment.static) diff --git a/icu_benchmarks/models/custom_metrics.py b/icu_benchmarks/models/custom_metrics.py index 271aa55c..eb0a5d23 100644 --- a/icu_benchmarks/models/custom_metrics.py +++ b/icu_benchmarks/models/custom_metrics.py @@ -138,12 +138,8 @@ def confusion_matrix(y_true: ndarray, y_pred: ndarray, normalize=False) -> torch confusion = sk_confusion_matrix(y_true, y_pred) if normalize: confusion = confusion / confusion.sum() - # confusion_tensor = torch.tensor(confusion) - # confusion = confusion.tolist() confusion_dict = {} for i in range(confusion.shape[0]): for j in range(confusion.shape[1]): confusion_dict[f"class_{i}_pred_{j}"] = confusion[i][j] - # logging.info(f"Confusion matrix: {confusion_dict}") - # dict = {"TP": confusion[0][0], "FP": confusion[0][1], "FN": confusion[1][0], "TN": confusion[1][1]} return confusion_dict diff --git a/icu_benchmarks/models/dl_models/rnn.py b/icu_benchmarks/models/dl_models/rnn.py index 7174e090..71a3da50 100644 --- a/icu_benchmarks/models/dl_models/rnn.py +++ b/icu_benchmarks/models/dl_models/rnn.py @@ -13,7 +13,7 @@ class RNNet(DLPredictionWrapper): def __init__(self, input_size, hidden_dim, layer_dim, num_classes, *args, **kwargs): super().__init__( - input_size=input_size, hidden_dim=hidden_dim, layer_dim=layer_dim, num_classes=num_classes, *args, **kwargs + *args, input_size=input_size, hidden_dim=hidden_dim, layer_dim=layer_dim, num_classes=num_classes,**kwargs ) self.hidden_dim = hidden_dim self.layer_dim = layer_dim @@ -39,7 +39,7 @@ class LSTMNet(DLPredictionWrapper): def __init__(self, input_size, hidden_dim, layer_dim, num_classes, *args, **kwargs): super().__init__( - input_size=input_size, hidden_dim=hidden_dim, layer_dim=layer_dim, num_classes=num_classes, *args, **kwargs + *args, input_size=input_size, hidden_dim=hidden_dim, layer_dim=layer_dim, num_classes=num_classes, **kwargs ) self.hidden_dim = hidden_dim self.layer_dim = layer_dim @@ -66,7 +66,7 @@ class GRUNet(DLPredictionWrapper): def __init__(self, input_size, hidden_dim, layer_dim, num_classes, *args, **kwargs): super().__init__( - input_size=input_size, hidden_dim=hidden_dim, layer_dim=layer_dim, num_classes=num_classes, *args, **kwargs + *args, input_size=input_size, hidden_dim=hidden_dim, layer_dim=layer_dim, num_classes=num_classes, **kwargs ) self.hidden_dim = hidden_dim self.layer_dim = layer_dim @@ -81,5 +81,4 @@ def forward(self, x): h0 = self.init_hidden(x) out, hn = self.rnn(x, h0) pred = self.logit(out) - return pred diff --git a/icu_benchmarks/models/dl_models/tcn.py b/icu_benchmarks/models/dl_models/tcn.py index 28547a57..8be71fea 100644 --- a/icu_benchmarks/models/dl_models/tcn.py +++ b/icu_benchmarks/models/dl_models/tcn.py @@ -17,10 +17,10 @@ class TemporalConvNet(DLPredictionWrapper): def __init__(self, input_size, num_channels, num_classes, *args, max_seq_length=0, kernel_size=2, dropout=0.0, **kwargs): super().__init__( + *args, input_size=input_size, num_channels=num_channels, num_classes=num_classes, - *args, max_seq_length=max_seq_length, kernel_size=kernel_size, dropout=dropout, diff --git a/icu_benchmarks/models/dl_models/transformer.py b/icu_benchmarks/models/dl_models/transformer.py index 95021379..0eb971d7 100644 --- a/icu_benchmarks/models/dl_models/transformer.py +++ b/icu_benchmarks/models/dl_models/transformer.py @@ -6,26 +6,36 @@ from icu_benchmarks.models.wrappers import DLPredictionWrapper +class BaseTransformer(DLPredictionWrapper): + _supported_run_modes = [RunMode.classification, RunMode.regression] + + def forward(self, x): + x = self.input_embedding(x) + if self.pos_encoder is not None: + x = self.pos_encoder(x) + x = self.tblocks(x) + pred = self.logit(x) + return pred + + @gin.configurable -class Transformer(DLPredictionWrapper): +class Transformer(BaseTransformer): """Transformer model as defined by the HiRID-Benchmark (https://github.com/ratschlab/HIRID-ICU-Benchmark).""" - _supported_run_modes = [RunMode.classification, RunMode.regression] - def __init__( - self, - input_size, - hidden, - heads, - ff_hidden_mult, - depth, - num_classes, - *args, - dropout=0.0, - l1_reg=0, - pos_encoding=True, - dropout_att=0.0, - **kwargs, + self, + input_size, + hidden, + heads, + ff_hidden_mult, + depth, + num_classes, + dropout=0.0, + l1_reg=0, + pos_encoding=True, + dropout_att=0.0, + *args, + **kwargs, ): super().__init__( input_size=input_size, @@ -66,35 +76,26 @@ def __init__( self.logit = nn.Linear(hidden, num_classes) self.l1_reg = l1_reg - def forward(self, x): - x = self.input_embedding(x) - if self.pos_encoder is not None: - x = self.pos_encoder(x) - x = self.tblocks(x) - pred = self.logit(x) - - return pred - @gin.configurable -class LocalTransformer(DLPredictionWrapper): +class LocalTransformer(BaseTransformer): _supported_run_modes = [RunMode.classification, RunMode.regression] def __init__( - self, - input_size, - hidden, - heads, - ff_hidden_mult, - depth, - num_classes, - *args, - dropout=0.0, - l1_reg=0, - pos_encoding=True, - local_context=1, - dropout_att=0.0, - **kwargs, + self, + input_size, + hidden, + heads, + ff_hidden_mult, + depth, + num_classes, + *args, + dropout=0.0, + l1_reg=0, + pos_encoding=True, + local_context=1, + dropout_att=0.0, + **kwargs, ): super().__init__( input_size=input_size, @@ -137,12 +138,3 @@ def __init__( self.tblocks = nn.Sequential(*tblocks) self.logit = nn.Linear(hidden, num_classes) self.l1_reg = l1_reg - - def forward(self, x): - x = self.input_embedding(x) - if self.pos_encoder is not None: - x = self.pos_encoder(x) - x = self.tblocks(x) - pred = self.logit(x) - - return pred diff --git a/icu_benchmarks/models/ml_models/catboost.py b/icu_benchmarks/models/ml_models/catboost.py index 2e7d63ed..95bff06d 100644 --- a/icu_benchmarks/models/ml_models/catboost.py +++ b/icu_benchmarks/models/ml_models/catboost.py @@ -8,10 +8,9 @@ class CBClassifier(MLWrapper): _supported_run_modes = [RunMode.classification] - def __init__(self, *args, **kwargs): - # self.model = self.set_model_args(cb.CatBoostClassifier, task_type="GPU" - # if not kwargs['cpu'] else "CPU", *args, **kwargs) - self.model = self.set_model_args(cb.CatBoostClassifier, task_type="CPU", *args, **kwargs) + def __init__(self, task_type="CPU", *args, **kwargs): + model_kwargs = {'task_type': task_type, **kwargs} + self.model = self.set_model_args(cb.CatBoostClassifier, *args, **model_kwargs) super().__init__(*args, **kwargs) def predict(self, features): diff --git a/icu_benchmarks/models/ml_models/xgboost.py b/icu_benchmarks/models/ml_models/xgboost.py index dfdaa841..118e2e6e 100644 --- a/icu_benchmarks/models/ml_models/xgboost.py +++ b/icu_benchmarks/models/ml_models/xgboost.py @@ -74,8 +74,6 @@ def test_step(self, dataset, _): # Save as: id, time (hours), ground truth, prediction 0, prediction 1 np.savetxt(os.path.join(self.logger.save_dir, "pred_indicators.csv"), pred_indicators, delimiter=",") logging.debug(f"Saved row indicators to {os.path.join(self.logger.save_dir,f'row_indicators.csv')}") - if self._explain_values and self.explainer is not None: - self.test_shap_values = self.explainer(test_rep) if self.mps: self.log("test/loss", np.float32(self.loss(test_label, test_pred)), sync_dist=True) self.log_metrics(np.float32(test_label), np.float32(test_pred), "test") diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index b5e96313..8fb08858 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -1,7 +1,7 @@ import logging from abc import ABC from typing import Dict, Any, List, Optional, Union - +from pathlib import Path import torchmetrics from sklearn.metrics import log_loss, mean_squared_error, average_precision_score, roc_auc_score @@ -46,6 +46,8 @@ class BaseModule(LightningModule): trained_columns = None # Type of run mode run_mode = None + debug = False + explain_features = False def forward(self, *args, **kwargs): raise NotImplementedError() @@ -62,8 +64,14 @@ def set_metrics(self, *args, **kwargs): def set_trained_columns(self, columns: List[str]): self.trained_columns = columns - def set_weight(self, weight, *args, **kwargs): - pass + def set_weight(self, weight, dataset): + """Set the weight for the loss function.""" + + if isinstance(weight, list): + weight = FloatTensor(weight).to(self.device) + elif weight == "balanced": + weight = FloatTensor(dataset.get_balance()).to(self.device) + self.loss_weights = weight def training_step(self, batch, batch_idx): return self.step_fn(batch, "train") @@ -253,14 +261,7 @@ def __init__( self.output_transform = None self.loss_weights = None - def set_weight(self, weight, dataset): - """Set the weight for the loss function.""" - if isinstance(weight, list): - weight = FloatTensor(weight).to(self.device) - elif weight == "balanced": - weight = FloatTensor(dataset.get_balance()).to(self.device) - self.loss_weights = weight def set_metrics(self, *args): """Set the evaluation metrics for the prediction model.""" @@ -373,10 +374,6 @@ def __init__(self, *args, run_mode=RunMode.classification, loss=log_loss, patien self.scaler = None self.check_supported_runmode(run_mode) self.run_mode = run_mode - # if loss.__name__ in ["average_precision_score", "roc_auc_score"]: - # self.loss = scorer_wrapper(average_precision_score) - # else: - # self.loss = self.loss self.loss = loss self.patience = patience self.mps = mps @@ -448,13 +445,18 @@ def validation_step(self, val_dataset, _): self.log_metrics(val_label, val_pred, "val") def test_step(self, dataset, _): - test_rep, test_label = dataset - test_rep, test_label = test_rep.squeeze().cpu().numpy(), test_label.squeeze().cpu().numpy() + test_rep, test_label, pred_indicators = dataset + test_rep, test_label, pred_indicators = ( + test_rep.squeeze().cpu().numpy(), + test_label.squeeze().cpu().numpy(), + pred_indicators.squeeze().cpu().numpy(), + ) self.set_metrics(test_label) test_pred = self.predict(test_rep) - # if self.explainer is not None: - # self.test_shap_values = self.explainer(test_rep) - # logging.info(f"Shap values: {self.test_shap_values}") + if self.debug: + self._save_model_outputs(pred_indicators, test_pred, test_label) + if self.explain_features: + self.explain_model(test_rep, test_label) if self.mps: self.log("test/loss", np.float32(self.loss(test_label, test_pred)), sync_dist=True) self.log_metrics(np.float32(test_label), np.float32(test_pred), "test") @@ -471,15 +473,12 @@ def predict(self, features): def log_metrics(self, label, pred, metric_type): """Log metrics to the PL logs.""" - if "Confusion_Matrix" in self.metrics.keys(): + if "Confusion_Matrix" in self.metrics: self.log_dict(confusion_matrix(self.label_transform(label), self.output_transform(pred)), sync_dist=True) self.log_dict( { - # MPS dependent type casting f"{metric_type}/{name}": ( metric(self.label_transform(label), self.output_transform(pred)) - if not self.mps - else metric(self.label_transform(label), self.output_transform(pred)) ) # For every metric for name, metric in self.metrics.items() @@ -489,7 +488,21 @@ def log_metrics(self, label, pred, metric_type): }, sync_dist=True, ) - + def _explain_model(self, test_rep, test_label): + if self.explainer is not None: + self.test_shap_values = self.explainer(test_rep) + else: + logging.warning("No explainer or explain_features values set.") + + def _save_model_outputs(self, pred_indicators, test_pred, test_label): + if len(pred_indicators.shape) > 1 and len(test_pred.shape) > 1 and pred_indicators.shape[1] == test_pred.shape[1]: + pred_indicators = np.hstack((pred_indicators, test_label.reshape(-1, 1))) + pred_indicators = np.hstack((pred_indicators, test_pred)) + # Save as: id, time (hours), ground truth, prediction 0, prediction 1 + np.savetxt(Path(self.logger.save_dir) / "pred_indicators.csv", pred_indicators, delimiter=",") + logging.debug(f"Saved row indicators to {Path(self.logger.save_dir) / f'row_indicators.csv'}") + else: + logging.warning("Could not save row indicators.") def configure_optimizers(self): return None diff --git a/icu_benchmarks/run_utils.py b/icu_benchmarks/run_utils.py index bf790caf..e35380e4 100644 --- a/icu_benchmarks/run_utils.py +++ b/icu_benchmarks/run_utils.py @@ -284,3 +284,18 @@ def get_config_files(config_dir: Path): logging.info(f"Found tasks: {tasks}") logging.info(f"Found models: {models}") return tasks, models + +def check_required_keys(vars, required_keys): + """ + Checks if all required keys are present in the vars dictionary. + + Args: + vars (dict): The dictionary to check. + required_keys (list): The list of required keys. + + Raises: + KeyError: If any required key is missing. + """ + missing_keys = [key for key in required_keys if key not in vars] + if missing_keys: + raise KeyError(f"Missing required keys in vars: {', '.join(missing_keys)}") \ No newline at end of file diff --git a/icu_benchmarks/tuning/hyperparameters.py b/icu_benchmarks/tuning/hyperparameters.py index 83af80aa..c835cf8d 100644 --- a/icu_benchmarks/tuning/hyperparameters.py +++ b/icu_benchmarks/tuning/hyperparameters.py @@ -22,7 +22,7 @@ @gin.configurable("tune_hyperparameters_deprecated") -def choose_and_bind_hyperparameters( +def choose_and_bind_hyperparameters_scikit_optimize( do_tune: bool, data_dir: Path, log_dir: Path, @@ -63,6 +63,8 @@ def choose_and_bind_hyperparameters( Raises: ValueError: If checkpoint is not None and the checkpoint does not exist. """ + logging.warning("This function is deprecated and will be removed in the future. " + "Use choose_and_bind_hyperparameters_optuna instead.") hyperparams = {} if len(scopes) == 0 or folds_to_tune_on is None: From d4e8845fccb20fcb024eb1e3dc3795debd90b9be Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 17 Oct 2024 16:58:35 +0200 Subject: [PATCH 199/207] Linting --- icu_benchmarks/cross_validation.py | 2 +- icu_benchmarks/data/preprocessor.py | 2 - icu_benchmarks/data/split_process_data.py | 9 +++- icu_benchmarks/models/dl_models/rnn.py | 2 +- .../models/dl_models/transformer.py | 54 +++++++++---------- icu_benchmarks/models/ml_models/catboost.py | 2 +- icu_benchmarks/models/ml_models/xgboost.py | 16 +++--- icu_benchmarks/models/train.py | 4 +- icu_benchmarks/models/wrappers.py | 8 ++- icu_benchmarks/run.py | 3 +- icu_benchmarks/run_utils.py | 6 +-- icu_benchmarks/tuning/hyperparameters.py | 6 ++- 12 files changed, 59 insertions(+), 55 deletions(-) diff --git a/icu_benchmarks/cross_validation.py b/icu_benchmarks/cross_validation.py index 0d6c64ea..e1563f9e 100644 --- a/icu_benchmarks/cross_validation.py +++ b/icu_benchmarks/cross_validation.py @@ -13,6 +13,7 @@ from icu_benchmarks.run_utils import log_full_line from icu_benchmarks.constants import RunMode + @gin.configurable def execute_repeated_cv( data_dir: Path, @@ -139,4 +140,3 @@ def execute_repeated_cv( log_full_line(f"FINISHED CV REPETITION {repetition}", level=logging.INFO, char="=", num_newlines=3) return agg_loss / (cv_repetitions_to_train * cv_folds_to_train) - diff --git a/icu_benchmarks/data/preprocessor.py b/icu_benchmarks/data/preprocessor.py index 5366b868..9f6103dd 100644 --- a/icu_benchmarks/data/preprocessor.py +++ b/icu_benchmarks/data/preprocessor.py @@ -44,8 +44,6 @@ def set_imputation_model(self, imputation_model): update_wandb_config({"imputation_model": self.imputation_model.__class__.__name__}) - - @gin.configurable("base_classification_preprocessor") class PolarsClassificationPreprocessor(Preprocessor): def __init__( diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index 6d059428..cf5a1401 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -318,8 +318,14 @@ def make_train_val( data_split[Split.test] = copy.deepcopy(data_split[Split.val]) return data_split + def _get_stays(data, id, polars): - return pl.Series(name=id, values=data[Segment.outcome][id].unique()) if polars else pd.Series(data[Segment.outcome][id].unique(), name=id) + return ( + pl.Series(name=id, values=data[Segment.outcome][id].unique()) + if polars + else pd.Series(data[Segment.outcome][id].unique(), name=id) + ) + def _get_labels(data, id, vars, polars): # Get labels from outcome data (takes the highest value (or True) in case seq2seq classification) @@ -328,6 +334,7 @@ def _get_labels(data, id, vars, polars): else: return data[Segment.outcome].groupby(id).max()[vars[Var.label]].reset_index(drop=True) + # Use these helper functions in both make_train_val and make_single_split diff --git a/icu_benchmarks/models/dl_models/rnn.py b/icu_benchmarks/models/dl_models/rnn.py index 71a3da50..4f0c65bc 100644 --- a/icu_benchmarks/models/dl_models/rnn.py +++ b/icu_benchmarks/models/dl_models/rnn.py @@ -13,7 +13,7 @@ class RNNet(DLPredictionWrapper): def __init__(self, input_size, hidden_dim, layer_dim, num_classes, *args, **kwargs): super().__init__( - *args, input_size=input_size, hidden_dim=hidden_dim, layer_dim=layer_dim, num_classes=num_classes,**kwargs + *args, input_size=input_size, hidden_dim=hidden_dim, layer_dim=layer_dim, num_classes=num_classes, **kwargs ) self.hidden_dim = hidden_dim self.layer_dim = layer_dim diff --git a/icu_benchmarks/models/dl_models/transformer.py b/icu_benchmarks/models/dl_models/transformer.py index 0eb971d7..e1f7fb73 100644 --- a/icu_benchmarks/models/dl_models/transformer.py +++ b/icu_benchmarks/models/dl_models/transformer.py @@ -23,19 +23,19 @@ class Transformer(BaseTransformer): """Transformer model as defined by the HiRID-Benchmark (https://github.com/ratschlab/HIRID-ICU-Benchmark).""" def __init__( - self, - input_size, - hidden, - heads, - ff_hidden_mult, - depth, - num_classes, - dropout=0.0, - l1_reg=0, - pos_encoding=True, - dropout_att=0.0, - *args, - **kwargs, + self, + input_size, + hidden, + heads, + ff_hidden_mult, + depth, + num_classes, + dropout=0.0, + l1_reg=0, + pos_encoding=True, + dropout_att=0.0, + *args, + **kwargs, ): super().__init__( input_size=input_size, @@ -82,20 +82,20 @@ class LocalTransformer(BaseTransformer): _supported_run_modes = [RunMode.classification, RunMode.regression] def __init__( - self, - input_size, - hidden, - heads, - ff_hidden_mult, - depth, - num_classes, - *args, - dropout=0.0, - l1_reg=0, - pos_encoding=True, - local_context=1, - dropout_att=0.0, - **kwargs, + self, + input_size, + hidden, + heads, + ff_hidden_mult, + depth, + num_classes, + *args, + dropout=0.0, + l1_reg=0, + pos_encoding=True, + local_context=1, + dropout_att=0.0, + **kwargs, ): super().__init__( input_size=input_size, diff --git a/icu_benchmarks/models/ml_models/catboost.py b/icu_benchmarks/models/ml_models/catboost.py index 95bff06d..4bbebea1 100644 --- a/icu_benchmarks/models/ml_models/catboost.py +++ b/icu_benchmarks/models/ml_models/catboost.py @@ -9,7 +9,7 @@ class CBClassifier(MLWrapper): _supported_run_modes = [RunMode.classification] def __init__(self, task_type="CPU", *args, **kwargs): - model_kwargs = {'task_type': task_type, **kwargs} + model_kwargs = {"task_type": task_type, **kwargs} self.model = self.set_model_args(cb.CatBoostClassifier, *args, **model_kwargs) super().__init__(*args, **kwargs) diff --git a/icu_benchmarks/models/ml_models/xgboost.py b/icu_benchmarks/models/ml_models/xgboost.py index 118e2e6e..7452141a 100644 --- a/icu_benchmarks/models/ml_models/xgboost.py +++ b/icu_benchmarks/models/ml_models/xgboost.py @@ -85,15 +85,15 @@ def test_step(self, dataset, _): def set_model_args(self, model, *args, **kwargs): """XGBoost signature does not include the hyperparams so we need to pass them manually.""" signature = inspect.signature(model.__init__).parameters - possible_hps = list(signature.keys()) - # Get passed keyword arguments - arguments = locals()["kwargs"] - # Get valid hyperparameters - hyperparams = arguments - logging.debug(f"Creating model with: {hyperparams}.") - return model(**hyperparams) + valid_params = signature.keys() + + # Filter out invalid arguments + valid_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + + logging.debug(f"Creating model with: {valid_kwargs}.") + return model(**valid_kwargs) def get_feature_importance(self): - if not hasattr(self.model, 'feature_importances_'): + if not hasattr(self.model, "feature_importances_"): raise ValueError("Model has not been fit yet. Call fit_model() before getting feature importances.") return self.model.feature_importances_ diff --git a/icu_benchmarks/models/train.py b/icu_benchmarks/models/train.py index e53d7700..7bc8e45a 100644 --- a/icu_benchmarks/models/train.py +++ b/icu_benchmarks/models/train.py @@ -145,7 +145,7 @@ def train_common( trainer = Trainer( max_epochs=epochs if model.requires_backprop else 1, - min_epochs=1, # We need at least one epoch to get results. + min_epochs=1, # We need at least one epoch to get results. callbacks=callbacks, precision=precision, accelerator="auto" if not cpu else "cpu", @@ -154,7 +154,7 @@ def train_common( benchmark=not reproducible, enable_progress_bar=verbose, logger=loggers, - num_sanity_val_steps=2, # Helps catch errors in the validation loop before training begins. + num_sanity_val_steps=2, # Helps catch errors in the validation loop before training begins. log_every_n_steps=5, ) if not eval_only: diff --git a/icu_benchmarks/models/wrappers.py b/icu_benchmarks/models/wrappers.py index 8fb08858..310f9fe6 100644 --- a/icu_benchmarks/models/wrappers.py +++ b/icu_benchmarks/models/wrappers.py @@ -261,8 +261,6 @@ def __init__( self.output_transform = None self.loss_weights = None - - def set_metrics(self, *args): """Set the evaluation metrics for the prediction model.""" @@ -477,9 +475,7 @@ def log_metrics(self, label, pred, metric_type): self.log_dict(confusion_matrix(self.label_transform(label), self.output_transform(pred)), sync_dist=True) self.log_dict( { - f"{metric_type}/{name}": ( - metric(self.label_transform(label), self.output_transform(pred)) - ) + f"{metric_type}/{name}": (metric(self.label_transform(label), self.output_transform(pred))) # For every metric for name, metric in self.metrics.items() # Filter out metrics that return a tuple (e.g. precision_recall_curve) @@ -488,6 +484,7 @@ def log_metrics(self, label, pred, metric_type): }, sync_dist=True, ) + def _explain_model(self, test_rep, test_label): if self.explainer is not None: self.test_shap_values = self.explainer(test_rep) @@ -503,6 +500,7 @@ def _save_model_outputs(self, pred_indicators, test_pred, test_label): logging.debug(f"Saved row indicators to {Path(self.logger.save_dir) / f'row_indicators.csv'}") else: logging.warning("Could not save row indicators.") + def configure_optimizers(self): return None diff --git a/icu_benchmarks/run.py b/icu_benchmarks/run.py index 187de3c1..1c5479a0 100644 --- a/icu_benchmarks/run.py +++ b/icu_benchmarks/run.py @@ -61,7 +61,8 @@ def main(my_args=tuple(sys.argv[1:])): if task not in tasks or model not in models: raise ValueError( f"Invalid task or model. Task: {task} {'not ' if task not in tasks else ''} found. " - f"Model: {model} {'not ' if model not in models else ''}found.") + f"Model: {model} {'not ' if model not in models else ''}found." + ) # Load task config gin.parse_config_file(f"configs/tasks/{task}.gin") mode = get_mode() diff --git a/icu_benchmarks/run_utils.py b/icu_benchmarks/run_utils.py index e35380e4..97b50f8c 100644 --- a/icu_benchmarks/run_utils.py +++ b/icu_benchmarks/run_utils.py @@ -15,8 +15,6 @@ from statistics import mean, pstdev from icu_benchmarks.models.utils import JsonResultLoggingEncoder from icu_benchmarks.wandb_utils import wandb_log -import os -import glob import polars as pl @@ -109,7 +107,6 @@ def aggregate_results(log_dir: Path, execution_time: timedelta = None): """ aggregated = {} shap_values_test = [] - # shap_values_train = [] for repetition in log_dir.iterdir(): if repetition.is_dir(): aggregated[repetition.name] = {} @@ -285,6 +282,7 @@ def get_config_files(config_dir: Path): logging.info(f"Found models: {models}") return tasks, models + def check_required_keys(vars, required_keys): """ Checks if all required keys are present in the vars dictionary. @@ -298,4 +296,4 @@ def check_required_keys(vars, required_keys): """ missing_keys = [key for key in required_keys if key not in vars] if missing_keys: - raise KeyError(f"Missing required keys in vars: {', '.join(missing_keys)}") \ No newline at end of file + raise KeyError(f"Missing required keys in vars: {', '.join(missing_keys)}") diff --git a/icu_benchmarks/tuning/hyperparameters.py b/icu_benchmarks/tuning/hyperparameters.py index c835cf8d..84027ddc 100644 --- a/icu_benchmarks/tuning/hyperparameters.py +++ b/icu_benchmarks/tuning/hyperparameters.py @@ -63,8 +63,9 @@ def choose_and_bind_hyperparameters_scikit_optimize( Raises: ValueError: If checkpoint is not None and the checkpoint does not exist. """ - logging.warning("This function is deprecated and will be removed in the future. " - "Use choose_and_bind_hyperparameters_optuna instead.") + logging.warning( + "This function is deprecated and will be removed in the future. " "Use choose_and_bind_hyperparameters_optuna instead." + ) hyperparams = {} if len(scopes) == 0 or folds_to_tune_on is None: @@ -243,6 +244,7 @@ def objective(trial, hyperparams_bounds, hyperparams_names): logging.info(f"Bounds: {hyperparams_bounds}, Names: {hyperparams_names}") for name, value in zip(hyperparams_names, hyperparams_bounds): if isinstance(value, tuple): + def suggest_int_param(trial, name, value): return trial.suggest_int(name, value[0], value[1], log=value[2] == "log" if len(value) == 3 else False) From 9a4ab8ede19b29409e484c7b937253f2a46d909d Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 17 Oct 2024 17:03:20 +0200 Subject: [PATCH 200/207] added checks --- icu_benchmarks/data/split_process_data.py | 6 ++++-- icu_benchmarks/models/ml_models/xgboost.py | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/icu_benchmarks/data/split_process_data.py b/icu_benchmarks/data/split_process_data.py index cf5a1401..a47c9729 100644 --- a/icu_benchmarks/data/split_process_data.py +++ b/icu_benchmarks/data/split_process_data.py @@ -39,6 +39,8 @@ def preprocess_data( complete_train: bool = False, runmode: RunMode = RunMode.classification, label: str = None, + required_var_types=["GROUP", "SEQUENCE", "LABEL"], + required_segments=[Segment.static, Segment.dynamic, Segment.outcome], ) -> dict[dict[pl.DataFrame]] or dict[dict[pd.DataFrame]]: """Perform loading, splitting, imputing and normalising of task data. @@ -67,8 +69,8 @@ def preprocess_data( """ cache_dir = data_dir / "cache" - required_keys = ["GROUP", "SEQUENCE", "LABEL"] - check_required_keys(vars, required_keys) + check_required_keys(vars, required_var_types) + check_required_keys(file_names, required_segments) if not use_static: file_names.pop(Segment.static) vars.pop(Segment.static) diff --git a/icu_benchmarks/models/ml_models/xgboost.py b/icu_benchmarks/models/ml_models/xgboost.py index 7452141a..3fa2e9de 100644 --- a/icu_benchmarks/models/ml_models/xgboost.py +++ b/icu_benchmarks/models/ml_models/xgboost.py @@ -14,6 +14,7 @@ from icu_benchmarks.constants import RunMode from icu_benchmarks.models.wrappers import MLWrapper + # Uncomment if needed in the future # from optuna.integration import XGBoostPruningCallback @@ -73,7 +74,7 @@ def test_step(self, dataset, _): pred_indicators = np.hstack((pred_indicators, test_pred)) # Save as: id, time (hours), ground truth, prediction 0, prediction 1 np.savetxt(os.path.join(self.logger.save_dir, "pred_indicators.csv"), pred_indicators, delimiter=",") - logging.debug(f"Saved row indicators to {os.path.join(self.logger.save_dir,f'row_indicators.csv')}") + logging.debug(f"Saved row indicators to {os.path.join(self.logger.save_dir, f'row_indicators.csv')}") if self.mps: self.log("test/loss", np.float32(self.loss(test_label, test_pred)), sync_dist=True) self.log_metrics(np.float32(test_label), np.float32(test_pred), "test") From 5004ae4d277e9cd98b519838f08b2b3ce62cf4b9 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 17 Oct 2024 17:05:36 +0200 Subject: [PATCH 201/207] Increase allowed complexity --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index acc5ed1f..38e2c473 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # the GitHub editor is 127 chars wide - flake8 . --count --max-complexity=20 --max-line-length=127 --statistics + flake8 . --count --max-complexity=30 --max-line-length=127 --statistics # - name: Test with pytest # run: python -m pytest ./tests/recipes # If we want to test running the tool later on From 619937c0635ff617a401c0ccb29440df7942ccf3 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 17 Oct 2024 17:06:40 +0200 Subject: [PATCH 202/207] docs --- icu_benchmarks/tuning/hyperparameters.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/icu_benchmarks/tuning/hyperparameters.py b/icu_benchmarks/tuning/hyperparameters.py index 84027ddc..c879bd62 100644 --- a/icu_benchmarks/tuning/hyperparameters.py +++ b/icu_benchmarks/tuning/hyperparameters.py @@ -199,7 +199,8 @@ def choose_and_bind_hyperparameters_optuna( """Choose hyperparameters to tune and bind them to gin. Uses Optuna for hyperparameter optimization. Args: - sampler: + plot: Whether to plot hyperparameter importances. + sampler: The sampler to use for hyperparameter optimization. wandb: Whether we use wandb or not. load_cache: Load cached data if available. generate_cache: Generate cache data. From 0133dbde6861d8aafb6aaaedd64b707a31d028b7 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 17 Oct 2024 17:44:15 +0200 Subject: [PATCH 203/207] Transformer refactor to reduce code duplication --- .../models/dl_models/transformer.py | 141 +++++------------- 1 file changed, 41 insertions(+), 100 deletions(-) diff --git a/icu_benchmarks/models/dl_models/transformer.py b/icu_benchmarks/models/dl_models/transformer.py index e1f7fb73..d5415613 100644 --- a/icu_benchmarks/models/dl_models/transformer.py +++ b/icu_benchmarks/models/dl_models/transformer.py @@ -8,49 +8,28 @@ class BaseTransformer(DLPredictionWrapper): _supported_run_modes = [RunMode.classification, RunMode.regression] - - def forward(self, x): - x = self.input_embedding(x) - if self.pos_encoder is not None: - x = self.pos_encoder(x) - x = self.tblocks(x) - pred = self.logit(x) - return pred - - -@gin.configurable -class Transformer(BaseTransformer): - """Transformer model as defined by the HiRID-Benchmark (https://github.com/ratschlab/HIRID-ICU-Benchmark).""" + """Refactored Transformer model as defined by the HiRID-Benchmark (https://github.com/ratschlab/HIRID-ICU-Benchmark).""" def __init__( - self, - input_size, - hidden, - heads, - ff_hidden_mult, - depth, - num_classes, - dropout=0.0, - l1_reg=0, - pos_encoding=True, - dropout_att=0.0, - *args, - **kwargs, - ): - super().__init__( - input_size=input_size, - hidden=hidden, - heads=heads, - ff_hidden_mult=ff_hidden_mult, - depth=depth, - num_classes=num_classes, + self, + block_class, + input_size, + hidden, + heads, + ff_hidden_mult, + depth, + num_classes, + dropout=0.0, + l1_reg=0, + pos_encoding=True, + dropout_att=0.0, + local_context=None, *args, - dropout=dropout, - l1_reg=l1_reg, - pos_encoding=pos_encoding, - dropout_att=dropout_att, **kwargs, - ) + ): + super().__init__(*args, **kwargs) + if local_context is not None and self._get_name() == "Transformer": + raise ValueError("Local context is only supported for LocalTransformer") hidden = hidden if hidden % 2 == 0 else hidden + 1 # Make sure hidden is even self.input_embedding = nn.Linear(input_size[2], hidden) # This acts as a time-distributed layer by defaults if pos_encoding: @@ -58,10 +37,10 @@ def __init__( else: self.pos_encoder = None - tblocks = [] - for i in range(depth): - tblocks.append( - TransformerBlock( + t_blocks = [] + for _ in range(depth): + t_blocks.append( + block_class( emb=hidden, hidden=hidden, heads=heads, @@ -69,72 +48,34 @@ def __init__( ff_hidden_mult=ff_hidden_mult, dropout=dropout, dropout_att=dropout_att, + **({"local_context": local_context} if local_context is not None else {}), ) ) - self.tblocks = nn.Sequential(*tblocks) + self.t_blocks = nn.Sequential(*t_blocks) self.logit = nn.Linear(hidden, num_classes) self.l1_reg = l1_reg + def forward(self, x): + x = self.input_embedding(x) + if self.pos_encoder is not None: + x = self.pos_encoder(x) + x = self.t_blocks(x) + pred = self.logit(x) + return pred + @gin.configurable -class LocalTransformer(BaseTransformer): - _supported_run_modes = [RunMode.classification, RunMode.regression] +class Transformer(BaseTransformer): + """Transformer model.""" - def __init__( - self, - input_size, - hidden, - heads, - ff_hidden_mult, - depth, - num_classes, - *args, - dropout=0.0, - l1_reg=0, - pos_encoding=True, - local_context=1, - dropout_att=0.0, - **kwargs, - ): - super().__init__( - input_size=input_size, - hidden=hidden, - heads=heads, - ff_hidden_mult=ff_hidden_mult, - depth=depth, - num_classes=num_classes, - *args, - dropout=dropout, - l1_reg=l1_reg, - pos_encoding=pos_encoding, - local_context=local_context, - dropout_att=dropout_att, - **kwargs, - ) + def __init__(self, *kwargs, **args): + super().__init__(TransformerBlock, *kwargs, **args) - hidden = hidden if hidden % 2 == 0 else hidden + 1 # Make sure hidden is even - self.input_embedding = nn.Linear(input_size[2], hidden) # This acts as a time-distributed layer by defaults - if pos_encoding: - self.pos_encoder = PositionalEncoding(hidden) - else: - self.pos_encoder = None - tblocks = [] - for i in range(depth): - tblocks.append( - LocalBlock( - emb=hidden, - hidden=hidden, - heads=heads, - mask=True, - ff_hidden_mult=ff_hidden_mult, - local_context=local_context, - dropout=dropout, - dropout_att=dropout_att, - ) - ) +@gin.configurable +class LocalTransformer(BaseTransformer): + """Transformer model with local context.""" - self.tblocks = nn.Sequential(*tblocks) - self.logit = nn.Linear(hidden, num_classes) - self.l1_reg = l1_reg + def __init__(self, *kwargs, **args): + super().__init__(LocalBlock, **args, *kwargs) From b3d7ad03af895d0900da6bc5899f1e666674cf8d Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 17 Oct 2024 17:48:06 +0200 Subject: [PATCH 204/207] transformer fix --- .../models/dl_models/transformer.py | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/icu_benchmarks/models/dl_models/transformer.py b/icu_benchmarks/models/dl_models/transformer.py index d5415613..ed7d4a2c 100644 --- a/icu_benchmarks/models/dl_models/transformer.py +++ b/icu_benchmarks/models/dl_models/transformer.py @@ -11,21 +11,21 @@ class BaseTransformer(DLPredictionWrapper): """Refactored Transformer model as defined by the HiRID-Benchmark (https://github.com/ratschlab/HIRID-ICU-Benchmark).""" def __init__( - self, - block_class, - input_size, - hidden, - heads, - ff_hidden_mult, - depth, - num_classes, - dropout=0.0, - l1_reg=0, - pos_encoding=True, - dropout_att=0.0, - local_context=None, - *args, - **kwargs, + self, + block_class, + input_size, + hidden, + heads, + ff_hidden_mult, + depth, + num_classes, + dropout=0.0, + l1_reg=0, + pos_encoding=True, + dropout_att=0.0, + local_context=None, + *args, + **kwargs, ): super().__init__(*args, **kwargs) if local_context is not None and self._get_name() == "Transformer": @@ -78,4 +78,4 @@ class LocalTransformer(BaseTransformer): """Transformer model with local context.""" def __init__(self, *kwargs, **args): - super().__init__(LocalBlock, **args, *kwargs) + super().__init__(LocalBlock, *kwargs, **args) From 23d2c682b0040abedd0a154018d8b31f36a4e720 Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 17 Oct 2024 17:48:20 +0200 Subject: [PATCH 205/207] optuna integration --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 69814bd3..e48f4d06 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,7 +25,7 @@ tqdm==4.66.3 einops==0.6.1 hydra-core==1.3 optuna==4.0.0 -optuna-integration==3.6.0 +optuna-integration==4.0.0 wandb==0.17.3 recipies==1.0 #Fixed version because of NumPy incompatibility and stale development status. From 18175d99edc17b2464d2bd5925bc26f655bae0dd Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 17 Oct 2024 17:49:36 +0200 Subject: [PATCH 206/207] removed superfluous test step (integrated in main wrapper now) --- icu_benchmarks/models/ml_models/xgboost.py | 26 +--------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/icu_benchmarks/models/ml_models/xgboost.py b/icu_benchmarks/models/ml_models/xgboost.py index 3fa2e9de..af8adb93 100644 --- a/icu_benchmarks/models/ml_models/xgboost.py +++ b/icu_benchmarks/models/ml_models/xgboost.py @@ -25,7 +25,7 @@ class XGBClassifier(MLWrapper): _explain_values = False def __init__(self, *args, **kwargs): - self.model = self.set_model_args(xgb.XGBClassifier, device="cpu", *args, **kwargs) + self.model = self.set_model_args(xgb.XGBClassifier, *args, **kwargs, device="cpu") super().__init__(*args, **kwargs) def predict(self, features): @@ -59,30 +59,6 @@ def fit_model(self, train_data, train_labels, val_data, val_labels): eval_score = mean(next(iter(self.model.evals_result_["validation_0"].values()))) return eval_score # , callbacks=callbacks) - def test_step(self, dataset, _): - # Pred indicators indicates the row id, and time in hours - test_rep, test_label, pred_indicators = dataset - test_rep, test_label, pred_indicators = ( - test_rep.squeeze().cpu().numpy(), - test_label.squeeze().cpu().numpy(), - pred_indicators.squeeze().cpu().numpy(), - ) - self.set_metrics(test_label) - test_pred = self.predict(test_rep) - if len(pred_indicators.shape) > 1 and len(test_pred.shape) > 1 and pred_indicators.shape[1] == test_pred.shape[1]: - pred_indicators = np.hstack((pred_indicators, test_label.reshape(-1, 1))) - pred_indicators = np.hstack((pred_indicators, test_pred)) - # Save as: id, time (hours), ground truth, prediction 0, prediction 1 - np.savetxt(os.path.join(self.logger.save_dir, "pred_indicators.csv"), pred_indicators, delimiter=",") - logging.debug(f"Saved row indicators to {os.path.join(self.logger.save_dir, f'row_indicators.csv')}") - if self.mps: - self.log("test/loss", np.float32(self.loss(test_label, test_pred)), sync_dist=True) - self.log_metrics(np.float32(test_label), np.float32(test_pred), "test") - else: - self.log("test/loss", self.loss(test_label, test_pred), sync_dist=True) - self.log_metrics(test_label, test_pred, "test") - logging.debug(f"Test loss: {self.loss(test_label, test_pred)}") - def set_model_args(self, model, *args, **kwargs): """XGBoost signature does not include the hyperparams so we need to pass them manually.""" signature = inspect.signature(model.__init__).parameters From 53729711955c888e0d6ec9be511e744274f5c6fc Mon Sep 17 00:00:00 2001 From: rvandewater Date: Thu, 17 Oct 2024 17:53:46 +0200 Subject: [PATCH 207/207] xgboost linting --- icu_benchmarks/models/ml_models/xgboost.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/icu_benchmarks/models/ml_models/xgboost.py b/icu_benchmarks/models/ml_models/xgboost.py index af8adb93..5ca738ac 100644 --- a/icu_benchmarks/models/ml_models/xgboost.py +++ b/icu_benchmarks/models/ml_models/xgboost.py @@ -1,10 +1,8 @@ import inspect import logging -import os from statistics import mean import gin -import numpy as np import shap import wandb import xgboost as xgb