Run_PREFER.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run PREFER"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook is used to test the benchmarking and property prediction framework."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### WARNING:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In order to run this notebook, please \n",
    "1) use the prefer-environment\n",
    "2) unpack the git submodules within the PREFER repo as described in the README.txt\n",
    "3) Change the config files as described in the README.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![caption](./prefer/docs/PREFER_scheme.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import warnings\n",
    "import pickle\n",
    "import numpy as np\n",
    "import time\n",
    "\n",
    "\n",
    "%load_ext autoreload\n",
    "# path to the main directory\n",
    "path_to_PREFER = 'path_to/PREFER/'\n",
    "# path to submodules\n",
    "path_to_cddd = 'path_to/PREFER/prefer/model_based_representations/models/cddd/'\n",
    "path_to_moler = 'path_to/PREFER/prefer/model_based_representations/models/molecule-generation/'\n",
    "sys.path.append(path_to_PREFER)\n",
    "sys.path.append(path_to_cddd)\n",
    "sys.path.append(path_to_moler)\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from prefer.utils.post_processing_and_optimization_helpers import create_heat_map\n",
    "from prefer.src.prefer_model_wrapper import PreferModelWrapper\n",
    "from prefer.utils.filtering import *\n",
    "from prefer.utils.automation import merge_table_metrics, data_preparation, generate_molecular_representations, run, create_comparison_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def PREFER_job(data_info):\n",
    "    # Prepare data\n",
    "    df = data_preparation(data_info)\n",
    "    # If time split you need to provide time column name\n",
    "    temporal_info_column_name = data_info['temporal_info_column_name']\n",
    "    if (data_info['split_type'] == 'temporal'):\n",
    "        if(not temporal_info_column_name):\n",
    "            raise ValueError(f'ERROR: if time split is required then you need to provide the temporal_info_column_name')\n",
    "    # Extract representations\n",
    "    representations = generate_molecular_representations(df, split_type = data_info['split_type'],\n",
    "                                   experiment_name = data_info['experiment_name'] ,\n",
    "                                   list_of_model_based_representations_paths = data_info['list_of_model_based_representations_paths'])\n",
    "\n",
    "    # Run PREFER\n",
    "    bench_list, dir_destination = run(representations, problem_type = data_info['problem_type'], model_instance = data_info['model_instance'])\n",
    "    # Evaluate results\n",
    "    merged = merge_table_metrics(bench_list)\n",
    "    merged.to_csv('merged.csv')\n",
    "    experiments_dict, tmp_dict = create_comparison_table(merged)\n",
    "    create_heat_map(experiments_dict, tmp_dict)\n",
    "    #create_heat_map_scaled(experiments_dict, tmp_dict)\n",
    "    return bench_list, merged, dir_destination"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set Data Info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# read yaml config file to set the data_info\n",
    "import yaml\n",
    "import json\n",
    "\n",
    "prefer_args = './config_files/config_PREFER_logD.yaml' # OR THE PATH TO YOUR CONFIG FILE\n",
    "a_yaml_file = open(prefer_args)\n",
    "parsed_yaml_file = yaml.load(a_yaml_file, Loader=yaml.FullLoader)\n",
    "\n",
    "path_to_df = parsed_yaml_file[\"path_to_df\"]\n",
    "experiment_name = parsed_yaml_file[\"experiment_name\"]\n",
    "id_column_name = parsed_yaml_file[\"id_column_name\"]\n",
    "smiles_column_name = parsed_yaml_file[\"smiles_column_name\"]\n",
    "properties_column_name = parsed_yaml_file[\"properties_column_name_list\"]\n",
    "problem_type = parsed_yaml_file[\"problem_type\"]\n",
    "splitting_strategy = parsed_yaml_file[\"splitting_strategy\"]\n",
    "\n",
    "if 'model_instance' in parsed_yaml_file:\n",
    "    model_instance = parsed_yaml_file[\"model_instance\"]\n",
    "else:\n",
    "    model_instance = None\n",
    "\n",
    "if \"temporal_info_column_name\" in parsed_yaml_file:\n",
    "    temporal_info_column_name = parsed_yaml_file[\"temporal_info_column_name\"]\n",
    "else:\n",
    "    temporal_info_column_name = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "data_info = {'path_to_data': path_to_df,\n",
    "             'experiment_name': experiment_name,\n",
    "             'id_column_name':id_column_name,\n",
    "             'model_instance' : model_instance,\n",
    "             'problem_type': problem_type,\n",
    "             'smiles_column_name':smiles_column_name,\n",
    "             'split_type': splitting_strategy,\n",
    "             'temporal_info_column_name': temporal_info_column_name,\n",
    "             'properties_column_name_list':properties_column_name, \n",
    "            'list_of_model_based_representations_paths': []}\n",
    "\n",
    "# To store the info related to model based representations\n",
    "dict_commands = dict()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute CDDD representations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### WARNING: Before running this please install the cddd-env-light as described in the README.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Preparing the shell commands to compute the data based molecular representations\n",
    "path_to_cddd_model = 'path_to/cddd/default_model'\n",
    "model_name = 'CDDD'\n",
    "# write commands that should run as subprocess\n",
    "# sometimes you may have a CommandNotFoundError since your shell has not been properly configured to use 'conda actovate'. \n",
    "# In this case you need to add at the beginning of the run_commands string: source path/to/conda.sh;\n",
    "# or try: . path/to/conda.sh;\n",
    "run_commands = f'conda activate cddd-env-prefer-light; PYTHONPATH=\"{path_to_cddd}:{path_to_moler}:{path_to_PREFER}:$PYTHONPATH\"; export PYTHONPATH; python compute_model_based_representations.py --prefer_args {prefer_args} --path_to_model {path_to_cddd_model} --model_name {model_name}'\n",
    "dict_commands[model_name] = dict()\n",
    "dict_commands[model_name]['run'] = run_commands\n",
    "dict_commands[model_name]['path_to_representations'] = f\"./{model_name}_representations_{experiment_name}\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "print('WARNING: in case of troubles with conda activate in the run_commands please follow the instructions in the comment above')\n",
    "!{run_commands}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# find path to the new CDDD representation found\n",
    "import datetime\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "cdddpath = f'./{model_name}_representations_{experiment_name}'\n",
    "files = [f for f in listdir(cdddpath) if isfile(join(cdddpath, f))]\n",
    "collect_dates = []\n",
    "mapping = {}\n",
    "for file in files:\n",
    "    date = file.split('_')[-1]\n",
    "    date = date.replace('.pkl','')\n",
    "    date = datetime.datetime.strptime(date, '%Y%m%d-%H%M%S')\n",
    "    collect_dates.append(date)\n",
    "    mapping[date] = file\n",
    "    \n",
    "collect_dates.sort()\n",
    "data_info['list_of_model_based_representations_paths'].append(f'{cdddpath}/{mapping[collect_dates[-1]]}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute MOLER representations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### WARNING: Before running this please install the moler-env as described in the README.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_to_moler_model = 'path_to/moler/'\n",
    "model_name = 'MOLER'\n",
    "# write commands that should run as subprocess\n",
    "# sometimes you may have a CommandNotFoundError since your shell has not been properly configured to use 'conda actovate'. \n",
    "# In this case you need to add at the beginning of the run_commands string: source path/to/conda.sh;\n",
    "# or try: . path/to/conda.sh;\n",
    "run_commands = f'conda activate moler-env-prefer-light; PYTHONPATH=\"{path_to_cddd}:{path_to_moler}:{path_to_PREFER}:$PYTHONPATH\"; export PYTHONPATH; python compute_model_based_representations.py --prefer_args {prefer_args} --path_to_model {path_to_moler_model} --model_name {model_name}'\n",
    "dict_commands[model_name] = dict()\n",
    "dict_commands[model_name]['run'] = run_commands\n",
    "dict_commands[model_name]['path_to_representations'] = f\"./{model_name}_representations_{experiment_name}\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "print('WARNING: in case of troubles with conda activate in the run_commands please follow the instructions in the comment above')\n",
    "!{run_commands}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# find path to the new MOLER representation found\n",
    "import datetime\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "molerpath = f'./{model_name}_representations_{experiment_name}'\n",
    "files = [f for f in listdir(molerpath) if isfile(join(molerpath, f))]\n",
    "collect_dates = []\n",
    "mapping = {}\n",
    "for file in files:\n",
    "    date = file.split('_')[-1]\n",
    "    date = date.replace('.pkl','')\n",
    "    date = datetime.datetime.strptime(date, '%Y%m%d-%H%M%S')\n",
    "    collect_dates.append(date)\n",
    "    mapping[date] = file\n",
    "    \n",
    "collect_dates.sort()\n",
    "data_info['list_of_model_based_representations_paths'].append(f'{molerpath}/{mapping[collect_dates[-1]]}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run PREFER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "bench_list, merged, dir_destination = PREFER_job(data_info)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save complete dataframe "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from prefer.utils.save_load import saving_procedure_autosklearn\n",
    "for bench in bench_list:\n",
    "    saving_procedure_autosklearn(bench, dir_destination)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Model Wrapper from Benchmarking object and use it to predict new samples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### For each combination of model and molecular representation create a PREFER-wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "timestr = time.strftime(\"%Y%m%d_%H%M%S\")\n",
    "path_to_model_dict = None\n",
    "model_based_representation = False\n",
    "for bench in bench_list:\n",
    "    representation_name = bench.representations[0]\n",
    "    print(f'Preparing wrapper for {representation_name}')\n",
    "    if representation_name == 'CDDD':\n",
    "        path_to_model_dict = dict()\n",
    "        model_based_representation = True\n",
    "        path_to_model_dict[representation_name] = path_to_cddd_model\n",
    "    if representation_name == 'MOLER':\n",
    "        path_to_model_dict = dict()\n",
    "        model_based_representation = True\n",
    "        path_to_model_dict[representation_name] = path_to_moler_model\n",
    "    \n",
    "    arg_dict = dict(\n",
    "        datapath=path_to_df,\n",
    "        friendly_model_name=experiment_name,\n",
    "        id_column_name=id_column_name,\n",
    "        smiles_column_name=smiles_column_name,\n",
    "        properties_column_name_list=properties_column_name,\n",
    "        problem_type=problem_type,  # Can be regression or classification\n",
    "        best_model_output_dir=dir_destination,\n",
    "        representations=[representation_name],\n",
    "        path_to_model=path_to_model_dict,  # this should be set\n",
    "        project_code=\"\",\n",
    "    )\n",
    "    \n",
    "    final_meta_data = arg_dict\n",
    "    bm_rep = representation_name\n",
    "    final_meta_data[\"best_model_representation\"] = bm_rep\n",
    "    final_meta_data[\"desirability_scores\"] = None\n",
    "    \n",
    "    \n",
    "    final_meta_data[\"rep_model_id\"] = bench.models_ids[representation_name]\n",
    "    model = bench.best_estimator[representation_name]\n",
    "    final_meta_data[\"features_scaling_type\"] = bench.features_scaling_type[representation_name]\n",
    "    final_meta_data[\"features_means_vect\"] = bench.features_means_vect[representation_name]\n",
    "    final_meta_data[\"features_stds_vect\"] = bench.features_stds_vect[representation_name]\n",
    "    \n",
    "    # add info needed to compute model based representations\n",
    "    if(model_based_representation):\n",
    "        \n",
    "        final_meta_data[\"prefer_path\"] = path_to_PREFER\n",
    "        final_meta_data[\"dict_commands\"] = dict_commands\n",
    "        \n",
    "    # Store info related to the probability threshold used (e.g. otpimized by GHOSTml) if classification task\n",
    "    if(problem_type == 'classification'):\n",
    "        final_meta_data[\"probability_threshold\"] = bench.metrics[representation_name]['prob_threshold']\n",
    "    \n",
    "    \n",
    "    # take all the train and the test set to refit the autosklearn model\n",
    "    Xtrain, ytrain, Xtest, ytest = bench.molecule_representations_obj_list[0].split()\n",
    "    X_fin = np.concatenate((Xtrain, Xtest), 0)\n",
    "    y_fin = np.concatenate((ytrain, ytest), 0)\n",
    "    print(\"Refitting AutoSklearn model...\")\n",
    "    model.refit(X_fin, y_fin)\n",
    "    print('Refitted!')\n",
    "    wrapper = PreferModelWrapper(model=model, metadata=final_meta_data)\n",
    "    # Save wrapper in final location\n",
    "    if not dir_destination.endswith('/'):\n",
    "        dir_destination = dir_destination+'/'\n",
    "    metadata_name = f\"{dir_destination}{experiment_name}_{representation_name}_{timestr}\"\n",
    "    print(f'Wrapper for {representation_name} model has been stored in {dir_destination}{experiment_name}_{representation_name}_{timestr}')\n",
    "    with open(metadata_name + \".pkl\", \"wb\") as output:\n",
    "        pickle.dump(wrapper, output)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load one wrapper related to one combination of model and molecular representation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "one_stored_wrapper_path = f'{dir_destination}{experiment_name}_{representation_name}_{timestr}'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f'{one_stored_wrapper_path}.pkl', 'rb') as f:\n",
    "    one_stored_wrapper = pickle.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load samples to predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_smiles_samples = list(bench_list[-1].df[representation_name].Smiles.values[0:5])\n",
    "test_smiles_samples"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predict with wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# problem IS that you pass the info of the entire original dataframe and not just the list of smiles you have. Need to r=fix this\n",
    "#import pandas as pd\n",
    "predictions = one_stored_wrapper.predict(test_smiles_samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate Overall Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "merged"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save merged\n",
    "merged.to_csv(f'{dir_destination}_merged_autosklearn_res.csv', index = False)\n",
    "merged.to_pickle(f'{dir_destination}_merged_autosklearn_res.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir_destination"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for bench in bench_list:\n",
    "    bench.plot_res()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inspect Results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get models with weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bench_list[0].representations[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from pprint import pprint\n",
    "for bench in bench_list:\n",
    "    repr_ = bench.representations[0]\n",
    "    print(f'-------------- Results for : {repr_} --------------')\n",
    "    print(f'-----------------------------------------------------')\n",
    "    pprint(bench.best_estimator[repr_].get_models_with_weights(), indent=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute PREFER table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for bench in bench_list:\n",
    "    pprint(bench.create_summary_table())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check the ensemble models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# detail of the models in the ensemble\n",
    "merged.loc['Prediction Model'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check feature preprocessing "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# detail of the feature preprocessor used \n",
    "\n",
    "for bench in bench_list:\n",
    "    repr_ = bench.representations[0]\n",
    "    print(f'-------------- Results for : {repr_} --------------')\n",
    "    print(f'-----------------------------------------------------')\n",
    "    pprint(bench.best_estimator[repr_].leaderboard(detailed = True, top_k= 50)['feature_preprocessors'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inspect all the evaluated models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bench_list[0].best_estimator[bench_list[0].representations[0]].leaderboard(detailed = True, ensemble_only= False, top_k= 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bench_list[1].best_estimator[bench_list[1].representations[0]].leaderboard(detailed = True, ensemble_only= False, top_k= 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "genchem-moler",
   "language": "python",
   "name": "genchem-moler"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}