Notebooks/Auxiliary/model_aggregated_3domainFeatures_family.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Aggregated model - family - test set: 3 domain features\n",
    "## Table of contents \n",
    "1. [Linear Regression](#LinearRegression)\n",
    "2. [MLP (Dense)](#MLP)\n",
    "3. [AE combined latent](#AE_combined)\n",
    "4. [AE OTU latent](#AE_latentOTU)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('../../Src/')\n",
    "from data import *\n",
    "from train_2 import *\n",
    "from transfer_learning import *\n",
    "from test_functions import *\n",
    "from layers import *\n",
    "from utils import *\n",
    "from loss import *\n",
    "from metric import *\n",
    "from results import *\n",
    "import tensorflow as tf\n",
    "import tensorflow.keras as keras\n",
    "from tensorflow.keras import layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_microbioma_train, df_microbioma_test, _, _, \\\n",
    "df_domain_train, df_domain_test, _, _, otu_columns, domain_columns = \\\n",
    "    read_df_with_transfer_learning_subset_fewerDomainFeatures( \\\n",
    "        metadata_names=['age','Temperature','Precipitation3Days'],\n",
    "        otu_filename='../../Datasets/Aggregated/otu_table_Family.csv',\n",
    "        metadata_filename='../../Datasets/Aggregated/metadata_table_all_80.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df_microbioma_train.shape[1])\n",
    "print(df_microbioma_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('TRAIN:')\n",
    "print('age:' + str(df_domain_train.loc[:,'age'].mean()))\n",
    "print('rain:' + str(df_domain_train.loc[:,'Precipitation3Days'].mean()))\n",
    "print('Tª:' + str(df_domain_train.loc[:,'Temperature'].mean()))\n",
    "    \n",
    "print('TEST:')\n",
    "print('age:' + str(df_domain_test.loc[:,'age'].mean()))\n",
    "print('rain:' + str(df_domain_test.loc[:,'Precipitation3Days'].mean()))\n",
    "print('Tª:' + str(df_domain_test.loc[:,'Temperature'].mean()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get numpy objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_microbioma_train = df_microbioma_train.to_numpy(dtype=np.float32)\n",
    "data_microbioma_test = df_microbioma_test.to_numpy(dtype=np.float32)\n",
    "data_domain_train = df_domain_train.to_numpy(dtype=np.float32)\n",
    "data_domain_test = df_domain_test.to_numpy(dtype=np.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute default error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Absolute abundance transformed to TSS (with epsilon=1E-6)\n",
    "def transform_to_rel_abundance(dataset):\n",
    "    epsilon=1E-6\n",
    "    sum_per_sample = dataset.sum(axis=1)\n",
    "    num_samples = sum_per_sample.shape\n",
    "    num_OTUs = np.shape(dataset)[-1] \n",
    "    sum_per_sample = sum_per_sample + (num_OTUs * epsilon)\n",
    "    dividend=dataset+epsilon\n",
    "    dataset_rel_abund = np.divide(dividend,sum_per_sample[:,None])\n",
    "    #display(Markdown(\"{}</p>\".format(np.array2string(actual_array,precision=6,floatmode='fixed'))))\n",
    "    #actual_array.sum(axis=1)\n",
    "    return dataset_rel_abund"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Pearson correlation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_microbioma_rel = transform_to_rel_abundance(data_microbioma_train)\n",
    "\n",
    "random_seed=347\n",
    "folds=5\n",
    "tf.random.set_seed(random_seed) # BGJ\n",
    "kf = KFold(n_splits=folds, random_state=random_seed, shuffle=True)\n",
    "tf.random.set_seed(random_seed)\n",
    "tot_cv_r = 0.0\n",
    "for train_index, test_index in kf.split(data_microbioma_rel):\n",
    "    m_train, m_test = data_microbioma_rel[train_index], data_microbioma_rel[test_index]\n",
    "    # prediction = average training samples\n",
    "    pred = data_microbioma_rel[train_index].mean(axis=0)\n",
    "    tot = 0.0\n",
    "    count = 0\n",
    "    for i,actual in enumerate(data_microbioma_rel[test_index]):\n",
    "        r, _ = scipy.stats.pearsonr(actual,pred)\n",
    "        if not np.isnan(r):\n",
    "            count += 1\n",
    "            tot += r\n",
    "    r_cv = tot/count\n",
    "    #print(r_cv)\n",
    "    tot_cv_r += r_cv\n",
    "tot_cv_r/folds     \n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Bray-Curtis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from skbio.diversity import beta_diversity\n",
    "\n",
    "data_microbioma_rel = transform_to_rel_abundance(data_microbioma_train)\n",
    "\n",
    "random_seed=347\n",
    "folds=5\n",
    "tf.random.set_seed(random_seed) # BGJ\n",
    "kf = KFold(n_splits=folds, random_state=random_seed, shuffle=True)\n",
    "tf.random.set_seed(random_seed)\n",
    "tot_cv = 0.0\n",
    "for train_index, test_index in kf.split(data_microbioma_rel):\n",
    "    m_train, m_test = data_microbioma_rel[train_index], data_microbioma_rel[test_index]\n",
    "    # prediction = average training samples\n",
    "    pred = data_microbioma_rel[train_index].mean(axis=0)\n",
    "    tot_bc = 0.0\n",
    "    for i,actual in enumerate(data_microbioma_rel[test_index]):\n",
    "        bc_dm = beta_diversity(\"braycurtis\", [actual,pred]) # Source: http://scikit-bio.org/docs/0.4.2/diversity.html\n",
    "        bc = bc_dm[0,1]\n",
    "        tot_bc += bc\n",
    "    bc_cv = tot_bc/(test_index.shape[0])\n",
    "    #print(bc_cv)\n",
    "    tot_cv += bc_cv\n",
    "tot_cv/folds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Linear regression <a name=\"LinearRegression\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model(shape_in, shape_out, output_transform):\n",
    "    in_layer = layers.Input(shape=(shape_in,))\n",
    "    net = in_layer\n",
    "    net = layers.Dense(shape_out, activation='linear')(net)\n",
    "    if output_transform is not None:\n",
    "        net = output_transform(net)\n",
    "    out_layer = net\n",
    "    \n",
    "    model = keras.Model(inputs=[in_layer], outputs=[out_layer], name='model')\n",
    "    return model\n",
    "\n",
    "def compile_model(model, optimizer, reconstruction_error, input_transform, output_transform):\n",
    "    metrics = get_experiment_metrics(input_transform, output_transform)[0][3:]\n",
    "    model.compile(optimizer=optimizer, loss=reconstruction_error, metrics=metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_fn():\n",
    "    m = model(shape_in=data_domain_train.shape[1],\n",
    "              shape_out=data_microbioma_train.shape[1],\n",
    "              output_transform=None)\n",
    "    \n",
    "    compile_model(model=m,\n",
    "                  optimizer=optimizers.Adam(lr=0.001),\n",
    "                  reconstruction_error=LossMeanSquaredErrorWrapper(CenterLogRatio(), None),\n",
    "                  input_transform=CenterLogRatio(),\n",
    "                  output_transform=None)\n",
    "    return m, None, m, None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "latent_space = 0\n",
    "results, modelsLR = train(model_fn,\n",
    "                        data_microbioma_train,\n",
    "                        data_domain_train,\n",
    "                        latent_space=latent_space,\n",
    "                        folds=5,\n",
    "                        epochs=100,\n",
    "                        batch_size=64,\n",
    "                        learning_rate_scheduler=None,\n",
    "                        verbose=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print_results(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = test_model(modelsLR, CenterLogRatio, None, otu_columns, data_microbioma_test, data_domain_test)\n",
    "#save_predictions(predictions, 'experiment_testSet_linear_regresion_3var.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. MLP (Dense) <a name=\"MLP\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model(shape_in, shape_out, output_transform, layers_list, activation_fn):\n",
    "    in_layer = layers.Input(shape=(shape_in,))\n",
    "    net = in_layer\n",
    "    for s in layers_list:\n",
    "        net = layers.Dense(s, activation=activation_fn)(net)\n",
    "    net = layers.Dense(shape_out, activation='linear')(net)\n",
    "    if output_transform is not None:\n",
    "        net = output_transform(net)\n",
    "    out_layer = net\n",
    "    \n",
    "    model = keras.Model(inputs=[in_layer], outputs=[out_layer], name='model')\n",
    "    return model\n",
    "\n",
    "def compile_model(model, optimizer, reconstruction_error, input_transform, output_transform):\n",
    "    metrics = get_experiment_metrics(input_transform, output_transform)[0][3:]\n",
    "    model.compile(optimizer=optimizer, loss=reconstruction_error, metrics=metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_fn():\n",
    "    m = model(shape_in=data_domain_train.shape[1],\n",
    "              shape_out=data_microbioma_train.shape[1],\n",
    "              output_transform=None,\n",
    "              layers_list=[128,512],\n",
    "              activation_fn='tanh')\n",
    "    \n",
    "    compile_model(model=m,\n",
    "                  optimizer=optimizers.Adam(lr=0.01),\n",
    "                  reconstruction_error=LossMeanSquaredErrorWrapper(CenterLogRatio(), None),\n",
    "                  input_transform=CenterLogRatio(),\n",
    "                  output_transform=None)\n",
    "    return m, None, m, None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "latent_space=0\n",
    "results, modelsMLP = train(model_fn,\n",
    "                        data_microbioma_train,\n",
    "                        data_domain_train,\n",
    "                        latent_space=latent_space,\n",
    "                        folds=5,\n",
    "                        epochs=100,\n",
    "                        batch_size=64,\n",
    "                        learning_rate_scheduler=None,\n",
    "                        verbose=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print_results(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = test_model(modelsMLP, CenterLogRatio, None, otu_columns, data_microbioma_test, data_domain_test)\n",
    "#save_predictions(predictions, 'experiment_testSet_MLP_3var.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Auto-encoder combined latent <a name=\"AE_combined\"></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### To create auto-encoder combined model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train the selected model (the best one from those with the smallest latent space (10)): no.351\n",
    "experiment_metrics, models, results = perform_experiment_2(cv_folds=5, \n",
    "                        epochs=100, \n",
    "                        batch_size=64, \n",
    "                        learning_rate=0.001, \n",
    "                        optimizer=optimizers.Adam,\n",
    "                        learning_rate_scheduler=None,\n",
    "                        input_transform=Percentage,\n",
    "                        output_transform=tf.keras.layers.Softmax,\n",
    "                        reconstruction_loss=MakeLoss(LossBrayCurtis, Percentage, None), \n",
    "                        latent_space=10, \n",
    "                        layers=[512,256],\n",
    "                        #layers=[128],\n",
    "                        activation='tanh', \n",
    "                        activation_latent='tanh', \n",
    "                        data_microbioma_train=data_microbioma_train,\n",
    "                        data_domain_train=data_domain_train,\n",
    "                        show_results=True, \n",
    "                        device='/CPU:0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = test_model_cv_predictions(models, Percentage, tf.keras.layers.Softmax, otu_columns, data_microbioma_test, data_domain_test)\n",
    "#save_predictions(predictions, 'experiment_aggregated_phylum_testSet_AE_combinedLatent_5CV_3var.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Auto-encoder OTU latent <a name=\"AE_latentOTU\"></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train the selected model (the best one from those with the smallest latent space (10)): no.351\n",
    "experiment_metrics, models, results = perform_experiment_2(cv_folds=0, \n",
    "                        epochs=100, \n",
    "                        batch_size=64, \n",
    "                        learning_rate=0.001, \n",
    "                        optimizer=optimizers.Adam,\n",
    "                        learning_rate_scheduler=None,\n",
    "                        input_transform=Percentage,\n",
    "                        output_transform=tf.keras.layers.Softmax,\n",
    "                        reconstruction_loss=MakeLoss(LossBrayCurtis, Percentage, None), \n",
    "                        latent_space=10, \n",
    "                        layers=[512,256],\n",
    "                        #layers=[128],\n",
    "                        activation='tanh', \n",
    "                        activation_latent='tanh', \n",
    "                        data_microbioma_train=data_microbioma_train,\n",
    "                        data_domain_train=None,\n",
    "                        show_results=True, \n",
    "                        device='/CPU:0')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### To get encoders and decoders to use in domain->latent model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model, encoder, _ ,decoder = models[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_domain_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### To predict latent space for samples in domain->latent model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "latent_train = encoder.predict(data_microbioma_train)\n",
    "latent_test = encoder.predict(data_microbioma_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### To build model to predict latent space "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_fn_latent():\n",
    "    in_layer = layers.Input(shape=(data_domain_train.shape[1],))\n",
    "    net = layers.Dense(128, activation='tanh')(in_layer)\n",
    "    net = layers.Dense(64, activation='tanh')(net)\n",
    "    net = layers.Dense(32, activation='tanh')(net)\n",
    "    net = layers.Dense(16, activation='tanh')(net)\n",
    "    out_layer = layers.Dense(latent_train.shape[1], activation=None)(net) # 'tanh already'\n",
    "    model = keras.Model(inputs=[in_layer], outputs=[out_layer], name='model')\n",
    "    model.compile(optimizer=optimizers.Adam(lr=0.001), loss=tf.keras.losses.MeanSquaredError(),\n",
    "                 metrics=[tf.keras.metrics.MeanSquaredError()])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_latent, model_latent = train_tl_noEnsemble(model_fn_latent,\n",
    "                            latent_train,\n",
    "                            latent_train,\n",
    "                            data_domain_train,\n",
    "                            data_domain_train,\n",
    "                            epochs=100,\n",
    "                            batch_size=16,\n",
    "                            verbose=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print_results_noEnsemble(result_latent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Test only Dense(domain->latent)\n",
    "predictions = test_model_tl_latent(model_latent, latent_test, data_domain_test)\n",
    "#save_predictions(predictions, 'experiment_testSet_domain-latent_AE_OTUlatent_3var.txt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Domain -> latent -> microbiome. Test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "predictions = test_model_tl_noEnsemble(model_latent, decoder, Percentage, tf.keras.layers.Softmax, otu_columns, data_microbioma_test, data_domain_test)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}