gtex_v8.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Direct ingestion of GTEx V8\n",
    "\n",
    "Expression Atlas only provides GTEx V7, released 2016-01-15. In this notebook, we ingest GTEx V8, released 2017-06-05. As of 2023-09-29, this is the latest available version. Release notes and differences between tissues can be found here: https://gtexportal.org/home/releaseInfoPage.\n",
    "\n",
    "## 1. Prepare the environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mDEPRECATION: flatbuffers 1.12.1-git20200711.33e2d80-dfsg1-0.6 has a non-standard version number. pip 23.3 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of flatbuffers or contact the author to suggest that they release a version with a conforming version number. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "! python3 -m venv env\n",
    "! source env/bin/activate\n",
    "! pip -q install --upgrade numpy pandas rpy2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The rpy2.ipython extension is already loaded. To reload it, use:\n",
      "  %reload_ext rpy2.ipython\n"
     ]
    }
   ],
   "source": [
    "import gzip\n",
    "import json\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "# Enables the %%R magic.\n",
    "%load_ext rpy2.ipython"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Download, ingest, and examine the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "! wget -q -O gtex_v8.gct.gz \"https://storage.googleapis.com/gtex_analysis_v8/rna_seq_data/GTEx_Analysis_2017-06-05_v8_RNASeQCv1.1.9_gene_median_tpm.gct.gz\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Adipose - Subcutaneous</th>\n",
       "      <th>Adipose - Visceral (Omentum)</th>\n",
       "      <th>Adrenal Gland</th>\n",
       "      <th>Artery - Aorta</th>\n",
       "      <th>Artery - Coronary</th>\n",
       "      <th>Artery - Tibial</th>\n",
       "      <th>Bladder</th>\n",
       "      <th>Brain - Amygdala</th>\n",
       "      <th>Brain - Anterior cingulate cortex (BA24)</th>\n",
       "      <th>Brain - Caudate (basal ganglia)</th>\n",
       "      <th>...</th>\n",
       "      <th>Skin - Not Sun Exposed (Suprapubic)</th>\n",
       "      <th>Skin - Sun Exposed (Lower leg)</th>\n",
       "      <th>Small Intestine - Terminal Ileum</th>\n",
       "      <th>Spleen</th>\n",
       "      <th>Stomach</th>\n",
       "      <th>Testis</th>\n",
       "      <th>Thyroid</th>\n",
       "      <th>Uterus</th>\n",
       "      <th>Vagina</th>\n",
       "      <th>Whole Blood</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ENSG00000223972</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.166403</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000227232</th>\n",
       "      <td>4.064030</td>\n",
       "      <td>3.371110</td>\n",
       "      <td>2.68549</td>\n",
       "      <td>4.04762</td>\n",
       "      <td>3.90076</td>\n",
       "      <td>3.639630</td>\n",
       "      <td>5.16375</td>\n",
       "      <td>1.43859</td>\n",
       "      <td>1.69285</td>\n",
       "      <td>1.566050</td>\n",
       "      <td>...</td>\n",
       "      <td>5.932980</td>\n",
       "      <td>6.13265</td>\n",
       "      <td>4.193780</td>\n",
       "      <td>5.926310</td>\n",
       "      <td>3.062480</td>\n",
       "      <td>4.702530</td>\n",
       "      <td>6.272550</td>\n",
       "      <td>7.190010</td>\n",
       "      <td>5.745540</td>\n",
       "      <td>2.64743</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000278267</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000243485</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.024264</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.054223</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000237613</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000198695</th>\n",
       "      <td>3517.220000</td>\n",
       "      <td>3598.090000</td>\n",
       "      <td>6637.79000</td>\n",
       "      <td>6374.49000</td>\n",
       "      <td>4972.73000</td>\n",
       "      <td>4635.000000</td>\n",
       "      <td>8275.20000</td>\n",
       "      <td>9631.77000</td>\n",
       "      <td>8319.14000</td>\n",
       "      <td>11516.600000</td>\n",
       "      <td>...</td>\n",
       "      <td>8919.460000</td>\n",
       "      <td>8113.22000</td>\n",
       "      <td>9764.490000</td>\n",
       "      <td>5477.360000</td>\n",
       "      <td>9628.750000</td>\n",
       "      <td>1676.590000</td>\n",
       "      <td>2520.410000</td>\n",
       "      <td>3644.000000</td>\n",
       "      <td>2237.430000</td>\n",
       "      <td>960.29100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000210194</th>\n",
       "      <td>11.903800</td>\n",
       "      <td>11.203400</td>\n",
       "      <td>24.41390</td>\n",
       "      <td>22.65940</td>\n",
       "      <td>20.58800</td>\n",
       "      <td>18.257600</td>\n",
       "      <td>27.50560</td>\n",
       "      <td>40.58860</td>\n",
       "      <td>31.35490</td>\n",
       "      <td>78.695300</td>\n",
       "      <td>...</td>\n",
       "      <td>32.932100</td>\n",
       "      <td>31.33400</td>\n",
       "      <td>35.456000</td>\n",
       "      <td>24.934900</td>\n",
       "      <td>33.496300</td>\n",
       "      <td>2.871770</td>\n",
       "      <td>5.795670</td>\n",
       "      <td>10.370100</td>\n",
       "      <td>5.724510</td>\n",
       "      <td>3.49736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000198727</th>\n",
       "      <td>23839.100000</td>\n",
       "      <td>25511.700000</td>\n",
       "      <td>28799.40000</td>\n",
       "      <td>12919.40000</td>\n",
       "      <td>15943.20000</td>\n",
       "      <td>13051.500000</td>\n",
       "      <td>23663.20000</td>\n",
       "      <td>32931.80000</td>\n",
       "      <td>28346.70000</td>\n",
       "      <td>40960.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>19058.200000</td>\n",
       "      <td>17223.10000</td>\n",
       "      <td>24521.300000</td>\n",
       "      <td>11578.600000</td>\n",
       "      <td>27193.100000</td>\n",
       "      <td>15337.800000</td>\n",
       "      <td>19868.500000</td>\n",
       "      <td>15025.000000</td>\n",
       "      <td>12803.400000</td>\n",
       "      <td>2305.49000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000210195</th>\n",
       "      <td>0.616205</td>\n",
       "      <td>0.697191</td>\n",
       "      <td>1.50928</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.544659</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>3.44476</td>\n",
       "      <td>3.47988</td>\n",
       "      <td>5.581470</td>\n",
       "      <td>...</td>\n",
       "      <td>0.884701</td>\n",
       "      <td>1.11364</td>\n",
       "      <td>0.815455</td>\n",
       "      <td>0.783492</td>\n",
       "      <td>0.803858</td>\n",
       "      <td>0.443781</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000210196</th>\n",
       "      <td>1.429970</td>\n",
       "      <td>1.192550</td>\n",
       "      <td>1.78041</td>\n",
       "      <td>1.57309</td>\n",
       "      <td>1.63648</td>\n",
       "      <td>1.950040</td>\n",
       "      <td>1.47692</td>\n",
       "      <td>5.67736</td>\n",
       "      <td>5.40242</td>\n",
       "      <td>7.071810</td>\n",
       "      <td>...</td>\n",
       "      <td>2.226800</td>\n",
       "      <td>1.88828</td>\n",
       "      <td>1.768040</td>\n",
       "      <td>0.948262</td>\n",
       "      <td>2.082370</td>\n",
       "      <td>1.057760</td>\n",
       "      <td>0.982817</td>\n",
       "      <td>0.808096</td>\n",
       "      <td>0.737385</td>\n",
       "      <td>0.00000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>56200 rows × 54 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Adipose - Subcutaneous  Adipose - Visceral (Omentum)  \\\n",
       "Name                                                                    \n",
       "ENSG00000223972                0.000000                      0.000000   \n",
       "ENSG00000227232                4.064030                      3.371110   \n",
       "ENSG00000278267                0.000000                      0.000000   \n",
       "ENSG00000243485                0.000000                      0.000000   \n",
       "ENSG00000237613                0.000000                      0.000000   \n",
       "...                                 ...                           ...   \n",
       "ENSG00000198695             3517.220000                   3598.090000   \n",
       "ENSG00000210194               11.903800                     11.203400   \n",
       "ENSG00000198727            23839.100000                  25511.700000   \n",
       "ENSG00000210195                0.616205                      0.697191   \n",
       "ENSG00000210196                1.429970                      1.192550   \n",
       "\n",
       "                 Adrenal Gland  Artery - Aorta  Artery - Coronary  \\\n",
       "Name                                                                \n",
       "ENSG00000223972        0.00000         0.00000            0.00000   \n",
       "ENSG00000227232        2.68549         4.04762            3.90076   \n",
       "ENSG00000278267        0.00000         0.00000            0.00000   \n",
       "ENSG00000243485        0.00000         0.00000            0.00000   \n",
       "ENSG00000237613        0.00000         0.00000            0.00000   \n",
       "...                        ...             ...                ...   \n",
       "ENSG00000198695     6637.79000      6374.49000         4972.73000   \n",
       "ENSG00000210194       24.41390        22.65940           20.58800   \n",
       "ENSG00000198727    28799.40000     12919.40000        15943.20000   \n",
       "ENSG00000210195        1.50928         0.00000            0.00000   \n",
       "ENSG00000210196        1.78041         1.57309            1.63648   \n",
       "\n",
       "                 Artery - Tibial      Bladder  Brain - Amygdala  \\\n",
       "Name                                                              \n",
       "ENSG00000223972         0.000000      0.00000           0.00000   \n",
       "ENSG00000227232         3.639630      5.16375           1.43859   \n",
       "ENSG00000278267         0.000000      0.00000           0.00000   \n",
       "ENSG00000243485         0.000000      0.00000           0.00000   \n",
       "ENSG00000237613         0.000000      0.00000           0.00000   \n",
       "...                          ...          ...               ...   \n",
       "ENSG00000198695      4635.000000   8275.20000        9631.77000   \n",
       "ENSG00000210194        18.257600     27.50560          40.58860   \n",
       "ENSG00000198727     13051.500000  23663.20000       32931.80000   \n",
       "ENSG00000210195         0.544659      0.00000           3.44476   \n",
       "ENSG00000210196         1.950040      1.47692           5.67736   \n",
       "\n",
       "                 Brain - Anterior cingulate cortex (BA24)  \\\n",
       "Name                                                        \n",
       "ENSG00000223972                                   0.00000   \n",
       "ENSG00000227232                                   1.69285   \n",
       "ENSG00000278267                                   0.00000   \n",
       "ENSG00000243485                                   0.00000   \n",
       "ENSG00000237613                                   0.00000   \n",
       "...                                                   ...   \n",
       "ENSG00000198695                                8319.14000   \n",
       "ENSG00000210194                                  31.35490   \n",
       "ENSG00000198727                               28346.70000   \n",
       "ENSG00000210195                                   3.47988   \n",
       "ENSG00000210196                                   5.40242   \n",
       "\n",
       "                 Brain - Caudate (basal ganglia)  ...  \\\n",
       "Name                                              ...   \n",
       "ENSG00000223972                         0.000000  ...   \n",
       "ENSG00000227232                         1.566050  ...   \n",
       "ENSG00000278267                         0.000000  ...   \n",
       "ENSG00000243485                         0.024264  ...   \n",
       "ENSG00000237613                         0.000000  ...   \n",
       "...                                          ...  ...   \n",
       "ENSG00000198695                     11516.600000  ...   \n",
       "ENSG00000210194                        78.695300  ...   \n",
       "ENSG00000198727                     40960.000000  ...   \n",
       "ENSG00000210195                         5.581470  ...   \n",
       "ENSG00000210196                         7.071810  ...   \n",
       "\n",
       "                 Skin - Not Sun Exposed (Suprapubic)  \\\n",
       "Name                                                   \n",
       "ENSG00000223972                             0.000000   \n",
       "ENSG00000227232                             5.932980   \n",
       "ENSG00000278267                             0.000000   \n",
       "ENSG00000243485                             0.000000   \n",
       "ENSG00000237613                             0.000000   \n",
       "...                                              ...   \n",
       "ENSG00000198695                          8919.460000   \n",
       "ENSG00000210194                            32.932100   \n",
       "ENSG00000198727                         19058.200000   \n",
       "ENSG00000210195                             0.884701   \n",
       "ENSG00000210196                             2.226800   \n",
       "\n",
       "                 Skin - Sun Exposed (Lower leg)  \\\n",
       "Name                                              \n",
       "ENSG00000223972                         0.00000   \n",
       "ENSG00000227232                         6.13265   \n",
       "ENSG00000278267                         0.00000   \n",
       "ENSG00000243485                         0.00000   \n",
       "ENSG00000237613                         0.00000   \n",
       "...                                         ...   \n",
       "ENSG00000198695                      8113.22000   \n",
       "ENSG00000210194                        31.33400   \n",
       "ENSG00000198727                     17223.10000   \n",
       "ENSG00000210195                         1.11364   \n",
       "ENSG00000210196                         1.88828   \n",
       "\n",
       "                 Small Intestine - Terminal Ileum        Spleen       Stomach  \\\n",
       "Name                                                                            \n",
       "ENSG00000223972                          0.000000      0.000000      0.000000   \n",
       "ENSG00000227232                          4.193780      5.926310      3.062480   \n",
       "ENSG00000278267                          0.000000      0.000000      0.000000   \n",
       "ENSG00000243485                          0.000000      0.000000      0.000000   \n",
       "ENSG00000237613                          0.000000      0.000000      0.000000   \n",
       "...                                           ...           ...           ...   \n",
       "ENSG00000198695                       9764.490000   5477.360000   9628.750000   \n",
       "ENSG00000210194                         35.456000     24.934900     33.496300   \n",
       "ENSG00000198727                      24521.300000  11578.600000  27193.100000   \n",
       "ENSG00000210195                          0.815455      0.783492      0.803858   \n",
       "ENSG00000210196                          1.768040      0.948262      2.082370   \n",
       "\n",
       "                       Testis       Thyroid        Uterus        Vagina  \\\n",
       "Name                                                                      \n",
       "ENSG00000223972      0.166403      0.000000      0.000000      0.000000   \n",
       "ENSG00000227232      4.702530      6.272550      7.190010      5.745540   \n",
       "ENSG00000278267      0.000000      0.000000      0.000000      0.000000   \n",
       "ENSG00000243485      0.054223      0.000000      0.000000      0.000000   \n",
       "ENSG00000237613      0.000000      0.000000      0.000000      0.000000   \n",
       "...                       ...           ...           ...           ...   \n",
       "ENSG00000198695   1676.590000   2520.410000   3644.000000   2237.430000   \n",
       "ENSG00000210194      2.871770      5.795670     10.370100      5.724510   \n",
       "ENSG00000198727  15337.800000  19868.500000  15025.000000  12803.400000   \n",
       "ENSG00000210195      0.443781      0.000000      0.000000      0.000000   \n",
       "ENSG00000210196      1.057760      0.982817      0.808096      0.737385   \n",
       "\n",
       "                 Whole Blood  \n",
       "Name                          \n",
       "ENSG00000223972      0.00000  \n",
       "ENSG00000227232      2.64743  \n",
       "ENSG00000278267      0.00000  \n",
       "ENSG00000243485      0.00000  \n",
       "ENSG00000237613      0.00000  \n",
       "...                      ...  \n",
       "ENSG00000198695    960.29100  \n",
       "ENSG00000210194      3.49736  \n",
       "ENSG00000198727   2305.49000  \n",
       "ENSG00000210195      0.00000  \n",
       "ENSG00000210196      0.00000  \n",
       "\n",
       "[56200 rows x 54 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"gtex_v8.gct.gz\", sep='\\t', skiprows=2)\n",
    "df.drop(columns=[\"Description\"], inplace=True)\n",
    "df[\"Name\"] = df[\"Name\"].str.split('.').str[0]\n",
    "df.set_index(keys=[\"Name\"], inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Filter out genes with very low expression\n",
    "\n",
    "Genes with very low expression tend to generate noisy data. One particular case is an expression vector where one tissue has expression of 0.1 TPM and all the other ones are 0.0 exactly. This causes the gene to be called as extremely specifically expressed, even though most of these issues are just false discoveries.\n",
    "\n",
    "A common threshold used for filtering out genes with very low expression is to filter out those where expression in all tissues is less than 1.0 TPM. This is also used by HPA, and supported by internal investigations into quality control."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Adipose - Subcutaneous</th>\n",
       "      <th>Adipose - Visceral (Omentum)</th>\n",
       "      <th>Adrenal Gland</th>\n",
       "      <th>Artery - Aorta</th>\n",
       "      <th>Artery - Coronary</th>\n",
       "      <th>Artery - Tibial</th>\n",
       "      <th>Bladder</th>\n",
       "      <th>Brain - Amygdala</th>\n",
       "      <th>Brain - Anterior cingulate cortex (BA24)</th>\n",
       "      <th>Brain - Caudate (basal ganglia)</th>\n",
       "      <th>...</th>\n",
       "      <th>Skin - Not Sun Exposed (Suprapubic)</th>\n",
       "      <th>Skin - Sun Exposed (Lower leg)</th>\n",
       "      <th>Small Intestine - Terminal Ileum</th>\n",
       "      <th>Spleen</th>\n",
       "      <th>Stomach</th>\n",
       "      <th>Testis</th>\n",
       "      <th>Thyroid</th>\n",
       "      <th>Uterus</th>\n",
       "      <th>Vagina</th>\n",
       "      <th>Whole Blood</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ENSG00000227232</th>\n",
       "      <td>4.064030</td>\n",
       "      <td>3.371110</td>\n",
       "      <td>2.685490</td>\n",
       "      <td>4.04762</td>\n",
       "      <td>3.900760</td>\n",
       "      <td>3.639630</td>\n",
       "      <td>5.16375</td>\n",
       "      <td>1.438590</td>\n",
       "      <td>1.692850</td>\n",
       "      <td>1.566050</td>\n",
       "      <td>...</td>\n",
       "      <td>5.932980</td>\n",
       "      <td>6.132650</td>\n",
       "      <td>4.193780</td>\n",
       "      <td>5.926310</td>\n",
       "      <td>3.062480</td>\n",
       "      <td>4.702530</td>\n",
       "      <td>6.272550</td>\n",
       "      <td>7.190010</td>\n",
       "      <td>5.745540</td>\n",
       "      <td>2.647430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000268903</th>\n",
       "      <td>3.241590</td>\n",
       "      <td>5.154330</td>\n",
       "      <td>3.668540</td>\n",
       "      <td>3.64269</td>\n",
       "      <td>3.601800</td>\n",
       "      <td>1.461070</td>\n",
       "      <td>3.50280</td>\n",
       "      <td>1.400810</td>\n",
       "      <td>1.330200</td>\n",
       "      <td>1.038280</td>\n",
       "      <td>...</td>\n",
       "      <td>5.855510</td>\n",
       "      <td>6.085610</td>\n",
       "      <td>4.685720</td>\n",
       "      <td>44.899900</td>\n",
       "      <td>3.606070</td>\n",
       "      <td>34.894400</td>\n",
       "      <td>9.503380</td>\n",
       "      <td>4.143990</td>\n",
       "      <td>8.724360</td>\n",
       "      <td>134.312000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000269981</th>\n",
       "      <td>5.137590</td>\n",
       "      <td>7.798020</td>\n",
       "      <td>6.503060</td>\n",
       "      <td>6.23226</td>\n",
       "      <td>5.241380</td>\n",
       "      <td>2.570040</td>\n",
       "      <td>5.45536</td>\n",
       "      <td>1.223390</td>\n",
       "      <td>1.031000</td>\n",
       "      <td>1.248690</td>\n",
       "      <td>...</td>\n",
       "      <td>9.923110</td>\n",
       "      <td>10.646100</td>\n",
       "      <td>8.385520</td>\n",
       "      <td>75.784400</td>\n",
       "      <td>6.307170</td>\n",
       "      <td>54.128100</td>\n",
       "      <td>16.278700</td>\n",
       "      <td>6.718230</td>\n",
       "      <td>14.570400</td>\n",
       "      <td>194.869000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000239906</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.166385</td>\n",
       "      <td>0.172620</td>\n",
       "      <td>0.112703</td>\n",
       "      <td>2.413640</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.114480</td>\n",
       "      <td>0.449414</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.233237</td>\n",
       "      <td>6.700720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000279928</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.059725</td>\n",
       "      <td>0.112722</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.076231</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.07154</td>\n",
       "      <td>0.034187</td>\n",
       "      <td>0.043519</td>\n",
       "      <td>0.044873</td>\n",
       "      <td>...</td>\n",
       "      <td>0.048327</td>\n",
       "      <td>0.062433</td>\n",
       "      <td>0.075272</td>\n",
       "      <td>0.227098</td>\n",
       "      <td>0.066219</td>\n",
       "      <td>3.686690</td>\n",
       "      <td>0.095636</td>\n",
       "      <td>0.034329</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.266855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000198695</th>\n",
       "      <td>3517.220000</td>\n",
       "      <td>3598.090000</td>\n",
       "      <td>6637.790000</td>\n",
       "      <td>6374.49000</td>\n",
       "      <td>4972.730000</td>\n",
       "      <td>4635.000000</td>\n",
       "      <td>8275.20000</td>\n",
       "      <td>9631.770000</td>\n",
       "      <td>8319.140000</td>\n",
       "      <td>11516.600000</td>\n",
       "      <td>...</td>\n",
       "      <td>8919.460000</td>\n",
       "      <td>8113.220000</td>\n",
       "      <td>9764.490000</td>\n",
       "      <td>5477.360000</td>\n",
       "      <td>9628.750000</td>\n",
       "      <td>1676.590000</td>\n",
       "      <td>2520.410000</td>\n",
       "      <td>3644.000000</td>\n",
       "      <td>2237.430000</td>\n",
       "      <td>960.291000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000210194</th>\n",
       "      <td>11.903800</td>\n",
       "      <td>11.203400</td>\n",
       "      <td>24.413900</td>\n",
       "      <td>22.65940</td>\n",
       "      <td>20.588000</td>\n",
       "      <td>18.257600</td>\n",
       "      <td>27.50560</td>\n",
       "      <td>40.588600</td>\n",
       "      <td>31.354900</td>\n",
       "      <td>78.695300</td>\n",
       "      <td>...</td>\n",
       "      <td>32.932100</td>\n",
       "      <td>31.334000</td>\n",
       "      <td>35.456000</td>\n",
       "      <td>24.934900</td>\n",
       "      <td>33.496300</td>\n",
       "      <td>2.871770</td>\n",
       "      <td>5.795670</td>\n",
       "      <td>10.370100</td>\n",
       "      <td>5.724510</td>\n",
       "      <td>3.497360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000198727</th>\n",
       "      <td>23839.100000</td>\n",
       "      <td>25511.700000</td>\n",
       "      <td>28799.400000</td>\n",
       "      <td>12919.40000</td>\n",
       "      <td>15943.200000</td>\n",
       "      <td>13051.500000</td>\n",
       "      <td>23663.20000</td>\n",
       "      <td>32931.800000</td>\n",
       "      <td>28346.700000</td>\n",
       "      <td>40960.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>19058.200000</td>\n",
       "      <td>17223.100000</td>\n",
       "      <td>24521.300000</td>\n",
       "      <td>11578.600000</td>\n",
       "      <td>27193.100000</td>\n",
       "      <td>15337.800000</td>\n",
       "      <td>19868.500000</td>\n",
       "      <td>15025.000000</td>\n",
       "      <td>12803.400000</td>\n",
       "      <td>2305.490000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000210195</th>\n",
       "      <td>0.616205</td>\n",
       "      <td>0.697191</td>\n",
       "      <td>1.509280</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.544659</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>3.444760</td>\n",
       "      <td>3.479880</td>\n",
       "      <td>5.581470</td>\n",
       "      <td>...</td>\n",
       "      <td>0.884701</td>\n",
       "      <td>1.113640</td>\n",
       "      <td>0.815455</td>\n",
       "      <td>0.783492</td>\n",
       "      <td>0.803858</td>\n",
       "      <td>0.443781</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000210196</th>\n",
       "      <td>1.429970</td>\n",
       "      <td>1.192550</td>\n",
       "      <td>1.780410</td>\n",
       "      <td>1.57309</td>\n",
       "      <td>1.636480</td>\n",
       "      <td>1.950040</td>\n",
       "      <td>1.47692</td>\n",
       "      <td>5.677360</td>\n",
       "      <td>5.402420</td>\n",
       "      <td>7.071810</td>\n",
       "      <td>...</td>\n",
       "      <td>2.226800</td>\n",
       "      <td>1.888280</td>\n",
       "      <td>1.768040</td>\n",
       "      <td>0.948262</td>\n",
       "      <td>2.082370</td>\n",
       "      <td>1.057760</td>\n",
       "      <td>0.982817</td>\n",
       "      <td>0.808096</td>\n",
       "      <td>0.737385</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>31910 rows × 54 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Adipose - Subcutaneous  Adipose - Visceral (Omentum)  \\\n",
       "Name                                                                    \n",
       "ENSG00000227232                4.064030                      3.371110   \n",
       "ENSG00000268903                3.241590                      5.154330   \n",
       "ENSG00000269981                5.137590                      7.798020   \n",
       "ENSG00000239906                0.000000                      0.000000   \n",
       "ENSG00000279928                0.000000                      0.059725   \n",
       "...                                 ...                           ...   \n",
       "ENSG00000198695             3517.220000                   3598.090000   \n",
       "ENSG00000210194               11.903800                     11.203400   \n",
       "ENSG00000198727            23839.100000                  25511.700000   \n",
       "ENSG00000210195                0.616205                      0.697191   \n",
       "ENSG00000210196                1.429970                      1.192550   \n",
       "\n",
       "                 Adrenal Gland  Artery - Aorta  Artery - Coronary  \\\n",
       "Name                                                                \n",
       "ENSG00000227232       2.685490         4.04762           3.900760   \n",
       "ENSG00000268903       3.668540         3.64269           3.601800   \n",
       "ENSG00000269981       6.503060         6.23226           5.241380   \n",
       "ENSG00000239906       0.000000         0.00000           0.000000   \n",
       "ENSG00000279928       0.112722         0.00000           0.076231   \n",
       "...                        ...             ...                ...   \n",
       "ENSG00000198695    6637.790000      6374.49000        4972.730000   \n",
       "ENSG00000210194      24.413900        22.65940          20.588000   \n",
       "ENSG00000198727   28799.400000     12919.40000       15943.200000   \n",
       "ENSG00000210195       1.509280         0.00000           0.000000   \n",
       "ENSG00000210196       1.780410         1.57309           1.636480   \n",
       "\n",
       "                 Artery - Tibial      Bladder  Brain - Amygdala  \\\n",
       "Name                                                              \n",
       "ENSG00000227232         3.639630      5.16375          1.438590   \n",
       "ENSG00000268903         1.461070      3.50280          1.400810   \n",
       "ENSG00000269981         2.570040      5.45536          1.223390   \n",
       "ENSG00000239906         0.000000      0.00000          0.000000   \n",
       "ENSG00000279928         0.000000      0.07154          0.034187   \n",
       "...                          ...          ...               ...   \n",
       "ENSG00000198695      4635.000000   8275.20000       9631.770000   \n",
       "ENSG00000210194        18.257600     27.50560         40.588600   \n",
       "ENSG00000198727     13051.500000  23663.20000      32931.800000   \n",
       "ENSG00000210195         0.544659      0.00000          3.444760   \n",
       "ENSG00000210196         1.950040      1.47692          5.677360   \n",
       "\n",
       "                 Brain - Anterior cingulate cortex (BA24)  \\\n",
       "Name                                                        \n",
       "ENSG00000227232                                  1.692850   \n",
       "ENSG00000268903                                  1.330200   \n",
       "ENSG00000269981                                  1.031000   \n",
       "ENSG00000239906                                  0.000000   \n",
       "ENSG00000279928                                  0.043519   \n",
       "...                                                   ...   \n",
       "ENSG00000198695                               8319.140000   \n",
       "ENSG00000210194                                 31.354900   \n",
       "ENSG00000198727                              28346.700000   \n",
       "ENSG00000210195                                  3.479880   \n",
       "ENSG00000210196                                  5.402420   \n",
       "\n",
       "                 Brain - Caudate (basal ganglia)  ...  \\\n",
       "Name                                              ...   \n",
       "ENSG00000227232                         1.566050  ...   \n",
       "ENSG00000268903                         1.038280  ...   \n",
       "ENSG00000269981                         1.248690  ...   \n",
       "ENSG00000239906                         0.000000  ...   \n",
       "ENSG00000279928                         0.044873  ...   \n",
       "...                                          ...  ...   \n",
       "ENSG00000198695                     11516.600000  ...   \n",
       "ENSG00000210194                        78.695300  ...   \n",
       "ENSG00000198727                     40960.000000  ...   \n",
       "ENSG00000210195                         5.581470  ...   \n",
       "ENSG00000210196                         7.071810  ...   \n",
       "\n",
       "                 Skin - Not Sun Exposed (Suprapubic)  \\\n",
       "Name                                                   \n",
       "ENSG00000227232                             5.932980   \n",
       "ENSG00000268903                             5.855510   \n",
       "ENSG00000269981                             9.923110   \n",
       "ENSG00000239906                             0.166385   \n",
       "ENSG00000279928                             0.048327   \n",
       "...                                              ...   \n",
       "ENSG00000198695                          8919.460000   \n",
       "ENSG00000210194                            32.932100   \n",
       "ENSG00000198727                         19058.200000   \n",
       "ENSG00000210195                             0.884701   \n",
       "ENSG00000210196                             2.226800   \n",
       "\n",
       "                 Skin - Sun Exposed (Lower leg)  \\\n",
       "Name                                              \n",
       "ENSG00000227232                        6.132650   \n",
       "ENSG00000268903                        6.085610   \n",
       "ENSG00000269981                       10.646100   \n",
       "ENSG00000239906                        0.172620   \n",
       "ENSG00000279928                        0.062433   \n",
       "...                                         ...   \n",
       "ENSG00000198695                     8113.220000   \n",
       "ENSG00000210194                       31.334000   \n",
       "ENSG00000198727                    17223.100000   \n",
       "ENSG00000210195                        1.113640   \n",
       "ENSG00000210196                        1.888280   \n",
       "\n",
       "                 Small Intestine - Terminal Ileum        Spleen       Stomach  \\\n",
       "Name                                                                            \n",
       "ENSG00000227232                          4.193780      5.926310      3.062480   \n",
       "ENSG00000268903                          4.685720     44.899900      3.606070   \n",
       "ENSG00000269981                          8.385520     75.784400      6.307170   \n",
       "ENSG00000239906                          0.112703      2.413640      0.000000   \n",
       "ENSG00000279928                          0.075272      0.227098      0.066219   \n",
       "...                                           ...           ...           ...   \n",
       "ENSG00000198695                       9764.490000   5477.360000   9628.750000   \n",
       "ENSG00000210194                         35.456000     24.934900     33.496300   \n",
       "ENSG00000198727                      24521.300000  11578.600000  27193.100000   \n",
       "ENSG00000210195                          0.815455      0.783492      0.803858   \n",
       "ENSG00000210196                          1.768040      0.948262      2.082370   \n",
       "\n",
       "                       Testis       Thyroid        Uterus        Vagina  \\\n",
       "Name                                                                      \n",
       "ENSG00000227232      4.702530      6.272550      7.190010      5.745540   \n",
       "ENSG00000268903     34.894400      9.503380      4.143990      8.724360   \n",
       "ENSG00000269981     54.128100     16.278700      6.718230     14.570400   \n",
       "ENSG00000239906      2.114480      0.449414      0.000000      0.233237   \n",
       "ENSG00000279928      3.686690      0.095636      0.034329      0.000000   \n",
       "...                       ...           ...           ...           ...   \n",
       "ENSG00000198695   1676.590000   2520.410000   3644.000000   2237.430000   \n",
       "ENSG00000210194      2.871770      5.795670     10.370100      5.724510   \n",
       "ENSG00000198727  15337.800000  19868.500000  15025.000000  12803.400000   \n",
       "ENSG00000210195      0.443781      0.000000      0.000000      0.000000   \n",
       "ENSG00000210196      1.057760      0.982817      0.808096      0.737385   \n",
       "\n",
       "                 Whole Blood  \n",
       "Name                          \n",
       "ENSG00000227232     2.647430  \n",
       "ENSG00000268903   134.312000  \n",
       "ENSG00000269981   194.869000  \n",
       "ENSG00000239906     6.700720  \n",
       "ENSG00000279928     0.266855  \n",
       "...                      ...  \n",
       "ENSG00000198695   960.291000  \n",
       "ENSG00000210194     3.497360  \n",
       "ENSG00000198727  2305.490000  \n",
       "ENSG00000210195     0.000000  \n",
       "ENSG00000210196     0.000000  \n",
       "\n",
       "[31910 rows x 54 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "THRESHOLD = 1.0\n",
    "max_expr_across_tissues = df.max(axis=1)\n",
    "expr_level_mask = max_expr_across_tissues >= THRESHOLD\n",
    "df = df[expr_level_mask]\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Compute Gini coefficient\n",
    "\n",
    "[Gini coefficient](https://en.wikipedia.org/wiki/Gini_coefficient) is a metric in economics which is used to characterise inequality in income, net worth, or other metrics. It ranges from 0 (perfect equality) to 1 (maximum inequality). It was also applied previously to the expression specificity problem."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adapted from https://github.com/oliviaguest/gini/blob/master/gini.py\n",
    "def gini(row):\n",
    "    \"\"\"Calculate the Gini coefficient of a Pandas row.\"\"\"\n",
    "    assert min(row) >= 0\n",
    "    if max(row) == 0:\n",
    "        return np.nan\n",
    "    # # Values cannot be 0:  (disabled because zeroes are filtered out already)\n",
    "    # array += 0.0000001\n",
    "    # Values must be sorted:\n",
    "    array = np.sort(row)\n",
    "    # Index per array element:\n",
    "    index = np.arange(1,array.shape[0]+1)\n",
    "    # Number of array elements:\n",
    "    n = array.shape[0]\n",
    "    # Gini coefficient:\n",
    "    return ((np.sum((2 * index - n  - 1) * array)) / (n * np.sum(array)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gini</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Name</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ENSG00000227232</th>\n",
       "      <td>0.278155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000268903</th>\n",
       "      <td>0.655462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000269981</th>\n",
       "      <td>0.666002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000239906</th>\n",
       "      <td>0.898719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000279928</th>\n",
       "      <td>0.685379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000198695</th>\n",
       "      <td>0.284763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000210194</th>\n",
       "      <td>0.355988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000198727</th>\n",
       "      <td>0.245582</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000210195</th>\n",
       "      <td>0.589682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000210196</th>\n",
       "      <td>0.418335</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>31910 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                     gini\n",
       "Name                     \n",
       "ENSG00000227232  0.278155\n",
       "ENSG00000268903  0.655462\n",
       "ENSG00000269981  0.666002\n",
       "ENSG00000239906  0.898719\n",
       "ENSG00000279928  0.685379\n",
       "...                   ...\n",
       "ENSG00000198695  0.284763\n",
       "ENSG00000210194  0.355988\n",
       "ENSG00000198727  0.245582\n",
       "ENSG00000210195  0.589682\n",
       "ENSG00000210196  0.418335\n",
       "\n",
       "[31910 rows x 1 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This is going to be our annotations array.\n",
    "a = df.iloc[:,:0].copy()\n",
    "a['gini'] = df.apply(gini, axis=1)\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD6CAYAAABNu5eFAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAASRUlEQVR4nO3df6zddX3H8edLioypTBwXUtuyMlM3gUyUu47MbcG5DcQ/iosudYsQQ1LHcNPEPwD/mG5Lk5pM3cgGriqhJAprpo5ugg6ZzhnBejFIKcjshMG1Da3iZnUJS8t7f5wv5KQ9vef0/jj39n6ej+TkfM/7+/me8zmfnvs63/P5fs9pqgpJUhtesNgdkCSNj6EvSQ0x9CWpIYa+JDXE0Jekhhj6ktSQoaGf5KeS7EzyrSS7k/xZV39ZkruTfKe7Pr1vm+uT7EnyaJJL+uoXJtnVrbshSRbmaUmSBsmw8/S7YH5RVf04ycnAV4F3A78LPF1VW5JcB5xeVdcmORe4DVgPvBz4IvDKqjqcZGe37X3AncANVXXXTI9/xhln1Nq1a+f0JCWpNffff//3q2riyPqKYRtW713hx93Nk7tLARuAi7v6NuDLwLVd/faqegZ4LMkeYH2Sx4HTqupegCS3ApcDM4b+2rVrmZqaGtZNSVKfJP81qD7SnH6Sk5I8AOwH7q6qrwNnVdU+gO76zK75KuDJvs2nu9qqbvnI+qDH25RkKsnUgQMHRumiJGkEI4V+VR2uqguA1fT22s+fofmgefqaoT7o8bZW1WRVTU5MHPXpRJI0S8d19k5V/Te9aZxLgaeSrATorvd3zaaBNX2brQb2dvXVA+qSpDEZ5eydiSQv7ZZPBX4L+DawA7iya3YlcEe3vAPYmOSUJOcA64Cd3RTQwSQXdQeHr+jbRpI0BkMP5AIrgW1JTqL3JrG9qv45yb3A9iRXAU8AbwWoqt1JtgMPA4eAa6rqcHdfVwO3AKfSO4A740FcSdL8GnrK5mKbnJwsz96RpOOT5P6qmjyy7jdyJakhhr4kNcTQl6SGjHIgV5I0Bmuv+9zzy49vedOCPIZ7+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1ZGjoJ1mT5EtJHkmyO8m7u/oHknwvyQPd5bK+ba5PsifJo0ku6atfmGRXt+6GJFmYpyVJGmTFCG0OAe+tqm8meQlwf5K7u3Ufqaq/7G+c5FxgI3Ae8HLgi0leWVWHgZuATcB9wJ3ApcBd8/NUJEnDDN3Tr6p9VfXNbvkg8AiwaoZNNgC3V9UzVfUYsAdYn2QlcFpV3VtVBdwKXD7XJyBJGt1xzeknWQu8Bvh6V3pXkgeT3Jzk9K62Cniyb7PprraqWz6yPuhxNiWZSjJ14MCB4+miJGkGI4d+khcDnwbeU1U/ojdV8wrgAmAf8KHnmg7YvGaoH12s2lpVk1U1OTExMWoXJUlDjBT6SU6mF/ifrKrPAFTVU1V1uKqeBT4GrO+aTwNr+jZfDezt6qsH1CVJYzLK2TsBPgE8UlUf7quv7Gv2ZuChbnkHsDHJKUnOAdYBO6tqH3AwyUXdfV4B3DFPz0OSNIJRzt55HfB2YFeSB7ra+4C3JbmA3hTN48A7Aapqd5LtwMP0zvy5pjtzB+Bq4BbgVHpn7XjmjiSN0dDQr6qvMng+/s4ZttkMbB5QnwLOP54OSpLmj9/IlaSGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaMjT0k6xJ8qUkjyTZneTdXf1lSe5O8p3u+vS+ba5PsifJo0ku6atfmGRXt+6GJFmYpyVJGmSUPf1DwHur6lXARcA1Sc4FrgPuqap1wD3dbbp1G4HzgEuBG5Oc1N3XTcAmYF13uXQen4skaYihoV9V+6rqm93yQeARYBWwAdjWNdsGXN4tbwBur6pnquoxYA+wPslK4LSqureqCri1bxtJ0hgc15x+krXAa4CvA2dV1T7ovTEAZ3bNVgFP9m023dVWdctH1gc9zqYkU0mmDhw4cDxdlCTNYOTQT/Ji4NPAe6rqRzM1HVCrGepHF6u2VtVkVU1OTEyM2kVJ0hAjhX6Sk+kF/ier6jNd+aluyobuen9XnwbW9G2+Gtjb1VcPqEuSxmSUs3cCfAJ4pKo+3LdqB3Blt3wlcEdffWOSU5KcQ++A7c5uCuhgkou6+7yibxtJ0hisGKHN64C3A7uSPNDV3gdsAbYnuQp4AngrQFXtTrIdeJjemT/XVNXhbrurgVuAU4G7uoskaUyGhn5VfZXB8/EAbzjGNpuBzQPqU8D5x9NBSdL88Ru5ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDhoZ+kpuT7E/yUF/tA0m+l+SB7nJZ37rrk+xJ8miSS/rqFybZ1a27IUnm/+lIkmYyyp7+LcClA+ofqaoLusudAEnOBTYC53Xb3JjkpK79TcAmYF13GXSfkqQFNDT0q+orwNMj3t8G4PaqeqaqHgP2AOuTrAROq6p7q6qAW4HLZ9lnSdIszWVO/11JHuymf07vaquAJ/vaTHe1Vd3ykfWBkmxKMpVk6sCBA3PooiSp32xD/ybgFcAFwD7gQ1190Dx9zVAfqKq2VtVkVU1OTEzMsouSpCPNKvSr6qmqOlxVzwIfA9Z3q6aBNX1NVwN7u/rqAXVJ0hjNKvS7OfrnvBl47syeHcDGJKckOYfeAdudVbUPOJjkou6snSuAO+bQb0nSLKwY1iDJbcDFwBlJpoH3AxcnuYDeFM3jwDsBqmp3ku3Aw8Ah4JqqOtzd1dX0zgQ6Fbiru0iSxmho6FfV2waUPzFD+83A5gH1KeD84+qdJGle+Y1cSWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDDH1JaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0JakhQ0M/yc1J9id5qK/2siR3J/lOd31637rrk+xJ8miSS/rqFybZ1a27IUnm/+lIkmYyyp7+LcClR9SuA+6pqnXAPd1tkpwLbATO67a5MclJ3TY3AZuAdd3lyPuUJC2woaFfVV8Bnj6ivAHY1i1vAy7vq99eVc9U1WPAHmB9kpXAaVV1b1UVcGvfNpKkMZntnP5ZVbUPoLs+s6uvAp7sazfd1VZ1y0fWB0qyKclUkqkDBw7MsouSpCPN94HcQfP0NUN9oKraWlWTVTU5MTExb52TpNbNNvSf6qZs6K73d/VpYE1fu9XA3q6+ekBdkjRGsw39HcCV3fKVwB199Y1JTklyDr0Dtju7KaCDSS7qztq5om8bSdKYrBjWIMltwMXAGUmmgfcDW4DtSa4CngDeClBVu5NsBx4GDgHXVNXh7q6upncm0KnAXd1FkjRGQ0O/qt52jFVvOEb7zcDmAfUp4Pzj6p0kaV75jVxJaoihL0kNMfQlqSGGviQ1xNCXpIYY+pLUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNcTQl6SGGPqS1BBDX5IaYuhLUkMMfUlqiKEvSQ0x9CWpIYa+JDXE0Jekhhj6ktQQQ1+SGmLoS1JDVix2BzQ7a6/73MD641veNLBNf11Suwx9SVpEx9qBWyiG/hI0lz30cb+AJJ1Y5hT6SR4HDgKHgUNVNZnkZcDfA2uBx4Hfq6ofdu2vB67q2v9JVX1hLo/fAqdoJM2n+djTf31Vfb/v9nXAPVW1Jcl13e1rk5wLbATOA14OfDHJK6vq8Dz0oQnztRc/yvEAScvTQpy9swHY1i1vAy7vq99eVc9U1WPAHmD9Ajy+JOkY5rqnX8C/JCng76pqK3BWVe0DqKp9Sc7s2q4C7uvbdrqrHSXJJmATwNlnnz3HLgqc65eWksX8e5xr6L+uqvZ2wX53km/P0DYDajWoYffmsRVgcnJyYBvNP48fSMvfnEK/qvZ21/uTfJbedM1TSVZ2e/krgf1d82lgTd/mq4G9c3l8LRzfAKTladahn+RFwAuq6mC3/DvAnwM7gCuBLd31Hd0mO4BPJfkwvQO564Cdc+i7xsQDv9LyMZc9/bOAzyZ57n4+VVWfT/INYHuSq4AngLcCVNXuJNuBh4FDwDWeuSNJ4zXr0K+q7wKvHlD/AfCGY2yzGdg828dczpbTgVanhqSly2/kal4spzctaTkz9Mdoue0BG/TSicfQXyStBOZye6OTjsdSfP0b+pI0j461Q7dUdvQMfS26pbg3JC1Xhr4Wxah7Pcd6Q/C7A9LsGPoam6Xy8VZqmaGvJcU3Bp2ITqTXraGvZcXjAxqXEyno+xn6C+xEfWEsRXMZS48BaD4sh79nQ38BLIcXxnLgv4Pmw3J7HRn6atp8nR007mklp7EW1nIL+n6GvtQZ5Q99lLCdrzbH27elaDHfnJzSG8zQlxbQ8b6R9JvPcFro8J3L/c/XtqNu3/qnJENfmqWF3vue6/0f7/ajTHWN81PJbO5nLs+5Fala2v8F7eTkZE1NTS12N4Zq8cWjpWGUYxHL6XFbMddPIUnur6rJI+vu6UsnuMUKXIP+xPSCxe6AJGl83NOfA/d0JJ1o3NOXpIYY+pLUEKd3jpNTOpJOZIb+CAx6ScuF0zuS1BBDX5Ia4vTOMTilI2k5ck9fkhoy9j39JJcCfw2cBHy8qraMuw/H4t69pOVurKGf5CTgb4HfBqaBbyTZUVUPj7Mf/Qx6SS0Z957+emBPVX0XIMntwAZgrKFv0Etq1bhDfxXwZN/taeBXjmyUZBOwqbv54ySPjqFvC+EM4PuL3YklyHEZzHE5WrNjkg/OuHqUcfm5QcVxh34G1I76Qf+q2gpsXfjuLKwkU4N+z7p1jstgjsvRHJPB5jIu4z57ZxpY03d7NbB3zH2QpGaNO/S/AaxLck6SFwIbgR1j7oMkNWus0ztVdSjJu4Av0Dtl8+aq2j3OPozZCT9FtUAcl8Ecl6M5JoPNelyW/P+RK0maP34jV5IaYuhLUkMM/XmQ5NIkjybZk+S6Aev/IMmD3eVrSV69GP0ct2Hj0tful5McTvKWcfZvMYwyJkkuTvJAkt1J/m3cfVwMI/wN/UySf0ryrW5c3rEY/RynJDcn2Z/koWOsT5IbujF7MMlrR7rjqvIyhwu9A9L/Cfw88ELgW8C5R7T5VeD0bvmNwNcXu99LYVz62v0rcCfwlsXu92KPCfBSet9QP7u7feZi93uJjMv7gA92yxPA08ALF7vvCzwuvwG8FnjoGOsvA+6i9/2ni0bNFff05+75n5aoqv8DnvtpiedV1deq6ofdzfvofT9huRs6Lp0/Bj4N7B9n5xbJKGPy+8BnquoJgKpyXHoKeEmSAC+mF/qHxtvN8aqqr9B7nseyAbi1eu4DXppk5bD7NfTnbtBPS6yaof1V9N6dl7uh45JkFfBm4KNj7NdiGuW18krg9CRfTnJ/kivG1rvFM8q4/A3wKnpf5twFvLuqnh1P95as480ewP9EZT6M9NMSAEleTy/0f21Be7Q0jDIufwVcW1WHeztwy94oY7ICuBB4A3AqcG+S+6rqPxa6c4tolHG5BHgA+E3gFcDdSf69qn60wH1bykbOnn6G/tyN9NMSSX4J+Djwxqr6wZj6tphGGZdJ4PYu8M8ALktyqKr+cSw9HL9RxmQa+H5V/QT4SZKvAK8GlnPojzIu7wC2VG8ye0+Sx4BfBHaOp4tL0qx+1sbpnbkb+tMSSc4GPgO8fZnvsfUbOi5VdU5Vra2qtcA/AH+0jAMfRvsZkjuAX0+yIslP0/sV2kfG3M9xG2VcnqD36YckZwG/AHx3rL1cenYAV3Rn8VwE/E9V7Ru2kXv6c1TH+GmJJH/Yrf8o8KfAzwI3dnu1h2qZ/3LgiOPSlFHGpKoeSfJ54EHgWXr/u9zAU/aWixFfK38B3JJkF71pjWuraln/5HKS24CLgTOSTAPvB06G58fkTnpn8OwB/pfep6Hh99ud+iNJaoDTO5LUEENfkhpi6EtSQwx9SWqIoS9JDTH0Jakhhr4kNeT/ARtkkQP7Z/GJAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(a.gini, bins=100)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. HPA specificity metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5.1. HPA categories and noise threshold\n",
    "\n",
    "HPA definitions of the tissue expression categories are available on this page: https://www.proteinatlas.org/about/assays+annotation#classification_rna\n",
    "\n",
    "There are two dimensions: **specificity** and **distribution.**\n",
    "\n",
    "They use the same threshold (nTPM > 1) for filtering out noise as we do in these calculations. Their definitions appear to be sensible, however, quite arbitrary, which is obvious from their use of cut-offs such as \"a third\" or \"four times\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gini</th>\n",
       "      <th>hpa_specificity</th>\n",
       "      <th>hpa_distribution</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ENSG00000227232</th>\n",
       "      <td>0.278155</td>\n",
       "      <td>Low tissue specificity</td>\n",
       "      <td>Detected in many</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000268903</th>\n",
       "      <td>0.655462</td>\n",
       "      <td>Tissue enhanced</td>\n",
       "      <td>Detected in many</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000269981</th>\n",
       "      <td>0.666002</td>\n",
       "      <td>Tissue enhanced</td>\n",
       "      <td>Detected in many</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000239906</th>\n",
       "      <td>0.898719</td>\n",
       "      <td>Tissue enhanced</td>\n",
       "      <td>Detected in some</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000279928</th>\n",
       "      <td>0.685379</td>\n",
       "      <td>Tissue enriched</td>\n",
       "      <td>Detected in single</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000198695</th>\n",
       "      <td>0.284763</td>\n",
       "      <td>Low tissue specificity</td>\n",
       "      <td>Detected in all</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000210194</th>\n",
       "      <td>0.355988</td>\n",
       "      <td>Low tissue specificity</td>\n",
       "      <td>Detected in all</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000198727</th>\n",
       "      <td>0.245582</td>\n",
       "      <td>Low tissue specificity</td>\n",
       "      <td>Detected in all</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000210195</th>\n",
       "      <td>0.589682</td>\n",
       "      <td>Tissue enhanced</td>\n",
       "      <td>Detected in many</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000210196</th>\n",
       "      <td>0.418335</td>\n",
       "      <td>Tissue enhanced</td>\n",
       "      <td>Detected in many</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>31910 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                     gini         hpa_specificity    hpa_distribution\n",
       "Name                                                                 \n",
       "ENSG00000227232  0.278155  Low tissue specificity    Detected in many\n",
       "ENSG00000268903  0.655462         Tissue enhanced    Detected in many\n",
       "ENSG00000269981  0.666002         Tissue enhanced    Detected in many\n",
       "ENSG00000239906  0.898719         Tissue enhanced    Detected in some\n",
       "ENSG00000279928  0.685379         Tissue enriched  Detected in single\n",
       "...                   ...                     ...                 ...\n",
       "ENSG00000198695  0.284763  Low tissue specificity     Detected in all\n",
       "ENSG00000210194  0.355988  Low tissue specificity     Detected in all\n",
       "ENSG00000198727  0.245582  Low tissue specificity     Detected in all\n",
       "ENSG00000210195  0.589682         Tissue enhanced    Detected in many\n",
       "ENSG00000210196  0.418335         Tissue enhanced    Detected in many\n",
       "\n",
       "[31910 rows x 3 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def hpa_specificity(row):\n",
    "    expr = sorted(row)\n",
    "    if expr[-1] < THRESHOLD:\n",
    "        return 'Not detected'\n",
    "    if expr[-2] == 0 or expr[-1]/expr[-2] >= 4.0:\n",
    "        return 'Tissue enriched'\n",
    "    for i in range(2, 6):\n",
    "        if expr[-i-1] == 0 or expr[-i] / expr[-i-1] >= 4.0:\n",
    "            return 'Group enriched'\n",
    "    mean = sum(expr) / len(row)\n",
    "    if 1 <= sum([e / mean >= 4.0 for e in expr]) <= 5:\n",
    "        return 'Tissue enhanced'\n",
    "    return 'Low tissue specificity'\n",
    "\n",
    "def hpa_distribution(row):\n",
    "    expr = sorted(row)\n",
    "    if expr[-1] < THRESHOLD:\n",
    "        return 'Not detected'\n",
    "    num_detected = sum([e > THRESHOLD for e in expr])\n",
    "    if num_detected == 1:\n",
    "        return 'Detected in single'\n",
    "    if num_detected < len(row) / 3:\n",
    "        return 'Detected in some'\n",
    "    if num_detected < len(row):\n",
    "        return 'Detected in many'\n",
    "    return 'Detected in all'\n",
    "\n",
    "a['hpa_specificity'] = df.apply(hpa_specificity, axis=1)\n",
    "a['hpa_distribution'] = df.apply(hpa_distribution, axis=1)\n",
    "a"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. Compute Adatiss specificity scores\n",
    "\n",
    "[Link to paper](https://academic.oup.com/bioinformatics/article/37/23/4469/6306407)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare Adatiss input file. Original column names cannot be ingested by Adatiss because they contain special characters. Hence temporary identifiers are used for Adatiss.\n",
    "column_map = {original_name: f\"tissue_{i}\" for i, original_name in enumerate(df.columns)}\n",
    "reverse_column_map = {v: k for k, v in column_map.items()}\n",
    "df.rename(columns=column_map).to_csv(\"adatiss_input.csv\", index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adatiss also requires a sample to tissue name map file.\n",
    "with open('adatiss_sample_phenotype.csv', 'w') as outfile:\n",
    "    outfile.write('sample_ID,tissue\\n')\n",
    "    for original_name, adatiss_name in column_map.items():\n",
    "        outfile.write(f'{adatiss_name},{original_name}\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Clone the Adatiss repository and apply a hot fix for a mathematical bug\n",
    "! rm -rf AdaTiSS && git clone -q https://github.com/mwgrassgreen/AdaTiSS\n",
    "! sed -i 's/if (length(y\\.1) >= 20)/if (length(y.1) >= 20 \\&\\& length(unique(y.1)) > 1)/g' AdaTiSS/R/AdaTiSS_fn.R"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] 500\n",
      "[1] 1000\n",
      "[1] 1500\n",
      "[1] 2000\n",
      "[1] 2500\n",
      "[1] 3000\n",
      "[1] 3500\n",
      "[1] 4000\n",
      "[1] 4500\n",
      "[1] 5000\n",
      "[1] 5500\n",
      "[1] 6000\n",
      "[1] 6500\n",
      "[1] 7000\n",
      "[1] 7500\n",
      "[1] 8000\n",
      "[1] 8500\n",
      "[1] 9000\n",
      "[1] 9500\n",
      "[1] 10000\n",
      "[1] 10500\n",
      "[1] 11000\n",
      "[1] 11500\n",
      "[1] 12000\n",
      "[1] 12500\n",
      "[1] 13000\n",
      "[1] 13500\n",
      "[1] 14000\n",
      "[1] 14500\n",
      "[1] 15000\n",
      "[1] 15500\n",
      "[1] 16000\n",
      "[1] 16500\n",
      "[1] 17000\n",
      "[1] 17500\n",
      "[1] 18000\n",
      "[1] 18500\n",
      "[1] 19000\n",
      "[1] 19500\n",
      "[1] 20000\n",
      "[1] 20500\n",
      "[1] 21000\n",
      "[1] 21500\n",
      "[1] 22000\n",
      "[1] 22500\n",
      "[1] 23000\n",
      "[1] 23500\n",
      "[1] 24000\n",
      "[1] 24500\n",
      "[1] 25000\n",
      "[1] 25500\n",
      "[1] 26000\n",
      "[1] 26500\n",
      "[1] 27000\n",
      "[1] 27500\n",
      "[1] 28000\n",
      "[1] 28500\n",
      "[1] 29000\n",
      "[1] 29500\n",
      "[1] 30000\n",
      "[1] 30500\n",
      "[1] 31000\n",
      "[1] 31500\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "In addition: Warning message:\n",
       "In (function (package, help, pos = 2, lib.loc = NULL, character.only = FALSE,  :\n",
       "  library ‘/usr/lib/R/site-library’ contains no packages\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%R\n",
    "\n",
    "# Run the computations\n",
    "\n",
    "source('./AdaTiSS/R/AdaTiSS_fn.R')\n",
    "dat.rna = read.csv(file='./adatiss_input.csv', row.names=1)\n",
    "X = preproc.filter.fn(dat.rna, dat.type = \"TPM or RPKM\", proc.zero = 'ceiled to 1', filter.col.prp = 1, exp.thres = 1)\n",
    "p.dat = read.csv('./adatiss_sample_phenotype.csv')\n",
    "tiss.abd = tiss.abd.fn(X, p.dat)\n",
    "result = AdaTiSS(X, tiss.abd=tiss.abd, adjust=TRUE, adjust.opt=0)\n",
    "write.table(result$ada.s, file = \"./adatiss_output.tsv\", sep = \"\\t\", quote = FALSE, row.names = TRUE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Adipose - Subcutaneous</th>\n",
       "      <th>Adipose - Visceral (Omentum)</th>\n",
       "      <th>Adrenal Gland</th>\n",
       "      <th>Artery - Aorta</th>\n",
       "      <th>Artery - Coronary</th>\n",
       "      <th>Artery - Tibial</th>\n",
       "      <th>Bladder</th>\n",
       "      <th>Brain - Amygdala</th>\n",
       "      <th>Brain - Anterior cingulate cortex (BA24)</th>\n",
       "      <th>Brain - Caudate (basal ganglia)</th>\n",
       "      <th>...</th>\n",
       "      <th>Skin - Not Sun Exposed (Suprapubic)</th>\n",
       "      <th>Skin - Sun Exposed (Lower leg)</th>\n",
       "      <th>Small Intestine - Terminal Ileum</th>\n",
       "      <th>Spleen</th>\n",
       "      <th>Stomach</th>\n",
       "      <th>Testis</th>\n",
       "      <th>Thyroid</th>\n",
       "      <th>Uterus</th>\n",
       "      <th>Vagina</th>\n",
       "      <th>Whole Blood</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ENSG00000227232</th>\n",
       "      <td>0.387857</td>\n",
       "      <td>0.044690</td>\n",
       "      <td>-0.372727</td>\n",
       "      <td>0.380429</td>\n",
       "      <td>0.312583</td>\n",
       "      <td>0.185383</td>\n",
       "      <td>0.827503</td>\n",
       "      <td>-1.518619</td>\n",
       "      <td>-1.219846</td>\n",
       "      <td>-1.362774</td>\n",
       "      <td>...</td>\n",
       "      <td>1.082426</td>\n",
       "      <td>1.143191</td>\n",
       "      <td>0.445551</td>\n",
       "      <td>1.080361</td>\n",
       "      <td>-0.131576</td>\n",
       "      <td>0.655744</td>\n",
       "      <td>1.184599</td>\n",
       "      <td>1.435200</td>\n",
       "      <td>1.023493</td>\n",
       "      <td>-0.398931</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000268903</th>\n",
       "      <td>-0.050832</td>\n",
       "      <td>0.496878</td>\n",
       "      <td>0.095292</td>\n",
       "      <td>0.086940</td>\n",
       "      <td>0.073609</td>\n",
       "      <td>-0.991954</td>\n",
       "      <td>0.040693</td>\n",
       "      <td>-1.041696</td>\n",
       "      <td>-1.102778</td>\n",
       "      <td>-1.395384</td>\n",
       "      <td>...</td>\n",
       "      <td>0.647509</td>\n",
       "      <td>0.693028</td>\n",
       "      <td>0.384310</td>\n",
       "      <td>3.053241</td>\n",
       "      <td>0.075008</td>\n",
       "      <td>2.755504</td>\n",
       "      <td>1.219419</td>\n",
       "      <td>0.239213</td>\n",
       "      <td>1.118411</td>\n",
       "      <td>4.347284</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000269981</th>\n",
       "      <td>0.004120</td>\n",
       "      <td>0.398468</td>\n",
       "      <td>0.226853</td>\n",
       "      <td>0.186657</td>\n",
       "      <td>0.023021</td>\n",
       "      <td>-0.650469</td>\n",
       "      <td>0.060835</td>\n",
       "      <td>-1.351962</td>\n",
       "      <td>-1.513653</td>\n",
       "      <td>-1.332618</td>\n",
       "      <td>...</td>\n",
       "      <td>0.626217</td>\n",
       "      <td>0.692679</td>\n",
       "      <td>0.467112</td>\n",
       "      <td>2.547492</td>\n",
       "      <td>0.197949</td>\n",
       "      <td>2.229452</td>\n",
       "      <td>1.094000</td>\n",
       "      <td>0.257616</td>\n",
       "      <td>0.989228</td>\n",
       "      <td>3.440013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000239906</th>\n",
       "      <td>-0.228142</td>\n",
       "      <td>-0.228142</td>\n",
       "      <td>-0.228142</td>\n",
       "      <td>-0.228142</td>\n",
       "      <td>-0.228142</td>\n",
       "      <td>-0.228142</td>\n",
       "      <td>-0.228142</td>\n",
       "      <td>-0.228142</td>\n",
       "      <td>-0.228142</td>\n",
       "      <td>-0.228142</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.228142</td>\n",
       "      <td>-0.228142</td>\n",
       "      <td>-0.228142</td>\n",
       "      <td>2.722479</td>\n",
       "      <td>-0.228142</td>\n",
       "      <td>2.279362</td>\n",
       "      <td>-0.228142</td>\n",
       "      <td>-0.228142</td>\n",
       "      <td>-0.228142</td>\n",
       "      <td>6.141721</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000279928</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>188.232611</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000198695</th>\n",
       "      <td>-0.837431</td>\n",
       "      <td>-0.799416</td>\n",
       "      <td>0.224678</td>\n",
       "      <td>0.156991</td>\n",
       "      <td>-0.258307</td>\n",
       "      <td>-0.375927</td>\n",
       "      <td>0.593400</td>\n",
       "      <td>0.847266</td>\n",
       "      <td>0.602256</td>\n",
       "      <td>1.146149</td>\n",
       "      <td>...</td>\n",
       "      <td>0.718778</td>\n",
       "      <td>0.560341</td>\n",
       "      <td>0.870152</td>\n",
       "      <td>-0.096669</td>\n",
       "      <td>0.846742</td>\n",
       "      <td>-2.076474</td>\n",
       "      <td>-1.394734</td>\n",
       "      <td>-0.778212</td>\n",
       "      <td>-1.593897</td>\n",
       "      <td>-3.008431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000210194</th>\n",
       "      <td>-0.803149</td>\n",
       "      <td>-0.890206</td>\n",
       "      <td>0.228050</td>\n",
       "      <td>0.120984</td>\n",
       "      <td>-0.016644</td>\n",
       "      <td>-0.189100</td>\n",
       "      <td>0.399229</td>\n",
       "      <td>0.957825</td>\n",
       "      <td>0.587267</td>\n",
       "      <td>1.908344</td>\n",
       "      <td>...</td>\n",
       "      <td>0.657724</td>\n",
       "      <td>0.586310</td>\n",
       "      <td>0.763737</td>\n",
       "      <td>0.258364</td>\n",
       "      <td>0.682111</td>\n",
       "      <td>-2.844500</td>\n",
       "      <td>-1.836432</td>\n",
       "      <td>-1.001166</td>\n",
       "      <td>-1.854168</td>\n",
       "      <td>-2.561568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000198727</th>\n",
       "      <td>0.155493</td>\n",
       "      <td>0.302684</td>\n",
       "      <td>0.565802</td>\n",
       "      <td>-1.174228</td>\n",
       "      <td>-0.717740</td>\n",
       "      <td>-1.152147</td>\n",
       "      <td>0.139417</td>\n",
       "      <td>0.856849</td>\n",
       "      <td>0.531410</td>\n",
       "      <td>1.330387</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.330358</td>\n",
       "      <td>-0.550126</td>\n",
       "      <td>0.216737</td>\n",
       "      <td>-1.412068</td>\n",
       "      <td>0.441226</td>\n",
       "      <td>-0.801769</td>\n",
       "      <td>-0.239977</td>\n",
       "      <td>-0.846495</td>\n",
       "      <td>-1.193806</td>\n",
       "      <td>-4.915172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000210195</th>\n",
       "      <td>-0.616127</td>\n",
       "      <td>-0.616127</td>\n",
       "      <td>0.102028</td>\n",
       "      <td>-0.616127</td>\n",
       "      <td>-0.616127</td>\n",
       "      <td>-0.616127</td>\n",
       "      <td>-0.616127</td>\n",
       "      <td>1.541751</td>\n",
       "      <td>1.559448</td>\n",
       "      <td>2.383715</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.616127</td>\n",
       "      <td>-0.428343</td>\n",
       "      <td>-0.616127</td>\n",
       "      <td>-0.616127</td>\n",
       "      <td>-0.616127</td>\n",
       "      <td>-0.616127</td>\n",
       "      <td>-0.616127</td>\n",
       "      <td>-0.616127</td>\n",
       "      <td>-0.616127</td>\n",
       "      <td>-0.616127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000210196</th>\n",
       "      <td>-0.531240</td>\n",
       "      <td>-0.808383</td>\n",
       "      <td>-0.196657</td>\n",
       "      <td>-0.385634</td>\n",
       "      <td>-0.325331</td>\n",
       "      <td>-0.057740</td>\n",
       "      <td>-0.481928</td>\n",
       "      <td>1.573486</td>\n",
       "      <td>1.497713</td>\n",
       "      <td>1.908741</td>\n",
       "      <td>...</td>\n",
       "      <td>0.144845</td>\n",
       "      <td>-0.106866</td>\n",
       "      <td>-0.207299</td>\n",
       "      <td>-1.077182</td>\n",
       "      <td>0.042482</td>\n",
       "      <td>-0.991467</td>\n",
       "      <td>-1.077182</td>\n",
       "      <td>-1.077182</td>\n",
       "      <td>-1.077182</td>\n",
       "      <td>-1.077182</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>31910 rows × 54 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Adipose - Subcutaneous  Adipose - Visceral (Omentum)  \\\n",
       "ENSG00000227232                0.387857                      0.044690   \n",
       "ENSG00000268903               -0.050832                      0.496878   \n",
       "ENSG00000269981                0.004120                      0.398468   \n",
       "ENSG00000239906               -0.228142                     -0.228142   \n",
       "ENSG00000279928                0.000000                      0.000000   \n",
       "...                                 ...                           ...   \n",
       "ENSG00000198695               -0.837431                     -0.799416   \n",
       "ENSG00000210194               -0.803149                     -0.890206   \n",
       "ENSG00000198727                0.155493                      0.302684   \n",
       "ENSG00000210195               -0.616127                     -0.616127   \n",
       "ENSG00000210196               -0.531240                     -0.808383   \n",
       "\n",
       "                 Adrenal Gland  Artery - Aorta  Artery - Coronary  \\\n",
       "ENSG00000227232      -0.372727        0.380429           0.312583   \n",
       "ENSG00000268903       0.095292        0.086940           0.073609   \n",
       "ENSG00000269981       0.226853        0.186657           0.023021   \n",
       "ENSG00000239906      -0.228142       -0.228142          -0.228142   \n",
       "ENSG00000279928       0.000000        0.000000           0.000000   \n",
       "...                        ...             ...                ...   \n",
       "ENSG00000198695       0.224678        0.156991          -0.258307   \n",
       "ENSG00000210194       0.228050        0.120984          -0.016644   \n",
       "ENSG00000198727       0.565802       -1.174228          -0.717740   \n",
       "ENSG00000210195       0.102028       -0.616127          -0.616127   \n",
       "ENSG00000210196      -0.196657       -0.385634          -0.325331   \n",
       "\n",
       "                 Artery - Tibial   Bladder  Brain - Amygdala  \\\n",
       "ENSG00000227232         0.185383  0.827503         -1.518619   \n",
       "ENSG00000268903        -0.991954  0.040693         -1.041696   \n",
       "ENSG00000269981        -0.650469  0.060835         -1.351962   \n",
       "ENSG00000239906        -0.228142 -0.228142         -0.228142   \n",
       "ENSG00000279928         0.000000  0.000000          0.000000   \n",
       "...                          ...       ...               ...   \n",
       "ENSG00000198695        -0.375927  0.593400          0.847266   \n",
       "ENSG00000210194        -0.189100  0.399229          0.957825   \n",
       "ENSG00000198727        -1.152147  0.139417          0.856849   \n",
       "ENSG00000210195        -0.616127 -0.616127          1.541751   \n",
       "ENSG00000210196        -0.057740 -0.481928          1.573486   \n",
       "\n",
       "                 Brain - Anterior cingulate cortex (BA24)  \\\n",
       "ENSG00000227232                                 -1.219846   \n",
       "ENSG00000268903                                 -1.102778   \n",
       "ENSG00000269981                                 -1.513653   \n",
       "ENSG00000239906                                 -0.228142   \n",
       "ENSG00000279928                                  0.000000   \n",
       "...                                                   ...   \n",
       "ENSG00000198695                                  0.602256   \n",
       "ENSG00000210194                                  0.587267   \n",
       "ENSG00000198727                                  0.531410   \n",
       "ENSG00000210195                                  1.559448   \n",
       "ENSG00000210196                                  1.497713   \n",
       "\n",
       "                 Brain - Caudate (basal ganglia)  ...  \\\n",
       "ENSG00000227232                        -1.362774  ...   \n",
       "ENSG00000268903                        -1.395384  ...   \n",
       "ENSG00000269981                        -1.332618  ...   \n",
       "ENSG00000239906                        -0.228142  ...   \n",
       "ENSG00000279928                         0.000000  ...   \n",
       "...                                          ...  ...   \n",
       "ENSG00000198695                         1.146149  ...   \n",
       "ENSG00000210194                         1.908344  ...   \n",
       "ENSG00000198727                         1.330387  ...   \n",
       "ENSG00000210195                         2.383715  ...   \n",
       "ENSG00000210196                         1.908741  ...   \n",
       "\n",
       "                 Skin - Not Sun Exposed (Suprapubic)  \\\n",
       "ENSG00000227232                             1.082426   \n",
       "ENSG00000268903                             0.647509   \n",
       "ENSG00000269981                             0.626217   \n",
       "ENSG00000239906                            -0.228142   \n",
       "ENSG00000279928                             0.000000   \n",
       "...                                              ...   \n",
       "ENSG00000198695                             0.718778   \n",
       "ENSG00000210194                             0.657724   \n",
       "ENSG00000198727                            -0.330358   \n",
       "ENSG00000210195                            -0.616127   \n",
       "ENSG00000210196                             0.144845   \n",
       "\n",
       "                 Skin - Sun Exposed (Lower leg)  \\\n",
       "ENSG00000227232                        1.143191   \n",
       "ENSG00000268903                        0.693028   \n",
       "ENSG00000269981                        0.692679   \n",
       "ENSG00000239906                       -0.228142   \n",
       "ENSG00000279928                        0.000000   \n",
       "...                                         ...   \n",
       "ENSG00000198695                        0.560341   \n",
       "ENSG00000210194                        0.586310   \n",
       "ENSG00000198727                       -0.550126   \n",
       "ENSG00000210195                       -0.428343   \n",
       "ENSG00000210196                       -0.106866   \n",
       "\n",
       "                 Small Intestine - Terminal Ileum    Spleen   Stomach  \\\n",
       "ENSG00000227232                          0.445551  1.080361 -0.131576   \n",
       "ENSG00000268903                          0.384310  3.053241  0.075008   \n",
       "ENSG00000269981                          0.467112  2.547492  0.197949   \n",
       "ENSG00000239906                         -0.228142  2.722479 -0.228142   \n",
       "ENSG00000279928                          0.000000  0.000000  0.000000   \n",
       "...                                           ...       ...       ...   \n",
       "ENSG00000198695                          0.870152 -0.096669  0.846742   \n",
       "ENSG00000210194                          0.763737  0.258364  0.682111   \n",
       "ENSG00000198727                          0.216737 -1.412068  0.441226   \n",
       "ENSG00000210195                         -0.616127 -0.616127 -0.616127   \n",
       "ENSG00000210196                         -0.207299 -1.077182  0.042482   \n",
       "\n",
       "                     Testis   Thyroid    Uterus    Vagina  Whole Blood  \n",
       "ENSG00000227232    0.655744  1.184599  1.435200  1.023493    -0.398931  \n",
       "ENSG00000268903    2.755504  1.219419  0.239213  1.118411     4.347284  \n",
       "ENSG00000269981    2.229452  1.094000  0.257616  0.989228     3.440013  \n",
       "ENSG00000239906    2.279362 -0.228142 -0.228142 -0.228142     6.141721  \n",
       "ENSG00000279928  188.232611  0.000000  0.000000  0.000000     0.000000  \n",
       "...                     ...       ...       ...       ...          ...  \n",
       "ENSG00000198695   -2.076474 -1.394734 -0.778212 -1.593897    -3.008431  \n",
       "ENSG00000210194   -2.844500 -1.836432 -1.001166 -1.854168    -2.561568  \n",
       "ENSG00000198727   -0.801769 -0.239977 -0.846495 -1.193806    -4.915172  \n",
       "ENSG00000210195   -0.616127 -0.616127 -0.616127 -0.616127    -0.616127  \n",
       "ENSG00000210196   -0.991467 -1.077182 -1.077182 -1.077182    -1.077182  \n",
       "\n",
       "[31910 rows x 54 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read the results.\n",
    "adatiss = pd.read_table(\"adatiss_output.tsv\")\n",
    "adatiss.rename(columns=reverse_column_map, inplace=True)\n",
    "adatiss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7. Pack the data for output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_name_mapping = pd.read_table(\"/home/ktsukanov/repositories/open-targets/curation/mappings/biosystem/gtex_v8_tissues.tsv\")\n",
    "name_mapping = {\n",
    "    gtex_tissue: ontology_code\n",
    "    for gtex_tissue, ontology_code\n",
    "    in df_name_mapping[[\"gtex_tissue_name\", \"ontology_code\"]].values.tolist()\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "for tissue in df.columns:\n",
    "    if tissue not in name_mapping:\n",
    "        raise AssertionError(f\"GTEx tissue {tissue} is not present in the provided mappings file\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pack Adatiss results for output.\n",
    "\n",
    "def pack_adatiss_row(row):\n",
    "    \"\"\"Given a row with Adatiss results for a given gene, pack them into the list of (tissue, z-score) values ready for output.\"\"\"\n",
    "    # Extract the column names and values from the row\n",
    "    cols = adatiss.columns\n",
    "    vals = row.values.tolist()\n",
    "\n",
    "    # Pack the values into a list of dictionaries\n",
    "    dicts = []\n",
    "    for col, val in zip(cols, vals):\n",
    "        dicts.append({\"bodyPartLevel\": \"tissue\", \"bodyPartName\": col, \"bodyPartId\": name_mapping[col], \"adatissScore\": round(val, 3)})\n",
    "\n",
    "    return dicts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gini</th>\n",
       "      <th>hpaSpecificity</th>\n",
       "      <th>hpaDistribution</th>\n",
       "      <th>adatissScores</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ENSG00000227232</th>\n",
       "      <td>0.278</td>\n",
       "      <td>Low tissue specificity</td>\n",
       "      <td>Detected in many</td>\n",
       "      <td>[{'bodyPartLevel': 'tissue', 'bodyPartName': '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000268903</th>\n",
       "      <td>0.655</td>\n",
       "      <td>Tissue enhanced</td>\n",
       "      <td>Detected in many</td>\n",
       "      <td>[{'bodyPartLevel': 'tissue', 'bodyPartName': '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000269981</th>\n",
       "      <td>0.666</td>\n",
       "      <td>Tissue enhanced</td>\n",
       "      <td>Detected in many</td>\n",
       "      <td>[{'bodyPartLevel': 'tissue', 'bodyPartName': '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000239906</th>\n",
       "      <td>0.899</td>\n",
       "      <td>Tissue enhanced</td>\n",
       "      <td>Detected in some</td>\n",
       "      <td>[{'bodyPartLevel': 'tissue', 'bodyPartName': '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000279928</th>\n",
       "      <td>0.685</td>\n",
       "      <td>Tissue enriched</td>\n",
       "      <td>Detected in single</td>\n",
       "      <td>[{'bodyPartLevel': 'tissue', 'bodyPartName': '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000198695</th>\n",
       "      <td>0.285</td>\n",
       "      <td>Low tissue specificity</td>\n",
       "      <td>Detected in all</td>\n",
       "      <td>[{'bodyPartLevel': 'tissue', 'bodyPartName': '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000210194</th>\n",
       "      <td>0.356</td>\n",
       "      <td>Low tissue specificity</td>\n",
       "      <td>Detected in all</td>\n",
       "      <td>[{'bodyPartLevel': 'tissue', 'bodyPartName': '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000198727</th>\n",
       "      <td>0.246</td>\n",
       "      <td>Low tissue specificity</td>\n",
       "      <td>Detected in all</td>\n",
       "      <td>[{'bodyPartLevel': 'tissue', 'bodyPartName': '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000210195</th>\n",
       "      <td>0.590</td>\n",
       "      <td>Tissue enhanced</td>\n",
       "      <td>Detected in many</td>\n",
       "      <td>[{'bodyPartLevel': 'tissue', 'bodyPartName': '...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ENSG00000210196</th>\n",
       "      <td>0.418</td>\n",
       "      <td>Tissue enhanced</td>\n",
       "      <td>Detected in many</td>\n",
       "      <td>[{'bodyPartLevel': 'tissue', 'bodyPartName': '...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>31910 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  gini          hpaSpecificity     hpaDistribution  \\\n",
       "Name                                                                 \n",
       "ENSG00000227232  0.278  Low tissue specificity    Detected in many   \n",
       "ENSG00000268903  0.655         Tissue enhanced    Detected in many   \n",
       "ENSG00000269981  0.666         Tissue enhanced    Detected in many   \n",
       "ENSG00000239906  0.899         Tissue enhanced    Detected in some   \n",
       "ENSG00000279928  0.685         Tissue enriched  Detected in single   \n",
       "...                ...                     ...                 ...   \n",
       "ENSG00000198695  0.285  Low tissue specificity     Detected in all   \n",
       "ENSG00000210194  0.356  Low tissue specificity     Detected in all   \n",
       "ENSG00000198727  0.246  Low tissue specificity     Detected in all   \n",
       "ENSG00000210195  0.590         Tissue enhanced    Detected in many   \n",
       "ENSG00000210196  0.418         Tissue enhanced    Detected in many   \n",
       "\n",
       "                                                     adatissScores  \n",
       "Name                                                                \n",
       "ENSG00000227232  [{'bodyPartLevel': 'tissue', 'bodyPartName': '...  \n",
       "ENSG00000268903  [{'bodyPartLevel': 'tissue', 'bodyPartName': '...  \n",
       "ENSG00000269981  [{'bodyPartLevel': 'tissue', 'bodyPartName': '...  \n",
       "ENSG00000239906  [{'bodyPartLevel': 'tissue', 'bodyPartName': '...  \n",
       "ENSG00000279928  [{'bodyPartLevel': 'tissue', 'bodyPartName': '...  \n",
       "...                                                            ...  \n",
       "ENSG00000198695  [{'bodyPartLevel': 'tissue', 'bodyPartName': '...  \n",
       "ENSG00000210194  [{'bodyPartLevel': 'tissue', 'bodyPartName': '...  \n",
       "ENSG00000198727  [{'bodyPartLevel': 'tissue', 'bodyPartName': '...  \n",
       "ENSG00000210195  [{'bodyPartLevel': 'tissue', 'bodyPartName': '...  \n",
       "ENSG00000210196  [{'bodyPartLevel': 'tissue', 'bodyPartName': '...  \n",
       "\n",
       "[31910 rows x 4 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Map annotation names.\n",
    "annotation_column_map = {\n",
    "    'gini': 'gini', \n",
    "    'hpa_specificity': 'hpaSpecificity',\n",
    "    'hpa_distribution': 'hpaDistribution'\n",
    "}\n",
    "columns_to_filter = list(annotation_column_map.keys())\n",
    "a_out = (\n",
    "    a\n",
    "    [columns_to_filter]\n",
    "    .rename(columns=annotation_column_map)\n",
    ")\n",
    "a_out['gini'] = a_out['gini'].round(3)\n",
    "a_out['adatissScores'] = adatiss.apply(pack_adatiss_row, axis=1)\n",
    "a_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Pack and output the data.\n",
    "\n",
    "def remove_adatiss_if_none(d):\n",
    "    if d['adatissScores'] != d['adatissScores']:  # using the fact that NaN != NaN\n",
    "        return {k: v for k, v in d.items() if k != 'adatissScores'}\n",
    "        print('removed')\n",
    "    else:\n",
    "        return d\n",
    "\n",
    "# Create a list to store JSON objects\n",
    "json_list = []\n",
    "\n",
    "# Iterate over each row index\n",
    "for idx in df.index:\n",
    "    # Create a dictionary for the current row\n",
    "    row_dict = {\n",
    "        \"ensemblGeneId\": idx,\n",
    "        \"expression\": [\n",
    "            {\n",
    "                \"bodyPartLevel\": \"tissue\",\n",
    "                \"bodyPartId\": key,\n",
    "                \"bodyPartName\": name_mapping[key],\n",
    "                \"tpm\": value,\n",
    "            }\n",
    "            for key, value\n",
    "            in df.loc[idx].to_dict().items()\n",
    "        ],\n",
    "        \"expressionSpecificity\": remove_adatiss_if_none(a_out.loc[idx].to_dict())\n",
    "    }\n",
    "    # Append the dictionary to the list\n",
    "    json_list.append(row_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "with gzip.open(\"expression_v2.1.jsonl.gz\", \"wt\", compresslevel=9) as f:\n",
    "    for obj in json_list:\n",
    "        f.write(json.dumps(obj))\n",
    "        f.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"ensemblGeneId\": \"ENSG00000227232\", \"expression\": [{\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Adipose - Subcutaneous\", \"bodyPartName\": \"UBERON:0002190\", \"tpm\": 4.06403}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Adipose - Visceral (Omentum)\", \"bodyPartName\": \"UBERON:0010414\", \"tpm\": 3.37111}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Adrenal Gland\", \"bodyPartName\": \"UBERON:0002369\", \"tpm\": 2.68549}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Artery - Aorta\", \"bodyPartName\": \"UBERON:0001496\", \"tpm\": 4.04762}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Artery - Coronary\", \"bodyPartName\": \"UBERON:0001621\", \"tpm\": 3.90076}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Artery - Tibial\", \"bodyPartName\": \"UBERON:0007610\", \"tpm\": 3.63963}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Bladder\", \"bodyPartName\": \"UBERON:0001255\", \"tpm\": 5.16375}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Brain - Amygdala\", \"bodyPartName\": \"UBERON:0001876\", \"tpm\": 1.43859}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Brain - Anterior cingulate cortex (BA24)\", \"bodyPartName\": \"UBERON:0009835\", \"tpm\": 1.69285}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Brain - Caudate (basal ganglia)\", \"bodyPartName\": \"UBERON:0001873\", \"tpm\": 1.56605}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Brain - Cerebellar Hemisphere\", \"bodyPartName\": \"UBERON:0002037\", \"tpm\": 4.99231}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Brain - Cerebellum\", \"bodyPartName\": \"UBERON:0002037\", \"tpm\": 5.72099}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Brain - Cortex\", \"bodyPartName\": \"UBERON:0001870\", \"tpm\": 2.48317}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Brain - Frontal Cortex (BA9)\", \"bodyPartName\": \"UBERON:0009834\", \"tpm\": 2.14667}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Brain - Hippocampus\", \"bodyPartName\": \"UBERON:0001954\", \"tpm\": 1.68599}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Brain - Hypothalamus\", \"bodyPartName\": \"UBERON:0001898\", \"tpm\": 1.74811}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Brain - Nucleus accumbens (basal ganglia)\", \"bodyPartName\": \"UBERON:0001882\", \"tpm\": 1.53899}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Brain - Putamen (basal ganglia)\", \"bodyPartName\": \"UBERON:0001874\", \"tpm\": 1.44167}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Brain - Spinal cord (cervical c-1)\", \"bodyPartName\": \"UBERON:0006469\", \"tpm\": 2.73049}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Brain - Substantia nigra\", \"bodyPartName\": \"UBERON:0002038\", \"tpm\": 1.74194}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Breast - Mammary Tissue\", \"bodyPartName\": \"UBERON:0008367\", \"tpm\": 4.43876}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Cells - Cultured fibroblasts\", \"bodyPartName\": \"EFO:0002009\", \"tpm\": 1.6786}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Cells - EBV-transformed lymphocytes\", \"bodyPartName\": \"EFO:0000572\", \"tpm\": 2.49477}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Cervix - Ectocervix\", \"bodyPartName\": \"UBERON:0012249\", \"tpm\": 5.62935}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Cervix - Endocervix\", \"bodyPartName\": \"UBERON:0000458\", \"tpm\": 7.09749}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Colon - Sigmoid\", \"bodyPartName\": \"UBERON:0001159\", \"tpm\": 4.64777}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Colon - Transverse\", \"bodyPartName\": \"UBERON:0001157\", \"tpm\": 3.59509}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Esophagus - Gastroesophageal Junction\", \"bodyPartName\": \"UBERON:0004550\", \"tpm\": 4.32641}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Esophagus - Mucosa\", \"bodyPartName\": \"UBERON:0006920\", \"tpm\": 3.11749}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Esophagus - Muscularis\", \"bodyPartName\": \"UBERON:0004648\", \"tpm\": 4.10335}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Fallopian Tube\", \"bodyPartName\": \"UBERON:0003889\", \"tpm\": 6.13409}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Heart - Atrial Appendage\", \"bodyPartName\": \"UBERON:0006631\", \"tpm\": 1.52031}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Heart - Left Ventricle\", \"bodyPartName\": \"UBERON:0006566\", \"tpm\": 0.924962}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Kidney - Cortex\", \"bodyPartName\": \"UBERON:0001225\", \"tpm\": 2.77081}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Kidney - Medulla\", \"bodyPartName\": \"UBERON:0001293\", \"tpm\": 2.21451}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Liver\", \"bodyPartName\": \"UBERON:0001114\", \"tpm\": 1.76541}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Lung\", \"bodyPartName\": \"UBERON:0008952\", \"tpm\": 4.50841}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Minor Salivary Gland\", \"bodyPartName\": \"UBERON:0006330\", \"tpm\": 3.52767}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Muscle - Skeletal\", \"bodyPartName\": \"UBERON:0011907\", \"tpm\": 1.41667}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Nerve - Tibial\", \"bodyPartName\": \"UBERON:0001323\", \"tpm\": 6.68531}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Ovary\", \"bodyPartName\": \"UBERON:0000992\", \"tpm\": 6.6341}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Pancreas\", \"bodyPartName\": \"UBERON:0001150\", \"tpm\": 1.80871}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Pituitary\", \"bodyPartName\": \"UBERON:0000007\", \"tpm\": 5.42546}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Prostate\", \"bodyPartName\": \"UBERON:0002367\", \"tpm\": 7.08318}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Skin - Not Sun Exposed (Suprapubic)\", \"bodyPartName\": \"UBERON:0036149\", \"tpm\": 5.93298}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Skin - Sun Exposed (Lower leg)\", \"bodyPartName\": \"UBERON:0004264\", \"tpm\": 6.13265}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Small Intestine - Terminal Ileum\", \"bodyPartName\": \"UBERON:0001211\", \"tpm\": 4.19378}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Spleen\", \"bodyPartName\": \"UBERON:0002106\", \"tpm\": 5.92631}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Stomach\", \"bodyPartName\": \"UBERON:0000945\", \"tpm\": 3.06248}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Testis\", \"bodyPartName\": \"UBERON:0000473\", \"tpm\": 4.70253}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Thyroid\", \"bodyPartName\": \"UBERON:0002046\", \"tpm\": 6.27255}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Uterus\", \"bodyPartName\": \"UBERON:0000995\", \"tpm\": 7.19001}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Vagina\", \"bodyPartName\": \"UBERON:0000996\", \"tpm\": 5.74554}, {\"bodyPartLevel\": \"tissue\", \"bodyPartId\": \"Whole Blood\", \"bodyPartName\": \"UBERON:0013756\", \"tpm\": 2.64743}], \"expressionSpecificity\": {\"gini\": 0.278, \"hpaSpecificity\": \"Low tissue specificity\", \"hpaDistribution\": \"Detected in many\", \"adatissScores\": [{\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Adipose - Subcutaneous\", \"bodyPartId\": \"UBERON:0002190\", \"adatissScore\": 0.388}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Adipose - Visceral (Omentum)\", \"bodyPartId\": \"UBERON:0010414\", \"adatissScore\": 0.045}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Adrenal Gland\", \"bodyPartId\": \"UBERON:0002369\", \"adatissScore\": -0.373}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Artery - Aorta\", \"bodyPartId\": \"UBERON:0001496\", \"adatissScore\": 0.38}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Artery - Coronary\", \"bodyPartId\": \"UBERON:0001621\", \"adatissScore\": 0.313}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Artery - Tibial\", \"bodyPartId\": \"UBERON:0007610\", \"adatissScore\": 0.185}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Bladder\", \"bodyPartId\": \"UBERON:0001255\", \"adatissScore\": 0.828}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Brain - Amygdala\", \"bodyPartId\": \"UBERON:0001876\", \"adatissScore\": -1.519}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Brain - Anterior cingulate cortex (BA24)\", \"bodyPartId\": \"UBERON:0009835\", \"adatissScore\": -1.22}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Brain - Caudate (basal ganglia)\", \"bodyPartId\": \"UBERON:0001873\", \"adatissScore\": -1.363}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Brain - Cerebellar Hemisphere\", \"bodyPartId\": \"UBERON:0002037\", \"adatissScore\": 0.766}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Brain - Cerebellum\", \"bodyPartId\": \"UBERON:0002037\", \"adatissScore\": 1.016}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Brain - Cortex\", \"bodyPartId\": \"UBERON:0001870\", \"adatissScore\": -0.517}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Brain - Frontal Cortex (BA9)\", \"bodyPartId\": \"UBERON:0009834\", \"adatissScore\": -0.784}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Brain - Hippocampus\", \"bodyPartId\": \"UBERON:0001954\", \"adatissScore\": -1.227}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Brain - Hypothalamus\", \"bodyPartId\": \"UBERON:0001898\", \"adatissScore\": -1.161}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Brain - Nucleus accumbens (basal ganglia)\", \"bodyPartId\": \"UBERON:0001882\", \"adatissScore\": -1.395}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Brain - Putamen (basal ganglia)\", \"bodyPartId\": \"UBERON:0001874\", \"adatissScore\": -1.515}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Brain - Spinal cord (cervical c-1)\", \"bodyPartId\": \"UBERON:0006469\", \"adatissScore\": -0.342}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Brain - Substantia nigra\", \"bodyPartId\": \"UBERON:0002038\", \"adatissScore\": -1.167}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Breast - Mammary Tissue\", \"bodyPartId\": \"UBERON:0008367\", \"adatissScore\": 0.55}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Cells - Cultured fibroblasts\", \"bodyPartId\": \"EFO:0002009\", \"adatissScore\": -1.235}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Cells - EBV-transformed lymphocytes\", \"bodyPartId\": \"EFO:0000572\", \"adatissScore\": -0.508}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Cervix - Ectocervix\", \"bodyPartId\": \"UBERON:0012249\", \"adatissScore\": 0.986}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Cervix - Endocervix\", \"bodyPartId\": \"UBERON:0000458\", \"adatissScore\": 1.411}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Colon - Sigmoid\", \"bodyPartId\": \"UBERON:0001159\", \"adatissScore\": 0.634}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Colon - Transverse\", \"bodyPartId\": \"UBERON:0001157\", \"adatissScore\": 0.163}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Esophagus - Gastroesophageal Junction\", \"bodyPartId\": \"UBERON:0004550\", \"adatissScore\": 0.503}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Esophagus - Mucosa\", \"bodyPartId\": \"UBERON:0006920\", \"adatissScore\": -0.099}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Esophagus - Muscularis\", \"bodyPartId\": \"UBERON:0004648\", \"adatissScore\": 0.406}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Fallopian Tube\", \"bodyPartId\": \"UBERON:0003889\", \"adatissScore\": 1.144}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Heart - Atrial Appendage\", \"bodyPartId\": \"UBERON:0006631\", \"adatissScore\": -1.417}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Heart - Left Ventricle\", \"bodyPartId\": \"UBERON:0006566\", \"adatissScore\": -2.186}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Kidney - Cortex\", \"bodyPartId\": \"UBERON:0001225\", \"adatissScore\": -0.315}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Kidney - Medulla\", \"bodyPartId\": \"UBERON:0001293\", \"adatissScore\": -0.727}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Liver\", \"bodyPartId\": \"UBERON:0001114\", \"adatissScore\": -1.143}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Lung\", \"bodyPartId\": \"UBERON:0008952\", \"adatissScore\": 0.578}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Minor Salivary Gland\", \"bodyPartId\": \"UBERON:0006330\", \"adatissScore\": 0.128}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Muscle - Skeletal\", \"bodyPartId\": \"UBERON:0011907\", \"adatissScore\": -1.547}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Nerve - Tibial\", \"bodyPartId\": \"UBERON:0001323\", \"adatissScore\": 1.302}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Ovary\", \"bodyPartId\": \"UBERON:0000992\", \"adatissScore\": 1.287}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Pancreas\", \"bodyPartId\": \"UBERON:0001150\", \"adatissScore\": -1.098}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Pituitary\", \"bodyPartId\": \"UBERON:0000007\", \"adatissScore\": 0.918}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Prostate\", \"bodyPartId\": \"UBERON:0002367\", \"adatissScore\": 1.408}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Skin - Not Sun Exposed (Suprapubic)\", \"bodyPartId\": \"UBERON:0036149\", \"adatissScore\": 1.082}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Skin - Sun Exposed (Lower leg)\", \"bodyPartId\": \"UBERON:0004264\", \"adatissScore\": 1.143}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Small Intestine - Terminal Ileum\", \"bodyPartId\": \"UBERON:0001211\", \"adatissScore\": 0.446}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Spleen\", \"bodyPartId\": \"UBERON:0002106\", \"adatissScore\": 1.08}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Stomach\", \"bodyPartId\": \"UBERON:0000945\", \"adatissScore\": -0.132}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Testis\", \"bodyPartId\": \"UBERON:0000473\", \"adatissScore\": 0.656}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Thyroid\", \"bodyPartId\": \"UBERON:0002046\", \"adatissScore\": 1.185}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Uterus\", \"bodyPartId\": \"UBERON:0000995\", \"adatissScore\": 1.435}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Vagina\", \"bodyPartId\": \"UBERON:0000996\", \"adatissScore\": 1.023}, {\"bodyPartLevel\": \"tissue\", \"bodyPartName\": \"Whole Blood\", \"bodyPartId\": \"UBERON:0013756\", \"adatissScore\": -0.399}]}}\n"
     ]
    }
   ],
   "source": [
    "# Output one example record.\n",
    "print(json.dumps(json_list[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ingest the data back and print the schema.\n",
    "df = spark.read.format(\"json\").option(\"compression\", \"gzip\").load(\"expression_v2.1.jsonl.gz\")\n",
    "df.printSchema()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}