0.1.0_Mouse_Atlas_Data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Mouse Atlas Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.12.3 cleaning vdj code\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "import importlib\n",
    "import himc_helper_functions as hf\n",
    "importlib.reload(hf)\n",
    "hf.get_version()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">> clustergrammer2 backend version 0.5.6\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from copy import deepcopy\n",
    "from clustergrammer2 import net\n",
    "df = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(24057, 1679)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['gex'] = pd.read_csv('data/primary/allen_brain_atlas/genes_counts.csv', index_col=0)\n",
    "df['gex'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A01101401</th>\n",
       "      <th>A01101402</th>\n",
       "      <th>A01101403</th>\n",
       "      <th>A01101404</th>\n",
       "      <th>A01101405</th>\n",
       "      <th>A01101406</th>\n",
       "      <th>A01101407</th>\n",
       "      <th>A01101408</th>\n",
       "      <th>A02271433</th>\n",
       "      <th>A02271434</th>\n",
       "      <th>...</th>\n",
       "      <th>Y11041422</th>\n",
       "      <th>Y11041423</th>\n",
       "      <th>Y11041424</th>\n",
       "      <th>Y11041425</th>\n",
       "      <th>Z11041426</th>\n",
       "      <th>Z11041427</th>\n",
       "      <th>Z11041428</th>\n",
       "      <th>Z11041429</th>\n",
       "      <th>Z11041430</th>\n",
       "      <th>Z11041431</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0610005C13Rik</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0610007C21Rik</th>\n",
       "      <td>992.00</td>\n",
       "      <td>2287.02</td>\n",
       "      <td>491.78</td>\n",
       "      <td>1932.0</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>130.03</td>\n",
       "      <td>2110.02</td>\n",
       "      <td>955.0</td>\n",
       "      <td>326.0</td>\n",
       "      <td>933.0</td>\n",
       "      <td>...</td>\n",
       "      <td>379.0</td>\n",
       "      <td>397.0</td>\n",
       "      <td>654.0</td>\n",
       "      <td>172.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>443.0</td>\n",
       "      <td>1457.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.08</td>\n",
       "      <td>2966.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0610007L01Rik</th>\n",
       "      <td>2.57</td>\n",
       "      <td>177.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.00</td>\n",
       "      <td>3040.99</td>\n",
       "      <td>101.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1042.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>411.0</td>\n",
       "      <td>471.0</td>\n",
       "      <td>692.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>115.00</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0610007N19Rik</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0610007P08Rik</th>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>17.03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1014.00</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1679 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               A01101401  A01101402  A01101403  A01101404  A01101405  \\\n",
       "0610005C13Rik       0.00       0.00       0.00        0.0        0.0   \n",
       "0610007C21Rik     992.00    2287.02     491.78     1932.0     1425.0   \n",
       "0610007L01Rik       2.57     177.00       0.00        1.0        2.0   \n",
       "0610007N19Rik       0.00       0.00       0.00        0.0        0.0   \n",
       "0610007P08Rik       0.00       0.00       0.00        0.0        0.0   \n",
       "\n",
       "               A01101406  A01101407  A01101408  A02271433  A02271434  \\\n",
       "0610005C13Rik       0.00       0.00        0.0        0.0        0.0   \n",
       "0610007C21Rik     130.03    2110.02      955.0      326.0      933.0   \n",
       "0610007L01Rik       3.00    3040.99      101.0        0.0     1042.0   \n",
       "0610007N19Rik       0.00       0.00        0.0        0.0        0.0   \n",
       "0610007P08Rik       0.00      17.03        0.0        0.0        0.0   \n",
       "\n",
       "                 ...      Y11041422  Y11041423  Y11041424  Y11041425  \\\n",
       "0610005C13Rik    ...            0.0        0.0        0.0        0.0   \n",
       "0610007C21Rik    ...          379.0      397.0      654.0      172.0   \n",
       "0610007L01Rik    ...            0.0      411.0      471.0      692.0   \n",
       "0610007N19Rik    ...            0.0        0.0        0.0        0.0   \n",
       "0610007P08Rik    ...            0.0        0.0        0.0        0.0   \n",
       "\n",
       "               Z11041426  Z11041427  Z11041428  Z11041429  Z11041430  \\\n",
       "0610005C13Rik        0.0        0.0        0.0        0.0       0.00   \n",
       "0610007C21Rik        0.0      443.0     1457.0        4.0       2.08   \n",
       "0610007L01Rik        0.0        0.0        0.0       12.0     115.00   \n",
       "0610007N19Rik        0.0        0.0        0.0        0.0       0.00   \n",
       "0610007P08Rik        0.0        2.0        0.0        0.0    1014.00   \n",
       "\n",
       "               Z11041431  \n",
       "0610005C13Rik        0.0  \n",
       "0610007C21Rik     2966.0  \n",
       "0610007L01Rik        2.0  \n",
       "0610007N19Rik        0.0  \n",
       "0610007P08Rik        0.0  \n",
       "\n",
       "[5 rows x 1679 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['gex'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1679, 16)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_meta = pd.read_csv('data/primary/allen_brain_atlas/cell_metadata.csv')\n",
    "df_meta.index = df['gex'].columns.tolist()\n",
    "df_meta.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['long_name',\n",
       " 'cre',\n",
       " 'collection_date',\n",
       " 'sequencing_type',\n",
       " 'total_reads',\n",
       " 'all_mapped_percent',\n",
       " 'mRNA_percent',\n",
       " 'genome_percent',\n",
       " 'ercc_percent',\n",
       " 'tdt_permillion',\n",
       " 'major_class',\n",
       " 'sub_class',\n",
       " 'major_dissection',\n",
       " 'layer_dissectoin',\n",
       " 'color_code',\n",
       " 'short_name']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_meta.columns.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Inhibitory     761\n",
       "Excitatory     756\n",
       "Glia           138\n",
       "Endothelial     21\n",
       "Unknown          3\n",
       "Name: major_class, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_meta['major_class'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(32,)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_meta['sub_class'].value_counts().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = hf.ini_meta_cell(df=df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gex-umi-sum</th>\n",
       "      <th>num_expressed_genes</th>\n",
       "      <th>gex-ribo-avg</th>\n",
       "      <th>gex-mito-avg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>A01101401</th>\n",
       "      <td>14030202.14</td>\n",
       "      <td>8266.40</td>\n",
       "      <td>1461.000686</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>A01101402</th>\n",
       "      <td>4803366.44</td>\n",
       "      <td>5954.36</td>\n",
       "      <td>638.585000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>A01101403</th>\n",
       "      <td>3073418.37</td>\n",
       "      <td>6276.43</td>\n",
       "      <td>293.652549</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>A01101404</th>\n",
       "      <td>12192899.59</td>\n",
       "      <td>8312.21</td>\n",
       "      <td>1527.811176</td>\n",
       "      <td>13.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>A01101405</th>\n",
       "      <td>13477232.54</td>\n",
       "      <td>8086.26</td>\n",
       "      <td>1589.521176</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           gex-umi-sum  num_expressed_genes  gex-ribo-avg  gex-mito-avg\n",
       "A01101401  14030202.14              8266.40   1461.000686           0.0\n",
       "A01101402   4803366.44              5954.36    638.585000           0.0\n",
       "A01101403   3073418.37              6276.43    293.652549           0.0\n",
       "A01101404  12192899.59              8312.21   1527.811176          13.0\n",
       "A01101405  13477232.54              8086.26   1589.521176           0.0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = hf.filter_ribo_mito_from_gex(df)\n",
    "\n",
    "keep_genes = [x for x in df['gex'].index.tolist() if 'mt_' not in x]\n",
    "df['gex'] = df['gex'].loc[keep_genes]\n",
    "df['meta_cell'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(23856, 1679)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['gex'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1679, 20)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['meta_cell'] = pd.concat([df['meta_cell'], df_meta], axis=1)\n",
    "df['meta_cell'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['gex-cat'] = deepcopy(net.umi_norm(df['gex']))\n",
    "cols = df['gex-cat'].columns\n",
    "\n",
    "add_cat_list = ['major_class', 'sub_class']\n",
    "\n",
    "new_cols = hf.add_cats_from_meta(add_cat_list=add_cat_list, df_meta=df_meta, barcodes=cols)\n",
    "df['gex-cat'].columns = new_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('A01101401', 'major_class: Inhibitory', 'sub_class: Vip')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['gex-cat'].columns.tolist()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['gex-cat-umi'] = net.umi_norm(df['gex-cat'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8e2d34ab77f74afa82b9d6cc502a386a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "ExampleWidget(network='{\"row_nodes\": [{\"name\": \"Malat1\", \"ini\": 100, \"clust\": 59, \"rank\": 41, \"rankvar\": 44, \"…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "net.load_df(df['gex-cat-umi'])\n",
    "net.filter_N_top(N_top=100, inst_rc='row', rank_type='var')\n",
    "net.normalize(axis='row', norm_type='zscore')\n",
    "net.clip(-5,5)\n",
    "net.widget()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "33"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_cats = [x[2] for x in df['gex-cat-umi'].columns.tolist()]\n",
    "len(pd.Series(list_cats).value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1513, 33)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_top_dims = 100\n",
    "df_sig, keep_genes_dict, df_gene_pval, all_fold_info = net.generate_signatures(df_ini=df['gex-cat-umi'], \n",
    "                                                                               category_level='sub_class', \n",
    "                                                                               num_top_dims=num_top_dims)\n",
    "df_sig.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e58e0a77f439466bb8bdec6bd00588e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "ExampleWidget(network='{\"row_nodes\": [{\"name\": \"0610011F06Rik\", \"ini\": 1513, \"clust\": 869, \"rank\": 1491, \"rank…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "net.load_df(df_sig)\n",
    "net.normalize(axis='row', norm_type='zscore')\n",
    "net.widget()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sig.columns = [x[0] for x in df_sig.columns.tolist()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Astrocyte',\n",
       " 'Chodl',\n",
       " 'Endothelial',\n",
       " 'L2/3',\n",
       " 'L2/3_Syt10',\n",
       " 'L4',\n",
       " 'L5',\n",
       " 'L5_Chrna6',\n",
       " 'L5a',\n",
       " 'L5a1',\n",
       " 'L5a2',\n",
       " 'L5b',\n",
       " 'L5b1',\n",
       " 'L5b2',\n",
       " 'L6a',\n",
       " 'L6a1',\n",
       " 'L6a2',\n",
       " 'L6b',\n",
       " 'Microglia',\n",
       " 'Ndnf',\n",
       " 'OPC',\n",
       " 'Oligodendrocyte',\n",
       " 'Pvalb',\n",
       " 'Sst',\n",
       " 'Sst_Cbln4',\n",
       " 'Sst_Chodl',\n",
       " 'Sst_Clstn2',\n",
       " 'Sst_Nmbr',\n",
       " 'Sst_Nr2f2',\n",
       " 'Th',\n",
       " 'Unknown',\n",
       " 'Vip',\n",
       " 'nan']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_sig.columns.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sig.to_csv('data/primary/allen_brain_atlas/sig_sub_class.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}