Postprocess ML1M final results.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('final_result.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df.loc[df['Model'] != 'UCB', 'C'] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>C</th>\n",
       "      <th>Clicks</th>\n",
       "      <th>Iterative</th>\n",
       "      <th>MAP@10</th>\n",
       "      <th>MAP@15</th>\n",
       "      <th>MAP@20</th>\n",
       "      <th>MAP@5</th>\n",
       "      <th>MAP@50</th>\n",
       "      <th>Model</th>\n",
       "      <th>NDCG</th>\n",
       "      <th>...</th>\n",
       "      <th>Precision@20</th>\n",
       "      <th>Precision@5</th>\n",
       "      <th>Precision@50</th>\n",
       "      <th>R-Precision</th>\n",
       "      <th>Recall@10</th>\n",
       "      <th>Recall@15</th>\n",
       "      <th>Recall@20</th>\n",
       "      <th>Recall@5</th>\n",
       "      <th>Recall@50</th>\n",
       "      <th>SampleFromAll</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.111811</td>\n",
       "      <td>True</td>\n",
       "      <td>0.526253</td>\n",
       "      <td>0.492304</td>\n",
       "      <td>0.465160</td>\n",
       "      <td>0.573244</td>\n",
       "      <td>0.361020</td>\n",
       "      <td>Greedy</td>\n",
       "      <td>0.234092</td>\n",
       "      <td>...</td>\n",
       "      <td>0.369291</td>\n",
       "      <td>0.527244</td>\n",
       "      <td>0.242772</td>\n",
       "      <td>0.170054</td>\n",
       "      <td>0.064956</td>\n",
       "      <td>0.086163</td>\n",
       "      <td>0.104316</td>\n",
       "      <td>0.038086</td>\n",
       "      <td>0.172504</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.071969</td>\n",
       "      <td>True</td>\n",
       "      <td>0.584978</td>\n",
       "      <td>0.554146</td>\n",
       "      <td>0.530175</td>\n",
       "      <td>0.627984</td>\n",
       "      <td>0.438852</td>\n",
       "      <td>Greedy</td>\n",
       "      <td>0.300699</td>\n",
       "      <td>...</td>\n",
       "      <td>0.447323</td>\n",
       "      <td>0.578898</td>\n",
       "      <td>0.328756</td>\n",
       "      <td>0.233194</td>\n",
       "      <td>0.075532</td>\n",
       "      <td>0.103532</td>\n",
       "      <td>0.129367</td>\n",
       "      <td>0.042262</td>\n",
       "      <td>0.236853</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.113386</td>\n",
       "      <td>False</td>\n",
       "      <td>0.515102</td>\n",
       "      <td>0.481702</td>\n",
       "      <td>0.454581</td>\n",
       "      <td>0.558651</td>\n",
       "      <td>0.349802</td>\n",
       "      <td>Greedy</td>\n",
       "      <td>0.224927</td>\n",
       "      <td>...</td>\n",
       "      <td>0.358346</td>\n",
       "      <td>0.517795</td>\n",
       "      <td>0.232189</td>\n",
       "      <td>0.162072</td>\n",
       "      <td>0.063442</td>\n",
       "      <td>0.084081</td>\n",
       "      <td>0.100961</td>\n",
       "      <td>0.037315</td>\n",
       "      <td>0.163881</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.064567</td>\n",
       "      <td>False</td>\n",
       "      <td>0.603015</td>\n",
       "      <td>0.572099</td>\n",
       "      <td>0.545774</td>\n",
       "      <td>0.643102</td>\n",
       "      <td>0.441806</td>\n",
       "      <td>Greedy</td>\n",
       "      <td>0.295396</td>\n",
       "      <td>...</td>\n",
       "      <td>0.452283</td>\n",
       "      <td>0.600945</td>\n",
       "      <td>0.315969</td>\n",
       "      <td>0.224289</td>\n",
       "      <td>0.078446</td>\n",
       "      <td>0.107450</td>\n",
       "      <td>0.130705</td>\n",
       "      <td>0.044024</td>\n",
       "      <td>0.227050</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.114173</td>\n",
       "      <td>True</td>\n",
       "      <td>0.522890</td>\n",
       "      <td>0.489380</td>\n",
       "      <td>0.461550</td>\n",
       "      <td>0.566651</td>\n",
       "      <td>0.357196</td>\n",
       "      <td>ThompsonSampling</td>\n",
       "      <td>0.230445</td>\n",
       "      <td>...</td>\n",
       "      <td>0.364961</td>\n",
       "      <td>0.526299</td>\n",
       "      <td>0.238520</td>\n",
       "      <td>0.166569</td>\n",
       "      <td>0.064805</td>\n",
       "      <td>0.085372</td>\n",
       "      <td>0.103225</td>\n",
       "      <td>0.037928</td>\n",
       "      <td>0.168838</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.074646</td>\n",
       "      <td>True</td>\n",
       "      <td>0.581251</td>\n",
       "      <td>0.552599</td>\n",
       "      <td>0.529044</td>\n",
       "      <td>0.620105</td>\n",
       "      <td>0.438197</td>\n",
       "      <td>ThompsonSampling</td>\n",
       "      <td>0.298932</td>\n",
       "      <td>...</td>\n",
       "      <td>0.444567</td>\n",
       "      <td>0.574803</td>\n",
       "      <td>0.326866</td>\n",
       "      <td>0.231580</td>\n",
       "      <td>0.075482</td>\n",
       "      <td>0.103864</td>\n",
       "      <td>0.128133</td>\n",
       "      <td>0.041696</td>\n",
       "      <td>0.235609</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.138583</td>\n",
       "      <td>False</td>\n",
       "      <td>0.481983</td>\n",
       "      <td>0.449039</td>\n",
       "      <td>0.423200</td>\n",
       "      <td>0.528824</td>\n",
       "      <td>0.330810</td>\n",
       "      <td>ThompsonSampling</td>\n",
       "      <td>0.218013</td>\n",
       "      <td>...</td>\n",
       "      <td>0.332835</td>\n",
       "      <td>0.473386</td>\n",
       "      <td>0.229197</td>\n",
       "      <td>0.160342</td>\n",
       "      <td>0.058559</td>\n",
       "      <td>0.077663</td>\n",
       "      <td>0.094293</td>\n",
       "      <td>0.033966</td>\n",
       "      <td>0.162357</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.081575</td>\n",
       "      <td>False</td>\n",
       "      <td>0.565218</td>\n",
       "      <td>0.538473</td>\n",
       "      <td>0.514621</td>\n",
       "      <td>0.596871</td>\n",
       "      <td>0.419726</td>\n",
       "      <td>ThompsonSampling</td>\n",
       "      <td>0.283906</td>\n",
       "      <td>...</td>\n",
       "      <td>0.427874</td>\n",
       "      <td>0.562835</td>\n",
       "      <td>0.306047</td>\n",
       "      <td>0.217942</td>\n",
       "      <td>0.075242</td>\n",
       "      <td>0.102352</td>\n",
       "      <td>0.124499</td>\n",
       "      <td>0.041393</td>\n",
       "      <td>0.221227</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.111811</td>\n",
       "      <td>True</td>\n",
       "      <td>0.525279</td>\n",
       "      <td>0.490594</td>\n",
       "      <td>0.463029</td>\n",
       "      <td>0.569685</td>\n",
       "      <td>0.358584</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.231484</td>\n",
       "      <td>...</td>\n",
       "      <td>0.365591</td>\n",
       "      <td>0.529134</td>\n",
       "      <td>0.239843</td>\n",
       "      <td>0.167661</td>\n",
       "      <td>0.064415</td>\n",
       "      <td>0.085435</td>\n",
       "      <td>0.103300</td>\n",
       "      <td>0.038104</td>\n",
       "      <td>0.170197</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.068504</td>\n",
       "      <td>True</td>\n",
       "      <td>0.585093</td>\n",
       "      <td>0.556429</td>\n",
       "      <td>0.533038</td>\n",
       "      <td>0.625381</td>\n",
       "      <td>0.440718</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.300370</td>\n",
       "      <td>...</td>\n",
       "      <td>0.450787</td>\n",
       "      <td>0.587402</td>\n",
       "      <td>0.328378</td>\n",
       "      <td>0.232209</td>\n",
       "      <td>0.076041</td>\n",
       "      <td>0.104809</td>\n",
       "      <td>0.130033</td>\n",
       "      <td>0.043136</td>\n",
       "      <td>0.236250</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.124724</td>\n",
       "      <td>False</td>\n",
       "      <td>0.497721</td>\n",
       "      <td>0.465597</td>\n",
       "      <td>0.439127</td>\n",
       "      <td>0.538073</td>\n",
       "      <td>0.340859</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.221483</td>\n",
       "      <td>...</td>\n",
       "      <td>0.345906</td>\n",
       "      <td>0.505197</td>\n",
       "      <td>0.231717</td>\n",
       "      <td>0.161826</td>\n",
       "      <td>0.061632</td>\n",
       "      <td>0.081396</td>\n",
       "      <td>0.098004</td>\n",
       "      <td>0.036337</td>\n",
       "      <td>0.164025</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.065669</td>\n",
       "      <td>False</td>\n",
       "      <td>0.589787</td>\n",
       "      <td>0.558427</td>\n",
       "      <td>0.533127</td>\n",
       "      <td>0.628047</td>\n",
       "      <td>0.431560</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.289742</td>\n",
       "      <td>...</td>\n",
       "      <td>0.442205</td>\n",
       "      <td>0.591811</td>\n",
       "      <td>0.309228</td>\n",
       "      <td>0.220333</td>\n",
       "      <td>0.076970</td>\n",
       "      <td>0.104879</td>\n",
       "      <td>0.128661</td>\n",
       "      <td>0.043666</td>\n",
       "      <td>0.222843</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.107402</td>\n",
       "      <td>True</td>\n",
       "      <td>0.529762</td>\n",
       "      <td>0.494289</td>\n",
       "      <td>0.465921</td>\n",
       "      <td>0.574787</td>\n",
       "      <td>0.360519</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.234211</td>\n",
       "      <td>...</td>\n",
       "      <td>0.365669</td>\n",
       "      <td>0.529449</td>\n",
       "      <td>0.242835</td>\n",
       "      <td>0.169797</td>\n",
       "      <td>0.065482</td>\n",
       "      <td>0.086041</td>\n",
       "      <td>0.103506</td>\n",
       "      <td>0.037990</td>\n",
       "      <td>0.172420</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.069764</td>\n",
       "      <td>True</td>\n",
       "      <td>0.585028</td>\n",
       "      <td>0.555514</td>\n",
       "      <td>0.531464</td>\n",
       "      <td>0.622304</td>\n",
       "      <td>0.439324</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.299273</td>\n",
       "      <td>...</td>\n",
       "      <td>0.447638</td>\n",
       "      <td>0.583307</td>\n",
       "      <td>0.326803</td>\n",
       "      <td>0.231684</td>\n",
       "      <td>0.076469</td>\n",
       "      <td>0.103795</td>\n",
       "      <td>0.129283</td>\n",
       "      <td>0.042747</td>\n",
       "      <td>0.235180</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.136693</td>\n",
       "      <td>False</td>\n",
       "      <td>0.489024</td>\n",
       "      <td>0.453850</td>\n",
       "      <td>0.426771</td>\n",
       "      <td>0.536955</td>\n",
       "      <td>0.333661</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.220358</td>\n",
       "      <td>...</td>\n",
       "      <td>0.333150</td>\n",
       "      <td>0.486614</td>\n",
       "      <td>0.231843</td>\n",
       "      <td>0.161950</td>\n",
       "      <td>0.058974</td>\n",
       "      <td>0.077487</td>\n",
       "      <td>0.094338</td>\n",
       "      <td>0.034643</td>\n",
       "      <td>0.164778</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.066614</td>\n",
       "      <td>False</td>\n",
       "      <td>0.595161</td>\n",
       "      <td>0.560732</td>\n",
       "      <td>0.533705</td>\n",
       "      <td>0.639942</td>\n",
       "      <td>0.430368</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.289373</td>\n",
       "      <td>...</td>\n",
       "      <td>0.439055</td>\n",
       "      <td>0.597480</td>\n",
       "      <td>0.307843</td>\n",
       "      <td>0.218939</td>\n",
       "      <td>0.076015</td>\n",
       "      <td>0.103672</td>\n",
       "      <td>0.127551</td>\n",
       "      <td>0.043945</td>\n",
       "      <td>0.222479</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.109606</td>\n",
       "      <td>True</td>\n",
       "      <td>0.527003</td>\n",
       "      <td>0.493367</td>\n",
       "      <td>0.465631</td>\n",
       "      <td>0.565979</td>\n",
       "      <td>0.360026</td>\n",
       "      <td>Greedy</td>\n",
       "      <td>0.232577</td>\n",
       "      <td>...</td>\n",
       "      <td>0.368740</td>\n",
       "      <td>0.528189</td>\n",
       "      <td>0.240693</td>\n",
       "      <td>0.168796</td>\n",
       "      <td>0.065605</td>\n",
       "      <td>0.086228</td>\n",
       "      <td>0.104268</td>\n",
       "      <td>0.038101</td>\n",
       "      <td>0.170786</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.071181</td>\n",
       "      <td>True</td>\n",
       "      <td>0.587461</td>\n",
       "      <td>0.556959</td>\n",
       "      <td>0.533029</td>\n",
       "      <td>0.629612</td>\n",
       "      <td>0.440974</td>\n",
       "      <td>Greedy</td>\n",
       "      <td>0.300903</td>\n",
       "      <td>...</td>\n",
       "      <td>0.447008</td>\n",
       "      <td>0.586772</td>\n",
       "      <td>0.328567</td>\n",
       "      <td>0.233200</td>\n",
       "      <td>0.076085</td>\n",
       "      <td>0.104751</td>\n",
       "      <td>0.128844</td>\n",
       "      <td>0.043013</td>\n",
       "      <td>0.236603</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.116693</td>\n",
       "      <td>False</td>\n",
       "      <td>0.517076</td>\n",
       "      <td>0.483949</td>\n",
       "      <td>0.456864</td>\n",
       "      <td>0.561307</td>\n",
       "      <td>0.349450</td>\n",
       "      <td>Greedy</td>\n",
       "      <td>0.223941</td>\n",
       "      <td>...</td>\n",
       "      <td>0.360157</td>\n",
       "      <td>0.517165</td>\n",
       "      <td>0.230299</td>\n",
       "      <td>0.160771</td>\n",
       "      <td>0.063660</td>\n",
       "      <td>0.085023</td>\n",
       "      <td>0.101639</td>\n",
       "      <td>0.037195</td>\n",
       "      <td>0.162421</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.065512</td>\n",
       "      <td>False</td>\n",
       "      <td>0.604492</td>\n",
       "      <td>0.572825</td>\n",
       "      <td>0.545824</td>\n",
       "      <td>0.643260</td>\n",
       "      <td>0.440746</td>\n",
       "      <td>Greedy</td>\n",
       "      <td>0.293908</td>\n",
       "      <td>...</td>\n",
       "      <td>0.449921</td>\n",
       "      <td>0.611969</td>\n",
       "      <td>0.314394</td>\n",
       "      <td>0.222587</td>\n",
       "      <td>0.078738</td>\n",
       "      <td>0.106988</td>\n",
       "      <td>0.130196</td>\n",
       "      <td>0.044718</td>\n",
       "      <td>0.225776</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.115748</td>\n",
       "      <td>True</td>\n",
       "      <td>0.520806</td>\n",
       "      <td>0.487883</td>\n",
       "      <td>0.461207</td>\n",
       "      <td>0.560504</td>\n",
       "      <td>0.357028</td>\n",
       "      <td>ThompsonSampling</td>\n",
       "      <td>0.229208</td>\n",
       "      <td>...</td>\n",
       "      <td>0.368346</td>\n",
       "      <td>0.520630</td>\n",
       "      <td>0.237165</td>\n",
       "      <td>0.165634</td>\n",
       "      <td>0.064439</td>\n",
       "      <td>0.085546</td>\n",
       "      <td>0.104020</td>\n",
       "      <td>0.037283</td>\n",
       "      <td>0.167809</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.074331</td>\n",
       "      <td>True</td>\n",
       "      <td>0.584398</td>\n",
       "      <td>0.555793</td>\n",
       "      <td>0.532612</td>\n",
       "      <td>0.622588</td>\n",
       "      <td>0.441039</td>\n",
       "      <td>ThompsonSampling</td>\n",
       "      <td>0.301010</td>\n",
       "      <td>...</td>\n",
       "      <td>0.450236</td>\n",
       "      <td>0.582362</td>\n",
       "      <td>0.328976</td>\n",
       "      <td>0.233393</td>\n",
       "      <td>0.076427</td>\n",
       "      <td>0.105077</td>\n",
       "      <td>0.130131</td>\n",
       "      <td>0.042678</td>\n",
       "      <td>0.237000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.145827</td>\n",
       "      <td>False</td>\n",
       "      <td>0.469968</td>\n",
       "      <td>0.438470</td>\n",
       "      <td>0.413812</td>\n",
       "      <td>0.514682</td>\n",
       "      <td>0.326388</td>\n",
       "      <td>ThompsonSampling</td>\n",
       "      <td>0.216521</td>\n",
       "      <td>...</td>\n",
       "      <td>0.327953</td>\n",
       "      <td>0.463307</td>\n",
       "      <td>0.229606</td>\n",
       "      <td>0.160180</td>\n",
       "      <td>0.057175</td>\n",
       "      <td>0.075850</td>\n",
       "      <td>0.092957</td>\n",
       "      <td>0.033376</td>\n",
       "      <td>0.162870</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.074803</td>\n",
       "      <td>False</td>\n",
       "      <td>0.574708</td>\n",
       "      <td>0.544007</td>\n",
       "      <td>0.518986</td>\n",
       "      <td>0.614031</td>\n",
       "      <td>0.423019</td>\n",
       "      <td>ThompsonSampling</td>\n",
       "      <td>0.286382</td>\n",
       "      <td>...</td>\n",
       "      <td>0.430472</td>\n",
       "      <td>0.574488</td>\n",
       "      <td>0.308315</td>\n",
       "      <td>0.219309</td>\n",
       "      <td>0.075209</td>\n",
       "      <td>0.101630</td>\n",
       "      <td>0.125030</td>\n",
       "      <td>0.042076</td>\n",
       "      <td>0.222528</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.110236</td>\n",
       "      <td>True</td>\n",
       "      <td>0.527201</td>\n",
       "      <td>0.493545</td>\n",
       "      <td>0.465934</td>\n",
       "      <td>0.571045</td>\n",
       "      <td>0.360642</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.232596</td>\n",
       "      <td>...</td>\n",
       "      <td>0.368976</td>\n",
       "      <td>0.529764</td>\n",
       "      <td>0.240724</td>\n",
       "      <td>0.168402</td>\n",
       "      <td>0.065421</td>\n",
       "      <td>0.086565</td>\n",
       "      <td>0.104643</td>\n",
       "      <td>0.038186</td>\n",
       "      <td>0.170632</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.068661</td>\n",
       "      <td>True</td>\n",
       "      <td>0.579510</td>\n",
       "      <td>0.551430</td>\n",
       "      <td>0.527689</td>\n",
       "      <td>0.622289</td>\n",
       "      <td>0.436190</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.298512</td>\n",
       "      <td>...</td>\n",
       "      <td>0.444488</td>\n",
       "      <td>0.574173</td>\n",
       "      <td>0.325039</td>\n",
       "      <td>0.231269</td>\n",
       "      <td>0.075554</td>\n",
       "      <td>0.104478</td>\n",
       "      <td>0.128955</td>\n",
       "      <td>0.042078</td>\n",
       "      <td>0.234622</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.124567</td>\n",
       "      <td>False</td>\n",
       "      <td>0.501465</td>\n",
       "      <td>0.467330</td>\n",
       "      <td>0.439659</td>\n",
       "      <td>0.547937</td>\n",
       "      <td>0.340115</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.221913</td>\n",
       "      <td>...</td>\n",
       "      <td>0.343465</td>\n",
       "      <td>0.499528</td>\n",
       "      <td>0.230866</td>\n",
       "      <td>0.161674</td>\n",
       "      <td>0.061223</td>\n",
       "      <td>0.080487</td>\n",
       "      <td>0.097138</td>\n",
       "      <td>0.035824</td>\n",
       "      <td>0.163728</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.073543</td>\n",
       "      <td>False</td>\n",
       "      <td>0.588767</td>\n",
       "      <td>0.560126</td>\n",
       "      <td>0.535224</td>\n",
       "      <td>0.624325</td>\n",
       "      <td>0.433708</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.290478</td>\n",
       "      <td>...</td>\n",
       "      <td>0.444252</td>\n",
       "      <td>0.584882</td>\n",
       "      <td>0.310835</td>\n",
       "      <td>0.221004</td>\n",
       "      <td>0.078189</td>\n",
       "      <td>0.105906</td>\n",
       "      <td>0.129215</td>\n",
       "      <td>0.043001</td>\n",
       "      <td>0.223938</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.115906</td>\n",
       "      <td>True</td>\n",
       "      <td>0.528105</td>\n",
       "      <td>0.493625</td>\n",
       "      <td>0.465620</td>\n",
       "      <td>0.571664</td>\n",
       "      <td>0.358754</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.230725</td>\n",
       "      <td>...</td>\n",
       "      <td>0.365276</td>\n",
       "      <td>0.534173</td>\n",
       "      <td>0.237953</td>\n",
       "      <td>0.166373</td>\n",
       "      <td>0.065149</td>\n",
       "      <td>0.086017</td>\n",
       "      <td>0.103363</td>\n",
       "      <td>0.038408</td>\n",
       "      <td>0.168420</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.071969</td>\n",
       "      <td>True</td>\n",
       "      <td>0.581809</td>\n",
       "      <td>0.553250</td>\n",
       "      <td>0.530117</td>\n",
       "      <td>0.621858</td>\n",
       "      <td>0.439006</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.299112</td>\n",
       "      <td>...</td>\n",
       "      <td>0.446535</td>\n",
       "      <td>0.580157</td>\n",
       "      <td>0.327181</td>\n",
       "      <td>0.231763</td>\n",
       "      <td>0.075717</td>\n",
       "      <td>0.104998</td>\n",
       "      <td>0.128965</td>\n",
       "      <td>0.042625</td>\n",
       "      <td>0.235331</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.118425</td>\n",
       "      <td>False</td>\n",
       "      <td>0.502868</td>\n",
       "      <td>0.469025</td>\n",
       "      <td>0.442047</td>\n",
       "      <td>0.550289</td>\n",
       "      <td>0.342571</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.222491</td>\n",
       "      <td>...</td>\n",
       "      <td>0.348031</td>\n",
       "      <td>0.499213</td>\n",
       "      <td>0.230772</td>\n",
       "      <td>0.161873</td>\n",
       "      <td>0.061214</td>\n",
       "      <td>0.081777</td>\n",
       "      <td>0.098346</td>\n",
       "      <td>0.035805</td>\n",
       "      <td>0.163747</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.073071</td>\n",
       "      <td>False</td>\n",
       "      <td>0.591915</td>\n",
       "      <td>0.560388</td>\n",
       "      <td>0.534474</td>\n",
       "      <td>0.627974</td>\n",
       "      <td>0.432671</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.291575</td>\n",
       "      <td>...</td>\n",
       "      <td>0.441181</td>\n",
       "      <td>0.589606</td>\n",
       "      <td>0.312661</td>\n",
       "      <td>0.222239</td>\n",
       "      <td>0.077560</td>\n",
       "      <td>0.104957</td>\n",
       "      <td>0.128418</td>\n",
       "      <td>0.043234</td>\n",
       "      <td>0.225659</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.111654</td>\n",
       "      <td>True</td>\n",
       "      <td>0.524014</td>\n",
       "      <td>0.491290</td>\n",
       "      <td>0.464612</td>\n",
       "      <td>0.567255</td>\n",
       "      <td>0.361376</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.234130</td>\n",
       "      <td>...</td>\n",
       "      <td>0.371181</td>\n",
       "      <td>0.523150</td>\n",
       "      <td>0.243465</td>\n",
       "      <td>0.170409</td>\n",
       "      <td>0.064895</td>\n",
       "      <td>0.087138</td>\n",
       "      <td>0.105072</td>\n",
       "      <td>0.037727</td>\n",
       "      <td>0.172647</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.078583</td>\n",
       "      <td>True</td>\n",
       "      <td>0.576672</td>\n",
       "      <td>0.549650</td>\n",
       "      <td>0.527544</td>\n",
       "      <td>0.611150</td>\n",
       "      <td>0.438133</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.298966</td>\n",
       "      <td>...</td>\n",
       "      <td>0.448189</td>\n",
       "      <td>0.572283</td>\n",
       "      <td>0.327717</td>\n",
       "      <td>0.232909</td>\n",
       "      <td>0.075861</td>\n",
       "      <td>0.104289</td>\n",
       "      <td>0.129390</td>\n",
       "      <td>0.041699</td>\n",
       "      <td>0.236168</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.119843</td>\n",
       "      <td>False</td>\n",
       "      <td>0.500598</td>\n",
       "      <td>0.467287</td>\n",
       "      <td>0.440137</td>\n",
       "      <td>0.545202</td>\n",
       "      <td>0.339985</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.221120</td>\n",
       "      <td>...</td>\n",
       "      <td>0.344173</td>\n",
       "      <td>0.500157</td>\n",
       "      <td>0.229323</td>\n",
       "      <td>0.160644</td>\n",
       "      <td>0.061371</td>\n",
       "      <td>0.081117</td>\n",
       "      <td>0.097355</td>\n",
       "      <td>0.035952</td>\n",
       "      <td>0.162560</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.072126</td>\n",
       "      <td>False</td>\n",
       "      <td>0.591531</td>\n",
       "      <td>0.559960</td>\n",
       "      <td>0.533218</td>\n",
       "      <td>0.630425</td>\n",
       "      <td>0.431529</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.290708</td>\n",
       "      <td>...</td>\n",
       "      <td>0.439606</td>\n",
       "      <td>0.589921</td>\n",
       "      <td>0.311150</td>\n",
       "      <td>0.221189</td>\n",
       "      <td>0.077702</td>\n",
       "      <td>0.104287</td>\n",
       "      <td>0.127799</td>\n",
       "      <td>0.043552</td>\n",
       "      <td>0.224387</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.105669</td>\n",
       "      <td>True</td>\n",
       "      <td>0.529986</td>\n",
       "      <td>0.495618</td>\n",
       "      <td>0.467760</td>\n",
       "      <td>0.574294</td>\n",
       "      <td>0.360304</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.232689</td>\n",
       "      <td>...</td>\n",
       "      <td>0.369291</td>\n",
       "      <td>0.531969</td>\n",
       "      <td>0.240378</td>\n",
       "      <td>0.168042</td>\n",
       "      <td>0.065864</td>\n",
       "      <td>0.086146</td>\n",
       "      <td>0.104585</td>\n",
       "      <td>0.038410</td>\n",
       "      <td>0.170240</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.074488</td>\n",
       "      <td>True</td>\n",
       "      <td>0.587036</td>\n",
       "      <td>0.558687</td>\n",
       "      <td>0.535466</td>\n",
       "      <td>0.624499</td>\n",
       "      <td>0.443645</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.301552</td>\n",
       "      <td>...</td>\n",
       "      <td>0.455039</td>\n",
       "      <td>0.587717</td>\n",
       "      <td>0.329165</td>\n",
       "      <td>0.233938</td>\n",
       "      <td>0.076812</td>\n",
       "      <td>0.105790</td>\n",
       "      <td>0.131339</td>\n",
       "      <td>0.043086</td>\n",
       "      <td>0.237375</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.136220</td>\n",
       "      <td>False</td>\n",
       "      <td>0.481126</td>\n",
       "      <td>0.446696</td>\n",
       "      <td>0.420786</td>\n",
       "      <td>0.531276</td>\n",
       "      <td>0.330039</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.217929</td>\n",
       "      <td>...</td>\n",
       "      <td>0.331339</td>\n",
       "      <td>0.478425</td>\n",
       "      <td>0.228913</td>\n",
       "      <td>0.160156</td>\n",
       "      <td>0.058212</td>\n",
       "      <td>0.077473</td>\n",
       "      <td>0.094203</td>\n",
       "      <td>0.034422</td>\n",
       "      <td>0.162435</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.077165</td>\n",
       "      <td>False</td>\n",
       "      <td>0.588204</td>\n",
       "      <td>0.557176</td>\n",
       "      <td>0.530770</td>\n",
       "      <td>0.622362</td>\n",
       "      <td>0.430469</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.291290</td>\n",
       "      <td>...</td>\n",
       "      <td>0.438110</td>\n",
       "      <td>0.592126</td>\n",
       "      <td>0.312787</td>\n",
       "      <td>0.222664</td>\n",
       "      <td>0.077028</td>\n",
       "      <td>0.103591</td>\n",
       "      <td>0.127372</td>\n",
       "      <td>0.043532</td>\n",
       "      <td>0.226214</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.112283</td>\n",
       "      <td>True</td>\n",
       "      <td>0.527834</td>\n",
       "      <td>0.494182</td>\n",
       "      <td>0.466214</td>\n",
       "      <td>0.570913</td>\n",
       "      <td>0.361679</td>\n",
       "      <td>Greedy</td>\n",
       "      <td>0.234272</td>\n",
       "      <td>...</td>\n",
       "      <td>0.368583</td>\n",
       "      <td>0.528189</td>\n",
       "      <td>0.243118</td>\n",
       "      <td>0.170468</td>\n",
       "      <td>0.065162</td>\n",
       "      <td>0.086251</td>\n",
       "      <td>0.104230</td>\n",
       "      <td>0.038029</td>\n",
       "      <td>0.172692</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.071181</td>\n",
       "      <td>True</td>\n",
       "      <td>0.584710</td>\n",
       "      <td>0.556805</td>\n",
       "      <td>0.533667</td>\n",
       "      <td>0.627596</td>\n",
       "      <td>0.442863</td>\n",
       "      <td>Greedy</td>\n",
       "      <td>0.302748</td>\n",
       "      <td>...</td>\n",
       "      <td>0.452205</td>\n",
       "      <td>0.581417</td>\n",
       "      <td>0.331402</td>\n",
       "      <td>0.235046</td>\n",
       "      <td>0.075902</td>\n",
       "      <td>0.105378</td>\n",
       "      <td>0.130605</td>\n",
       "      <td>0.042425</td>\n",
       "      <td>0.238585</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.114803</td>\n",
       "      <td>False</td>\n",
       "      <td>0.513036</td>\n",
       "      <td>0.481656</td>\n",
       "      <td>0.455088</td>\n",
       "      <td>0.554331</td>\n",
       "      <td>0.349208</td>\n",
       "      <td>Greedy</td>\n",
       "      <td>0.224046</td>\n",
       "      <td>...</td>\n",
       "      <td>0.359843</td>\n",
       "      <td>0.516535</td>\n",
       "      <td>0.230677</td>\n",
       "      <td>0.160431</td>\n",
       "      <td>0.063707</td>\n",
       "      <td>0.085091</td>\n",
       "      <td>0.101573</td>\n",
       "      <td>0.037190</td>\n",
       "      <td>0.163037</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.076378</td>\n",
       "      <td>False</td>\n",
       "      <td>0.595517</td>\n",
       "      <td>0.564098</td>\n",
       "      <td>0.538944</td>\n",
       "      <td>0.632252</td>\n",
       "      <td>0.439039</td>\n",
       "      <td>Greedy</td>\n",
       "      <td>0.293472</td>\n",
       "      <td>...</td>\n",
       "      <td>0.450866</td>\n",
       "      <td>0.595906</td>\n",
       "      <td>0.314835</td>\n",
       "      <td>0.223492</td>\n",
       "      <td>0.077774</td>\n",
       "      <td>0.105100</td>\n",
       "      <td>0.130249</td>\n",
       "      <td>0.043554</td>\n",
       "      <td>0.226528</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.117008</td>\n",
       "      <td>True</td>\n",
       "      <td>0.525001</td>\n",
       "      <td>0.490894</td>\n",
       "      <td>0.463235</td>\n",
       "      <td>0.568840</td>\n",
       "      <td>0.359046</td>\n",
       "      <td>ThompsonSampling</td>\n",
       "      <td>0.231828</td>\n",
       "      <td>...</td>\n",
       "      <td>0.366378</td>\n",
       "      <td>0.523465</td>\n",
       "      <td>0.239906</td>\n",
       "      <td>0.167759</td>\n",
       "      <td>0.064726</td>\n",
       "      <td>0.085969</td>\n",
       "      <td>0.103613</td>\n",
       "      <td>0.037706</td>\n",
       "      <td>0.170192</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.067874</td>\n",
       "      <td>True</td>\n",
       "      <td>0.581656</td>\n",
       "      <td>0.553456</td>\n",
       "      <td>0.529875</td>\n",
       "      <td>0.620121</td>\n",
       "      <td>0.437545</td>\n",
       "      <td>ThompsonSampling</td>\n",
       "      <td>0.298398</td>\n",
       "      <td>...</td>\n",
       "      <td>0.446220</td>\n",
       "      <td>0.585512</td>\n",
       "      <td>0.326016</td>\n",
       "      <td>0.231154</td>\n",
       "      <td>0.075979</td>\n",
       "      <td>0.103968</td>\n",
       "      <td>0.128813</td>\n",
       "      <td>0.042780</td>\n",
       "      <td>0.234999</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.142835</td>\n",
       "      <td>False</td>\n",
       "      <td>0.477151</td>\n",
       "      <td>0.446095</td>\n",
       "      <td>0.420699</td>\n",
       "      <td>0.520252</td>\n",
       "      <td>0.330588</td>\n",
       "      <td>ThompsonSampling</td>\n",
       "      <td>0.218301</td>\n",
       "      <td>...</td>\n",
       "      <td>0.330709</td>\n",
       "      <td>0.472126</td>\n",
       "      <td>0.231402</td>\n",
       "      <td>0.161324</td>\n",
       "      <td>0.058444</td>\n",
       "      <td>0.077767</td>\n",
       "      <td>0.093447</td>\n",
       "      <td>0.033874</td>\n",
       "      <td>0.164143</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.077480</td>\n",
       "      <td>False</td>\n",
       "      <td>0.574355</td>\n",
       "      <td>0.542765</td>\n",
       "      <td>0.517239</td>\n",
       "      <td>0.616016</td>\n",
       "      <td>0.419498</td>\n",
       "      <td>ThompsonSampling</td>\n",
       "      <td>0.284068</td>\n",
       "      <td>...</td>\n",
       "      <td>0.427480</td>\n",
       "      <td>0.575748</td>\n",
       "      <td>0.305575</td>\n",
       "      <td>0.217160</td>\n",
       "      <td>0.074705</td>\n",
       "      <td>0.100830</td>\n",
       "      <td>0.124400</td>\n",
       "      <td>0.042160</td>\n",
       "      <td>0.220205</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.109606</td>\n",
       "      <td>True</td>\n",
       "      <td>0.525718</td>\n",
       "      <td>0.492048</td>\n",
       "      <td>0.464924</td>\n",
       "      <td>0.568730</td>\n",
       "      <td>0.360389</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.232576</td>\n",
       "      <td>...</td>\n",
       "      <td>0.369134</td>\n",
       "      <td>0.530394</td>\n",
       "      <td>0.240441</td>\n",
       "      <td>0.168870</td>\n",
       "      <td>0.065466</td>\n",
       "      <td>0.086283</td>\n",
       "      <td>0.104424</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.170946</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.071969</td>\n",
       "      <td>True</td>\n",
       "      <td>0.589553</td>\n",
       "      <td>0.559992</td>\n",
       "      <td>0.535471</td>\n",
       "      <td>0.634037</td>\n",
       "      <td>0.441306</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.300450</td>\n",
       "      <td>...</td>\n",
       "      <td>0.449606</td>\n",
       "      <td>0.586457</td>\n",
       "      <td>0.326614</td>\n",
       "      <td>0.232205</td>\n",
       "      <td>0.076348</td>\n",
       "      <td>0.105174</td>\n",
       "      <td>0.130091</td>\n",
       "      <td>0.042764</td>\n",
       "      <td>0.235437</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.120945</td>\n",
       "      <td>False</td>\n",
       "      <td>0.503073</td>\n",
       "      <td>0.467434</td>\n",
       "      <td>0.439832</td>\n",
       "      <td>0.547260</td>\n",
       "      <td>0.340646</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.221735</td>\n",
       "      <td>...</td>\n",
       "      <td>0.342520</td>\n",
       "      <td>0.500472</td>\n",
       "      <td>0.230299</td>\n",
       "      <td>0.160662</td>\n",
       "      <td>0.061360</td>\n",
       "      <td>0.080069</td>\n",
       "      <td>0.096709</td>\n",
       "      <td>0.035960</td>\n",
       "      <td>0.163274</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.065512</td>\n",
       "      <td>False</td>\n",
       "      <td>0.603547</td>\n",
       "      <td>0.568282</td>\n",
       "      <td>0.540083</td>\n",
       "      <td>0.648598</td>\n",
       "      <td>0.434668</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.292063</td>\n",
       "      <td>...</td>\n",
       "      <td>0.440709</td>\n",
       "      <td>0.607559</td>\n",
       "      <td>0.310583</td>\n",
       "      <td>0.220656</td>\n",
       "      <td>0.077385</td>\n",
       "      <td>0.104293</td>\n",
       "      <td>0.127849</td>\n",
       "      <td>0.044404</td>\n",
       "      <td>0.224241</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.113543</td>\n",
       "      <td>True</td>\n",
       "      <td>0.526552</td>\n",
       "      <td>0.492951</td>\n",
       "      <td>0.465744</td>\n",
       "      <td>0.571234</td>\n",
       "      <td>0.361402</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.233917</td>\n",
       "      <td>...</td>\n",
       "      <td>0.369055</td>\n",
       "      <td>0.525984</td>\n",
       "      <td>0.242394</td>\n",
       "      <td>0.169859</td>\n",
       "      <td>0.064699</td>\n",
       "      <td>0.086374</td>\n",
       "      <td>0.104451</td>\n",
       "      <td>0.037769</td>\n",
       "      <td>0.171989</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.067559</td>\n",
       "      <td>True</td>\n",
       "      <td>0.590620</td>\n",
       "      <td>0.559556</td>\n",
       "      <td>0.535278</td>\n",
       "      <td>0.632478</td>\n",
       "      <td>0.442847</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.301790</td>\n",
       "      <td>...</td>\n",
       "      <td>0.452047</td>\n",
       "      <td>0.587717</td>\n",
       "      <td>0.328945</td>\n",
       "      <td>0.233065</td>\n",
       "      <td>0.076369</td>\n",
       "      <td>0.104740</td>\n",
       "      <td>0.130566</td>\n",
       "      <td>0.043056</td>\n",
       "      <td>0.236939</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.120945</td>\n",
       "      <td>False</td>\n",
       "      <td>0.495547</td>\n",
       "      <td>0.462847</td>\n",
       "      <td>0.436364</td>\n",
       "      <td>0.538808</td>\n",
       "      <td>0.338963</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.221139</td>\n",
       "      <td>...</td>\n",
       "      <td>0.343386</td>\n",
       "      <td>0.495748</td>\n",
       "      <td>0.231118</td>\n",
       "      <td>0.161412</td>\n",
       "      <td>0.060806</td>\n",
       "      <td>0.080957</td>\n",
       "      <td>0.097258</td>\n",
       "      <td>0.035641</td>\n",
       "      <td>0.163723</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.077323</td>\n",
       "      <td>False</td>\n",
       "      <td>0.583506</td>\n",
       "      <td>0.553779</td>\n",
       "      <td>0.529223</td>\n",
       "      <td>0.623601</td>\n",
       "      <td>0.429209</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.288131</td>\n",
       "      <td>...</td>\n",
       "      <td>0.440787</td>\n",
       "      <td>0.582047</td>\n",
       "      <td>0.307843</td>\n",
       "      <td>0.219146</td>\n",
       "      <td>0.075704</td>\n",
       "      <td>0.104312</td>\n",
       "      <td>0.127545</td>\n",
       "      <td>0.042621</td>\n",
       "      <td>0.221758</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.108976</td>\n",
       "      <td>True</td>\n",
       "      <td>0.532562</td>\n",
       "      <td>0.497095</td>\n",
       "      <td>0.468646</td>\n",
       "      <td>0.579097</td>\n",
       "      <td>0.361963</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.234778</td>\n",
       "      <td>...</td>\n",
       "      <td>0.369370</td>\n",
       "      <td>0.535118</td>\n",
       "      <td>0.242992</td>\n",
       "      <td>0.170195</td>\n",
       "      <td>0.065349</td>\n",
       "      <td>0.086544</td>\n",
       "      <td>0.104180</td>\n",
       "      <td>0.038507</td>\n",
       "      <td>0.172365</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.073386</td>\n",
       "      <td>True</td>\n",
       "      <td>0.584895</td>\n",
       "      <td>0.554922</td>\n",
       "      <td>0.530435</td>\n",
       "      <td>0.630042</td>\n",
       "      <td>0.438070</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.300171</td>\n",
       "      <td>...</td>\n",
       "      <td>0.443780</td>\n",
       "      <td>0.576063</td>\n",
       "      <td>0.327622</td>\n",
       "      <td>0.232329</td>\n",
       "      <td>0.075408</td>\n",
       "      <td>0.103908</td>\n",
       "      <td>0.128242</td>\n",
       "      <td>0.042269</td>\n",
       "      <td>0.235917</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.131339</td>\n",
       "      <td>False</td>\n",
       "      <td>0.491214</td>\n",
       "      <td>0.455074</td>\n",
       "      <td>0.428213</td>\n",
       "      <td>0.537848</td>\n",
       "      <td>0.333429</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.219392</td>\n",
       "      <td>...</td>\n",
       "      <td>0.334567</td>\n",
       "      <td>0.488189</td>\n",
       "      <td>0.229921</td>\n",
       "      <td>0.160885</td>\n",
       "      <td>0.058934</td>\n",
       "      <td>0.077588</td>\n",
       "      <td>0.094712</td>\n",
       "      <td>0.034748</td>\n",
       "      <td>0.163279</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.071181</td>\n",
       "      <td>False</td>\n",
       "      <td>0.593755</td>\n",
       "      <td>0.559785</td>\n",
       "      <td>0.531694</td>\n",
       "      <td>0.636226</td>\n",
       "      <td>0.428316</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.288724</td>\n",
       "      <td>...</td>\n",
       "      <td>0.433937</td>\n",
       "      <td>0.593071</td>\n",
       "      <td>0.308063</td>\n",
       "      <td>0.219173</td>\n",
       "      <td>0.076426</td>\n",
       "      <td>0.102891</td>\n",
       "      <td>0.126010</td>\n",
       "      <td>0.043055</td>\n",
       "      <td>0.222905</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>87 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      C    Clicks  Iterative    MAP@10    MAP@15    MAP@20     MAP@5  \\\n",
       "0   0.0  0.111811       True  0.526253  0.492304  0.465160  0.573244   \n",
       "1   0.0  0.071969       True  0.584978  0.554146  0.530175  0.627984   \n",
       "2   0.0  0.113386      False  0.515102  0.481702  0.454581  0.558651   \n",
       "3   0.0  0.064567      False  0.603015  0.572099  0.545774  0.643102   \n",
       "4   0.0  0.114173       True  0.522890  0.489380  0.461550  0.566651   \n",
       "5   0.0  0.074646       True  0.581251  0.552599  0.529044  0.620105   \n",
       "6   0.0  0.138583      False  0.481983  0.449039  0.423200  0.528824   \n",
       "7   0.0  0.081575      False  0.565218  0.538473  0.514621  0.596871   \n",
       "8   1.0  0.111811       True  0.525279  0.490594  0.463029  0.569685   \n",
       "9   1.0  0.068504       True  0.585093  0.556429  0.533038  0.625381   \n",
       "10  1.0  0.124724      False  0.497721  0.465597  0.439127  0.538073   \n",
       "11  1.0  0.065669      False  0.589787  0.558427  0.533127  0.628047   \n",
       "12  2.0  0.107402       True  0.529762  0.494289  0.465921  0.574787   \n",
       "13  2.0  0.069764       True  0.585028  0.555514  0.531464  0.622304   \n",
       "14  2.0  0.136693      False  0.489024  0.453850  0.426771  0.536955   \n",
       "15  2.0  0.066614      False  0.595161  0.560732  0.533705  0.639942   \n",
       "16  0.0  0.109606       True  0.527003  0.493367  0.465631  0.565979   \n",
       "17  0.0  0.071181       True  0.587461  0.556959  0.533029  0.629612   \n",
       "18  0.0  0.116693      False  0.517076  0.483949  0.456864  0.561307   \n",
       "19  0.0  0.065512      False  0.604492  0.572825  0.545824  0.643260   \n",
       "20  0.0  0.115748       True  0.520806  0.487883  0.461207  0.560504   \n",
       "21  0.0  0.074331       True  0.584398  0.555793  0.532612  0.622588   \n",
       "22  0.0  0.145827      False  0.469968  0.438470  0.413812  0.514682   \n",
       "23  0.0  0.074803      False  0.574708  0.544007  0.518986  0.614031   \n",
       "24  1.0  0.110236       True  0.527201  0.493545  0.465934  0.571045   \n",
       "25  1.0  0.068661       True  0.579510  0.551430  0.527689  0.622289   \n",
       "26  1.0  0.124567      False  0.501465  0.467330  0.439659  0.547937   \n",
       "27  1.0  0.073543      False  0.588767  0.560126  0.535224  0.624325   \n",
       "28  2.0  0.115906       True  0.528105  0.493625  0.465620  0.571664   \n",
       "29  2.0  0.071969       True  0.581809  0.553250  0.530117  0.621858   \n",
       "..  ...       ...        ...       ...       ...       ...       ...   \n",
       "57  0.5  0.118425      False  0.502868  0.469025  0.442047  0.550289   \n",
       "58  0.5  0.073071      False  0.591915  0.560388  0.534474  0.627974   \n",
       "59  1.0  0.111654       True  0.524014  0.491290  0.464612  0.567255   \n",
       "60  1.0  0.078583       True  0.576672  0.549650  0.527544  0.611150   \n",
       "61  1.0  0.119843      False  0.500598  0.467287  0.440137  0.545202   \n",
       "62  1.0  0.072126      False  0.591531  0.559960  0.533218  0.630425   \n",
       "63  2.0  0.105669       True  0.529986  0.495618  0.467760  0.574294   \n",
       "64  2.0  0.074488       True  0.587036  0.558687  0.535466  0.624499   \n",
       "65  2.0  0.136220      False  0.481126  0.446696  0.420786  0.531276   \n",
       "66  2.0  0.077165      False  0.588204  0.557176  0.530770  0.622362   \n",
       "67  0.0  0.112283       True  0.527834  0.494182  0.466214  0.570913   \n",
       "68  0.0  0.071181       True  0.584710  0.556805  0.533667  0.627596   \n",
       "69  0.0  0.114803      False  0.513036  0.481656  0.455088  0.554331   \n",
       "70  0.0  0.076378      False  0.595517  0.564098  0.538944  0.632252   \n",
       "71  0.0  0.117008       True  0.525001  0.490894  0.463235  0.568840   \n",
       "72  0.0  0.067874       True  0.581656  0.553456  0.529875  0.620121   \n",
       "73  0.0  0.142835      False  0.477151  0.446095  0.420699  0.520252   \n",
       "74  0.0  0.077480      False  0.574355  0.542765  0.517239  0.616016   \n",
       "75  0.5  0.109606       True  0.525718  0.492048  0.464924  0.568730   \n",
       "76  0.5  0.071969       True  0.589553  0.559992  0.535471  0.634037   \n",
       "77  0.5  0.120945      False  0.503073  0.467434  0.439832  0.547260   \n",
       "78  0.5  0.065512      False  0.603547  0.568282  0.540083  0.648598   \n",
       "79  1.0  0.113543       True  0.526552  0.492951  0.465744  0.571234   \n",
       "80  1.0  0.067559       True  0.590620  0.559556  0.535278  0.632478   \n",
       "81  1.0  0.120945      False  0.495547  0.462847  0.436364  0.538808   \n",
       "82  1.0  0.077323      False  0.583506  0.553779  0.529223  0.623601   \n",
       "83  2.0  0.108976       True  0.532562  0.497095  0.468646  0.579097   \n",
       "84  2.0  0.073386       True  0.584895  0.554922  0.530435  0.630042   \n",
       "85  2.0  0.131339      False  0.491214  0.455074  0.428213  0.537848   \n",
       "86  2.0  0.071181      False  0.593755  0.559785  0.531694  0.636226   \n",
       "\n",
       "      MAP@50             Model      NDCG  ...  Precision@20  Precision@5  \\\n",
       "0   0.361020            Greedy  0.234092  ...      0.369291     0.527244   \n",
       "1   0.438852            Greedy  0.300699  ...      0.447323     0.578898   \n",
       "2   0.349802            Greedy  0.224927  ...      0.358346     0.517795   \n",
       "3   0.441806            Greedy  0.295396  ...      0.452283     0.600945   \n",
       "4   0.357196  ThompsonSampling  0.230445  ...      0.364961     0.526299   \n",
       "5   0.438197  ThompsonSampling  0.298932  ...      0.444567     0.574803   \n",
       "6   0.330810  ThompsonSampling  0.218013  ...      0.332835     0.473386   \n",
       "7   0.419726  ThompsonSampling  0.283906  ...      0.427874     0.562835   \n",
       "8   0.358584               UCB  0.231484  ...      0.365591     0.529134   \n",
       "9   0.440718               UCB  0.300370  ...      0.450787     0.587402   \n",
       "10  0.340859               UCB  0.221483  ...      0.345906     0.505197   \n",
       "11  0.431560               UCB  0.289742  ...      0.442205     0.591811   \n",
       "12  0.360519               UCB  0.234211  ...      0.365669     0.529449   \n",
       "13  0.439324               UCB  0.299273  ...      0.447638     0.583307   \n",
       "14  0.333661               UCB  0.220358  ...      0.333150     0.486614   \n",
       "15  0.430368               UCB  0.289373  ...      0.439055     0.597480   \n",
       "16  0.360026            Greedy  0.232577  ...      0.368740     0.528189   \n",
       "17  0.440974            Greedy  0.300903  ...      0.447008     0.586772   \n",
       "18  0.349450            Greedy  0.223941  ...      0.360157     0.517165   \n",
       "19  0.440746            Greedy  0.293908  ...      0.449921     0.611969   \n",
       "20  0.357028  ThompsonSampling  0.229208  ...      0.368346     0.520630   \n",
       "21  0.441039  ThompsonSampling  0.301010  ...      0.450236     0.582362   \n",
       "22  0.326388  ThompsonSampling  0.216521  ...      0.327953     0.463307   \n",
       "23  0.423019  ThompsonSampling  0.286382  ...      0.430472     0.574488   \n",
       "24  0.360642               UCB  0.232596  ...      0.368976     0.529764   \n",
       "25  0.436190               UCB  0.298512  ...      0.444488     0.574173   \n",
       "26  0.340115               UCB  0.221913  ...      0.343465     0.499528   \n",
       "27  0.433708               UCB  0.290478  ...      0.444252     0.584882   \n",
       "28  0.358754               UCB  0.230725  ...      0.365276     0.534173   \n",
       "29  0.439006               UCB  0.299112  ...      0.446535     0.580157   \n",
       "..       ...               ...       ...  ...           ...          ...   \n",
       "57  0.342571               UCB  0.222491  ...      0.348031     0.499213   \n",
       "58  0.432671               UCB  0.291575  ...      0.441181     0.589606   \n",
       "59  0.361376               UCB  0.234130  ...      0.371181     0.523150   \n",
       "60  0.438133               UCB  0.298966  ...      0.448189     0.572283   \n",
       "61  0.339985               UCB  0.221120  ...      0.344173     0.500157   \n",
       "62  0.431529               UCB  0.290708  ...      0.439606     0.589921   \n",
       "63  0.360304               UCB  0.232689  ...      0.369291     0.531969   \n",
       "64  0.443645               UCB  0.301552  ...      0.455039     0.587717   \n",
       "65  0.330039               UCB  0.217929  ...      0.331339     0.478425   \n",
       "66  0.430469               UCB  0.291290  ...      0.438110     0.592126   \n",
       "67  0.361679            Greedy  0.234272  ...      0.368583     0.528189   \n",
       "68  0.442863            Greedy  0.302748  ...      0.452205     0.581417   \n",
       "69  0.349208            Greedy  0.224046  ...      0.359843     0.516535   \n",
       "70  0.439039            Greedy  0.293472  ...      0.450866     0.595906   \n",
       "71  0.359046  ThompsonSampling  0.231828  ...      0.366378     0.523465   \n",
       "72  0.437545  ThompsonSampling  0.298398  ...      0.446220     0.585512   \n",
       "73  0.330588  ThompsonSampling  0.218301  ...      0.330709     0.472126   \n",
       "74  0.419498  ThompsonSampling  0.284068  ...      0.427480     0.575748   \n",
       "75  0.360389               UCB  0.232576  ...      0.369134     0.530394   \n",
       "76  0.441306               UCB  0.300450  ...      0.449606     0.586457   \n",
       "77  0.340646               UCB  0.221735  ...      0.342520     0.500472   \n",
       "78  0.434668               UCB  0.292063  ...      0.440709     0.607559   \n",
       "79  0.361402               UCB  0.233917  ...      0.369055     0.525984   \n",
       "80  0.442847               UCB  0.301790  ...      0.452047     0.587717   \n",
       "81  0.338963               UCB  0.221139  ...      0.343386     0.495748   \n",
       "82  0.429209               UCB  0.288131  ...      0.440787     0.582047   \n",
       "83  0.361963               UCB  0.234778  ...      0.369370     0.535118   \n",
       "84  0.438070               UCB  0.300171  ...      0.443780     0.576063   \n",
       "85  0.333429               UCB  0.219392  ...      0.334567     0.488189   \n",
       "86  0.428316               UCB  0.288724  ...      0.433937     0.593071   \n",
       "\n",
       "    Precision@50  R-Precision  Recall@10  Recall@15  Recall@20  Recall@5  \\\n",
       "0       0.242772     0.170054   0.064956   0.086163   0.104316  0.038086   \n",
       "1       0.328756     0.233194   0.075532   0.103532   0.129367  0.042262   \n",
       "2       0.232189     0.162072   0.063442   0.084081   0.100961  0.037315   \n",
       "3       0.315969     0.224289   0.078446   0.107450   0.130705  0.044024   \n",
       "4       0.238520     0.166569   0.064805   0.085372   0.103225  0.037928   \n",
       "5       0.326866     0.231580   0.075482   0.103864   0.128133  0.041696   \n",
       "6       0.229197     0.160342   0.058559   0.077663   0.094293  0.033966   \n",
       "7       0.306047     0.217942   0.075242   0.102352   0.124499  0.041393   \n",
       "8       0.239843     0.167661   0.064415   0.085435   0.103300  0.038104   \n",
       "9       0.328378     0.232209   0.076041   0.104809   0.130033  0.043136   \n",
       "10      0.231717     0.161826   0.061632   0.081396   0.098004  0.036337   \n",
       "11      0.309228     0.220333   0.076970   0.104879   0.128661  0.043666   \n",
       "12      0.242835     0.169797   0.065482   0.086041   0.103506  0.037990   \n",
       "13      0.326803     0.231684   0.076469   0.103795   0.129283  0.042747   \n",
       "14      0.231843     0.161950   0.058974   0.077487   0.094338  0.034643   \n",
       "15      0.307843     0.218939   0.076015   0.103672   0.127551  0.043945   \n",
       "16      0.240693     0.168796   0.065605   0.086228   0.104268  0.038101   \n",
       "17      0.328567     0.233200   0.076085   0.104751   0.128844  0.043013   \n",
       "18      0.230299     0.160771   0.063660   0.085023   0.101639  0.037195   \n",
       "19      0.314394     0.222587   0.078738   0.106988   0.130196  0.044718   \n",
       "20      0.237165     0.165634   0.064439   0.085546   0.104020  0.037283   \n",
       "21      0.328976     0.233393   0.076427   0.105077   0.130131  0.042678   \n",
       "22      0.229606     0.160180   0.057175   0.075850   0.092957  0.033376   \n",
       "23      0.308315     0.219309   0.075209   0.101630   0.125030  0.042076   \n",
       "24      0.240724     0.168402   0.065421   0.086565   0.104643  0.038186   \n",
       "25      0.325039     0.231269   0.075554   0.104478   0.128955  0.042078   \n",
       "26      0.230866     0.161674   0.061223   0.080487   0.097138  0.035824   \n",
       "27      0.310835     0.221004   0.078189   0.105906   0.129215  0.043001   \n",
       "28      0.237953     0.166373   0.065149   0.086017   0.103363  0.038408   \n",
       "29      0.327181     0.231763   0.075717   0.104998   0.128965  0.042625   \n",
       "..           ...          ...        ...        ...        ...       ...   \n",
       "57      0.230772     0.161873   0.061214   0.081777   0.098346  0.035805   \n",
       "58      0.312661     0.222239   0.077560   0.104957   0.128418  0.043234   \n",
       "59      0.243465     0.170409   0.064895   0.087138   0.105072  0.037727   \n",
       "60      0.327717     0.232909   0.075861   0.104289   0.129390  0.041699   \n",
       "61      0.229323     0.160644   0.061371   0.081117   0.097355  0.035952   \n",
       "62      0.311150     0.221189   0.077702   0.104287   0.127799  0.043552   \n",
       "63      0.240378     0.168042   0.065864   0.086146   0.104585  0.038410   \n",
       "64      0.329165     0.233938   0.076812   0.105790   0.131339  0.043086   \n",
       "65      0.228913     0.160156   0.058212   0.077473   0.094203  0.034422   \n",
       "66      0.312787     0.222664   0.077028   0.103591   0.127372  0.043532   \n",
       "67      0.243118     0.170468   0.065162   0.086251   0.104230  0.038029   \n",
       "68      0.331402     0.235046   0.075902   0.105378   0.130605  0.042425   \n",
       "69      0.230677     0.160431   0.063707   0.085091   0.101573  0.037190   \n",
       "70      0.314835     0.223492   0.077774   0.105100   0.130249  0.043554   \n",
       "71      0.239906     0.167759   0.064726   0.085969   0.103613  0.037706   \n",
       "72      0.326016     0.231154   0.075979   0.103968   0.128813  0.042780   \n",
       "73      0.231402     0.161324   0.058444   0.077767   0.093447  0.033874   \n",
       "74      0.305575     0.217160   0.074705   0.100830   0.124400  0.042160   \n",
       "75      0.240441     0.168870   0.065466   0.086283   0.104424  0.038284   \n",
       "76      0.326614     0.232205   0.076348   0.105174   0.130091  0.042764   \n",
       "77      0.230299     0.160662   0.061360   0.080069   0.096709  0.035960   \n",
       "78      0.310583     0.220656   0.077385   0.104293   0.127849  0.044404   \n",
       "79      0.242394     0.169859   0.064699   0.086374   0.104451  0.037769   \n",
       "80      0.328945     0.233065   0.076369   0.104740   0.130566  0.043056   \n",
       "81      0.231118     0.161412   0.060806   0.080957   0.097258  0.035641   \n",
       "82      0.307843     0.219146   0.075704   0.104312   0.127545  0.042621   \n",
       "83      0.242992     0.170195   0.065349   0.086544   0.104180  0.038507   \n",
       "84      0.327622     0.232329   0.075408   0.103908   0.128242  0.042269   \n",
       "85      0.229921     0.160885   0.058934   0.077588   0.094712  0.034748   \n",
       "86      0.308063     0.219173   0.076426   0.102891   0.126010  0.043055   \n",
       "\n",
       "    Recall@50  SampleFromAll  \n",
       "0    0.172504           True  \n",
       "1    0.236853          False  \n",
       "2    0.163881           True  \n",
       "3    0.227050          False  \n",
       "4    0.168838           True  \n",
       "5    0.235609          False  \n",
       "6    0.162357           True  \n",
       "7    0.221227          False  \n",
       "8    0.170197           True  \n",
       "9    0.236250          False  \n",
       "10   0.164025           True  \n",
       "11   0.222843          False  \n",
       "12   0.172420           True  \n",
       "13   0.235180          False  \n",
       "14   0.164778           True  \n",
       "15   0.222479          False  \n",
       "16   0.170786           True  \n",
       "17   0.236603          False  \n",
       "18   0.162421           True  \n",
       "19   0.225776          False  \n",
       "20   0.167809           True  \n",
       "21   0.237000          False  \n",
       "22   0.162870           True  \n",
       "23   0.222528          False  \n",
       "24   0.170632           True  \n",
       "25   0.234622          False  \n",
       "26   0.163728           True  \n",
       "27   0.223938          False  \n",
       "28   0.168420           True  \n",
       "29   0.235331          False  \n",
       "..        ...            ...  \n",
       "57   0.163747           True  \n",
       "58   0.225659          False  \n",
       "59   0.172647           True  \n",
       "60   0.236168          False  \n",
       "61   0.162560           True  \n",
       "62   0.224387          False  \n",
       "63   0.170240           True  \n",
       "64   0.237375          False  \n",
       "65   0.162435           True  \n",
       "66   0.226214          False  \n",
       "67   0.172692           True  \n",
       "68   0.238585          False  \n",
       "69   0.163037           True  \n",
       "70   0.226528          False  \n",
       "71   0.170192           True  \n",
       "72   0.234999          False  \n",
       "73   0.164143           True  \n",
       "74   0.220205          False  \n",
       "75   0.170946           True  \n",
       "76   0.235437          False  \n",
       "77   0.163274           True  \n",
       "78   0.224241          False  \n",
       "79   0.171989           True  \n",
       "80   0.236939          False  \n",
       "81   0.163723           True  \n",
       "82   0.221758          False  \n",
       "83   0.172365           True  \n",
       "84   0.235917          False  \n",
       "85   0.163279           True  \n",
       "86   0.222905          False  \n",
       "\n",
       "[87 rows x 22 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#df = df.drop(list(range(1, 174, 2))).reset_index(drop=True)\n",
    "#df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df.to_csv('final_result.csv', sep='\\t', encoding='utf-8', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Iterative</th>\n",
       "      <th>MAP@5</th>\n",
       "      <th>MAP@10</th>\n",
       "      <th>MAP@15</th>\n",
       "      <th>MAP@20</th>\n",
       "      <th>MAP@50</th>\n",
       "      <th>NDCG</th>\n",
       "      <th>Precision@5</th>\n",
       "      <th>Precision@10</th>\n",
       "      <th>Precision@15</th>\n",
       "      <th>Precision@20</th>\n",
       "      <th>Precision@50</th>\n",
       "      <th>R-Precision</th>\n",
       "      <th>Recall@5</th>\n",
       "      <th>Recall@10</th>\n",
       "      <th>Recall@15</th>\n",
       "      <th>Recall@20</th>\n",
       "      <th>Recall@50</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>False</td>\n",
       "      <td>0.559470</td>\n",
       "      <td>0.516241</td>\n",
       "      <td>0.483123</td>\n",
       "      <td>0.455906</td>\n",
       "      <td>0.349279</td>\n",
       "      <td>0.224324</td>\n",
       "      <td>0.519055</td>\n",
       "      <td>0.446677</td>\n",
       "      <td>0.398068</td>\n",
       "      <td>0.358630</td>\n",
       "      <td>0.230961</td>\n",
       "      <td>0.160961</td>\n",
       "      <td>0.037405</td>\n",
       "      <td>0.063629</td>\n",
       "      <td>0.084639</td>\n",
       "      <td>0.101207</td>\n",
       "      <td>0.163064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>True</td>\n",
       "      <td>0.567882</td>\n",
       "      <td>0.525302</td>\n",
       "      <td>0.491568</td>\n",
       "      <td>0.463978</td>\n",
       "      <td>0.360013</td>\n",
       "      <td>0.233142</td>\n",
       "      <td>0.525827</td>\n",
       "      <td>0.456693</td>\n",
       "      <td>0.403990</td>\n",
       "      <td>0.367224</td>\n",
       "      <td>0.241811</td>\n",
       "      <td>0.169505</td>\n",
       "      <td>0.037910</td>\n",
       "      <td>0.065046</td>\n",
       "      <td>0.085826</td>\n",
       "      <td>0.103835</td>\n",
       "      <td>0.171665</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Iterative     MAP@5    MAP@10    MAP@15    MAP@20    MAP@50      NDCG  \\\n",
       "0      False  0.559470  0.516241  0.483123  0.455906  0.349279  0.224324   \n",
       "1       True  0.567882  0.525302  0.491568  0.463978  0.360013  0.233142   \n",
       "\n",
       "   Precision@5  Precision@10  Precision@15  Precision@20  Precision@50  \\\n",
       "0     0.519055      0.446677      0.398068      0.358630      0.230961   \n",
       "1     0.525827      0.456693      0.403990      0.367224      0.241811   \n",
       "\n",
       "   R-Precision  Recall@5  Recall@10  Recall@15  Recall@20  Recall@50  \n",
       "0     0.160961  0.037405   0.063629   0.084639   0.101207   0.163064  \n",
       "1     0.169505  0.037910   0.065046   0.085826   0.103835   0.171665  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[(df['Model'] == 'Greedy') & (df['SampleFromAll'] == True)].groupby('Iterative', as_index=False)['MAP@5', 'MAP@10', 'MAP@15', 'MAP@20', 'MAP@50', 'NDCG',\n",
    "'Precision@5', 'Precision@10', 'Precision@15', 'Precision@20', 'Precision@50', 'R-Precision', \n",
    "'Recall@5', 'Recall@10', 'Recall@15', 'Recall@20', 'Recall@50'].agg({'MAP@5': 'mean', 'MAP@10': 'mean', \n",
    "'MAP@15': 'mean', 'MAP@20': 'mean', 'MAP@50': 'mean', 'NDCG': 'mean', 'Precision@5': 'mean', \n",
    "'Precision@10': 'mean', 'Precision@15': 'mean', 'Precision@20': 'mean', 'Precision@50': 'mean', \n",
    "'R-Precision': 'mean', 'Recall@5': 'mean', 'Recall@10': 'mean', 'Recall@15': 'mean', 'Recall@20': 'mean',\n",
    "                                                                     'Recall@50': 'mean'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>C</th>\n",
       "      <th>Clicks</th>\n",
       "      <th>Iterative</th>\n",
       "      <th>MAP@10</th>\n",
       "      <th>MAP@15</th>\n",
       "      <th>MAP@20</th>\n",
       "      <th>MAP@5</th>\n",
       "      <th>MAP@50</th>\n",
       "      <th>Model</th>\n",
       "      <th>NDCG</th>\n",
       "      <th>...</th>\n",
       "      <th>Precision@20</th>\n",
       "      <th>Precision@5</th>\n",
       "      <th>Precision@50</th>\n",
       "      <th>R-Precision</th>\n",
       "      <th>Recall@10</th>\n",
       "      <th>Recall@15</th>\n",
       "      <th>Recall@20</th>\n",
       "      <th>Recall@5</th>\n",
       "      <th>Recall@50</th>\n",
       "      <th>SampleFromAll</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.111811</td>\n",
       "      <td>True</td>\n",
       "      <td>0.525279</td>\n",
       "      <td>0.490594</td>\n",
       "      <td>0.463029</td>\n",
       "      <td>0.569685</td>\n",
       "      <td>0.358584</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.231484</td>\n",
       "      <td>...</td>\n",
       "      <td>0.365591</td>\n",
       "      <td>0.529134</td>\n",
       "      <td>0.239843</td>\n",
       "      <td>0.167661</td>\n",
       "      <td>0.064415</td>\n",
       "      <td>0.085435</td>\n",
       "      <td>0.103300</td>\n",
       "      <td>0.038104</td>\n",
       "      <td>0.170197</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.107402</td>\n",
       "      <td>True</td>\n",
       "      <td>0.529762</td>\n",
       "      <td>0.494289</td>\n",
       "      <td>0.465921</td>\n",
       "      <td>0.574787</td>\n",
       "      <td>0.360519</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.234211</td>\n",
       "      <td>...</td>\n",
       "      <td>0.365669</td>\n",
       "      <td>0.529449</td>\n",
       "      <td>0.242835</td>\n",
       "      <td>0.169797</td>\n",
       "      <td>0.065482</td>\n",
       "      <td>0.086041</td>\n",
       "      <td>0.103506</td>\n",
       "      <td>0.037990</td>\n",
       "      <td>0.172420</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.110236</td>\n",
       "      <td>True</td>\n",
       "      <td>0.527201</td>\n",
       "      <td>0.493545</td>\n",
       "      <td>0.465934</td>\n",
       "      <td>0.571045</td>\n",
       "      <td>0.360642</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.232596</td>\n",
       "      <td>...</td>\n",
       "      <td>0.368976</td>\n",
       "      <td>0.529764</td>\n",
       "      <td>0.240724</td>\n",
       "      <td>0.168402</td>\n",
       "      <td>0.065421</td>\n",
       "      <td>0.086565</td>\n",
       "      <td>0.104643</td>\n",
       "      <td>0.038186</td>\n",
       "      <td>0.170632</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.115906</td>\n",
       "      <td>True</td>\n",
       "      <td>0.528105</td>\n",
       "      <td>0.493625</td>\n",
       "      <td>0.465620</td>\n",
       "      <td>0.571664</td>\n",
       "      <td>0.358754</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.230725</td>\n",
       "      <td>...</td>\n",
       "      <td>0.365276</td>\n",
       "      <td>0.534173</td>\n",
       "      <td>0.237953</td>\n",
       "      <td>0.166373</td>\n",
       "      <td>0.065149</td>\n",
       "      <td>0.086017</td>\n",
       "      <td>0.103363</td>\n",
       "      <td>0.038408</td>\n",
       "      <td>0.168420</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.112598</td>\n",
       "      <td>True</td>\n",
       "      <td>0.527571</td>\n",
       "      <td>0.493828</td>\n",
       "      <td>0.465776</td>\n",
       "      <td>0.570215</td>\n",
       "      <td>0.360066</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.233006</td>\n",
       "      <td>...</td>\n",
       "      <td>0.367717</td>\n",
       "      <td>0.525354</td>\n",
       "      <td>0.241039</td>\n",
       "      <td>0.168971</td>\n",
       "      <td>0.065309</td>\n",
       "      <td>0.086076</td>\n",
       "      <td>0.104054</td>\n",
       "      <td>0.037704</td>\n",
       "      <td>0.171186</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.114488</td>\n",
       "      <td>True</td>\n",
       "      <td>0.521709</td>\n",
       "      <td>0.489267</td>\n",
       "      <td>0.462627</td>\n",
       "      <td>0.565648</td>\n",
       "      <td>0.359316</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.231974</td>\n",
       "      <td>...</td>\n",
       "      <td>0.368189</td>\n",
       "      <td>0.526929</td>\n",
       "      <td>0.240693</td>\n",
       "      <td>0.168680</td>\n",
       "      <td>0.064332</td>\n",
       "      <td>0.086233</td>\n",
       "      <td>0.104184</td>\n",
       "      <td>0.038000</td>\n",
       "      <td>0.170516</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.114961</td>\n",
       "      <td>True</td>\n",
       "      <td>0.524862</td>\n",
       "      <td>0.490835</td>\n",
       "      <td>0.463927</td>\n",
       "      <td>0.567948</td>\n",
       "      <td>0.358909</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.231157</td>\n",
       "      <td>...</td>\n",
       "      <td>0.369213</td>\n",
       "      <td>0.532913</td>\n",
       "      <td>0.239055</td>\n",
       "      <td>0.167247</td>\n",
       "      <td>0.065024</td>\n",
       "      <td>0.086001</td>\n",
       "      <td>0.104475</td>\n",
       "      <td>0.038315</td>\n",
       "      <td>0.169400</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.111654</td>\n",
       "      <td>True</td>\n",
       "      <td>0.524014</td>\n",
       "      <td>0.491290</td>\n",
       "      <td>0.464612</td>\n",
       "      <td>0.567255</td>\n",
       "      <td>0.361376</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.234130</td>\n",
       "      <td>...</td>\n",
       "      <td>0.371181</td>\n",
       "      <td>0.523150</td>\n",
       "      <td>0.243465</td>\n",
       "      <td>0.170409</td>\n",
       "      <td>0.064895</td>\n",
       "      <td>0.087138</td>\n",
       "      <td>0.105072</td>\n",
       "      <td>0.037727</td>\n",
       "      <td>0.172647</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.105669</td>\n",
       "      <td>True</td>\n",
       "      <td>0.529986</td>\n",
       "      <td>0.495618</td>\n",
       "      <td>0.467760</td>\n",
       "      <td>0.574294</td>\n",
       "      <td>0.360304</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.232689</td>\n",
       "      <td>...</td>\n",
       "      <td>0.369291</td>\n",
       "      <td>0.531969</td>\n",
       "      <td>0.240378</td>\n",
       "      <td>0.168042</td>\n",
       "      <td>0.065864</td>\n",
       "      <td>0.086146</td>\n",
       "      <td>0.104585</td>\n",
       "      <td>0.038410</td>\n",
       "      <td>0.170240</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>0.5</td>\n",
       "      <td>0.109606</td>\n",
       "      <td>True</td>\n",
       "      <td>0.525718</td>\n",
       "      <td>0.492048</td>\n",
       "      <td>0.464924</td>\n",
       "      <td>0.568730</td>\n",
       "      <td>0.360389</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.232576</td>\n",
       "      <td>...</td>\n",
       "      <td>0.369134</td>\n",
       "      <td>0.530394</td>\n",
       "      <td>0.240441</td>\n",
       "      <td>0.168870</td>\n",
       "      <td>0.065466</td>\n",
       "      <td>0.086283</td>\n",
       "      <td>0.104424</td>\n",
       "      <td>0.038284</td>\n",
       "      <td>0.170946</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.113543</td>\n",
       "      <td>True</td>\n",
       "      <td>0.526552</td>\n",
       "      <td>0.492951</td>\n",
       "      <td>0.465744</td>\n",
       "      <td>0.571234</td>\n",
       "      <td>0.361402</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.233917</td>\n",
       "      <td>...</td>\n",
       "      <td>0.369055</td>\n",
       "      <td>0.525984</td>\n",
       "      <td>0.242394</td>\n",
       "      <td>0.169859</td>\n",
       "      <td>0.064699</td>\n",
       "      <td>0.086374</td>\n",
       "      <td>0.104451</td>\n",
       "      <td>0.037769</td>\n",
       "      <td>0.171989</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.108976</td>\n",
       "      <td>True</td>\n",
       "      <td>0.532562</td>\n",
       "      <td>0.497095</td>\n",
       "      <td>0.468646</td>\n",
       "      <td>0.579097</td>\n",
       "      <td>0.361963</td>\n",
       "      <td>UCB</td>\n",
       "      <td>0.234778</td>\n",
       "      <td>...</td>\n",
       "      <td>0.369370</td>\n",
       "      <td>0.535118</td>\n",
       "      <td>0.242992</td>\n",
       "      <td>0.170195</td>\n",
       "      <td>0.065349</td>\n",
       "      <td>0.086544</td>\n",
       "      <td>0.104180</td>\n",
       "      <td>0.038507</td>\n",
       "      <td>0.172365</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      C    Clicks  Iterative    MAP@10    MAP@15    MAP@20     MAP@5  \\\n",
       "8   1.0  0.111811       True  0.525279  0.490594  0.463029  0.569685   \n",
       "12  2.0  0.107402       True  0.529762  0.494289  0.465921  0.574787   \n",
       "24  1.0  0.110236       True  0.527201  0.493545  0.465934  0.571045   \n",
       "28  2.0  0.115906       True  0.528105  0.493625  0.465620  0.571664   \n",
       "40  1.0  0.112598       True  0.527571  0.493828  0.465776  0.570215   \n",
       "44  2.0  0.114488       True  0.521709  0.489267  0.462627  0.565648   \n",
       "55  0.5  0.114961       True  0.524862  0.490835  0.463927  0.567948   \n",
       "59  1.0  0.111654       True  0.524014  0.491290  0.464612  0.567255   \n",
       "63  2.0  0.105669       True  0.529986  0.495618  0.467760  0.574294   \n",
       "75  0.5  0.109606       True  0.525718  0.492048  0.464924  0.568730   \n",
       "79  1.0  0.113543       True  0.526552  0.492951  0.465744  0.571234   \n",
       "83  2.0  0.108976       True  0.532562  0.497095  0.468646  0.579097   \n",
       "\n",
       "      MAP@50 Model      NDCG  ...  Precision@20  Precision@5  Precision@50  \\\n",
       "8   0.358584   UCB  0.231484  ...      0.365591     0.529134      0.239843   \n",
       "12  0.360519   UCB  0.234211  ...      0.365669     0.529449      0.242835   \n",
       "24  0.360642   UCB  0.232596  ...      0.368976     0.529764      0.240724   \n",
       "28  0.358754   UCB  0.230725  ...      0.365276     0.534173      0.237953   \n",
       "40  0.360066   UCB  0.233006  ...      0.367717     0.525354      0.241039   \n",
       "44  0.359316   UCB  0.231974  ...      0.368189     0.526929      0.240693   \n",
       "55  0.358909   UCB  0.231157  ...      0.369213     0.532913      0.239055   \n",
       "59  0.361376   UCB  0.234130  ...      0.371181     0.523150      0.243465   \n",
       "63  0.360304   UCB  0.232689  ...      0.369291     0.531969      0.240378   \n",
       "75  0.360389   UCB  0.232576  ...      0.369134     0.530394      0.240441   \n",
       "79  0.361402   UCB  0.233917  ...      0.369055     0.525984      0.242394   \n",
       "83  0.361963   UCB  0.234778  ...      0.369370     0.535118      0.242992   \n",
       "\n",
       "    R-Precision  Recall@10  Recall@15  Recall@20  Recall@5  Recall@50  \\\n",
       "8      0.167661   0.064415   0.085435   0.103300  0.038104   0.170197   \n",
       "12     0.169797   0.065482   0.086041   0.103506  0.037990   0.172420   \n",
       "24     0.168402   0.065421   0.086565   0.104643  0.038186   0.170632   \n",
       "28     0.166373   0.065149   0.086017   0.103363  0.038408   0.168420   \n",
       "40     0.168971   0.065309   0.086076   0.104054  0.037704   0.171186   \n",
       "44     0.168680   0.064332   0.086233   0.104184  0.038000   0.170516   \n",
       "55     0.167247   0.065024   0.086001   0.104475  0.038315   0.169400   \n",
       "59     0.170409   0.064895   0.087138   0.105072  0.037727   0.172647   \n",
       "63     0.168042   0.065864   0.086146   0.104585  0.038410   0.170240   \n",
       "75     0.168870   0.065466   0.086283   0.104424  0.038284   0.170946   \n",
       "79     0.169859   0.064699   0.086374   0.104451  0.037769   0.171989   \n",
       "83     0.170195   0.065349   0.086544   0.104180  0.038507   0.172365   \n",
       "\n",
       "    SampleFromAll  \n",
       "8            True  \n",
       "12           True  \n",
       "24           True  \n",
       "28           True  \n",
       "40           True  \n",
       "44           True  \n",
       "55           True  \n",
       "59           True  \n",
       "63           True  \n",
       "75           True  \n",
       "79           True  \n",
       "83           True  \n",
       "\n",
       "[12 rows x 22 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[(df['Model'] == 'UCB') & (df['Iterative'] == True) & (df['SampleFromAll'] == True)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>C</th>\n",
       "      <th>Iterative</th>\n",
       "      <th>SampleFromAll</th>\n",
       "      <th>MAP@5</th>\n",
       "      <th>MAP@10</th>\n",
       "      <th>MAP@15</th>\n",
       "      <th>MAP@20</th>\n",
       "      <th>MAP@50</th>\n",
       "      <th>NDCG</th>\n",
       "      <th>...</th>\n",
       "      <th>Precision@10</th>\n",
       "      <th>Precision@15</th>\n",
       "      <th>Precision@20</th>\n",
       "      <th>Precision@50</th>\n",
       "      <th>R-Precision</th>\n",
       "      <th>Recall@5</th>\n",
       "      <th>Recall@10</th>\n",
       "      <th>Recall@15</th>\n",
       "      <th>Recall@20</th>\n",
       "      <th>Recall@50</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Greedy</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.636387</td>\n",
       "      <td>0.599495</td>\n",
       "      <td>0.568758</td>\n",
       "      <td>0.542806</td>\n",
       "      <td>0.439841</td>\n",
       "      <td>0.293907</td>\n",
       "      <td>...</td>\n",
       "      <td>0.539150</td>\n",
       "      <td>0.489134</td>\n",
       "      <td>0.450472</td>\n",
       "      <td>0.314765</td>\n",
       "      <td>0.223278</td>\n",
       "      <td>0.044158</td>\n",
       "      <td>0.078547</td>\n",
       "      <td>0.106479</td>\n",
       "      <td>0.130231</td>\n",
       "      <td>0.226431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Greedy</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.559470</td>\n",
       "      <td>0.516241</td>\n",
       "      <td>0.483123</td>\n",
       "      <td>0.455906</td>\n",
       "      <td>0.349279</td>\n",
       "      <td>0.224324</td>\n",
       "      <td>...</td>\n",
       "      <td>0.446677</td>\n",
       "      <td>0.398068</td>\n",
       "      <td>0.358630</td>\n",
       "      <td>0.230961</td>\n",
       "      <td>0.160961</td>\n",
       "      <td>0.037405</td>\n",
       "      <td>0.063629</td>\n",
       "      <td>0.084639</td>\n",
       "      <td>0.101207</td>\n",
       "      <td>0.163064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Greedy</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>0.625411</td>\n",
       "      <td>0.584062</td>\n",
       "      <td>0.555060</td>\n",
       "      <td>0.531863</td>\n",
       "      <td>0.440809</td>\n",
       "      <td>0.301108</td>\n",
       "      <td>...</td>\n",
       "      <td>0.522677</td>\n",
       "      <td>0.482016</td>\n",
       "      <td>0.449843</td>\n",
       "      <td>0.329398</td>\n",
       "      <td>0.233702</td>\n",
       "      <td>0.042423</td>\n",
       "      <td>0.075868</td>\n",
       "      <td>0.104571</td>\n",
       "      <td>0.130009</td>\n",
       "      <td>0.237275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Greedy</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.567882</td>\n",
       "      <td>0.525302</td>\n",
       "      <td>0.491568</td>\n",
       "      <td>0.463978</td>\n",
       "      <td>0.360013</td>\n",
       "      <td>0.233142</td>\n",
       "      <td>...</td>\n",
       "      <td>0.456693</td>\n",
       "      <td>0.403990</td>\n",
       "      <td>0.367224</td>\n",
       "      <td>0.241811</td>\n",
       "      <td>0.169505</td>\n",
       "      <td>0.037910</td>\n",
       "      <td>0.065046</td>\n",
       "      <td>0.085826</td>\n",
       "      <td>0.103835</td>\n",
       "      <td>0.171665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ThompsonSampling</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.613557</td>\n",
       "      <td>0.573595</td>\n",
       "      <td>0.542864</td>\n",
       "      <td>0.517744</td>\n",
       "      <td>0.421410</td>\n",
       "      <td>0.285719</td>\n",
       "      <td>...</td>\n",
       "      <td>0.510961</td>\n",
       "      <td>0.464357</td>\n",
       "      <td>0.428661</td>\n",
       "      <td>0.307414</td>\n",
       "      <td>0.218842</td>\n",
       "      <td>0.042046</td>\n",
       "      <td>0.074824</td>\n",
       "      <td>0.101652</td>\n",
       "      <td>0.124709</td>\n",
       "      <td>0.222046</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ThompsonSampling</td>\n",
       "      <td>0.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.518442</td>\n",
       "      <td>0.474580</td>\n",
       "      <td>0.442955</td>\n",
       "      <td>0.418123</td>\n",
       "      <td>0.329114</td>\n",
       "      <td>0.217527</td>\n",
       "      <td>...</td>\n",
       "      <td>0.407276</td>\n",
       "      <td>0.363066</td>\n",
       "      <td>0.331449</td>\n",
       "      <td>0.230362</td>\n",
       "      <td>0.160909</td>\n",
       "      <td>0.033785</td>\n",
       "      <td>0.057846</td>\n",
       "      <td>0.076937</td>\n",
       "      <td>0.093807</td>\n",
       "      <td>0.163376</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>ThompsonSampling</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>0.621474</td>\n",
       "      <td>0.582742</td>\n",
       "      <td>0.554001</td>\n",
       "      <td>0.530491</td>\n",
       "      <td>0.439252</td>\n",
       "      <td>0.299699</td>\n",
       "      <td>...</td>\n",
       "      <td>0.522268</td>\n",
       "      <td>0.479895</td>\n",
       "      <td>0.447606</td>\n",
       "      <td>0.327843</td>\n",
       "      <td>0.232349</td>\n",
       "      <td>0.042478</td>\n",
       "      <td>0.075856</td>\n",
       "      <td>0.104130</td>\n",
       "      <td>0.129169</td>\n",
       "      <td>0.236139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ThompsonSampling</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.561318</td>\n",
       "      <td>0.520612</td>\n",
       "      <td>0.487964</td>\n",
       "      <td>0.460984</td>\n",
       "      <td>0.357428</td>\n",
       "      <td>0.230464</td>\n",
       "      <td>...</td>\n",
       "      <td>0.453512</td>\n",
       "      <td>0.403633</td>\n",
       "      <td>0.366031</td>\n",
       "      <td>0.238929</td>\n",
       "      <td>0.167140</td>\n",
       "      <td>0.037526</td>\n",
       "      <td>0.064626</td>\n",
       "      <td>0.085784</td>\n",
       "      <td>0.103524</td>\n",
       "      <td>0.169395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>UCB</td>\n",
       "      <td>0.5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.638286</td>\n",
       "      <td>0.597731</td>\n",
       "      <td>0.564335</td>\n",
       "      <td>0.537278</td>\n",
       "      <td>0.433669</td>\n",
       "      <td>0.291819</td>\n",
       "      <td>...</td>\n",
       "      <td>0.531654</td>\n",
       "      <td>0.479213</td>\n",
       "      <td>0.440945</td>\n",
       "      <td>0.311622</td>\n",
       "      <td>0.221447</td>\n",
       "      <td>0.043819</td>\n",
       "      <td>0.077472</td>\n",
       "      <td>0.104625</td>\n",
       "      <td>0.128134</td>\n",
       "      <td>0.224950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>UCB</td>\n",
       "      <td>0.5</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.548774</td>\n",
       "      <td>0.502971</td>\n",
       "      <td>0.468229</td>\n",
       "      <td>0.440940</td>\n",
       "      <td>0.341609</td>\n",
       "      <td>0.222113</td>\n",
       "      <td>...</td>\n",
       "      <td>0.430551</td>\n",
       "      <td>0.381365</td>\n",
       "      <td>0.345276</td>\n",
       "      <td>0.230535</td>\n",
       "      <td>0.161268</td>\n",
       "      <td>0.035883</td>\n",
       "      <td>0.061287</td>\n",
       "      <td>0.080923</td>\n",
       "      <td>0.097527</td>\n",
       "      <td>0.163510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>UCB</td>\n",
       "      <td>0.5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>0.631832</td>\n",
       "      <td>0.589725</td>\n",
       "      <td>0.560081</td>\n",
       "      <td>0.535568</td>\n",
       "      <td>0.440719</td>\n",
       "      <td>0.300095</td>\n",
       "      <td>...</td>\n",
       "      <td>0.527874</td>\n",
       "      <td>0.484199</td>\n",
       "      <td>0.449449</td>\n",
       "      <td>0.326047</td>\n",
       "      <td>0.231266</td>\n",
       "      <td>0.043138</td>\n",
       "      <td>0.076818</td>\n",
       "      <td>0.105220</td>\n",
       "      <td>0.130043</td>\n",
       "      <td>0.234936</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>UCB</td>\n",
       "      <td>0.5</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.568339</td>\n",
       "      <td>0.525290</td>\n",
       "      <td>0.491442</td>\n",
       "      <td>0.464425</td>\n",
       "      <td>0.359649</td>\n",
       "      <td>0.231867</td>\n",
       "      <td>...</td>\n",
       "      <td>0.457402</td>\n",
       "      <td>0.405039</td>\n",
       "      <td>0.369173</td>\n",
       "      <td>0.239748</td>\n",
       "      <td>0.168059</td>\n",
       "      <td>0.038300</td>\n",
       "      <td>0.065245</td>\n",
       "      <td>0.086142</td>\n",
       "      <td>0.104449</td>\n",
       "      <td>0.170173</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>UCB</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.626317</td>\n",
       "      <td>0.587666</td>\n",
       "      <td>0.557223</td>\n",
       "      <td>0.531858</td>\n",
       "      <td>0.431091</td>\n",
       "      <td>0.289660</td>\n",
       "      <td>...</td>\n",
       "      <td>0.526677</td>\n",
       "      <td>0.478824</td>\n",
       "      <td>0.440961</td>\n",
       "      <td>0.309928</td>\n",
       "      <td>0.220472</td>\n",
       "      <td>0.043156</td>\n",
       "      <td>0.076987</td>\n",
       "      <td>0.104670</td>\n",
       "      <td>0.128068</td>\n",
       "      <td>0.223323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>UCB</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.541444</td>\n",
       "      <td>0.498199</td>\n",
       "      <td>0.465248</td>\n",
       "      <td>0.438419</td>\n",
       "      <td>0.339709</td>\n",
       "      <td>0.221170</td>\n",
       "      <td>...</td>\n",
       "      <td>0.430394</td>\n",
       "      <td>0.380997</td>\n",
       "      <td>0.344110</td>\n",
       "      <td>0.230431</td>\n",
       "      <td>0.161174</td>\n",
       "      <td>0.035883</td>\n",
       "      <td>0.061284</td>\n",
       "      <td>0.080895</td>\n",
       "      <td>0.097398</td>\n",
       "      <td>0.163288</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>UCB</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>0.624056</td>\n",
       "      <td>0.583961</td>\n",
       "      <td>0.555210</td>\n",
       "      <td>0.531770</td>\n",
       "      <td>0.440146</td>\n",
       "      <td>0.300145</td>\n",
       "      <td>...</td>\n",
       "      <td>0.522803</td>\n",
       "      <td>0.481491</td>\n",
       "      <td>0.449181</td>\n",
       "      <td>0.327761</td>\n",
       "      <td>0.232511</td>\n",
       "      <td>0.042550</td>\n",
       "      <td>0.075978</td>\n",
       "      <td>0.104674</td>\n",
       "      <td>0.129830</td>\n",
       "      <td>0.236125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>UCB</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.569887</td>\n",
       "      <td>0.526123</td>\n",
       "      <td>0.492442</td>\n",
       "      <td>0.465019</td>\n",
       "      <td>0.360414</td>\n",
       "      <td>0.233027</td>\n",
       "      <td>...</td>\n",
       "      <td>0.455874</td>\n",
       "      <td>0.406341</td>\n",
       "      <td>0.368504</td>\n",
       "      <td>0.241493</td>\n",
       "      <td>0.169060</td>\n",
       "      <td>0.037898</td>\n",
       "      <td>0.064948</td>\n",
       "      <td>0.086318</td>\n",
       "      <td>0.104304</td>\n",
       "      <td>0.171330</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>UCB</td>\n",
       "      <td>2.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.626614</td>\n",
       "      <td>0.587225</td>\n",
       "      <td>0.555322</td>\n",
       "      <td>0.528814</td>\n",
       "      <td>0.427787</td>\n",
       "      <td>0.288738</td>\n",
       "      <td>...</td>\n",
       "      <td>0.522520</td>\n",
       "      <td>0.472504</td>\n",
       "      <td>0.435701</td>\n",
       "      <td>0.308769</td>\n",
       "      <td>0.219830</td>\n",
       "      <td>0.043310</td>\n",
       "      <td>0.076412</td>\n",
       "      <td>0.103246</td>\n",
       "      <td>0.126687</td>\n",
       "      <td>0.223222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>UCB</td>\n",
       "      <td>2.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>0.531462</td>\n",
       "      <td>0.484496</td>\n",
       "      <td>0.450016</td>\n",
       "      <td>0.423740</td>\n",
       "      <td>0.331025</td>\n",
       "      <td>0.218124</td>\n",
       "      <td>...</td>\n",
       "      <td>0.411055</td>\n",
       "      <td>0.364556</td>\n",
       "      <td>0.332441</td>\n",
       "      <td>0.229109</td>\n",
       "      <td>0.160291</td>\n",
       "      <td>0.034507</td>\n",
       "      <td>0.058493</td>\n",
       "      <td>0.077485</td>\n",
       "      <td>0.094213</td>\n",
       "      <td>0.162616</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>UCB</td>\n",
       "      <td>2.0</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>0.625411</td>\n",
       "      <td>0.585460</td>\n",
       "      <td>0.556253</td>\n",
       "      <td>0.532480</td>\n",
       "      <td>0.440413</td>\n",
       "      <td>0.299984</td>\n",
       "      <td>...</td>\n",
       "      <td>0.524126</td>\n",
       "      <td>0.482058</td>\n",
       "      <td>0.448583</td>\n",
       "      <td>0.327515</td>\n",
       "      <td>0.232289</td>\n",
       "      <td>0.042706</td>\n",
       "      <td>0.076199</td>\n",
       "      <td>0.104748</td>\n",
       "      <td>0.129541</td>\n",
       "      <td>0.235747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>UCB</td>\n",
       "      <td>2.0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>0.573098</td>\n",
       "      <td>0.528425</td>\n",
       "      <td>0.493979</td>\n",
       "      <td>0.466115</td>\n",
       "      <td>0.360171</td>\n",
       "      <td>0.232875</td>\n",
       "      <td>...</td>\n",
       "      <td>0.457291</td>\n",
       "      <td>0.405879</td>\n",
       "      <td>0.367559</td>\n",
       "      <td>0.240970</td>\n",
       "      <td>0.168617</td>\n",
       "      <td>0.038263</td>\n",
       "      <td>0.065235</td>\n",
       "      <td>0.086196</td>\n",
       "      <td>0.103964</td>\n",
       "      <td>0.170792</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               Model    C  Iterative  SampleFromAll     MAP@5    MAP@10  \\\n",
       "0             Greedy  0.0      False          False  0.636387  0.599495   \n",
       "1             Greedy  0.0      False           True  0.559470  0.516241   \n",
       "2             Greedy  0.0       True          False  0.625411  0.584062   \n",
       "3             Greedy  0.0       True           True  0.567882  0.525302   \n",
       "4   ThompsonSampling  0.0      False          False  0.613557  0.573595   \n",
       "5   ThompsonSampling  0.0      False           True  0.518442  0.474580   \n",
       "6   ThompsonSampling  0.0       True          False  0.621474  0.582742   \n",
       "7   ThompsonSampling  0.0       True           True  0.561318  0.520612   \n",
       "8                UCB  0.5      False          False  0.638286  0.597731   \n",
       "9                UCB  0.5      False           True  0.548774  0.502971   \n",
       "10               UCB  0.5       True          False  0.631832  0.589725   \n",
       "11               UCB  0.5       True           True  0.568339  0.525290   \n",
       "12               UCB  1.0      False          False  0.626317  0.587666   \n",
       "13               UCB  1.0      False           True  0.541444  0.498199   \n",
       "14               UCB  1.0       True          False  0.624056  0.583961   \n",
       "15               UCB  1.0       True           True  0.569887  0.526123   \n",
       "16               UCB  2.0      False          False  0.626614  0.587225   \n",
       "17               UCB  2.0      False           True  0.531462  0.484496   \n",
       "18               UCB  2.0       True          False  0.625411  0.585460   \n",
       "19               UCB  2.0       True           True  0.573098  0.528425   \n",
       "\n",
       "      MAP@15    MAP@20    MAP@50      NDCG  ...  Precision@10  Precision@15  \\\n",
       "0   0.568758  0.542806  0.439841  0.293907  ...      0.539150      0.489134   \n",
       "1   0.483123  0.455906  0.349279  0.224324  ...      0.446677      0.398068   \n",
       "2   0.555060  0.531863  0.440809  0.301108  ...      0.522677      0.482016   \n",
       "3   0.491568  0.463978  0.360013  0.233142  ...      0.456693      0.403990   \n",
       "4   0.542864  0.517744  0.421410  0.285719  ...      0.510961      0.464357   \n",
       "5   0.442955  0.418123  0.329114  0.217527  ...      0.407276      0.363066   \n",
       "6   0.554001  0.530491  0.439252  0.299699  ...      0.522268      0.479895   \n",
       "7   0.487964  0.460984  0.357428  0.230464  ...      0.453512      0.403633   \n",
       "8   0.564335  0.537278  0.433669  0.291819  ...      0.531654      0.479213   \n",
       "9   0.468229  0.440940  0.341609  0.222113  ...      0.430551      0.381365   \n",
       "10  0.560081  0.535568  0.440719  0.300095  ...      0.527874      0.484199   \n",
       "11  0.491442  0.464425  0.359649  0.231867  ...      0.457402      0.405039   \n",
       "12  0.557223  0.531858  0.431091  0.289660  ...      0.526677      0.478824   \n",
       "13  0.465248  0.438419  0.339709  0.221170  ...      0.430394      0.380997   \n",
       "14  0.555210  0.531770  0.440146  0.300145  ...      0.522803      0.481491   \n",
       "15  0.492442  0.465019  0.360414  0.233027  ...      0.455874      0.406341   \n",
       "16  0.555322  0.528814  0.427787  0.288738  ...      0.522520      0.472504   \n",
       "17  0.450016  0.423740  0.331025  0.218124  ...      0.411055      0.364556   \n",
       "18  0.556253  0.532480  0.440413  0.299984  ...      0.524126      0.482058   \n",
       "19  0.493979  0.466115  0.360171  0.232875  ...      0.457291      0.405879   \n",
       "\n",
       "    Precision@20  Precision@50  R-Precision  Recall@5  Recall@10  Recall@15  \\\n",
       "0       0.450472      0.314765     0.223278  0.044158   0.078547   0.106479   \n",
       "1       0.358630      0.230961     0.160961  0.037405   0.063629   0.084639   \n",
       "2       0.449843      0.329398     0.233702  0.042423   0.075868   0.104571   \n",
       "3       0.367224      0.241811     0.169505  0.037910   0.065046   0.085826   \n",
       "4       0.428661      0.307414     0.218842  0.042046   0.074824   0.101652   \n",
       "5       0.331449      0.230362     0.160909  0.033785   0.057846   0.076937   \n",
       "6       0.447606      0.327843     0.232349  0.042478   0.075856   0.104130   \n",
       "7       0.366031      0.238929     0.167140  0.037526   0.064626   0.085784   \n",
       "8       0.440945      0.311622     0.221447  0.043819   0.077472   0.104625   \n",
       "9       0.345276      0.230535     0.161268  0.035883   0.061287   0.080923   \n",
       "10      0.449449      0.326047     0.231266  0.043138   0.076818   0.105220   \n",
       "11      0.369173      0.239748     0.168059  0.038300   0.065245   0.086142   \n",
       "12      0.440961      0.309928     0.220472  0.043156   0.076987   0.104670   \n",
       "13      0.344110      0.230431     0.161174  0.035883   0.061284   0.080895   \n",
       "14      0.449181      0.327761     0.232511  0.042550   0.075978   0.104674   \n",
       "15      0.368504      0.241493     0.169060  0.037898   0.064948   0.086318   \n",
       "16      0.435701      0.308769     0.219830  0.043310   0.076412   0.103246   \n",
       "17      0.332441      0.229109     0.160291  0.034507   0.058493   0.077485   \n",
       "18      0.448583      0.327515     0.232289  0.042706   0.076199   0.104748   \n",
       "19      0.367559      0.240970     0.168617  0.038263   0.065235   0.086196   \n",
       "\n",
       "    Recall@20  Recall@50  \n",
       "0    0.130231   0.226431  \n",
       "1    0.101207   0.163064  \n",
       "2    0.130009   0.237275  \n",
       "3    0.103835   0.171665  \n",
       "4    0.124709   0.222046  \n",
       "5    0.093807   0.163376  \n",
       "6    0.129169   0.236139  \n",
       "7    0.103524   0.169395  \n",
       "8    0.128134   0.224950  \n",
       "9    0.097527   0.163510  \n",
       "10   0.130043   0.234936  \n",
       "11   0.104449   0.170173  \n",
       "12   0.128068   0.223323  \n",
       "13   0.097398   0.163288  \n",
       "14   0.129830   0.236125  \n",
       "15   0.104304   0.171330  \n",
       "16   0.126687   0.223222  \n",
       "17   0.094213   0.162616  \n",
       "18   0.129541   0.235747  \n",
       "19   0.103964   0.170792  \n",
       "\n",
       "[20 rows x 21 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby(['Model', 'C', 'Iterative', 'SampleFromAll'], as_index=False)['MAP@5', 'MAP@10', 'MAP@15', 'MAP@20', 'MAP@50', 'NDCG',\n",
    "'Precision@5', 'Precision@10', 'Precision@15', 'Precision@20', 'Precision@50', 'R-Precision', 'Recall@5',\n",
    "'Recall@10', 'Recall@15', 'Recall@20', 'Recall@50'].agg({'MAP@5': 'mean', 'MAP@10': 'mean', 'MAP@15': 'mean', \n",
    "'MAP@20': 'mean', 'MAP@50': 'mean', 'NDCG': 'mean', 'Precision@5': 'mean', 'Precision@10': 'mean', \n",
    "'Precision@15': 'mean', 'Precision@20': 'mean', 'Precision@50': 'mean', 'R-Precision': 'mean', 'Recall@5': 'mean',\n",
    "'Recall@10': 'mean', 'Recall@15': 'mean', 'Recall@20': 'mean', 'Recall@50': 'mean'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "TensorFlow-GPU",
   "language": "python",
   "name": "tf-gpu"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}