statistics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calculate Basic Statistics of the Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gzip\n",
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "pd.options.display.float_format = '{:,}'.format"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Specify your directory here:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DIR = './'\n",
    "DIR_GENRE = './genre/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**This function will calculate the number of records in the input file**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_lines(file_name):\n",
    "    print('counting file:', file_name)\n",
    "    count = 0\n",
    "    with gzip.open(file_name) as fin:\n",
    "        for l in fin:\n",
    "            count += 1\n",
    "    print('done!')\n",
    "    return count"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Calculate basic statistics of the book graph**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "counting file: ./goodreads_books.json.gz\n",
      "done!\n",
      "counting file: ./goodreads_book_works.json.gz\n",
      "done!\n",
      "counting file: ./goodreads_book_authors.json.gz\n",
      "done!\n",
      "counting file: ./goodreads_book_series.json.gz\n",
      "done!\n"
     ]
    }
   ],
   "source": [
    "n_book = count_lines(os.path.join(DIR, 'goodreads_books.json.gz'))\n",
    "n_work = count_lines(os.path.join(DIR, 'goodreads_book_works.json.gz'))\n",
    "n_author = count_lines(os.path.join(DIR, 'goodreads_book_authors.json.gz'))\n",
    "n_series = count_lines(os.path.join(DIR, 'goodreads_book_series.json.gz'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th># book</th>\n",
       "      <td>2,360,655.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th># work</th>\n",
       "      <td>1,521,962.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th># author</th>\n",
       "      <td>829,529.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th># series</th>\n",
       "      <td>400,390.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               count\n",
       "# book   2,360,655.0\n",
       "# work   1,521,962.0\n",
       "# author   829,529.0\n",
       "# series   400,390.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df_book_stats = pd.DataFrame([n_book, n_work, n_author, n_series], dtype = float, \n",
    "                             columns = ['count'],\n",
    "                             index = ['# book', '# work', '# author', '# series'])\n",
    "display(df_book_stats)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**This function will extract statistics from interaction files**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_interactions(file_name):\n",
    "    print('counting file:', file_name)\n",
    "    n_shelve, n_read, n_rate, n_review = 0, 0, 0, 0\n",
    "    user_set = set()\n",
    "    print('current line:', end='')\n",
    "    with gzip.open(file_name) as fin:\n",
    "        for l in fin:\n",
    "            d = json.loads(l)\n",
    "            if n_shelve % 1000000 == 0:\n",
    "                print(n_shelve, end=',')\n",
    "            n_shelve += 1\n",
    "            n_read += int(d['is_read'])\n",
    "            n_rate += int(d['rating'] > 0)\n",
    "            n_review += int(d['review_text_incomplete'] != '')\n",
    "            user_set.add(d['user_id'])\n",
    "    print('complete')\n",
    "    print('done!')\n",
    "    return n_shelve, n_read, n_rate, n_review, len(user_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Now we can calculate basic statistics for each genre:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "genre_list = ['children', 'comics_graphic', 'fantasy_paranormal', 'history_biography',\n",
    "              'mystery_thriller_crime', 'poetry', 'romance', 'young_adult']\n",
    "\n",
    "def count_all_genres(genre_list):\n",
    "    res = []\n",
    "    for g in genre_list:\n",
    "        n_book = count_lines(os.path.join(DIR_GENRE, 'goodreads_books_'+g+'.json.gz'))\n",
    "        n_shelve, n_read, n_rate, n_review, n_user = count_interactions(\n",
    "            os.path.join(DIR_GENRE, 'goodreads_interactions_'+g+'.json.gz'))\n",
    "        res.append([n_book, n_user, n_shelve, n_read, n_rate, n_review])\n",
    "    df_stats_by_genre = pd.DataFrame(res, dtype = float, \n",
    "                                     columns = ['# book', '# user', '# shelve', '# read', '# rate', '# review'],\n",
    "                                     index = genre_list)\n",
    "    return df_stats_by_genre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "counting file: ./genre/goodreads_books_children.json.gz\n",
      "done!\n",
      "counting file: ./genre/goodreads_interactions_children.json.gz\n",
      "current line:0,1000000,2000000,3000000,4000000,5000000,6000000,7000000,8000000,9000000,10000000,complete\n",
      "done!\n",
      "counting file: ./genre/goodreads_books_comics_graphic.json.gz\n",
      "done!\n",
      "counting file: ./genre/goodreads_interactions_comics_graphic.json.gz\n",
      "current line:0,1000000,2000000,3000000,4000000,5000000,6000000,7000000,complete\n",
      "done!\n",
      "counting file: ./genre/goodreads_books_fantasy_paranormal.json.gz\n",
      "done!\n",
      "counting file: ./genre/goodreads_interactions_fantasy_paranormal.json.gz\n",
      "current line:0,1000000,2000000,3000000,4000000,5000000,6000000,7000000,8000000,9000000,10000000,11000000,12000000,13000000,14000000,15000000,16000000,17000000,18000000,19000000,20000000,21000000,22000000,23000000,24000000,25000000,26000000,27000000,28000000,29000000,30000000,31000000,32000000,33000000,34000000,35000000,36000000,37000000,38000000,39000000,40000000,41000000,42000000,43000000,44000000,45000000,46000000,47000000,48000000,49000000,50000000,51000000,52000000,53000000,54000000,55000000,complete\n",
      "done!\n",
      "counting file: ./genre/goodreads_books_history_biography.json.gz\n",
      "done!\n",
      "counting file: ./genre/goodreads_interactions_history_biography.json.gz\n",
      "current line:0,1000000,2000000,3000000,4000000,5000000,6000000,7000000,8000000,9000000,10000000,11000000,12000000,13000000,14000000,15000000,16000000,17000000,18000000,19000000,20000000,21000000,22000000,23000000,24000000,25000000,26000000,27000000,28000000,29000000,30000000,31000000,complete\n",
      "done!\n",
      "counting file: ./genre/goodreads_books_mystery_thriller_crime.json.gz\n",
      "done!\n",
      "counting file: ./genre/goodreads_interactions_mystery_thriller_crime.json.gz\n",
      "current line:0,1000000,2000000,3000000,4000000,5000000,6000000,7000000,8000000,9000000,10000000,11000000,12000000,13000000,14000000,15000000,16000000,17000000,18000000,19000000,20000000,21000000,22000000,23000000,24000000,complete\n",
      "done!\n",
      "counting file: ./genre/goodreads_books_poetry.json.gz\n",
      "done!\n",
      "counting file: ./genre/goodreads_interactions_poetry.json.gz\n",
      "current line:0,1000000,2000000,complete\n",
      "done!\n",
      "counting file: ./genre/goodreads_books_romance.json.gz\n",
      "done!\n",
      "counting file: ./genre/goodreads_interactions_romance.json.gz\n",
      "current line:0,1000000,2000000,3000000,4000000,5000000,6000000,7000000,8000000,9000000,10000000,11000000,12000000,13000000,14000000,15000000,16000000,17000000,18000000,19000000,20000000,21000000,22000000,23000000,24000000,25000000,26000000,27000000,28000000,29000000,30000000,31000000,32000000,33000000,34000000,35000000,36000000,37000000,38000000,39000000,40000000,41000000,42000000,complete\n",
      "done!\n",
      "counting file: ./genre/goodreads_books_young_adult.json.gz\n",
      "done!\n",
      "counting file: ./genre/goodreads_interactions_young_adult.json.gz\n",
      "current line:0,1000000,2000000,3000000,4000000,5000000,6000000,7000000,8000000,9000000,10000000,11000000,12000000,13000000,14000000,15000000,16000000,17000000,18000000,19000000,20000000,21000000,22000000,23000000,24000000,25000000,26000000,27000000,28000000,29000000,30000000,31000000,32000000,33000000,34000000,complete\n",
      "done!\n"
     ]
    }
   ],
   "source": [
    "df_stats_by_genre = count_all_genres(genre_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>children</th>\n",
       "      <th>comics_graphic</th>\n",
       "      <th>fantasy_paranormal</th>\n",
       "      <th>history_biography</th>\n",
       "      <th>mystery_thriller_crime</th>\n",
       "      <th>poetry</th>\n",
       "      <th>romance</th>\n",
       "      <th>young_adult</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th># book</th>\n",
       "      <td>124,082.0</td>\n",
       "      <td>89,411.0</td>\n",
       "      <td>258,585.0</td>\n",
       "      <td>302,935.0</td>\n",
       "      <td>219,235.0</td>\n",
       "      <td>36,514.0</td>\n",
       "      <td>335,449.0</td>\n",
       "      <td>93,398.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th># user</th>\n",
       "      <td>542,145.0</td>\n",
       "      <td>342,415.0</td>\n",
       "      <td>726,932.0</td>\n",
       "      <td>761,215.0</td>\n",
       "      <td>676,075.0</td>\n",
       "      <td>377,799.0</td>\n",
       "      <td>655,454.0</td>\n",
       "      <td>644,686.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th># shelve</th>\n",
       "      <td>10,059,349.0</td>\n",
       "      <td>7,347,630.0</td>\n",
       "      <td>55,397,550.0</td>\n",
       "      <td>31,479,229.0</td>\n",
       "      <td>24,799,896.0</td>\n",
       "      <td>2,734,350.0</td>\n",
       "      <td>42,792,856.0</td>\n",
       "      <td>34,919,254.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th># read</th>\n",
       "      <td>6,626,989.0</td>\n",
       "      <td>4,764,133.0</td>\n",
       "      <td>27,904,041.0</td>\n",
       "      <td>13,436,575.0</td>\n",
       "      <td>12,524,984.0</td>\n",
       "      <td>1,313,610.0</td>\n",
       "      <td>21,174,642.0</td>\n",
       "      <td>15,722,749.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th># rate</th>\n",
       "      <td>6,384,470.0</td>\n",
       "      <td>4,514,094.0</td>\n",
       "      <td>26,193,771.0</td>\n",
       "      <td>12,379,895.0</td>\n",
       "      <td>11,715,518.0</td>\n",
       "      <td>1,229,059.0</td>\n",
       "      <td>19,701,197.0</td>\n",
       "      <td>14,731,908.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th># review</th>\n",
       "      <td>736,682.0</td>\n",
       "      <td>544,371.0</td>\n",
       "      <td>3,444,043.0</td>\n",
       "      <td>2,074,497.0</td>\n",
       "      <td>1,856,053.0</td>\n",
       "      <td>155,414.0</td>\n",
       "      <td>3,585,643.0</td>\n",
       "      <td>2,405,359.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th># shelve/book</th>\n",
       "      <td>81.07</td>\n",
       "      <td>82.18</td>\n",
       "      <td>214.23</td>\n",
       "      <td>103.91</td>\n",
       "      <td>113.12</td>\n",
       "      <td>74.88</td>\n",
       "      <td>127.57</td>\n",
       "      <td>373.88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th># shelve/user</th>\n",
       "      <td>18.55</td>\n",
       "      <td>21.46</td>\n",
       "      <td>76.21</td>\n",
       "      <td>41.35</td>\n",
       "      <td>36.68</td>\n",
       "      <td>7.24</td>\n",
       "      <td>65.29</td>\n",
       "      <td>54.16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th># read/book</th>\n",
       "      <td>53.41</td>\n",
       "      <td>53.28</td>\n",
       "      <td>107.91</td>\n",
       "      <td>44.35</td>\n",
       "      <td>57.13</td>\n",
       "      <td>35.98</td>\n",
       "      <td>63.12</td>\n",
       "      <td>168.34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th># read/user</th>\n",
       "      <td>12.22</td>\n",
       "      <td>13.91</td>\n",
       "      <td>38.39</td>\n",
       "      <td>17.65</td>\n",
       "      <td>18.53</td>\n",
       "      <td>3.48</td>\n",
       "      <td>32.31</td>\n",
       "      <td>24.39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th># rate/book</th>\n",
       "      <td>51.45</td>\n",
       "      <td>50.49</td>\n",
       "      <td>101.3</td>\n",
       "      <td>40.87</td>\n",
       "      <td>53.44</td>\n",
       "      <td>33.66</td>\n",
       "      <td>58.73</td>\n",
       "      <td>157.73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th># rate/user</th>\n",
       "      <td>11.78</td>\n",
       "      <td>13.18</td>\n",
       "      <td>36.03</td>\n",
       "      <td>16.26</td>\n",
       "      <td>17.33</td>\n",
       "      <td>3.25</td>\n",
       "      <td>30.06</td>\n",
       "      <td>22.85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th># review/book</th>\n",
       "      <td>5.94</td>\n",
       "      <td>6.09</td>\n",
       "      <td>13.32</td>\n",
       "      <td>6.85</td>\n",
       "      <td>8.47</td>\n",
       "      <td>4.26</td>\n",
       "      <td>10.69</td>\n",
       "      <td>25.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th># review/user</th>\n",
       "      <td>1.36</td>\n",
       "      <td>1.59</td>\n",
       "      <td>4.74</td>\n",
       "      <td>2.73</td>\n",
       "      <td>2.75</td>\n",
       "      <td>0.41</td>\n",
       "      <td>5.47</td>\n",
       "      <td>3.73</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  children  comics_graphic  fantasy_paranormal  \\\n",
       "# book           124,082.0        89,411.0           258,585.0   \n",
       "# user           542,145.0       342,415.0           726,932.0   \n",
       "# shelve      10,059,349.0     7,347,630.0        55,397,550.0   \n",
       "# read         6,626,989.0     4,764,133.0        27,904,041.0   \n",
       "# rate         6,384,470.0     4,514,094.0        26,193,771.0   \n",
       "# review         736,682.0       544,371.0         3,444,043.0   \n",
       "# shelve/book        81.07           82.18              214.23   \n",
       "# shelve/user        18.55           21.46               76.21   \n",
       "# read/book          53.41           53.28              107.91   \n",
       "# read/user          12.22           13.91               38.39   \n",
       "# rate/book          51.45           50.49               101.3   \n",
       "# rate/user          11.78           13.18               36.03   \n",
       "# review/book         5.94            6.09               13.32   \n",
       "# review/user         1.36            1.59                4.74   \n",
       "\n",
       "               history_biography  mystery_thriller_crime      poetry  \\\n",
       "# book                 302,935.0               219,235.0    36,514.0   \n",
       "# user                 761,215.0               676,075.0   377,799.0   \n",
       "# shelve            31,479,229.0            24,799,896.0 2,734,350.0   \n",
       "# read              13,436,575.0            12,524,984.0 1,313,610.0   \n",
       "# rate              12,379,895.0            11,715,518.0 1,229,059.0   \n",
       "# review             2,074,497.0             1,856,053.0   155,414.0   \n",
       "# shelve/book             103.91                  113.12       74.88   \n",
       "# shelve/user              41.35                   36.68        7.24   \n",
       "# read/book                44.35                   57.13       35.98   \n",
       "# read/user                17.65                   18.53        3.48   \n",
       "# rate/book                40.87                   53.44       33.66   \n",
       "# rate/user                16.26                   17.33        3.25   \n",
       "# review/book               6.85                    8.47        4.26   \n",
       "# review/user               2.73                    2.75        0.41   \n",
       "\n",
       "                   romance  young_adult  \n",
       "# book           335,449.0     93,398.0  \n",
       "# user           655,454.0    644,686.0  \n",
       "# shelve      42,792,856.0 34,919,254.0  \n",
       "# read        21,174,642.0 15,722,749.0  \n",
       "# rate        19,701,197.0 14,731,908.0  \n",
       "# review       3,585,643.0  2,405,359.0  \n",
       "# shelve/book       127.57       373.88  \n",
       "# shelve/user        65.29        54.16  \n",
       "# read/book          63.12       168.34  \n",
       "# read/user          32.31        24.39  \n",
       "# rate/book          58.73       157.73  \n",
       "# rate/user          30.06        22.85  \n",
       "# review/book        10.69        25.75  \n",
       "# review/user         5.47         3.73  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for _t in ['# shelve', '# read', '# rate', '# review']:\n",
    "    df_stats_by_genre[_t+'/'+'book'] = df_stats_by_genre[_t]/df_stats_by_genre['# book']\n",
    "    df_stats_by_genre[_t+'/'+'user'] = df_stats_by_genre[_t]/df_stats_by_genre['# user']\n",
    "display(df_stats_by_genre.round(2).transpose())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}