quora-question-pairs-training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Quora question pairs: training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Import packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "from __future__ import print_function\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import datetime, time, json\n",
    "from keras.models import Model\n",
    "from keras.layers import Input, TimeDistributed, Dense, Lambda, concatenate, Dropout, BatchNormalization\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.regularizers import l2\n",
    "from keras.callbacks import Callback, ModelCheckpoint\n",
    "from keras import backend as K\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Initialize global variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "Q1_TRAINING_DATA_FILE = 'q1_train.npy'\n",
    "Q2_TRAINING_DATA_FILE = 'q2_train.npy'\n",
    "LABEL_TRAINING_DATA_FILE = 'label_train.npy'\n",
    "WORD_EMBEDDING_MATRIX_FILE = 'word_embedding_matrix.npy'\n",
    "NB_WORDS_DATA_FILE = 'nb_words.json'\n",
    "MODEL_WEIGHTS_FILE = 'question_pairs_weights.h5'\n",
    "MAX_SEQUENCE_LENGTH = 25\n",
    "EMBEDDING_DIM = 300\n",
    "VALIDATION_SPLIT = 0.1\n",
    "TEST_SPLIT = 0.1\n",
    "RNG_SEED = 13371447\n",
    "NB_EPOCHS = 25\n",
    "DROPOUT = 0.1\n",
    "BATCH_SIZE = 32"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Load the dataset, embedding matrix and word count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "q1_data = np.load(open(Q1_TRAINING_DATA_FILE, 'rb'))\n",
    "q2_data = np.load(open(Q2_TRAINING_DATA_FILE, 'rb'))\n",
    "labels = np.load(open(LABEL_TRAINING_DATA_FILE, 'rb'))\n",
    "word_embedding_matrix = np.load(open(WORD_EMBEDDING_MATRIX_FILE, 'rb'))\n",
    "with open(NB_WORDS_DATA_FILE, 'r') as f:\n",
    "    nb_words = json.load(f)['nb_words']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Partition the dataset into train and test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "X = np.stack((q1_data, q2_data), axis=1)\n",
    "y = labels\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=TEST_SPLIT, random_state=RNG_SEED)\n",
    "Q1_train = X_train[:,0]\n",
    "Q2_train = X_train[:,1]\n",
    "Q1_test = X_test[:,0]\n",
    "Q2_test = X_test[:,1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Define the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "question1 = Input(shape=(MAX_SEQUENCE_LENGTH,))\n",
    "question2 = Input(shape=(MAX_SEQUENCE_LENGTH,))\n",
    "\n",
    "q1 = Embedding(nb_words + 1, \n",
    "                 EMBEDDING_DIM, \n",
    "                 weights=[word_embedding_matrix], \n",
    "                 input_length=MAX_SEQUENCE_LENGTH, \n",
    "                 trainable=False)(question1)\n",
    "q1 = TimeDistributed(Dense(EMBEDDING_DIM, activation='relu'))(q1)\n",
    "q1 = Lambda(lambda x: K.max(x, axis=1), output_shape=(EMBEDDING_DIM, ))(q1)\n",
    "\n",
    "q2 = Embedding(nb_words + 1, \n",
    "                 EMBEDDING_DIM, \n",
    "                 weights=[word_embedding_matrix], \n",
    "                 input_length=MAX_SEQUENCE_LENGTH, \n",
    "                 trainable=False)(question2)\n",
    "q2 = TimeDistributed(Dense(EMBEDDING_DIM, activation='relu'))(q2)\n",
    "q2 = Lambda(lambda x: K.max(x, axis=1), output_shape=(EMBEDDING_DIM, ))(q2)\n",
    "\n",
    "merged = concatenate([q1,q2])\n",
    "merged = Dense(200, activation='relu')(merged)\n",
    "merged = Dropout(DROPOUT)(merged)\n",
    "merged = BatchNormalization()(merged)\n",
    "merged = Dense(200, activation='relu')(merged)\n",
    "merged = Dropout(DROPOUT)(merged)\n",
    "merged = BatchNormalization()(merged)\n",
    "merged = Dense(200, activation='relu')(merged)\n",
    "merged = Dropout(DROPOUT)(merged)\n",
    "merged = BatchNormalization()(merged)\n",
    "merged = Dense(200, activation='relu')(merged)\n",
    "merged = Dropout(DROPOUT)(merged)\n",
    "merged = BatchNormalization()(merged)\n",
    "\n",
    "is_duplicate = Dense(1, activation='sigmoid')(merged)\n",
    "\n",
    "model = Model(inputs=[question1,question2], outputs=is_duplicate)\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input_1 (InputLayer)             (None, 25)            0                                            \n",
      "____________________________________________________________________________________________________\n",
      "input_2 (InputLayer)             (None, 25)            0                                            \n",
      "____________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)          (None, 25, 300)       28679100    input_1[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)          (None, 25, 300)       28679100    input_2[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "time_distributed_1 (TimeDistribu (None, 25, 300)       90300       embedding_1[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "time_distributed_2 (TimeDistribu (None, 25, 300)       90300       embedding_2[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "lambda_1 (Lambda)                (None, 300)           0           time_distributed_1[0][0]         \n",
      "____________________________________________________________________________________________________\n",
      "lambda_2 (Lambda)                (None, 300)           0           time_distributed_2[0][0]         \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)      (None, 600)           0           lambda_1[0][0]                   \n",
      "                                                                   lambda_2[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_3 (Dense)                  (None, 200)           120200      concatenate_1[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)              (None, 200)           0           dense_3[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNorm (None, 200)           800         dropout_1[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_4 (Dense)                  (None, 200)           40200       batch_normalization_1[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)              (None, 200)           0           dense_4[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_2 (BatchNorm (None, 200)           800         dropout_2[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_5 (Dense)                  (None, 200)           40200       batch_normalization_2[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)              (None, 200)           0           dense_5[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNorm (None, 200)           800         dropout_3[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_6 (Dense)                  (None, 200)           40200       batch_normalization_3[0][0]      \n",
      "____________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)              (None, 200)           0           dense_6[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "batch_normalization_4 (BatchNorm (None, 200)           800         dropout_4[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_7 (Dense)                  (None, 1)             201         batch_normalization_4[0][0]      \n",
      "====================================================================================================\n",
      "Total params: 57,783,001\n",
      "Trainable params: 423,201\n",
      "Non-trainable params: 57,359,800\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Train the model, checkpointing weights with best validation accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting training at 2017-06-01 19:36:54.250699\n",
      "Train on 327474 samples, validate on 36387 samples\n",
      "Epoch 1/25\n",
      "142s - loss: 0.5421 - acc: 0.7264 - val_loss: 0.4948 - val_acc: 0.7613\n",
      "Epoch 2/25\n",
      "142s - loss: 0.4894 - acc: 0.7606 - val_loss: 0.4681 - val_acc: 0.7718\n",
      "Epoch 3/25\n",
      "141s - loss: 0.4564 - acc: 0.7797 - val_loss: 0.4600 - val_acc: 0.7798\n",
      "Epoch 4/25\n",
      "141s - loss: 0.4345 - acc: 0.7936 - val_loss: 0.4401 - val_acc: 0.7857\n",
      "Epoch 5/25\n",
      "141s - loss: 0.4170 - acc: 0.8058 - val_loss: 0.4334 - val_acc: 0.7945\n",
      "Epoch 6/25\n",
      "142s - loss: 0.4012 - acc: 0.8154 - val_loss: 0.4218 - val_acc: 0.7974\n",
      "Epoch 7/25\n",
      "141s - loss: 0.3862 - acc: 0.8243 - val_loss: 0.4179 - val_acc: 0.8037\n",
      "Epoch 8/25\n",
      "143s - loss: 0.3802 - acc: 0.8279 - val_loss: 0.4135 - val_acc: 0.8038\n",
      "Epoch 9/25\n",
      "142s - loss: 0.3671 - acc: 0.8357 - val_loss: 0.4099 - val_acc: 0.8046\n",
      "Epoch 10/25\n",
      "141s - loss: 0.3528 - acc: 0.8431 - val_loss: 0.4145 - val_acc: 0.8005\n",
      "Epoch 11/25\n",
      "140s - loss: 0.3431 - acc: 0.8484 - val_loss: 0.4152 - val_acc: 0.8011\n",
      "Epoch 12/25\n",
      "141s - loss: 0.3334 - acc: 0.8539 - val_loss: 0.4197 - val_acc: 0.7997\n",
      "Epoch 13/25\n",
      "139s - loss: 0.3267 - acc: 0.8583 - val_loss: 0.4078 - val_acc: 0.8069\n",
      "Epoch 14/25\n",
      "137s - loss: 0.3181 - acc: 0.8623 - val_loss: 0.4204 - val_acc: 0.8005\n",
      "Epoch 15/25\n",
      "137s - loss: 0.3132 - acc: 0.8646 - val_loss: 0.4248 - val_acc: 0.7983\n",
      "Epoch 16/25\n",
      "139s - loss: 0.3062 - acc: 0.8688 - val_loss: 0.4139 - val_acc: 0.8083\n",
      "Epoch 17/25\n",
      "138s - loss: 0.3028 - acc: 0.8706 - val_loss: 0.4086 - val_acc: 0.8066\n",
      "Epoch 18/25\n",
      "137s - loss: 0.2997 - acc: 0.8714 - val_loss: 0.4211 - val_acc: 0.8080\n",
      "Epoch 19/25\n",
      "138s - loss: 0.2948 - acc: 0.8740 - val_loss: 0.4561 - val_acc: 0.7823\n",
      "Epoch 20/25\n",
      "138s - loss: 0.2940 - acc: 0.8745 - val_loss: 0.4214 - val_acc: 0.8064\n",
      "Epoch 21/25\n",
      "138s - loss: 0.2983 - acc: 0.8727 - val_loss: 0.4150 - val_acc: 0.8078\n",
      "Epoch 22/25\n",
      "137s - loss: 0.2963 - acc: 0.8741 - val_loss: 0.4318 - val_acc: 0.7995\n",
      "Epoch 23/25\n",
      "138s - loss: 0.2865 - acc: 0.8785 - val_loss: 0.4405 - val_acc: 0.7981\n",
      "Epoch 24/25\n",
      "137s - loss: 0.2826 - acc: 0.8810 - val_loss: 0.4314 - val_acc: 0.8067\n",
      "Epoch 25/25\n",
      "136s - loss: 0.2755 - acc: 0.8840 - val_loss: 0.4200 - val_acc: 0.8042\n",
      "Training ended at 2017-06-01 20:35:11.907055\n",
      "Minutes elapsed: 58.294262\n"
     ]
    }
   ],
   "source": [
    "print(\"Starting training at\", datetime.datetime.now())\n",
    "t0 = time.time()\n",
    "callbacks = [ModelCheckpoint(MODEL_WEIGHTS_FILE, monitor='val_acc', save_best_only=True)]\n",
    "history = model.fit([Q1_train, Q2_train],\n",
    "                    y_train,\n",
    "                    epochs=NB_EPOCHS,\n",
    "                    validation_split=VALIDATION_SPLIT,\n",
    "                    verbose=2,\n",
    "                    batch_size=BATCH_SIZE,\n",
    "                    callbacks=callbacks)\n",
    "t1 = time.time()\n",
    "print(\"Training ended at\", datetime.datetime.now())\n",
    "print(\"Minutes elapsed: %f\" % ((t1 - t0) / 60.))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Plot training and validation accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfYAAAFHCAYAAABTfW5sAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl0HOWZ7/Hvo8W2NkvCwgu2JcsYjFmMgbCEwUFALpgt\nBgIEGwMGEiYkTCYzJBNghpglTAY4k0yWeyEDJhgYIGxhCEuAQAxkY7VYvQC2vFu2ZcuWZFlL93v/\nqJbUklu2Fld1d/Xvc06dqre6VP32020//dZTXWXOOURERCQcspLdAREREdl7lNhFRERCRIldREQk\nRJTYRUREQkSJXUREJESU2EVERELE18RuZvPNrNbMPtjNNj83s0/NrNrMpvnZHxERkbDze8T+a+C0\n3h40s9OB/Z1zBwB/D9ztc39ERERCzdfE7pz7E7B1N5vMBB6IbfsmUGxmo/zsk4iISJglu8Y+Flgd\n114bWyciIiIDkOzEbgnW6Rq3IiIiA5ST5OdfA4yPa48D1iXa0MyU8EVEJKM45xINgHcriBG7kXhk\nDvAMcCmAmR0H1DvnanvbkXNO0x6mefPmJb0PmTYp5op5JkyKefDTQPk6Yjezh4EqYISZrQLmAUMA\n55z7b+fc82Z2hpl9BjQBl/vZn0xQU1OT7C5kHMU8eIp58BTz9OFrYnfOze7DNtf42QcREZFMkuyT\n52Qvmzt3brK7kHEU8+Ap5sFTzNOHDeY4fpDMzKVLX0VERAbLzHApevKcryZMmICZaUrBacKECcn+\neARi4cKFye5CxlHMg6eYp49k/9xt0FauXDmoswfFP2b9/qIpIiKDlPaH4mOHKpLQI9kTvTciIgOX\nsYfiRUREpIsSu8ggqfYYPMU8eIp5+lBiT3FXX301t912217fVkREwkk1dp9VVlYyf/58Tj755GR3\nJXCp/t6IiKQy1djTUCQSSXYXREQkZJTYfXTppZeyatUqzjrrLIYPH86dd95JVlYW9913HxUVFZxy\nyikAXHjhhYwZM4bS0lKqqqr45JNPOvdx+eWX88Mf/hCA1157jfHjx/OTn/yEUaNGMXbsWO6///4B\nbbtlyxbOPvtsiouLOfbYY7nxxhuZPn26/0EJIdUeg6eYB08xTx9K7D564IEHKC8v57nnnmP79u1c\neOGFALz++ussWbKEF198EYAzzjiDzz//nI0bN3LkkUdy8cUX97rPDRs20NDQwLp167j33nv59re/\nzbZt2/q97be+9S2KiorYuHEj999/PwsWLNDvzkVEQiD0id1s70yDEV9nNjNuvvlm8vLyGDp0KOBd\ngzk/P5/c3Fx++MMf8v7779PQ0JBwX0OGDOHGG28kOzub008/ncLCQpYuXdqvbaPRKE899RS33HIL\nQ4cOZcqUKVx22WWDe5EZrKqqKtldyDiKefAU8/QR+sTu3N6Z9qZx48Z1LkejUa677jomTZpESUkJ\nlZWVmBmbN29O+LcjRowgK6vrbcvPz6exsbFf227atIlIJNKtH+PHjx/syxIRkRQQ+sSebIkOb8ev\ne/jhh/nd737Hq6++Sn19PTU1NTjnfD2bfN999yUnJ4c1a9Z0rlu9erVvzxd2qj0GTzEPnmKePpTY\nfTZ69GiWL18OkDBhNzQ0MHToUEpLS2lqauL666/3vdadlZXFeeedx0033URzczNLlizhgQce8PU5\nRUQkGErsPrvuuuu49dZb2WeffXjyySd3SdqXXnop5eXljB07lkMPPZTjjz++X/vvz5eA+G1/8Ytf\nUF9fz5gxY7jsssuYPXt2Z81f+ke1x+Ap5sFTzNOHLlAjgPcFpLa2ll//+td7bZ96b0REBk4XqJF+\nWbp0KR9++CEAb731FvPnz+e8885Lcq/Sk2qPwVPMg6eYp4+0vx+7DExDQwOzZs1i/fr1jBw5ku9/\n//ucffbZye6WiIgMkg7Fi2/03oiIDJwOxYuIiIgSu8hgqfYYPMU8eIp5+lBiFxERCRHV2MU3em9E\nRAZONXYRERFRYk9FHfdS73DooYfy+uuv92nb/rr66qu57bbbBvz3otpjMijmwVPM04d+x56i4i//\n+tFHH/V5291ZsGAB9957L2+88UbnurvuumtgHRQRkUFxDnbuhKYmaGzcdT5QSuwZxDnn+w1mMpGu\noR08xTx4mRjzaBSam2HHjq55xxTf7rncW6JONB8yBAoKoLCw+7ygYOD9VmL30e23384777zD448/\n3rnuu9/9LgDTpk3jjjvuYM2aNYwcOZJ/+Zd/4aqrrkq4n8rKSubPn8/JJ5/Mzp07+eY3v8kzzzzD\nfvvtx9y5c3d5znvuuYeNGzdSXl7Oj370I8455xyWLFnC1VdfTXt7O0VFReTm5rJlyxYuv/xyxo8f\nzy233ALAPffcwx133MHWrVs54YQTuOuuuxgzZgzg3RXurrvu4j//8z+pq6tj1qxZ/PKXv/QhciIi\ng9PcDFu3dp/q63tft23brkm6tRXy8rwpP79r2l07Lw/KymDChMQJO36enw85u8nCAx2HKbH7aNas\nWdx66600NjZSWFhINBrlscce4+mnn6auro7nnnuOyspK3njjDWbMmMExxxzDtGnTdrvPm266iRUr\nVrBixQoaGxuZMWNGt8cnTZrEn//8Z0aNGsXjjz/OnDlz+PzzzznooIO4++67mT9/fq/1+ldffZUb\nbriBP/zhDxx88MFce+21XHTRRbz22mud2zz33HO8++671NfXc9RRR/GVr3yFU089dfDBSmMLFy7M\nyNFMMmVazJ2DlpbEI8Q9TW1t3shzMFMkAps2LWS//aoYOhSGDoVhw+hc3t2UaLusLC9ptrXtfr6n\nbVpaYPv2xMnaOSgtTTyVlEB5ORx+eNe64cO9hBufoIcNG3hyTabQJ3a7ee+8K25e/3+2VV5ezpFH\nHsnTTz/NnDlzeOWVVygoKOCYY47ptt306dM59dRTeeONN/aY2B9//HHuvvtuiouLKS4u5jvf+Q63\n3npr5+Nf/epXO5cvuOAC/v3f/5233nqrT9eBf/jhh7nyyis5/PDDAfjxj39MaWkpq1atory8HIDr\nr7+eoqIiioqKOOmkk6iurs74xC7pKRLxRmmNjV6CaGnxEkbHcvyUaH2ide3t3n53N+1pm/Z2b+qZ\noHNzu48S+zKVlnqHerOyBj999BFMntz1Wnfu7P7aGxpg8+ZdY9Jzu5YW78vCkCHelJvbt3nH8rBh\nUFTU1S4u9hJ1z+Sdl5fsT1jyhD6xDyQh702zZs3ikUceYc6cOTzyyCPMnj0bgBdeeIFbbrmFZcuW\nEY1GaW5uZurUqXvc37p16xg3blxnu6KiotvjDzzwAD/96U+pqakBoKmpic2bN/epr+vWreOoo47q\nbBcUFDBixAjWrl3bmdhHjRrV+Xh+fj6NgznDIyQyaeSYKqqqqnDOS3jxh1Pr67svJ1rXsdzU5CWI\noiJvFDlkyK4jy57reraHDfNGeh2P5eZCdnbvU07O7h/v2CYnxxs9dhzizcvb/SHbIJxzTlVyOyB9\nFvrEnmwXXHAB3/ve91i7di2//e1vefPNN2ltbeX888/noYceYubMmWRlZXHuuef26WIuY8aMYfXq\n1UyZMgWAlStXdj62atUqrrrqKv74xz/yxS9+EYAjjjiic797OnFuv/3267a/pqYm6urqun2REOmr\nlhZvRNyRTLdt85Jpc7M3imtu7r68p3n88o4d3v5yc7sOrfacl5TA+PFw2GGJtykq8kaiImGjxO6z\nsrIyTjzxRC6//HImTpzIgQceSGNjI62trZSVlZGVlcULL7zASy+9xGGHHbbH/V144YX8+Mc/5phj\njqGxsbHbyWtNTU1kZWVRVlZGNBplwYIF3X4qN2rUKNasWUNbWxu5ubm77Hv27NnMmjWL2bNnM3ny\nZG644QaOO+64Qf1OPhOEsd7bMRqOP7EoPkH3XE60rr29K8EWF3tTYWFX7bLjpKSO5eLi7u2e8/jl\n995byBlnePVeCUYYP+dhpcQegNmzZ3PZZZdx5513AlBYWMjPf/5zLrjgAlpbWzn77LOZOXNmr38f\nP9KeN28e3/zmN6msrGTs2LFcfvnl/OxnPwNgypQpXHvttRx33HFkZ2dz6aWXcsIJJ3T+7cknn8wh\nhxzC6NGjyc7OZuPGjd2e5+STT+bWW2/lvPPOo76+nuOPP55HH300YT8StSW1dNSQ488E7nmYenfr\nsrO7j347EnTH8j77wMSJu67vWM7L8+/EoxUrUFIX6YWuFS++0Xuzd7W2eicnbdoEGzd2nydabmjw\nDjcnOgzd23L8fNiwZL9ikcw20GvFK7GLb/Te7N6OHV6ijp96S9KbNnmJuqwM9t3Xm0aO7D7vuVxS\n4o26RSQ9KbFLysmU92bhwoUcf3wVdXW7JupE06ZN3jwS8RJwWVn3qbeEXVqqk706qN4bPMU8eANN\n7Kqxi+zB9u2wZg2sXp14vnKldwb4iBG7JumyMqishKOP3nV9QUF6XvxCRFKbRuzim3R4bxoaek/a\nHcvt7d7PpsaNSzwfO9YbTStJi8jepEPxknKS/d60tcG6dbBqVeJp9WrvhLTdJe1x45S0RSQ5lNgl\n5fj53jjn/SyrZ6KOb9fWwqhR3jWhe07jx3vTPvsMPmmr9hg8xTx4innwMrbGXlFRod9Tp6iel7vt\nqeMiKNu3932qq+tK3Dk53RN1eTlMndq1br/9vCuTiYhkkrQfsUtqiUS8uvTy5d60YoU3bdmya5Ju\naPAuMjJ8eN+n0tKuRF5cnOxXKyLin4w9FC/Bq6/vStzxCXz5cu9w+L77elckq6z05hMmeGeBdyTn\n4mJvXlSU/BtbiIikKiV2AfZOHSwS8X7C9fnniRN4e7uXsOOnjiReUZF5VyxT7TF4innwFPPgpWyN\n3cxmAP8FZAHznXO393h8PLAAKIltc71z7gW/+5XpolHvkPmnn3rTsmVdyzU13klnkyZ1Je7zz+9K\n4CNG6CxxEZFU5euI3cyygGXAKcA64G3gIufckrhtfgW855z7lZlNAZ53zlUm2JdG7P3kHKxf3z1p\nd0zLl3v16gMOgAMP9OYd0/77Z96oWzJPJBph9fbVbGnewpDsIQzNHurNc4Z2a+dk5fh2gm4kGqE1\n0kpbtM2bR7x50dAi9snbx5fnDJpzjoiLdHt9+bn5FAwpSHbXUl6qjtiPAT51zq0EMLNHgZnAkrht\nosDw2HIJsNbnPoVOe7uXrN9/Hz78sCuRf/aZd5vM+KR98cXefNIk78pn6co5R9RFibooERchEo30\ne56Xm0fJsBKKhxYzNCdzbxUWdVE2Nm1k7fa1rG1Yy5rta1i7fS1rGrx5Q2sDIwtGMrpgNGOKxjC6\ncHTnNKbQa+fl5iX7ZSTknKOuuY6lm5eyrG4Zy+qWsbTOW/586+eU5ZdRll9Ga6SV1kgrLe0t3jzS\n0tmOumi3hN/bl4Dc7FyiLtotgfVM2D3bQOc+c7NzvXlWLttbtjO9YjpXTLuCsw48i9zs5P28o6Gl\ngacWP8Wznz5LY2tjv15fR9uwbq9zR9sOCocUUllSyYSSCd2mypJKKkoqyM/NT9pr7gvnHLVNtazY\nuoIV9Su65vUrOr+85Ofmk5eT17ncs52X2/tjg3n9fo/Yvwqc5py7KtaeAxzjnPtO3DajgZeAUiAf\n+LJzblGCfWnEjnfi2vvvd00ffACffOL9tOvww6GwcCEzZlR1JvLhw/e8z4FyzrF151Y2NG7oNq1v\nWM+Gpq52bWMtbdG2zt+0O1zn33e0d/dYfLsjkUddFMPIzsomy7LItmyys7L7NW9ub2bbzm1s3bmV\nnKycziRfMqykc4pvFw8rTri++m/VTD9xute3aKTbF47O5bj1u3sM2OU/+fjljseyLbtPo8iW9hbW\nNazrnrC3r2FtQ1cS39C4geFDhzNu+DjGFo3tnI8d7i0XDSli045N3vva8T43xb3XjRsYmjO0W6Lv\nmfhHF3pfCoYPHc7Q7KGDHgH3rPfuaNvBZ1s+60rgW5Z1LkddlMllk5k8YjIHjjiQA0ccyOQRk5m0\nz6Q+jRoj0Qht0baEST++3RppJduyE75fvbWzsxLfpaeptYknPnmC+Yvms7RuKZdMvYQrj7iSKftO\nGVTc+qot0sbLy1/mwQ8e5PlPn6dqQhUHNx7M9BOn9/m17e51dnyZrKmvYcXWFdTU13jTNm++sn4l\nxcOKuxJ+8QQqS7u+BFQUVwTyZXLbzm3dkvbyrcs72zX1NRQMKaCypJKJpROpLKns7GNeTh472nZ0\nTs3tzd3bbbF2e9xygm3X/POa1Dt5zszOB07tkdiPds79Y9w2/wTgnPupmR2HV4c/JMG+MiqxR6Pe\nyWvxSfz9972fjR12mJfEO6bDDvNG5jC4E1xaI600tTbR1NZEU2vTLkm7Z8Le0LiBvJy8zv+0RxeO\nZnTB6G7/sY8uHM2owlEMzfZGxB3/oRu2S3t3j3W0O5J5XxNbXzjnaG5vpn5nPdt2bqN+Z33ntK1l\nD+2d29i6eCs5++d0fsHIsixvOStuOcH6ROuiLkpbpI22aFu3UU/PEVDHKHJ3iX/zjs3U76xndOFo\nL1kPH8u4onGdCbsjee9XtB/DcgZee3HOsa1lW7dE3/mZaVzfbXl7y3baIm0MyxnGsJxh5OXmefOc\nvN6Xc/J22Xb5ouVkVWZ1JvBNOzYxsXRiZ9KOT+Bl+WVpfa2LZXXL+PWiX7Pg/QVUlFRw5RFX8rVD\nvkbR0KK9+jzOOd5Z9w4PfvAgv/n4N+xfuj9zps7hwkMupCy/LNCT56IuyobGDV0JP25aUb+CVdtW\nUTqslIqSCgqHFHb77Hf8m8jNyu11faJ1bdE2aupruiXv1kgrlaWVuyTvjiMNe/s96Cklz4qPJeqb\nnHMzYu3rABd/Ap2ZfYQ3ql8ba38OHOuc29xjX+6yyy5jwoQJAJSUlDBt2rTOD9rChQsB0rLd0AD3\n37+Qzz+H5uaqWBJfSHExHHtsFYcfDtnZC5k0CWbNqiIrq/vfR6IRnnj+CdY3rmfElBFsb9nOB29+\nwM72nex7yL40tTbx+aLP2dm+k/wD8mlqa2LDhxtobm/GTXA0tTbRsKwBgKIDiygYUoDVGIVDCpn8\nhcmMLhjNzs92sk/ePpxYdSKjC0dTU11D6bBSTvvyaUmPXya2X/3jq7RH2/niCV+kLdrGwoULaY+0\n84W/+wKtkVb++sZfGT50OOfMOIfsrOyk9ze+HXVRXnrlJVojrRz1xaNobm/mjdfeoKW9hYOPOZid\n7Tt5689v0RppZeIRE2lua+bDtz6kNdLKmEPH0NzeTE11DWMKx3DmqWdy4IgDWbFoBdlZ2Snx+vxq\nR6IRmsc1M3/RfF5+5WVOKD+Bf7v03/i78X/Ha6+9NuD9r9i6gh898CNeXv4yQ/YfwiVTL2HS9kmM\nHT42pV5/fPvVP75K3Y46xk4dy462Hbz7l3dpi7Zx0BcOoi3SxgdvfkB7tJ39j9yf1kgri99eTMRF\nGD91PG3RNj577zMi0QijDh1FW6SNVe+vIjsrm+OnH09lSSVbFm9hTOEYZs6YiZkF9vo6lmtqagBY\nsGBBSib2bGAp3slz64G3gFnOucVx2zwHPOacWxA7ee5l59y4BPsKxYi9pcUbeb/9dtdUUwOHHNJ9\nFD51avcLsNTvrGfFVu9QUMfUcWho1bZVjMgfwcTSiVQUVzB86HAKcgsoGFKQcF44pDDhY0OyhyQt\nLiLSd7WNtTz4wYPMXzSfqItyxbQruPTwSxlTNKZPf7+1eSuPffwYD334EEs2L+Frh3yNOVPncOzY\nY9P66EbYpOSIHTp/7vYzun7u9h9mdjPwtnPu2VgyvwcoxDuR7vvOuVcS7CftEnsk4tW/45P4J594\nZ6EffXTXdOihQFYbq7atSpi4l29dTlu0rfNQ0MTSid2m+HrTQv3WNHCKefAUc49zjr+t+Rv3LbqP\nJxY/wfTy6VxxxBWcecCZu5xw19LewnOfPsdDHzzEKyte4bT9T+OSqZdw2qTT+vSlXjEPXsom9r0l\n1RO7c15NPD6JL3o/wsiKOqZ8oZaKQ2rZt7KWvLJa6ts2UttUS21jbed8045NjCkcs0vS7kjkfa0T\n6h9f8BTz4Cnmu2psbew84e7Tuk+5ZOolXHHEFdQ11/Hg+w/yxOInmDpqKnMOm8P5B59P8bD+XZNZ\nMQ+eEnuAtrds58NVK3nxb2tYtKyWpWtrWbWlluyijRSMqiV7eC07c2ppjGyheGgxowpHMapgFKMK\nRzEyf2S39qiCUYwsGMl+Rfsl9SctIhIey+qWcd+i+1jw/gL2yduHS6ZewuzDZlNeXJ7srkk/KLHv\nJR1nY66sX8mqbatYuS02r1/J0tpVrN6+ktZIG9RXUJo9jv2Gj2bS6FEcWjmSSWO6J+yy/DIlaxER\nGRAl9j7a2b6zM1F3S9yx+ZrtaygZVkJFcQX7FVQQ2VLOpk8rWPZ2OYXtFcw4vpxzT9uHqiojLwWv\nyaHDZcFTzIOnmAdPMQ9eql55LmnqdtSxZPMSFm9ezOJNi1lSt4TFmxazrmEdY4ePpaK4gvLiciqK\nK5hePp05JXMYP7ycbavH8+qLebzwELzyHpxwAlx8Bpz+D97V2kRERFJZWo/Yoy7K6m2rWbx5sZfE\n4xJ4S6SFg8oOYkrZFKaUTfGW951CZUllt8Pj27fDH/4Azz8Pv/+9d3/wM86A00+HqirIT+2rGoqI\nSEhlxKH4xz9+vNsofGndUkqGlXQl7rIpTNnXWx5TOCbhWeTOwccfe4n8hRfgnXfg+OO7kvkBB+jO\nZSIiknwZkdhnPjKz2+j7oLKDGD60bxdDb2+HBx+E227zfl/ekchPOim9b4bSk+pgwVPMg6eYB08x\nD15G1Nifvujpfv9NJAKPPgo33wxjxsB998H06RqVi4hIOKXViL0/fY1G4amnYN487w5nt94Kp5yi\nhC4iIukhI0bsfeEcPPss3HgjZGfDnXd6h9yV0EVEJBNkJbsDe4tz8OKLcNxxcMMNcNNN3olxZ5yR\nWUk9/i5BEgzFPHiKefAU8/QRihH7woXeCH3TJq+WfsEFkBWarywiIiJ9l9Y19r/8xUvoNTVeLX32\nbMgJxVcVERHJdAOtsafluPbdd71D7LNmedOSJXDppUrqIiIiaZXYP/gAzj0XvvIVOPNMWLYMvv51\nyNV9VjqpDhY8xTx4innwFPP0kVaJ/dRT4Utfgs8+g29/27v8q4iIiHRJqxp7Q4OjsDDZPREREfFf\nRlxSNl36KiIiMlgZdfKc9E51sOAp5sFTzIOnmKcPJXYREZEQ0aF4ERGRFKRD8SIiIqLEHjaqgwVP\nMQ+eYh48xTx9KLGLiIiEiGrsIiIiKUg1dhEREVFiDxvVwYKnmAdPMQ+eYp4+lNhFRERCRDV2ERGR\nFKQau4iIiCixh43qYMFTzIOnmAdPMU8fSuwiIiIhohq7iIhIClKNXURERJTYw0Z1sOAp5sFTzIOn\nmKcPJXYREZEQUY1dREQkBanGLiIiIkrsYaM6WPAU8+Ap5sFTzNOHEruIiEiIqMYuIiKSglRjFxER\nESX2sFEdLHiKefAU8+Ap5ulDiV1ERCREVGMXERFJQaqxi4iIiBJ72KgOFjzFPHiKefAU8/The2I3\nsxlmtsTMlpnZD3rZ5kIz+9jMPjSzh/zuk4iISFj5WmM3syxgGXAKsA54G7jIObckbptJwG+Ak5xz\n282szDm3OcG+VGMXEZGMkao19mOAT51zK51zbcCjwMwe23wD+L/Oue0AiZK6iIiI9I3fiX0ssDqu\nvSa2Lt6BwGQz+5OZ/cXMTvO5T6GmOljwFPPgKebBU8zTR47P+090CKHn8fQcYBLwJaAceMPMDukY\nwcebO3cuEyZMAKCkpIRp06ZRVVUFdH3oMr3dIVX6o7bafrSrq6tTqj+Z0K6urk6p/oSx3bFcU1PD\nYPhdYz8OuMk5NyPWvg5wzrnb47a5C/irc+6BWPsPwA+cc+/22Jdq7CIikjFStcb+NjDJzCrMbAhw\nEfBMj22eBk4GMLMy4ABguc/9EhERCSVfE7tzLgJcA7wEfAw86pxbbGY3m9lZsW1eBOrM7GPgFeB7\nzrmtfvYrzOIP6UgwFPPgKebBU8zTh981dpxzvwcm91g3r0f7WuBav/siIiISdrpWvIiISApK1Rq7\niIiIBEiJPWRUBwueYh48xTx4inn66FNiN7MnzezM2CViRUREJEX1qcZuZl8GLgeOAx4H7o+/3nsQ\nVGMXEZFMMtAae79OnjOzYmAW8K94l4q9B3godh14Xymxi4hIJvH95DkzGwHMBb4OLAJ+BhwJvNzf\nJxX/qA4WPMU8eIp58BTz9NGn37Gb2VPAQcCDwNnOufWxh35jZu/41TkRERHpn77W2E92zr0aQH92\n1wcdihcRkYzh96H4KWZWEvdkpWb2rf4+mYiIiPirr4n9G865+o5G7Fru3/CnSzIYqoMFTzEPnmIe\nPMU8ffQ1sWeZWefhADPLBob40yUREREZqL7W2O8EJgB3Aw74JrA6dvOWQKjGLiIimcTX37HHrjj3\n98ApgOHdhvXe2G1ZA6HELiIimcTXk+ecc1Hn3F3OufOdc191zv0qyKQufac6WPAU8+Ap5sFTzNNH\nX3/HfgDwY+BgYFjHeufcRJ/6JSIiIgPQ10PxfwLmAT8Fzsa7bnyWc+6H/navWx90KF5ERDKG379j\nz3POvYL3RWClc+4m4Mz+PpmIiIj4q6+JfWfsBLpPzewaMzsXKPSxXzJAqoMFTzEPnmIePMU8ffQ1\nsX8XyAe+AxwFzAEu86tTIiIiMjB7rLHHLkZzu3Pue8F0qdd+qMYuIiIZw7cae+xnbScMqFciIiIS\nqL4eil9kZs+Y2SVmdl7H5GvPZEBUBwueYh48xTx4inn66NPv2PF+u14HnBy3zgFP7fUeiYiIyID1\n6XfsqUA1dhERySQDrbH39cpzv8YboXfjnLuiv08oIiIi/ulrjf1Z4LnY9AowHGj0q1MycKqDBU8x\nD55iHjzFPH30acTunHsyvm1mjwB/8qVHIiIiMmADqrGb2WTgOefcpL3fpV6fUzV2ERHJGH7X2Bvo\nXmPfAPyaoG9fAAANi0lEQVSgv08mIiIi/urr/diLnHPD46YDex6el9SgOljwFPPgKebBU8zTR58S\nu5mda2bFce0SMzvHv26JiIjIQPT1fuzVzrlpPdYtcs4d4VvPdu2DauwiIpIx/L4fe6Lt+nrVOhER\nEQlIXxP7O2b2EzPb38wmmtlPgXf97JgMjOpgwVPMg6eYB08xTx99Tez/ALQCvwEeA5qBb/vVKRER\nERkYXSteREQkBflaYzezl82sJK5damYv9vfJRERExF99PRRf5pyr72g457YCI/3pkgyG6mDBU8yD\np5gHTzFPH31N7FEzK+9omNkEEtztTURERJKrr79jnwH8N/BabNWXgKucc4EdjleNXUREMslAa+x9\nPnnOzEYCVwHVwDBgo3Pu9f4+4UApsYuISCbx++S5r+Pdh/3a2PQgcFN/n0z8pzpY8BTz4CnmwVPM\n00dfa+z/CBwNrHTOnQQcAdTv/k9EREQkaH2tsb/tnDvazKqBY51zLWb2sXPuEP+72NkHHYoXEZGM\n4ev92IE1sd+xPw28bGZbgZX9fTIRERHxV1/vx36uc67eOXcTcCMwH9BtW1OQ6mDBU8yDp5gHTzFP\nH32tsXdyzr3mnHvGOdfal+3NbIaZLTGzZWb2g91sd76ZRc3syP72SURERDy+XivezLKAZcApwDrg\nbeAi59ySHtsVAs8BucA1zrn3EuxLNXYREckYft+PfaCOAT51zq10zrUBjwIzE2x3K3A70OJzf0RE\nRELN78Q+Flgd114TW9fJzKYB45xzz/vcl4ygOljwFPPgKebBU8zTR1/Pih+oRIcQOo+nm5kBPwUu\n28PfADB37lwmTJgAQElJCdOmTaOqqgro+tBlertDqvRHbbX9aFdXV6dUfzKhXV1dnVL9CWO7Y7mm\npobB8LvGfhxwk3NuRqx9HeCcc7fH2sOBz4BGvIQ+GqgDvtKzzq4au4iIZBLfrxU/EGaWDSzFO3lu\nPfAWMMs5t7iX7f8I/LNzblGCx5TYRUQkY6TkyXPOuQhwDfAS8DHwqHNusZndbGZnJfoTdnMoXvYs\n/pCOBEMxD55iHjzFPH34XWPHOfd7YHKPdfN62fZkv/sjIiISZr4eit+bdCheREQySUoeihcREZFg\nKbGHjOpgwVPMg6eYB08xTx9K7CIiIiGiGruIiEgKUo1dRERElNjDRnWw4CnmwVPMg6eYpw8ldhER\nkRBRjV1ERCQFqcYuIiIiSuxhozpY8BTz4CnmwVPM04cSu4iISIioxi4iIpKCVGMXERERJfawUR0s\neIp58BTz4Cnm6UOJXUREJERUYxcREUlBqrGLiIiIEnvYqA4WPMU8eIp58BTz9KHELiIiEiKqsYuI\niKQg1dhFREREiT1sVAcLnmIePMU8eIp5+lBiFxERCRHV2EVERFKQauwiIiKixB42qoMFTzEPnmIe\nPMU8fSixi4iIhIhq7CIiIilINXYRERFRYg8b1cGCp5gHTzEPnmKePpTYRUREQkQ1dhERkRSkGruI\niIgosYeN6mDBU8yDp5gHTzFPH0rsIiIiIaIau4iISApSjV1ERESU2MNGdbDgKebBU8yDp5inDyV2\nERGREFGNXUREJAWpxi4iIiJK7GGjOljwFPPgKebBU8zThxK7iIhIiKjGLiIikoJUYxcRERH/E7uZ\nzTCzJWa2zMx+kODxfzKzj82s2sxeNrPxfvcpzFQHC55iHjzFPHiKefrwNbGbWRbwS+A04BBglpkd\n1GOz94CjnHPTgCeBO/3sk4iISJj5WmM3s+OAec6502Pt6wDnnLu9l+2nAb9wzk1P8Jhq7CIikjFS\ntcY+Flgd114TW9ebK4EXfO2RiIhIiPmd2BN900g47DazOcBR6FD8oKgOFjzFPHiKefAU8/SR4/P+\n1wDlce1xwLqeG5nZl4HrgS8559p629ncuXOZMGECACUlJUybNo2qqiqg60OX6e0OqdIftdX2o11d\nXZ1S/cmEdnV1dUr1J4ztjuWamhoGw+8aezawFDgFWA+8Bcxyzi2O2+YI4HHgNOfc57vZl2rsIiKS\nMVKyxu6ciwDXAC8BHwOPOucWm9nNZnZWbLM7gALgcTNbZGZP+9knERGRMPP9d+zOud875yY75w5w\nzv1HbN0859yzseX/45wb45w70jl3hHPuHL/7FGbxh3QkGIp58BTz4Cnm6UNXnhMREQkRXSteREQk\nBaVkjV1ERESCpcQeMqqDBU8xD55iHjzFPH0osYuIiISIauwiIiIpSDV2ERERUWIPG9XBgqeYB08x\nD55inj6U2EVEREJENXYREZEUpBq7iIiIKLGHjepgwVPMg6eYB08xTx9K7CIiIiGiGruIiEgKUo1d\nRERElNjDRnWw4CnmwVPMg6eYpw8ldhERkRBRjV1ERCQFqcYuIiIiSuxhozpY8BTz4CnmwVPM04cS\nu4iISIioxi4iIpKCVGMXERERJfawUR0seIp58BTz4Cnm6UOJXUREJERUYxcREUlBqrGLiIiIEnvY\nqA4WPMU8eIp58BTz9KHELiIiEiKqsYuIiKQg1dhFREREiT1sVAcLnmIePMU8eIp5+lBiFxERCRHV\n2EVERFKQauwiIiKixB42qoMFTzEPnmIePMU8fSixi4iIhIhq7CIiIilINXYRERFRYg8b1cGCp5gH\nTzEPnmKePpTYRUREQkQ1dhERkRSkGruIiIgosYeN6mDBU8yDp5gHTzFPH0rsIiIiIaIau4iISApS\njV1ERET8T+xmNsPMlpjZMjP7QYLHh5jZo2b2qZn91czK/e5TmKkOFjzFPHiKefAU8/Tha2I3syzg\nl8BpwCHALDM7qMdmVwJbnHMHAP8F3OFnn8Kuuro62V3IOIp58BTz4Cnm6cPvEfsxwKfOuZXOuTbg\nUWBmj21mAgtiy08Ap/jcp1Crr69PdhcyjmIePMU8eIp5+vA7sY8FVse118TWJdzGORcB6s1sH5/7\nJSIiEkp+J/ZEZ/P1PLW95zaWYBvpo5qammR3IeMo5sFTzIOnmKcPX3/uZmbHATc552bE2tcBzjl3\ne9w2L8S2edPMsoH1zrmRCfalZC8iIhllID93y/GjI3HeBiaZWQWwHrgImNVjm98BlwFvAhcAryba\n0UBenIiISKbxNbE75yJmdg3wEt5h//nOucVmdjPwtnPuWWA+8KCZfQrU4SV/ERERGYC0ufKciIiI\n7JmuPBciZlZjZu+b2SIzeyvZ/QkjM5tvZrVm9kHculIze8nMlprZi2ZWnMw+hk0vMZ9nZmvM7L3Y\nNCOZfQwbMxtnZq+a2Sdm9qGZfSe2Xp91nySI+T/E1vf7s64Re4iY2XLgKOfc1mT3JazM7ASgEXjA\nOTc1tu52oM45d0fs6oqlzrnrktnPMOkl5vOABufcT5LauZAys9HAaOdctZkVAu/iXXPkcvRZ98Vu\nYv41+vlZ14g9XAy9p75yzv0J6PnFKf4iSwuAcwLtVMj1EnNI/HNa2Quccxucc9Wx5UZgMTAOfdZ9\n00vMO6770q/PupJAuDjgRTN728y+kezOZJCRzrla8P5xAvsmuT+Z4ttmVm1m9+qQsH/MbAIwDfgb\nMEqfdf/FxfzN2Kp+fdaV2MPleOfcF4Az8D4IJyS7QyI++X/A/s65acAGQIfkfRA7JPwE8I+xUaRq\ntz5LEPN+f9aV2EMk9g0a59wm4Ld41+oX/9Wa2SjorJNtTHJ/Qs85t8l1nSB0D3B0MvsTRmaWg5dg\nHnTO/W9stT7rPkoU84F81pXYQ8LM8mPf9DCzAuBU4KPk9iq0jO41r2eAubHly4D/7fkHMmjdYh5L\nKh3OQ591P9wHfOKc+1ncOn3W/bVLzAfyWddZ8SFhZpV4o3SHd+Gh/3HO/UdyexU+ZvYwUAWMAGqB\necDTwOPAeGAVcIFzTrfC2kt6iflJeDXIKFAD/H1H7VcGz8z+Dngd+BDv/xQH3AC8BTyGPut73W5i\nPpt+ftaV2EVEREJEh+JFRERCRIldREQkRJTYRUREQkSJXUREJESU2EVEREJEiV1ERCRElNhFZNDM\n7EQz+12y+yEiSuwisvfoohgiKUCJXSSDmNnFZvammb1nZneZWZaZNZjZT8zsIzN72cxGxLadZmZ/\njd1V6smOu0qZ2f6x7arN7J3YVQ8BiszscTNbbGYPJu1FimQ4JXaRDGFmBwFfw7sL4JF4l6i8GMgH\n3nLOHYp3Sct5sT9ZAHw/dlepj+LW/w/wi9j644H1sfXTgO8ABwP7m9nx/r8qEekpJ9kdEJHAnAIc\nCbxtZgYMw7v2ehTv+t8ADwFPmtlwoNg596fY+gXAY7EbDY11zj0D4JxrBfB2x1vOufWxdjUwAfhL\nAK9LROIosYtkDgMWOOf+tdtKsxt7bOfitk+0j960xC1H0P8vIkmhQ/EimeMV4Hwz2xfAzErNrBzI\nBs6PbXMx8Cfn3HZgS+yOUwCXAK855xqA1WY2M7aPIWaWF+irEJHd0jdqkQzhnFtsZv8GvGRmWUAr\ncA3QBBwTG7nX4tXhwbvf9q9iiXs5cHls/SXAf5vZLbF9XJDo6fx7JSKyO7ptq0iGM7MG51xRsvsh\nInuHDsWLiL7di4SIRuwiIiIhohG7iIhIiCixi4iIhIgSu4iISIgosYuIiISIEruIiEiIKLGLiIiE\nyP8H39S5XeHsry0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f09d3623390>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "acc = pd.DataFrame({'epoch': [ i + 1 for i in history.epoch ],\n",
    "                    'training': history.history['acc'],\n",
    "                    'validation': history.history['val_acc']})\n",
    "ax = acc.iloc[:,:].plot(x='epoch', figsize={5,8}, grid=True)\n",
    "ax.set_ylabel(\"accuracy\")\n",
    "ax.set_ylim([0.0,1.0]);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Print best validation accuracy and epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Maximum accuracy at epoch 16 = 0.8083\n"
     ]
    }
   ],
   "source": [
    "max_val_acc, idx = max((val, idx) for (idx, val) in enumerate(history.history['val_acc']))\n",
    "print('Maximum accuracy at epoch', '{:d}'.format(idx+1), '=', '{:.4f}'.format(max_val_acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Evaluate the model with best validation accuracy on the test partition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss = 0.4102, accuracy = 0.8110\n"
     ]
    }
   ],
   "source": [
    "model.load_weights(MODEL_WEIGHTS_FILE)\n",
    "loss, accuracy = model.evaluate([Q1_test, Q2_test], y_test, verbose=0)\n",
    "print('loss = {0:.4f}, accuracy = {1:.4f}'.format(loss, accuracy))"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}