vanilla_mobilenet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "from keras import optimizers\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.callbacks import ReduceLROnPlateau, EarlyStopping\n",
    "from keras.applications.mobilenet import preprocess_input\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "from mobilenet import get_mobilenet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = '/home/dongsung/study/machine_learning/distillation/data'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 0 images belonging to 0 classes.\n",
      "Found 0 images belonging to 0 classes.\n"
     ]
    }
   ],
   "source": [
    "data_generator = ImageDataGenerator(\n",
    "    data_format='channels_last',\n",
    "    preprocessing_function=preprocess_input\n",
    ")\n",
    "\n",
    "train_generator = data_generator.flow_from_directory(\n",
    "    data_dir + 'train', \n",
    "    target_size=(224, 224),\n",
    "    batch_size=64\n",
    ")\n",
    "\n",
    "val_generator = data_generator.flow_from_directory(\n",
    "    data_dir + 'val', shuffle=False,\n",
    "    target_size=(224, 224),\n",
    "    batch_size=64\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 224, 224, 3)       0         \n",
      "_________________________________________________________________\n",
      "conv1_pad (ZeroPadding2D)    (None, 225, 225, 3)       0         \n",
      "_________________________________________________________________\n",
      "conv1 (Conv2D)               (None, 112, 112, 8)       216       \n",
      "_________________________________________________________________\n",
      "conv1_bn (BatchNormalization (None, 112, 112, 8)       32        \n",
      "_________________________________________________________________\n",
      "conv1_relu (ReLU)            (None, 112, 112, 8)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_1 (DepthwiseConv2D)  (None, 112, 112, 8)       72        \n",
      "_________________________________________________________________\n",
      "conv_dw_1_bn (BatchNormaliza (None, 112, 112, 8)       32        \n",
      "_________________________________________________________________\n",
      "conv_dw_1_relu (ReLU)        (None, 112, 112, 8)       0         \n",
      "_________________________________________________________________\n",
      "conv_pw_1 (Conv2D)           (None, 112, 112, 16)      128       \n",
      "_________________________________________________________________\n",
      "conv_pw_1_bn (BatchNormaliza (None, 112, 112, 16)      64        \n",
      "_________________________________________________________________\n",
      "conv_pw_1_relu (ReLU)        (None, 112, 112, 16)      0         \n",
      "_________________________________________________________________\n",
      "conv_pad_2 (ZeroPadding2D)   (None, 113, 113, 16)      0         \n",
      "_________________________________________________________________\n",
      "conv_dw_2 (DepthwiseConv2D)  (None, 56, 56, 16)        144       \n",
      "_________________________________________________________________\n",
      "conv_dw_2_bn (BatchNormaliza (None, 56, 56, 16)        64        \n",
      "_________________________________________________________________\n",
      "conv_dw_2_relu (ReLU)        (None, 56, 56, 16)        0         \n",
      "_________________________________________________________________\n",
      "conv_pw_2 (Conv2D)           (None, 56, 56, 32)        512       \n",
      "_________________________________________________________________\n",
      "conv_pw_2_bn (BatchNormaliza (None, 56, 56, 32)        128       \n",
      "_________________________________________________________________\n",
      "conv_pw_2_relu (ReLU)        (None, 56, 56, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv_dw_3 (DepthwiseConv2D)  (None, 56, 56, 32)        288       \n",
      "_________________________________________________________________\n",
      "conv_dw_3_bn (BatchNormaliza (None, 56, 56, 32)        128       \n",
      "_________________________________________________________________\n",
      "conv_dw_3_relu (ReLU)        (None, 56, 56, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv_pw_3 (Conv2D)           (None, 56, 56, 32)        1024      \n",
      "_________________________________________________________________\n",
      "conv_pw_3_bn (BatchNormaliza (None, 56, 56, 32)        128       \n",
      "_________________________________________________________________\n",
      "conv_pw_3_relu (ReLU)        (None, 56, 56, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv_pad_4 (ZeroPadding2D)   (None, 57, 57, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv_dw_4 (DepthwiseConv2D)  (None, 28, 28, 32)        288       \n",
      "_________________________________________________________________\n",
      "conv_dw_4_bn (BatchNormaliza (None, 28, 28, 32)        128       \n",
      "_________________________________________________________________\n",
      "conv_dw_4_relu (ReLU)        (None, 28, 28, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv_pw_4 (Conv2D)           (None, 28, 28, 64)        2048      \n",
      "_________________________________________________________________\n",
      "conv_pw_4_bn (BatchNormaliza (None, 28, 28, 64)        256       \n",
      "_________________________________________________________________\n",
      "conv_pw_4_relu (ReLU)        (None, 28, 28, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv_dw_5 (DepthwiseConv2D)  (None, 28, 28, 64)        576       \n",
      "_________________________________________________________________\n",
      "conv_dw_5_bn (BatchNormaliza (None, 28, 28, 64)        256       \n",
      "_________________________________________________________________\n",
      "conv_dw_5_relu (ReLU)        (None, 28, 28, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv_pw_5 (Conv2D)           (None, 28, 28, 64)        4096      \n",
      "_________________________________________________________________\n",
      "conv_pw_5_bn (BatchNormaliza (None, 28, 28, 64)        256       \n",
      "_________________________________________________________________\n",
      "conv_pw_5_relu (ReLU)        (None, 28, 28, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv_pad_6 (ZeroPadding2D)   (None, 29, 29, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv_dw_6 (DepthwiseConv2D)  (None, 14, 14, 64)        576       \n",
      "_________________________________________________________________\n",
      "conv_dw_6_bn (BatchNormaliza (None, 14, 14, 64)        256       \n",
      "_________________________________________________________________\n",
      "conv_dw_6_relu (ReLU)        (None, 14, 14, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv_pw_6 (Conv2D)           (None, 14, 14, 128)       8192      \n",
      "_________________________________________________________________\n",
      "conv_pw_6_bn (BatchNormaliza (None, 14, 14, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_pw_6_relu (ReLU)        (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_7 (DepthwiseConv2D)  (None, 14, 14, 128)       1152      \n",
      "_________________________________________________________________\n",
      "conv_dw_7_bn (BatchNormaliza (None, 14, 14, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_dw_7_relu (ReLU)        (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_pw_7 (Conv2D)           (None, 14, 14, 128)       16384     \n",
      "_________________________________________________________________\n",
      "conv_pw_7_bn (BatchNormaliza (None, 14, 14, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_pw_7_relu (ReLU)        (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_8 (DepthwiseConv2D)  (None, 14, 14, 128)       1152      \n",
      "_________________________________________________________________\n",
      "conv_dw_8_bn (BatchNormaliza (None, 14, 14, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_dw_8_relu (ReLU)        (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_pw_8 (Conv2D)           (None, 14, 14, 128)       16384     \n",
      "_________________________________________________________________\n",
      "conv_pw_8_bn (BatchNormaliza (None, 14, 14, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_pw_8_relu (ReLU)        (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_9 (DepthwiseConv2D)  (None, 14, 14, 128)       1152      \n",
      "_________________________________________________________________\n",
      "conv_dw_9_bn (BatchNormaliza (None, 14, 14, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_dw_9_relu (ReLU)        (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_pw_9 (Conv2D)           (None, 14, 14, 128)       16384     \n",
      "_________________________________________________________________\n",
      "conv_pw_9_bn (BatchNormaliza (None, 14, 14, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_pw_9_relu (ReLU)        (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_10 (DepthwiseConv2D) (None, 14, 14, 128)       1152      \n",
      "_________________________________________________________________\n",
      "conv_dw_10_bn (BatchNormaliz (None, 14, 14, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_dw_10_relu (ReLU)       (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_pw_10 (Conv2D)          (None, 14, 14, 128)       16384     \n",
      "_________________________________________________________________\n",
      "conv_pw_10_bn (BatchNormaliz (None, 14, 14, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_pw_10_relu (ReLU)       (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_11 (DepthwiseConv2D) (None, 14, 14, 128)       1152      \n",
      "_________________________________________________________________\n",
      "conv_dw_11_bn (BatchNormaliz (None, 14, 14, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_dw_11_relu (ReLU)       (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_pw_11 (Conv2D)          (None, 14, 14, 128)       16384     \n",
      "_________________________________________________________________\n",
      "conv_pw_11_bn (BatchNormaliz (None, 14, 14, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_pw_11_relu (ReLU)       (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_pad_12 (ZeroPadding2D)  (None, 15, 15, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_12 (DepthwiseConv2D) (None, 7, 7, 128)         1152      \n",
      "_________________________________________________________________\n",
      "conv_dw_12_bn (BatchNormaliz (None, 7, 7, 128)         512       \n",
      "_________________________________________________________________\n",
      "conv_dw_12_relu (ReLU)       (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv_pw_12 (Conv2D)          (None, 7, 7, 256)         32768     \n",
      "_________________________________________________________________\n",
      "conv_pw_12_bn (BatchNormaliz (None, 7, 7, 256)         1024      \n",
      "_________________________________________________________________\n",
      "conv_pw_12_relu (ReLU)       (None, 7, 7, 256)         0         \n",
      "_________________________________________________________________\n",
      "conv_dw_13 (DepthwiseConv2D) (None, 7, 7, 256)         2304      \n",
      "_________________________________________________________________\n",
      "conv_dw_13_bn (BatchNormaliz (None, 7, 7, 256)         1024      \n",
      "_________________________________________________________________\n",
      "conv_dw_13_relu (ReLU)       (None, 7, 7, 256)         0         \n",
      "_________________________________________________________________\n",
      "conv_pw_13 (Conv2D)          (None, 7, 7, 256)         65536     \n",
      "_________________________________________________________________\n",
      "conv_pw_13_bn (BatchNormaliz (None, 7, 7, 256)         1024      \n",
      "_________________________________________________________________\n",
      "conv_pw_13_relu (ReLU)       (None, 7, 7, 256)         0         \n",
      "_________________________________________________________________\n",
      "global_average_pooling2d_1 ( (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 256)               65792     \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 256)               0         \n",
      "=================================================================\n",
      "Total params: 284,336\n",
      "Trainable params: 65,792\n",
      "Non-trainable params: 218,544\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = get_mobilenet(224, alpha=0.25, weight_decay=1e-5, dropout=0.1)\n",
    "model.count_params()\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://github.com/fchollet/deep-learning-models/releases/download/v0.6/mobilenet_2_5_224_tf.h5\n",
      "2113536/2108140 [==============================] - 1s 0us/step\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         (None, 224, 224, 3)       0         \n",
      "_________________________________________________________________\n",
      "conv1_pad (ZeroPadding2D)    (None, 225, 225, 3)       0         \n",
      "_________________________________________________________________\n",
      "conv1 (Conv2D)               (None, 112, 112, 8)       216       \n",
      "_________________________________________________________________\n",
      "conv1_bn (BatchNormalization (None, 112, 112, 8)       32        \n",
      "_________________________________________________________________\n",
      "conv1_relu (ReLU)            (None, 112, 112, 8)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_1 (DepthwiseConv2D)  (None, 112, 112, 8)       72        \n",
      "_________________________________________________________________\n",
      "conv_dw_1_bn (BatchNormaliza (None, 112, 112, 8)       32        \n",
      "_________________________________________________________________\n",
      "conv_dw_1_relu (ReLU)        (None, 112, 112, 8)       0         \n",
      "_________________________________________________________________\n",
      "conv_pw_1 (Conv2D)           (None, 112, 112, 16)      128       \n",
      "_________________________________________________________________\n",
      "conv_pw_1_bn (BatchNormaliza (None, 112, 112, 16)      64        \n",
      "_________________________________________________________________\n",
      "conv_pw_1_relu (ReLU)        (None, 112, 112, 16)      0         \n",
      "_________________________________________________________________\n",
      "conv_pad_2 (ZeroPadding2D)   (None, 113, 113, 16)      0         \n",
      "_________________________________________________________________\n",
      "conv_dw_2 (DepthwiseConv2D)  (None, 56, 56, 16)        144       \n",
      "_________________________________________________________________\n",
      "conv_dw_2_bn (BatchNormaliza (None, 56, 56, 16)        64        \n",
      "_________________________________________________________________\n",
      "conv_dw_2_relu (ReLU)        (None, 56, 56, 16)        0         \n",
      "_________________________________________________________________\n",
      "conv_pw_2 (Conv2D)           (None, 56, 56, 32)        512       \n",
      "_________________________________________________________________\n",
      "conv_pw_2_bn (BatchNormaliza (None, 56, 56, 32)        128       \n",
      "_________________________________________________________________\n",
      "conv_pw_2_relu (ReLU)        (None, 56, 56, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv_dw_3 (DepthwiseConv2D)  (None, 56, 56, 32)        288       \n",
      "_________________________________________________________________\n",
      "conv_dw_3_bn (BatchNormaliza (None, 56, 56, 32)        128       \n",
      "_________________________________________________________________\n",
      "conv_dw_3_relu (ReLU)        (None, 56, 56, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv_pw_3 (Conv2D)           (None, 56, 56, 32)        1024      \n",
      "_________________________________________________________________\n",
      "conv_pw_3_bn (BatchNormaliza (None, 56, 56, 32)        128       \n",
      "_________________________________________________________________\n",
      "conv_pw_3_relu (ReLU)        (None, 56, 56, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv_pad_4 (ZeroPadding2D)   (None, 57, 57, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv_dw_4 (DepthwiseConv2D)  (None, 28, 28, 32)        288       \n",
      "_________________________________________________________________\n",
      "conv_dw_4_bn (BatchNormaliza (None, 28, 28, 32)        128       \n",
      "_________________________________________________________________\n",
      "conv_dw_4_relu (ReLU)        (None, 28, 28, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv_pw_4 (Conv2D)           (None, 28, 28, 64)        2048      \n",
      "_________________________________________________________________\n",
      "conv_pw_4_bn (BatchNormaliza (None, 28, 28, 64)        256       \n",
      "_________________________________________________________________\n",
      "conv_pw_4_relu (ReLU)        (None, 28, 28, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv_dw_5 (DepthwiseConv2D)  (None, 28, 28, 64)        576       \n",
      "_________________________________________________________________\n",
      "conv_dw_5_bn (BatchNormaliza (None, 28, 28, 64)        256       \n",
      "_________________________________________________________________\n",
      "conv_dw_5_relu (ReLU)        (None, 28, 28, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv_pw_5 (Conv2D)           (None, 28, 28, 64)        4096      \n",
      "_________________________________________________________________\n",
      "conv_pw_5_bn (BatchNormaliza (None, 28, 28, 64)        256       \n",
      "_________________________________________________________________\n",
      "conv_pw_5_relu (ReLU)        (None, 28, 28, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv_pad_6 (ZeroPadding2D)   (None, 29, 29, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv_dw_6 (DepthwiseConv2D)  (None, 14, 14, 64)        576       \n",
      "_________________________________________________________________\n",
      "conv_dw_6_bn (BatchNormaliza (None, 14, 14, 64)        256       \n",
      "_________________________________________________________________\n",
      "conv_dw_6_relu (ReLU)        (None, 14, 14, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv_pw_6 (Conv2D)           (None, 14, 14, 128)       8192      \n",
      "_________________________________________________________________\n",
      "conv_pw_6_bn (BatchNormaliza (None, 14, 14, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_pw_6_relu (ReLU)        (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_7 (DepthwiseConv2D)  (None, 14, 14, 128)       1152      \n",
      "_________________________________________________________________\n",
      "conv_dw_7_bn (BatchNormaliza (None, 14, 14, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_dw_7_relu (ReLU)        (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_pw_7 (Conv2D)           (None, 14, 14, 128)       16384     \n",
      "_________________________________________________________________\n",
      "conv_pw_7_bn (BatchNormaliza (None, 14, 14, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_pw_7_relu (ReLU)        (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_8 (DepthwiseConv2D)  (None, 14, 14, 128)       1152      \n",
      "_________________________________________________________________\n",
      "conv_dw_8_bn (BatchNormaliza (None, 14, 14, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_dw_8_relu (ReLU)        (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_pw_8 (Conv2D)           (None, 14, 14, 128)       16384     \n",
      "_________________________________________________________________\n",
      "conv_pw_8_bn (BatchNormaliza (None, 14, 14, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_pw_8_relu (ReLU)        (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_9 (DepthwiseConv2D)  (None, 14, 14, 128)       1152      \n",
      "_________________________________________________________________\n",
      "conv_dw_9_bn (BatchNormaliza (None, 14, 14, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_dw_9_relu (ReLU)        (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_pw_9 (Conv2D)           (None, 14, 14, 128)       16384     \n",
      "_________________________________________________________________\n",
      "conv_pw_9_bn (BatchNormaliza (None, 14, 14, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_pw_9_relu (ReLU)        (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_10 (DepthwiseConv2D) (None, 14, 14, 128)       1152      \n",
      "_________________________________________________________________\n",
      "conv_dw_10_bn (BatchNormaliz (None, 14, 14, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_dw_10_relu (ReLU)       (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_pw_10 (Conv2D)          (None, 14, 14, 128)       16384     \n",
      "_________________________________________________________________\n",
      "conv_pw_10_bn (BatchNormaliz (None, 14, 14, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_pw_10_relu (ReLU)       (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_11 (DepthwiseConv2D) (None, 14, 14, 128)       1152      \n",
      "_________________________________________________________________\n",
      "conv_dw_11_bn (BatchNormaliz (None, 14, 14, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_dw_11_relu (ReLU)       (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_pw_11 (Conv2D)          (None, 14, 14, 128)       16384     \n",
      "_________________________________________________________________\n",
      "conv_pw_11_bn (BatchNormaliz (None, 14, 14, 128)       512       \n",
      "_________________________________________________________________\n",
      "conv_pw_11_relu (ReLU)       (None, 14, 14, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_pad_12 (ZeroPadding2D)  (None, 15, 15, 128)       0         \n",
      "_________________________________________________________________\n",
      "conv_dw_12 (DepthwiseConv2D) (None, 7, 7, 128)         1152      \n",
      "_________________________________________________________________\n",
      "conv_dw_12_bn (BatchNormaliz (None, 7, 7, 128)         512       \n",
      "_________________________________________________________________\n",
      "conv_dw_12_relu (ReLU)       (None, 7, 7, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv_pw_12 (Conv2D)          (None, 7, 7, 256)         32768     \n",
      "_________________________________________________________________\n",
      "conv_pw_12_bn (BatchNormaliz (None, 7, 7, 256)         1024      \n",
      "_________________________________________________________________\n",
      "conv_pw_12_relu (ReLU)       (None, 7, 7, 256)         0         \n",
      "_________________________________________________________________\n",
      "conv_dw_13 (DepthwiseConv2D) (None, 7, 7, 256)         2304      \n",
      "_________________________________________________________________\n",
      "conv_dw_13_bn (BatchNormaliz (None, 7, 7, 256)         1024      \n",
      "_________________________________________________________________\n",
      "conv_dw_13_relu (ReLU)       (None, 7, 7, 256)         0         \n",
      "_________________________________________________________________\n",
      "conv_pw_13 (Conv2D)          (None, 7, 7, 256)         65536     \n",
      "_________________________________________________________________\n",
      "conv_pw_13_bn (BatchNormaliz (None, 7, 7, 256)         1024      \n",
      "_________________________________________________________________\n",
      "conv_pw_13_relu (ReLU)       (None, 7, 7, 256)         0         \n",
      "_________________________________________________________________\n",
      "global_average_pooling2d_2 ( (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "reshape_1 (Reshape)          (None, 1, 1, 256)         0         \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 1, 1, 256)         0         \n",
      "_________________________________________________________________\n",
      "conv_preds (Conv2D)          (None, 1, 1, 1000)        257000    \n",
      "_________________________________________________________________\n",
      "act_softmax (Activation)     (None, 1, 1, 1000)        0         \n",
      "_________________________________________________________________\n",
      "reshape_2 (Reshape)          (None, 1000)              0         \n",
      "=================================================================\n",
      "Total params: 475,544\n",
      "Trainable params: 470,072\n",
      "Non-trainable params: 5,472\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "from keras.applications.mobilenet import MobileNet\n",
    "\n",
    "input_shape = (224, 224, 3)\n",
    "model2 = MobileNet(\n",
    "        include_top=True, weights='imagenet', \n",
    "        input_shape=input_shape, alpha=0.25)\n",
    "model2.count_params()\n",
    "\n",
    "model2.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(\n",
    "    optimizer=optimizers.SGD(lr=1e-2, momentum=0.9, nesterov=True), \n",
    "    loss='categorical_crossentropy', metrics=['accuracy', 'top_k_categorical_accuracy']\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dongsung/tensorflow_virtualenv/lib/python3.5/site-packages/keras/callbacks.py:1065: UserWarning: `epsilon` argument is deprecated and will be removed, use `min_delta` instead.\n",
      "  warnings.warn('`epsilon` argument is deprecated and '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n"
     ]
    },
    {
     "ename": "ZeroDivisionError",
     "evalue": "integer division or modulo by zero",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mZeroDivisionError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-942c0c336c54>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      6\u001b[0m         \u001b[0mEarlyStopping\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmonitor\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'val_acc'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpatience\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmin_delta\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.01\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     ],\n\u001b[0;32m----> 8\u001b[0;31m     \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_generator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m80\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mworkers\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m )\n",
      "\u001b[0;32m~/tensorflow_virtualenv/lib/python3.5/site-packages/keras/legacy/interfaces.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     89\u001b[0m                 warnings.warn('Update your `' + object_name + '` call to the ' +\n\u001b[1;32m     90\u001b[0m                               'Keras 2 API: ' + signature, stacklevel=2)\n\u001b[0;32m---> 91\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     92\u001b[0m         \u001b[0mwrapper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_function\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tensorflow_virtualenv/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m   1416\u001b[0m             \u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1417\u001b[0m             \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1418\u001b[0;31m             initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m   1419\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1420\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0minterfaces\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlegacy_generator_methods_support\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tensorflow_virtualenv/lib/python3.5/site-packages/keras/engine/training_generator.py\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(model, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)\u001b[0m\n\u001b[1;32m    179\u001b[0m             \u001b[0mbatch_index\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    180\u001b[0m             \u001b[0;32mwhile\u001b[0m \u001b[0msteps_done\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 181\u001b[0;31m                 \u001b[0mgenerator_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_generator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    182\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    183\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgenerator_output\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'__len__'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tensorflow_virtualenv/lib/python3.5/site-packages/keras/utils/data_utils.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    707\u001b[0m                     \u001b[0;34m\"`use_multiprocessing=False, workers > 1`.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    708\u001b[0m                     \"For more information see issue #1638.\")\n\u001b[0;32m--> 709\u001b[0;31m             \u001b[0msix\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreraise\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexc_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/tensorflow_virtualenv/lib/python3.5/site-packages/six.py\u001b[0m in \u001b[0;36mreraise\u001b[0;34m(tp, value, tb)\u001b[0m\n\u001b[1;32m    691\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mtb\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    692\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwith_traceback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 693\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    694\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    695\u001b[0m             \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tensorflow_virtualenv/lib/python3.5/site-packages/keras/utils/data_utils.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    683\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    684\u001b[0m             \u001b[0;32mwhile\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_running\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 685\u001b[0;31m                 \u001b[0minputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mqueue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mblock\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    686\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mqueue\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtask_done\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    687\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0minputs\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/multiprocessing/pool.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    606\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_value\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    607\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 608\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_value\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    609\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    610\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_set\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.5/multiprocessing/pool.py\u001b[0m in \u001b[0;36mworker\u001b[0;34m(inqueue, outqueue, initializer, initargs, maxtasks, wrap_exception)\u001b[0m\n\u001b[1;32m    117\u001b[0m         \u001b[0mjob\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtask\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    118\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 119\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    120\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    121\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mwrap_exception\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tensorflow_virtualenv/lib/python3.5/site-packages/keras/utils/data_utils.py\u001b[0m in \u001b[0;36mnext_sample\u001b[0;34m(uid)\u001b[0m\n\u001b[1;32m    624\u001b[0m         \u001b[0mThe\u001b[0m \u001b[0mnext\u001b[0m \u001b[0mvalue\u001b[0m \u001b[0mof\u001b[0m \u001b[0mgenerator\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0muid\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    625\u001b[0m     \"\"\"\n\u001b[0;32m--> 626\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0msix\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_SHARED_SEQUENCES\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0muid\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    627\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    628\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tensorflow_virtualenv/lib/python3.5/site-packages/keras_preprocessing/image.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1524\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1525\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__next__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1526\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1527\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1528\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_get_batches_of_transformed_samples\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex_array\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tensorflow_virtualenv/lib/python3.5/site-packages/keras_preprocessing/image.py\u001b[0m in \u001b[0;36mnext\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1965\u001b[0m         \"\"\"\n\u001b[1;32m   1966\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1967\u001b[0;31m             \u001b[0mindex_array\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex_generator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1968\u001b[0m         \u001b[0;31m# The transformation of images is not under thread lock\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1969\u001b[0m         \u001b[0;31m# so it can be done in parallel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/tensorflow_virtualenv/lib/python3.5/site-packages/keras_preprocessing/image.py\u001b[0m in \u001b[0;36m_flow_index\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1509\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_set_index_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1510\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1511\u001b[0;31m             \u001b[0mcurrent_index\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_index\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1512\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0mcurrent_index\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1513\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_index\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mZeroDivisionError\u001b[0m: integer division or modulo by zero"
     ]
    }
   ],
   "source": [
    "model.fit_generator(\n",
    "    train_generator, \n",
    "    steps_per_epoch=400, epochs=30, verbose=1,\n",
    "    callbacks=[\n",
    "        ReduceLROnPlateau(monitor='val_acc', factor=0.1, patience=2, epsilon=0.007),\n",
    "        EarlyStopping(monitor='val_acc', patience=4, min_delta=0.01)\n",
    "    ],\n",
    "    validation_data=val_generator, validation_steps=80, workers=4\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loss/epoch plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xt8XWWd7/HPL/f7TpukbZrd+5XSNIVWKJZRBHRKRWAE\nxAEVHM9BR1RwvME4Rx2OR3Cc0dHxggiMoIAyBUdkQBClItIWW6D3UnqhNCVt07TNpUma2+/8sVbS\nNE3bnTY7O8n+vl+v9dprr/Ws1V8K7bfPetZ6lrk7IiIiACmJLkBERAYPhYKIiHRRKIiISBeFgoiI\ndFEoiIhIF4WCiIh0USiIiEgXhYKIiHRRKIiISJe0RBfQV8XFxT5x4sRElyEiMqSsWrVqn7uXnKxd\n3EPBzFKBlcAud7+0x75M4AFgHlADXOPub5zofBMnTmTlypVxqlZEZHgysx2xtBuIy0c3AxuPs+9j\nwAF3nwp8B/jmANQjIiLHEddQMLMo8F7gnuM0uRy4P1xfAlxkZhbPmkRE5Pji3VP4d+CLQMdx9pcB\nOwHcvQ2oBYp6NjKzG81spZmtrK6ujletIiJJL25jCmZ2KbDX3VeZ2QWncy53vxu4G2D+/Pma61tE\n+qy1tZXKykqam5sTXUpcZWVlEY1GSU9PP6Xj4znQvBC4zMwWA1lAgZn93N0/1K3NLmAcUGlmaUCE\nYMBZRKRfVVZWkp+fz8SJExmuV6ndnZqaGiorK5k0adIpnSNul4/c/TZ3j7r7ROCDwB96BALA48D1\n4fpVYRv1BESk3zU3N1NUVDRsAwHAzCgqKjqt3tCAP6dgZrcDK939ceBe4GdmtgXYTxAeIiJxMZwD\nodPp/owDEgruvhRYGq5/pdv2ZuDqgajhtd31PPZyJZ+5aBq5mUPumT0RkQGRNNNc7NzfyI+f38b6\nt+oSXYqIJKGDBw/ywx/+sM/HLV68mIMHD8ahot4lTSjMGRcBYE3lwP3mioh0Ol4otLW1nfC4J598\nksLCwniVdYykuY4yKj+L0kgWayprE12KiCShW2+9la1btzJ37lzS09PJyspixIgRbNq0ic2bN3PF\nFVewc+dOmpubufnmm7nxxhuBI1P7NDQ0cMkll3D++efz4osvUlZWxq9//Wuys7P7tc6kCQWA8rII\na3cpFESS3T//Zj0b+vlS8qyxBXz1fWced/+dd97JunXrePXVV1m6dCnvfe97WbduXdeto/fddx8j\nR46kqamJt73tbVx55ZUUFR39LO/rr7/Oww8/zE9+8hM+8IEP8Oijj/KhD/W8qfP0JM3lI4CKcYVs\n33eI2qbWRJciIknunHPOOepZgu9973tUVFSwYMECdu7cyeuvv37MMZMmTWLu3LkAzJs3jzfeeKPf\n60q6ngLA2spazp9WnOBqRCRRTvQv+oGSm5vbtb506VKeffZZli1bRk5ODhdccEGvzxpkZmZ2raem\nptLU1NTvdSVVT2FONBxs3qXBZhEZWPn5+dTX1/e6r7a2lhEjRpCTk8OmTZtYvnz5AFd3RFL1FApz\nMhg/Moc1OzWuICIDq6ioiIULFzJ79myys7MZPXp0175FixZx1113ccYZZzBjxgwWLFiQsDqTKhQg\n6C288qZ6CiIy8B566KFet2dmZvLUU0/1uq9z3KC4uJh169Z1bf/85z/f7/VBkl0+AqiIFrLrYBP7\nGg4nuhQRkUEn6UKhPHpksFlERI6WdKEwuyyCGazWk80iIsdIulDIy0xjSkmenmwWEelF0oUCBIPN\naypr0asbRESOlpyhUBZhX8NhqmqH92v5RET6KjlDYVww46AuIYnIYJWXl5eQXzcpQ2FWaQFpKaZp\ntEVEeki6h9cAstJTmT46XzOmisiAufXWWxk3bhw33XQTAF/72tdIS0vjueee48CBA7S2tvL1r3+d\nyy+/PKF1JmUoAFSMi/Dk2t24e1K8t1VEunnqVti9tn/POaYcLrnzuLuvueYabrnllq5QeOSRR3j6\n6af5zGc+Q0FBAfv27WPBggVcdtllCf07KSkvHwGUlxVS29TKjprGRJciIkngrLPOYu/evbz11lus\nXr2aESNGMGbMGP7xH/+ROXPmcPHFF7Nr1y727NmT0DqTtqdwZMbUWiYW556ktYgMKyf4F308XX31\n1SxZsoTdu3dzzTXX8OCDD1JdXc2qVatIT09n4sSJvU6ZPZCStqcwY0w+GWkprNmpwWYRGRjXXHMN\nv/jFL1iyZAlXX301tbW1jBo1ivT0dJ577jl27NiR6BKTt6eQnprCrNIC1miwWUQGyJlnnkl9fT1l\nZWWUlpZy3XXX8b73vY/y8nLmz5/PzJkzE11i8oYCQEU0wn+tqqS9w0lN0WCziMTf2rVHBriLi4tZ\ntmxZr+0aGhoGqqSjJO3lI4DyaCGNLe1sq07Mb76IyGATt1Awsywze8nMVpvZejP7517a3GBm1Wb2\narj8r3jV05uKcLB5tZ5sFhEB4ttTOAxc6O4VwFxgkZn19o65X7r73HC5J471HGNySR45Gal6slkk\nSSTDJJin+zPGLRQ80HldJj1cBtV/kdQUY3ZZRHMgiSSBrKwsampqhnUwuDs1NTVkZWWd8jniOtBs\nZqnAKmAq8AN3X9FLsyvN7B3AZuCz7r4znjX1NKcswgPLd9DS1kFGWlIPsYgMa9FolMrKSqqrqxNd\nSlxlZWURjUZP+fi4hoK7twNzzawQ+JWZzXb3dd2a/AZ42N0Pm9nHgfuBC3uex8xuBG4EGD9+fL/W\nOGdcIS0vbGfznnpml0X69dwiMnikp6czadKkRJcx6A3IP43d/SDwHLCox/Yadz8cfr0HmHec4+92\n9/nuPr+kpKRfa+scbNYlJBGR+N59VBL2EDCzbODdwKYebUq7fb0M2Biveo5n/MgcItnprN2lwWYR\nkXhePioF7g/HFVKAR9z9CTO7HVjp7o8DnzGzy4A2YD9wQxzr6ZWZMScaYfVO9RREROIWCu6+Bjir\nl+1f6bZ+G3BbvGqIVXlZhB8/v43m1nay0lMTXY6ISMLodhtgTrSQ9g5nQ1VdoksREUkohQLdptHW\njKkikuQUCkBpJIvivEzNmCoiSU+hQDDYXBHVk80iIgqFUHk0wtbqBhoOtyW6FBGRhFEohCqihbjD\nOl1CEpEkplAIlYeDzWt1CUlEkphCIVScl0lZYTarNY22iCQxhUI3czTYLCJJTqHQTXk0wpv7GznY\n2JLoUkREEkKh0E1FtBDQjKkikrwUCt10vk9hre5AEpEkpVDoJpKdzqTiXFZrugsRSVIKhR7KyyLq\nKYhI0lIo9DAnGqGqtpm99c2JLkVEZMApFHqoGBcONuulOyKShBQKPZw5toAUQzOmikhSUij0kJOR\nxrRR+azRk80ikoQUCr0oj0ZYW1mLuye6FBGRAaVQ6EVFNELNoRZ2HWxKdCkiIgNKodCL8vDJZs2Y\nKiLJRqHQizNK80lPNVYrFEQkySgUepGZlsrMMQUabBaRpKNQOI7yaPBkc0eHBptFJHkoFI6jIhqh\nvrmNN2oOJboUEZEBE7dQMLMsM3vJzFab2Xoz++de2mSa2S/NbIuZrTCzifGqp6/Ky8LBZj3EJiJJ\nJJ49hcPAhe5eAcwFFpnZgh5tPgYccPepwHeAb8axnj6ZPjqPzLQUVmu6CxFJInELBQ80hF/Tw6Xn\nBfrLgfvD9SXARWZm8aqpL9JSUzhzbAFrd2mwWUSSR1zHFMws1cxeBfYCv3P3FT2alAE7Ady9DagF\niuJZU1/MiRayblcdbe0diS5FRGRAxDUU3L3d3ecCUeAcM5t9KucxsxvNbKWZrayuru7fIk+gYlyE\nptZ2tlQ3nLyxiMgwMCB3H7n7QeA5YFGPXbuAcQBmlgZEgJpejr/b3ee7+/ySkpJ4l9ulc7BZ72wW\nkWQRz7uPSsysMFzPBt4NbOrR7HHg+nD9KuAPPohmoZtcnEteZpoeYhORpJEWx3OXAvebWSpB+Dzi\n7k+Y2e3ASnd/HLgX+JmZbQH2Ax+MYz19lpJizC4r0BxIIpI04hYK7r4GOKuX7V/ptt4MXB2vGvpD\nRbSQ//zzG7S0dZCRpmf9RGR4099yJ1EejdDS3sFru+sTXYqISNwpFE6iIpxGe7XGFUQkCSgUTiI6\nIpsROekabBaRpKBQOAkzozxaqNtSRSQpKBRiUBGN8PreBppa2hNdiohIXCkUYlBeFqG9w9lQpd6C\niAxvCoUYVIwLB5s1Y6qIDHMKhRiMLshidEGm3q0gIsOeQiFG5WWFui1VRIY9hUKMKqIRtlUfoq65\nNdGliIjEjUIhRuXRCADrdAlJRIYxhUKM5kQ1jbaIDH8KhRiNzM0gOiJbM6aKyLCmUOiDiqgGm0Vk\neFMo9MGcaITKA03sP9SS6FJEROJCodAHnYPNmhxPRIarPoeCmaWYWUE8ihnsyss6Q0HjCiIyPMUU\nCmb2kJkVmFkusA7YYGZfiG9pg09+VjqTS3IVCiIybMXaU5jl7nXAFcBTwCTgw3GrahCriBbq8pGI\nDFuxhkK6maUThMLj7t4KePzKGrzKyyLsrT/MnrrmRJciItLvYg2FHwNvALnA82Y2AaiLV1GDWcW4\nYFxh9U71FkRk+IkpFNz9e+5e5u6LPbADeFecaxuUZpVGSE0xzZgqIsNSrAPNN4cDzWZm95rZy8CF\nca5tUMrOSGXaqDxWa7BZRIahWC8f/V040PweYATBIPOdcatqkKuIFrK28iDuSTmsIiLDWKyhYOHn\nYuBn7r6+27akUx6NcKCxlcoDTYkuRUSkX8UaCqvM7BmCUHjazPKBjviVNbhVhDOmah4kERluYg2F\njwG3Am9z90YgA/joiQ4ws3Fm9pyZbTCz9WZ2cy9tLjCzWjN7NVy+0uefoC8ON/TLaWaMyScjNUUz\nporIsJMWSyN37zCzKHCtmQH80d1/c5LD2oDPufvLYc9ilZn9zt039Gj3J3e/tM+V99WGx+HxT8HH\nn4cRE0/rVBlpKZxRmq+egogMO7HefXQncDOwIVw+Y2bfONEx7l7l7i+H6/XARqDs9Mo9DWXzoL0N\nfntbv5xuTrSQdbvq6OjQYLOIDB+xXj5aDLzb3e9z9/uARUDM/7o3s4nAWcCKXnafZ2arzewpMzvz\nOMffaGYrzWxldXV1rL/s0SJl8M4vwmtPwuZnTu0c3ZRHIzQcbmPbvkOnfS4RkcGiL7OkFnZbj8R6\nkJnlAY8Ct4S3tXb3MjDB3SuA/wD+u7dzuPvd7j7f3eeXlJT0oeQeFnwSiqfDU1+E1tObpqJzsHnt\nLl1CEpHhI9ZQuAN4xcx+amb3A6uA/3eyg8L5kh4FHnT3x3rud/c6d28I158kmGOpOObq+yotAy75\nFziwHV78j9M61ZSSXLLTU1m9U4PNIjJ8xDrNxcPAAuAxgr/kz3P3X57oGAtGpO8FNrr7t4/TZkzY\nDjM7J6ynJvbyT8GUd8Gsy+FP/wYH3zzl06SlpjC7rEAzporIsHLCUDCzszsXoBSoDJex4bYTWUjw\n5POF3W45XWxmnzCzT4RtrgLWmdlq4HvAB30gHhP+62+A2WkPOpeXFbL+rTra2pP2kQ0RGWZOdkvq\nv51gn3OC+Y/c/QVO8tSzu38f+P5Jauh/kSi84/Pw+9thy7Mw9eJTOk3FuAj3/bmDzXsamDU2KV9G\nJyLDzAlDwd2H70yo530KXnkQnvwifHIZpGX2+RRzug02KxREZDiI9TmF9/eyXGRmo+JdYNykZcLi\nf4H9W2HZqXVWJozMIT8rTTOmisiwEdMTzQTTXJwHPBd+v4DgDqRJZna7u/8sDrXF39SLYeal8Py/\nQvkHoHBcnw5PSTHmRCOa7kJEho1Yb0lNA85w9yvd/UpgFsGYwrnAl+JV3IBYdAe4wzNfPqXDy8sK\n2bS7jsNt7f1cmIjIwIs1FMa5+55u3/eG2/YDrf1f1gAqHA9/9TnY8GvY+tzJ2/dQEY3Q2u5srKqP\nQ3EiIgMr1lBYamZPmNn1ZnY98Hi4LRcY+jfqv/3TMGISPPkFaGvp06Hl0eDh7rV6XkFEhoFYQ+Em\n4D+BueFyP3CTux8aFncopWcFTzrXvA7Lf9inQ8sKsynKzdBgs4gMC7FOne1m9gLQQjCW8NKAPGQ2\nkKa/B2Yshj/+C5RfHUygFwMzDTaLyPAR6y2pHwBeIngC+QPACjO7Kp6FJcSiO8Db4Zl/6tNh5dFC\nXt9bT2NLW5wKExEZGLFePvoywVvXrnf3jwDnAP8nfmUlyIiJcP5nYf1jsP35mA+riEbocFj/Vs9J\nYEVEhpZYQyHF3fd2+17Th2OHloU3Q+GEYNC5PbYbqzoHm1fv1GCziAxtsf7F/lsze9rMbjCzG4D/\nAZ6MX1kJlJ4Nl3wTqjfBirtiOmRUfhbTR+dxz5+2U11/OM4FiojET6xTZ38BuBuYEy53u/vQfmjt\nRGZcAtP+GpbeCXVVMR3ynWvmcqCxhU899LJmTRWRISvmS0Du/qi7/0O4/CqeRQ0Kl9wZXD76XWxD\nJ2eOjXDH+8tZsX0/dz61Kc7FiYjEx8nep1BvZnW9LPVmNrxHVUdODsYX1v4XvPFCTIe8/+wo1583\ngXte2M5vVr8V5wJFRPrfCUPB3fPdvaCXJd/dh/9c0ed/FiLj+zTo/OX3zmLehBF8cckaXtutqS9E\nZGgZnncQ9ZeMnODZhb0b4KWfxHZIWgo/vO5s8rLS+MTPV1HbNLSnhhKR5KJQOJmZ7w2m2F56B9Tv\nOXl7YHRBFj+87mx27m/kc4+8SkfH8Hr4W0SGL4XCyZgF8yK1NcPvvhLzYW+bOJJ/eu8ZPLtxLz94\nbkscCxQR6T8KhVgUTQlmUl3zC9ixLObDrn/7RK6YO5ZvP7uZ517be/IDREQSTKEQq7/6HBRE4cnP\nQ3tscxyZGXe8fw4zxxRwyy9e5c2axjgXKSJyehQKscrIhUXfgD3rYOW9MR+WnZHKXR86G3fn4z9f\nRVOL3tAmIoOXQqEvzrgMJr8L/vD/oCH2y0ETinL57t+exabddXz5V2sZbrOOi8jwoVDoCzNY/C1o\nbYRnv9anQ981YxS3XDSdx17ZxQPLdsSnPhGR06RQ6KviaXDeTfDqg7DzpT4d+ukLp3LRzFH83yc2\nsPKN/XEqUETk1MUtFMxsnJk9Z2YbzGy9md3cSxszs++Z2RYzW2NmZ8ernn71ji9AQRn8z+egI/Yx\ngpQU49vXzCU6Ipu/f/Bl9tY1x7FIEZG+i2dPoQ34nLvPAhYAN5nZrB5tLgGmhcuNwI/iWE//ycyD\n93wddq+Blff16dBIdjp3fXgeDc1tfPLBl2lp04yqIjJ4xC0U3L3K3V8O1+uBjUDPFx9fDjzggeVA\noZmVxqumfnXm38Ckd8Af/i8c2tenQ2eOKeDOK8tZueMA33hyY5wKFBHpuwEZUzCzicBZwIoeu8qA\nnd2+V3JscAxOZrD4X6HlUJ8HnQEun1vG3y2cxE9ffINfvVLZ//WJiJyCuIeCmeUBjwK3uPspTbdt\nZjea2UozW1ldXd2/BZ6Okhmw4O/hlZ9B5co+H37b4pmcM2kktz22lg16v7OIDAJxDQUzSycIhAfd\n/bFemuwCxnX7Hg23HcXd73b3+e4+v6SkJD7Fnqp3fgnyS/s86AyQnprCD649m0h2Oh//+UoONrbE\nqUgRkdjE8+4jA+4FNrr7t4/T7HHgI+FdSAuAWneP7f2Xg0VmfjDoXPUqPPtVaO3bHUUl+Zn88Lp5\n7K5t5pZfakZVEUmsePYUFgIfBi40s1fDZbGZfcLMPhG2eRLYBmwBfgJ8Mo71xM/sK6HiWnjxP+AH\n58DG30AfnlqeN2EEX3nfmSx9rZp///3rcSxUROTEbKhNuTB//nxfubLv1+8HxLal8NStUL0RJr0T\nLvkmjDojpkPdnS8sWcOSVZXc85H5XDxrdHxrFZGkYmar3H3+ydrpieb+NPkC+MQLcMm3oGo1/Ghh\n8CrPxpM/vWxmfP2K2cwuK+Czj7zK9n2H4l6uiEhPCoX+lpoG594In34Z5t0Af7kH/mNe8HmSgeis\n9FR+dN08UlOMT/xsFY0tsU3RLSLSXxQK8ZJbBJd+Gz7+Jxh9ZnB30o/fAdv/dMLDxo3M4XsfPIvN\ne+v50qOaUVVEBpZCId7GzIbrfwMfeACa6+D+S+GRj8DBN497yDuml/D598zgN6vf4t4Xtg9gsSKS\n7BQKA8EMZl0On3oJ3vVl2PwMfP9twXsZWnofO/jkBVN4z6zR3PHUJpZvqxnggkUkWSkUBlJ6Nrzz\ni/DplTDzUnj+X4JwWLvkmFtYzYx/+0AFE0bm8KmHXqaqtilBRYtIMlEoJEIkClfdCx/9LeQUwaMf\ng/+8JLhjqZv8rHR+/OF5NLa0878fWMmOGt2RJCLxpVBIpAnnwY1L4X3fg32vw4/fCY9/5qhZV6eN\nzue7HzyL7dWHeM93nueHS7fQ2q7ptkUkPvTw2mDRdBCe/xasuAvSc+GCL8E5N0JqOgBVtU187fH1\nPL1+DzNG5/ON95czb8KIBBctIkNFrA+vKRQGm+rN8PRtsOVZKJ4Oi+6AqRd37X5m/W6++vh6dtc1\nc+054/nioplEstMTWLCIDAV6onmoKpkO1y2Bv/0ldLTBz6+Ehz4I+7YA8J4zx/C7f3gnN7x9Ig+/\n9CYXf/uPPLHmLT3PICL9Qj2FwaztcHA56Y/fgpZ6KK2AGYthxiUwZg5rdtXyj79ay7pddbxrRgm3\nXz6bcSNzEl21iAxCunw0nNTvgVcfhM2/hZ0vAQ4FZTB9Ee3TFnH/7vH86+/fwB0+++5p/N3CSaSl\nqhMoIkcoFIarhmp4/Rl47UnY+gdobYSMPJrGv5OHDp7J9ysnM6Y0yh3vL2fuuMJEVysig4RCIRm0\nNsP254OA2PxbqK/CSWG1zeCp1rnkzbmMGy57N/lZGogWSXYKhWTjHrz97bWnaN/4JKl71wKwg7F0\nTF/EpIVXQ/ScYBZXEUk6CoVkd3AnO5c/xp6V/82c1tVkWDsdWSNImf7XwUD11IuCV4mKSFJQKAgA\nre0d/GzpWlYvfYwLU1bx1xlryGqthdQMmPhXQUBMXwSF4xJdqojEkUJBjrJzfyP/9N/reGHzbq4a\n9Rafn7iNkspnYf/WoEHuKCiZESzFM46s540OZnkVkSFNoSDHcHf+Z20VX3t8A/sPHeajCyfxubON\nnB1/gD0bYN9rwRPVh2uPHJQVCUNiOpTMPBIYkXGQotteRYYKhYIcV21TK9/87SYeWvEmYyNZ3H75\nbC6eNTrY6Q71u8OACJd9m6F6ExyqPnKS9BwomhoERcn0MCxmwshJXfM1icjgoVCQk1q1Yz+3PbaW\nzXsaqIhGuPbc8byvYiw5Gce5Q6lx/5GAqN58JDhqdx5pk5IOIycfufzUGRT5pcElKt39JJIQCgWJ\nSUtbB7/4y5v8bNkOXt/bQF5mGlecNZZrz5nArLEFsZ3kcAPUvH5sz2L/dvD2I+0sJQiGgtIgJPLH\nQP7Y8DP8XjAWskdoHEOknykUpE/cnVU7DvDQijd5Ym0VLW0dzB1XGPQe5owlOyO17ydtOww1W+Hg\nDqivCi5L1b0VfNZXBUtjL68aTc04NjB6C5LMvNP/wUWShEJBTtnBxhYefXkXD63YwdbqQ+RnpfE3\nZ5Vx7bnjmTkmxt5DrNoOhyHRLSjqq6Cu6kiQ1FdBS8Oxx2bkh2HR2dPoHhxhkOSNhrTM/q1ZZAhK\neCiY2X3ApcBed5/dy/4LgF8D28NNj7n77Sc7r0Jh4Lg7f3njAA+t2MGT63bT0tbB2eMLufbcCVw6\np5Ss9FPoPZyqw/W99zTq3oKGPUdCpKP12GNzio4NjM5LVZ3fc0sgZQB/HpEBNhhC4R1AA/DACULh\n8+5+aV/Oq1BIjAOHWnj05UoeeulNtlUfoiArjfefHeXac8czffQgeTLaPRgM797j6AyQ7j2PQ3vB\ne7zS1FKDXsVRoVEajG9k5IVLbvCZ2W09Iw/SMhLz84r0QcJDISxiIvCEQmH4cHdWbN/PQyve5Lfr\ndtPS3sH8CSO49tzxLC4f4N7DqWpvC4LhmNDY3W3bW9B0ILbzpaQHIZGZH4ZF7tFBclSIdN+XExyb\nmhZ+pvfyPfUE+9I0IC8xGyqh8ChQCbxFEBDrT3ZOhcLgsf9QC4+uquThl95k275DRLLTef/ZZVx3\n7nimjhokvYfT0doMzbXBeEZLA7QcCpeG4I6rzvXu+w7XH92uc9/hBmhr6v8aU3qExDHhkXEK653n\nOsF6Slq4pIThlBp+Tw2Wo76nBXeeHfU99Ujbo45PCc6fnqPA62dDIRQKgA53bzCzxcB33X3acc5z\nI3AjwPjx4+ft2LEjbjVL37k7y7bV8NCKN3l6/W5a251zJo7k2nPHs2j2mKHRexgIHe3dwiIMjo42\naG8NxkLaW3t8b+u2vTU4/nj7jvre45ynvN6SuN+rtGzIGxVc0uv6HA35o4/eljtKl+9iNOhDoZe2\nbwDz3X3fidqppzC41TQcZknYe3ijppHCnHQunVPK+VOLWTC5iMIc/QEeMtyDIOoMiK7gaQmCwzuC\n/d4eBFFHey/f28J23b+3H6dt+L3tMDTug4a9wU0EDXuDS3pN+3uvM3tkj/AYdWR8qHugJPnzL4M+\nFMxsDLDH3d3MzgGWABP8JAUpFIaGjg5n+bYaHnzpTZ7btJfGlnbMYFZpAQunFnPelCLOmTiS3Ew9\n4SwxamsJplpp2NNt2Xtkvb7b9rbmY49PSQ9CIjM/uJxlKUFIdK13LqnH35+S2qOt9TjOgjAFwHtZ\nD78fs87R2493jlmXw1kfOqXfvlhDIW5/Is3sYeACoNjMKoGvAukA7n4XcBXw92bWBjQBHzxZIMjQ\nkZJivH1qMW+fWkxrewdrKg/y5y01vLh1Hz/98xvc/fw20lKMinGFLJxSxHlTijlrfKEuNcnxpWVA\npCxYTsQ9GNtp2AsNu48Oj/o90Hoo6L24h58d3Xo93b539pS8NejBHLWvx/HdjzUDwh5Jb+tdvZUT\nrdP7OVoOnfZv48no4TUZcM2t7ax84wAvbt3Hi1trWFN5kA6HzLQU3jZxJOdNKeLtU4ooL4uQlqqZ\nWEX6Q8KB/CxyAAAMgElEQVR7CiLHk5WeyvnTijl/WjEAdc2tvLRtPy9uDXoS33r6NQDyM9M4d/JI\nzptSzMKpRUwflU9KSvJeExYZCAoFSbiCrHQunjW6a/rufQ2HWb6tJgiJLft4duNeAIpyM1gwpYiF\nU4p5+5QiJhTlYEk8cCgSD7p8JIPeroNNLAsD4s9b97Gn7jAAYyNZnDclGLReMHkk0RE5Ca5UZPAa\nFHcfxYNCIbm5O9v3HeLPW2tYtnUfy7bWcKAxmO8oOiKbBZOLOG9yEQumFFFWmJ3gakUGD4WCJIWO\nDmfz3nqWb61h+bb9rNh+JCTGjcxmwaQiFigkRBQKkpw6Q2LZ1hqWb6thxfb9HOwREsHlpiLGKiQk\niSgURAhC4rU99SzfdmxIjB+Zw4LJI4OehEJChjmFgkgvOjqcTbuPDonapqND4rwpRZw7SSEhw4tC\nQSQG3UNi2bYaVmyroa65DYAJRTmUl0WYUpLH5JJcppTkMak4V1NzyJCkUBA5Be0dzqbddSzftp/l\n22p4bXc9Ow80HjU9TWkkqyskJhfnMrkkjymj8igtyNLDdTJoKRRE+klzazs7ahrZVt3A1uoGtlUf\n6vqsP9zW1S4rPYVJxXlMKQmDQr0LGUQ0zYVIP8lKT2XGmHxmjDn6xUHuTnXDYbbuPcS2fUfCYk1l\nLf+ztqrX3sXk7qGh3oUMQgoFkVNkZozKz2JUfhbnTSk6at/xehf//cquY3oXk4uDgOjsWXSGR3aG\nZoyVgadQEImDWHoX3cPi1Z0HeGLNW0f1LsoKs5kyKhi36AyNqSV5lORnas4niRuFgsgAOlnvYvu+\nICQ6L0ltrW7gL9v309Ta3tUuPzOta6C7ew9jfFEOmWnqXcjpUSiIDBJZ6amcUVrAGaUFR23v6HB2\n1zV39So6lxe31vDYK7u62qVY8KxF5yWo8SNzKBuRTVlh8JmnwW6Jgf4vERnkUlKMsYXZjC3M7noH\nRaeGw21s7xYWncHxpy37aGnrOKptJDudssLsMCiyiYafnd9H5mbospQoFESGsrzMNMqjEcqjkaO2\nd3QEYxeVB5rYdbCJXQea2HWwkV0HmthRc4gXt+zjUEv7Ucdkp6cytjCLshE5vYbG6IIsUnWn1LCn\nUBAZhlJSjNEFWYwuyGLehBHH7Hd3aptae4TGkc/1u2qpOdRy1DFpKcaYSFYQFIXZjIlkURrJYkwk\nO/zMYmROhm6xHeIUCiJJyMwozMmgMCeD2WWRXts0tbQHQdGjp7HrYBMrtu9nT10zbR1HP/yakZrC\n6EgmpQVBaIyJZDGmIKsrNEoj2ZTkZ6rHMYgpFESkV9kZqUwdlcfUUXm97u/ocPYdOszu2maqapu7\nfTZRVdvM6sqDPL2+mcM9xjZSU4xR+ZlHehoF2YyJZHb1OEojWYyNZKvHkSAKBRE5JSkpR26vnRPt\nvY27c7CxNQiLuqYe4dHMa7vrWfpaNY09xjdyM4LnPM4oLWBmaQFnhM985GelD8BPltwUCiISN2bG\niNwMRuRmMGtsQa9t3J36w23sCcOi8kATm/fUs7Gqjt+sfosHV7zZ1XbcyGxmjglu2z0jDI3xI3PU\nq+hHCgURSSgzoyArnYKsdKaNPvYJ8KraZjZW1bFpdz0bqurYVFXH7zfuoXM4Iycjlemj88NnPILP\nGWPyKVCv4pRollQRGXKaW9vZvKeeTVX1bNxdx8aqOjZW1Xe9MAmCaUK6B8XMMflMKMpN2kFuzZIq\nIsNWVnoqc6KFzIkWdm1zD578PhIU9WyqquO51/bSHnYrstNTGRPJIicjlZyMVLIz0sjNSCU7I5Xc\njLRwW7CeHbbJCbcfs56ZRnZ66mmHjLvT4cG7PDrcaevwYL3Dafcjn23tTn5WGoU5Gaf1651M3ELB\nzO4DLgX2uvvsXvYb8F1gMdAI3ODuL8erHhEZ3syM0kg2pZFs3jVzVNf25tZ2tuxt6OpNVDccpvFw\nG40t7dQ2tVJ1sInGlnYaW4JtPe+WOpnMtBRyw4DISk/BnSN/sbuf4C97aOvooKMPF2s+8c4p3HrJ\nzD7V11fx7Cn8FPg+8MBx9l8CTAuXc4EfhZ8iIv0mKz2V2WWR4z6P0VN7h9PY0kZTSzuNLe0c6rbe\nGRzd15vCNo0t7Rxu7SAlxUg1wk8jNeXIktLje6pZt3bBMWk92qVYuC3FOGNM74P1/SluoeDuz5vZ\nxBM0uRx4wINBjeVmVmhmpe5eFa+aREROJjXFyM9KT9rbX1MS+GuXATu7fa8Mtx3DzG40s5VmtrK6\nunpAihMRSUaJDIWYufvd7j7f3eeXlJQkuhwRkWErkaGwCxjX7Xs03CYiIgmSyFB4HPiIBRYAtRpP\nEBFJrHjekvowcAFQbGaVwFeBdAB3vwt4kuB21C0Et6R+NF61iIhIbOJ599HfnmS/AzfF69cXEZG+\nGxIDzSIiMjAUCiIi0mXITYhnZtXAjlM8vBjY14/lDCTVnhiqPTGGau2Due4J7n7Se/qHXCicDjNb\nGcssgYORak8M1Z4YQ7X2oVp3d7p8JCIiXRQKIiLSJdlC4e5EF3AaVHtiqPbEGKq1D9W6uyTVmIKI\niJxYsvUURETkBJImFMxskZm9ZmZbzOzWRNcTKzMbZ2bPmdkGM1tvZjcnuqa+MLNUM3vFzJ5IdC19\nEb7fY4mZbTKzjWZ2XqJripWZfTb8f2WdmT1sZlmJrul4zOw+M9trZuu6bRtpZr8zs9fDzxGJrPF4\njlP7t8L/Z9aY2a/MrPBE5xiMkiIUzCwV+AHB295mAX9rZrMSW1XM2oDPufssYAFw0xCqHeBmYGOi\nizgF3wV+6+4zgQqGyM9gZmXAZ4D54WtwU4EPJraqE/opsKjHtluB37v7NOD34ffB6KccW/vvgNnu\nPgfYDNw20EWdrqQIBeAcYIu7b3P3FuAXBG9+G/Tcvarz3dXuXk/wl1OvLyMabMwsCrwXuCfRtfSF\nmUWAdwD3Arh7i7sfTGxVfZIGZJtZGpADvJXgeo7L3Z8H9vfYfDlwf7h+P3DFgBYVo95qd/dn3L0t\n/Lqc4JUAQ0qyhELMb3kbzMLXm54FrEhsJTH7d+CLQN/ehJ54k4Bq4D/DS1/3mFluoouKhbvvAv4V\neBOoIpiS/pnEVtVno7tNo78bGJ3IYk7D3wFPJbqIvkqWUBjyzCwPeBS4xd3rEl3PyZjZpcBed1+V\n6FpOQRpwNvAjdz8LOMTgvYRxlPD6++UEwTYWyDWzDyW2qlMXzqY85G6RNLMvE1z6fTDRtfRVsoTC\nkH7Lm5mlEwTCg+7+WKLridFC4DIze4Pgct2FZvbzxJYUs0qg0t07e2RLCEJiKLgY2O7u1e7eCjwG\nvD3BNfXVHjMrBQg/9ya4nj4xsxuAS4HrfAje858sofAXYJqZTTKzDIKBt8cTXFNMzMwIrm1vdPdv\nJ7qeWLn7be4edfeJBL/ff3D3IfEvVnffDew0sxnhpouADQksqS/eBBaYWU74/85FDJFB8m4eB64P\n168Hfp3AWvrEzBYRXDK9zN0bE13PqUiKUAgHfj4FPE3wB+QRd1+f2KpithD4MMG/tF8Nl8WJLioJ\nfBp40MzWAHOBbyS4npiEvZslwMvAWoI/44P2KdvwDY3LgBlmVmlmHwPuBN5tZq8T9HzuTGSNx3Oc\n2r8P5AO/C/+s3pXQIk+BnmgWEZEuSdFTEBGR2CgURESki0JBRES6KBRERKSLQkFERLooFEQGkJld\nMNRmjJXkolAQEZEuCgWRXpjZh8zspfABpB+H74VoMLPvhO8q+L2ZlYRt55rZ8m5z6I8It081s2fN\nbLWZvWxmU8LT53V7V8OD4ZPHIoOCQkGkBzM7A7gGWOjuc4F24DogF1jp7mcCfwS+Gh7yAPClcA79\ntd22Pwj8wN0rCOYf6pz58yzgFoJ3e0wmeGpdZFBIS3QBIoPQRcA84C/hP+KzCSZl6wB+Gbb5OfBY\n+O6FQnf/Y7j9fuC/zCwfKHP3XwG4ezNAeL6X3L0y/P4qMBF4If4/lsjJKRREjmXA/e5+1FuzzOz/\n9Gh3qnPEHO623o7+HMogostHIsf6PXCVmY2CrncGTyD483JV2OZa4AV3rwUOmNlfhds/DPwxfEte\npZldEZ4j08xyBvSnEDkF+heKSA/uvsHM/gl4xsxSgFbgJoKX7ZwT7ttLMO4AwfTOd4V/6W8DPhpu\n/zDwYzO7PTzH1QP4Y4icEs2SKhIjM2tw97xE1yEST7p8JCIiXdRTEBGRLuopiIhIF4WCiIh0USiI\niEgXhYKIiHRRKIiISBeFgoiIdPn/ft2jjoeXnkQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe7e1ff8240>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(model.history.history['loss'], label='train');\n",
    "plt.plot(model.history.history['val_loss'], label='val');\n",
    "plt.legend();\n",
    "plt.xlabel('epoch');\n",
    "plt.ylabel('logloss');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xt8nVWd7/HPL7fm0jS3pmnatE1p0xuFthKgWLQgouUO\nowKKHEEFZ7jqOHPEGY8iOiMzZwYPzqAMw4A3LiKiVkRuCkWkYAuJQEubltIm6TVNc23u2b/zx7OT\npmnS7rbZ2Un29/167dd+Lmvv/ALN+j3PWs9ay9wdERERgIRYByAiIiOHkoKIiPRSUhARkV5KCiIi\n0ktJQUREeikpiIhILyUFERHppaQgIiK9lBRERKRXUqwDOFoTJ0704uLiWIchIjKqvP7663vdPf9I\n5UZdUiguLmbt2rWxDkNEZFQxs22RlFPzkYiI9FJSEBGRXkoKIiLSa9T1KQyks7OT6upq2traYh1K\nVKWmplJUVERycnKsQxGRMWpMJIXq6moyMzMpLi7GzGIdTlS4O7W1tVRXVzNz5sxYhyMiY9SYaD5q\na2sjLy9vzCYEADMjLy9vzN8NiUhsjYmkAIzphNAjHn5HEYmtMdF8JCIy3EIhZ+/+dnbWt7GzoY2d\nDa3UtXSSYJBoRmKiBe8JB14JfffD2wkJRlKfc0nhY4lmJCRAUkICiQmQYMa03HQmjh8X1d9LSWEI\n1NfX8/DDD3PDDTcc1efOP/98Hn74YbKzs6MUmYgci54Kf1dDGzvq29jV0Bqu+IPKf2dDG7sb2+js\nHt417r916UKuXjojqj9DSWEI1NfX8/3vf/+QpNDV1UVS0uD/iZ966qlohyYi/YRCTu3+jqDCb2g9\n6H1nfbA9UIWfkpjA5KxUCrNSKZ2RQ2F2GoVZqRRm9bynkpOeggPdISfkTlfIg+2Q0+3Be+8xD967\nw+eCctAVCoXPcdD5UMgpKRgf9f8+SgpD4LbbbuPdd99l8eLFJCcnk5qaSk5ODhs2bKCiooJLL72U\nqqoq2trauPXWW7n++uuBA1N2NDc3c95553HmmWfyyiuvMHXqVH7961+TlpYW499MZPTb29zOa1v2\nsXrLXl7bso9ttS10dIcOKtNT4U8OV/iTs9KYkp3K5AmpTMlOY3JWKnkZKRH36yUmjN7+vzGXFL75\nm3Ws39E4pN+5YMoEvnHRiYOev/POO3n77bcpLy/nxRdf5IILLuDtt9/ufXT0gQceIDc3l9bWVk49\n9VQ+9rGPkZeXd9B3bNq0iUceeYT//u//5vLLL+cXv/gFn/70p4f09xCJB7XN7bz23j5e3VLLq1tq\nqdjdDEBGSiKnzszlnPkFvVf2hVlpFGankpueQsIorsiH0phLCiPBaaeddtBYgu9973v88pe/BKCq\nqopNmzYdkhRmzpzJ4sWLATjllFPYunXrsMUrMprV7e/gtfdqeXXLPla/W8vG3U0ApKckcmpxLpct\nKeKMWXksnDKBpMQx88Bl1Iy5pHC4K/rhkpGR0bv94osv8vzzz7N69WrS09M566yzBhxrMG7cgScK\nEhMTaW1tHZZYRUab+pYOXnsvSACvbqllw64gCaQlJ1JanMPFi6dwxqw8TpqaRbKSwFEbc0khFjIz\nM2lqahrwXENDAzk5OaSnp7NhwwZeffXVYY5OZHRraOnsvRN4dUst7+xqxB1SkxMonZHL332kMJwE\nsklJUhI4XkoKQyAvL49ly5axcOFC0tLSKCgo6D23YsUK7r33XubPn8/cuXNZunRpDCMVGfka2zr5\nczgBrN5Sy/qdQRIYl5TAKTNy+NsPz2HprDwWFSkJRIO5D+9ztsertLTU+y+y88477zB//vwYRTS8\n4ul3lbHJ3dnb3EHlvhaq9rVQ2edVta+FXY1tuENKUgLvm57NGSdMZOkJuSyens24pMRYhz9qmdnr\n7l56pHK6UxCRIdfW2T1ghR+8t9La2X1Q+ckTUpmem877Z02kOC+d0uJclkzPJjVZSWC4KSmIyFEL\nhZya5vagwq9tOeSqf09T+0Hl01MSmZ6bzoy8DD5Qks/03HSm56YzLTedopw0Vf4jiJKCiAyqszvE\nttr9VOxuZtPuZir2NLF5dzNba/fT3nVgAJgZFE5IZVpuOsvnhCv9vKDSn56bflQDvyS2lBREhI6u\nEFtr9wcV/+4mNu9pZtOeJt7bu793ugczmJaTTsmk8XxwzkSm52X0XvFPyU5Ve/8YoaQgEkfau7rZ\nureFit1NbNrTzOY9TVTsbmbr3v10hQ5U/tNz0ymZlMk58wuYUzCekkmZzMofT1qKKv6xTklBZAxq\n7+rmvb1Bs8/m3UHFv2lPE1trW+gOV/4JBjPyMpg9aTwfPbGAkkmZlBSMZ1b+eLXxxzElhRgYP348\nzc3NsQ5DxoBQyKmua2XDrkY27mpiw+4mNu4Kmn16Kv/EBGNGXtDsc97CQkrCV/4n5Geo8pdDKCmI\njBL79nf0Vv4bdzWxYVcTFbubaOk48Hjn9Nx05k7O5LyFkykpyGROwXhmTsxQe79ETElhCNx2221M\nmzaNG2+8EYDbb7+dpKQkXnjhBerq6ujs7OTb3/42l1xySYwjldGgrbObTbubDySA3UECqOnzmGdu\nRgpzCzK5vHQa8yZnMndyJnMKMskYpz9pOT5R/RdkZiuAu4FE4H53v7Pf+e8CZ4d304FJ7n58y5D9\n7jbY9dZxfcUhJp8E59056OkrrriCL37xi71J4bHHHuOZZ57hlltuYcKECezdu5elS5dy8cUX67E8\n6dUdcir3tbBxVyMbwlf/G3c1sbV2P+GWH8YlJVBSMJ4PluT3Vv7zCjPJHz9O/5YkKqKWFMwsEbgH\nOBeoBtaY2Up3X99Txt2/1Kf8zcCSaMUTTUuWLGHPnj3s2LGDmpoacnJymDx5Ml/60pd46aWXSEhI\nYPv27ezevZvJkyfHOlwZJu5OfUsn1XWtbK9vobquNfwKtrfVtvSO7DWDGeGmnwsXTelNAMV5GaN6\nwRYZfaJ5p3AasNndtwCY2aPAJcD6Qcp/EvjGcf/Uw1zRR9MnPvEJHn/8cXbt2sUVV1zBQw89RE1N\nDa+//jrJyckUFxcPOGW2jF7uTl1LZ28l3/O+vU/lv7/j4Okcxo9LoignjaKcNM6Ylcf8yROYOzl4\n6ic9RU0/EnvR/Fc4Fajqs18NnD5QQTObAcwE/jDI+euB6wGmT58+tFEOkSuuuILrrruOvXv3smrV\nKh577DEmTZpEcnIyL7zwAtu2bYt1iHKU3IO1fA9U9AdX/tV1h87hk5maRFFOMJr3/bPzKMpJZ2p2\nkASm5aQzIS1JzT4yoo2US5MrgcfdvXugk+5+H3AfBLOkDmdgkTrxxBNpampi6tSpFBYWctVVV3HR\nRRdx0kknUVpayrx582IdokRgd2MbqypqWFVRw8ub9tLQ2nnQ+ay0ZIpy0jghP5jDp+eqvygnnak5\naWSlJccocpGhEc2ksB2Y1me/KHxsIFcCN0YxlmHx1lsHOrgnTpzI6tWrByynMQojR3tXN69vretN\nBD2reBVMGMdHFhRw4pQJFOWkU5SbxtTsNDJTVenL2BbNpLAGKDGzmQTJ4ErgU/0Lmdk8IAcYuAYV\nGWLbavcHSWBjDau31NLS0U1yonFqcS5fPW8ey+fmM7cgU808EpeilhTcvcvMbgKeIXgk9QF3X2dm\ndwBr3X1luOiVwKM+2lb7kVGjpaOL1e/Wsqqihpcqatha2wIEA70+fkoRy+fks/SEPD3jL0KU+xTc\n/SngqX7Hvt5v//Yh+llj/spOeTMy7k7F7mZWVexhVUUNa96ro6M7RFpyImfMyuPaZTNZPief4okZ\nsQ5VZMQZE5dGqamp1NbWkpeXN2YTg7tTW1tLampqrEMZkRpaOnl5815WVezhpYq97GoMHv+dW5DJ\nNcuKWT4nn9LiHE33IHIEYyIpFBUVUV1dTU1NTaxDiarU1FSKiopiHcaIEAo5b21v4MWNNby0qYay\nyjpCDhNSkzizZCLL5+TzwTn5FGalxTpUkVFlTCSF5ORkZs6cGeswJMr2Nrfzx001vLixhj9u2su+\n/R2YwclTs7jp7Nksn5vPoqJskhITYh2qyKg1JpKCjE1d3SHKq+pZVREkgre2NwCQl5HC8jn5nDU3\nnzNnTyRv/LgYRyoydigpyIjSO3hsYw1/3FRDY1sXCQZLpufw5XPnsHxuPgunZJGg+YBEokJJQWKq\nszvE2j6Dx97Z2QjApMxxfPTEyZw1dxJnzp5IVroGjYkMByUFGXbb61tZtbGGFzfu4ZV3a2lu7yIp\nwThlRg5fWTGP5XPymV+owWMisaCkIFHX3tXNmvfqeHFjMG5g055gmo8pWalctGgKy+fks2x2nqaQ\nEBkBlBQkKtyd1e/W8tPXtvHChhpaO7tJSUzgtJm5XHHqNJbPyWf2pPG6GxAZYZQUZEg1tnXyxOvV\n/OTVbbxbs5/s9GQ+dspUzp47iTNm5WnNAJERTn+hMiQ27GrkJ6u38cuy7bR0dLNoWjb/9olFXHhy\nIanJGkUsMlooKcgx6+gK8fS6Xfx09Tb+vHUf45ISuHjRFK4+YwYnFx3fUtsiEhtKCnLUdja08vBr\nlTzy5yr2NrczPTedfzh/Hp84ZRo5GSmxDk9EjoOSgkTE3Xnl3Vp+vHorz7+zh5A7H5o7iU+fMYPl\nJfkaTCZjjzuEuqCrHbo7Dry6erbbobvz4POhbkhICr8S+2wPsJ+Y1O/8AGUsAYb5YQwlBTmsxrZO\nfhHuON5Ss5+c9GSu+8AJXHX6dKblpsc6PJFDuUNbAzTthMYd4fedwXvTTuho7lOxdxym0u8ARsB0\n9QnJB5LEin+G9/2vqP44JQUZ0Ds7G/nx6m38qmw7rZ3dLJ6WzV2XL+L8k9RxLDHU3QnNu8OV/I5+\n732SQGfLoZ9Ny4HMQhg3AZJSYNx4SBwHicmQNA4SU4JXUvhYYvhYUkq/c333e7bDn0lIDO4WQt3B\nXUbvq//+QK8BPtPdefD+xLlR/0+spCC9OrpC/O7tnfxk9TbWbqtjXFIClyyewtVLizmpKCvW4cnh\nuENXG3Tsh/amoFLsCjdvhDrDV75938PboT7b3R3Q3dWnKaTPdt/PdHcCHm4CSQ6uZBOTD95PSBz8\nXG+zySDnHGjeNXDFv7+GQ67eE1MgczJkToHCk2HOCphQGCSAzMID28maRj0SSgrCjvqg4/jRNZXs\nbe6gOC+dr10wn4+fUkR2ujqOh1QodKCS7moPmjI69h+ozHu2O5oPPtez3d588H7fsh4amhgtXKEn\nphx476nAE1OCihsLJ42epNN1IMGEuvtsdx5fLGm5MGFKUKkXLgoq/szJB45NmALpecPe7j6WKSnE\nse6Qc++qd/nucxV0u3POvElcfUYxH5g9Mf46jt2htQ4atwdNEA3V4fbn/QNcKfe/iu53RX3IlXmf\n8959lIEZpIyHlIyguSMlI9gfP+nAds/5nv1x4yE5vU8zSMoAFXtyv3NJB44nDGHzoHu/JNF1mGQS\n3ofg98sshGStNDjclBTi1J7GNr70WDl/2lzLhScX8pUV88Zux7E7tNVDQ7jCb6w+eLtxR/Dq3w5t\nCZCcMUhF2tPcEd5Ozjr4fEK/Sjcxqc92+LNJ4/pV+P0q95TxQZPHaL4KNgv/7klqvhkllBTi0Isb\n9/Dlx/7C/o4u/uVjJ3F56bTROwdRz5Mmfa/wD9reEewPVOH3ND8ULISSj0LW1GB/QlGwnTEp3FQi\nEj/0Lz6OdHSF+LdnN3LfS1uYNzmTRz+5lJKCzFiHFZn2ZqjdDHs3Qe2mA+/73gva0/uyBBgfbncu\nWAAlHwm2s6bChPBrfIEqfJEB6K8iTlTWtnDzI2/wl+oGPr10Ol+7YMHIe7Q0FIKGqgOVfm8C2Bw8\nfdLLIGcG5JXAjGWQVXSgss+aGiQEVfgix0R/OXHgN3/ZwT888RZm8IOr3sd5JxXGNqC2xgOV/d6K\nA9v73g0eq+yRmhVU/Ccsh7zZMLEk2M89QR2QIlGipDCGtXZ0883frOPRNVW8b3o2d1+5ZPg6k0Mh\naKgMX/FXHHzl37z7QDlLhJzioMKfdfaBin9iCWTkj+5OVpFRSElhjNqwq5GbHy5jc00zN5w1iy+d\nO4fkxISh/0Fd7VD7LuzdCDUV4QSwMbjy72o9UC4tN6joZ58LE2cfqPhzZgajQkVkRFBSGGPcnYf/\nXMkdv1lPZmoyP/ns6ZxZMvH4v7itIbjSr9l4cAKoe6/PoCmD7GnBUPyZy2HinAOvjLzjj0FEok5J\nYQxpaO3ktl+8ye/e3sUH5+Tz759YRH7muMi/wB2adoWv9PslgOZdB8olpkDuLJi8EBZ+DPLnBhV/\n3mxIGaNjHUTihJLCGPH6tjpueaSM3Y1tfPW8eVz3gROOPCq5vQk2Px+89mwIrvzbGw+cHzchqOxn\nfQjy5wR3APlzIXuGnu4RGaOi+pdtZiuAu4FE4H53v3OAMpcDtxPMcvUXd/9UNGMaa0Ih596X3uXf\nn62gMCuVn//1GSyZnjP4B5p2Q8XvYMNvYcuLwdQLaTkw+SQ4+fJwxR9OAJmT1dErEmeilhTMLBG4\nBzgXqAbWmNlKd1/fp0wJ8FVgmbvXmdmkaMUzFu1pauNvf/YXXt68lwtOLuQ7f3USE1KTDy1Y+y5s\neDJIBFV/Bjy42j/1Oph3AUw7XVf+IgJE907hNGCzu28BMLNHgUuA9X3KXAfc4+51AO6+J4rxjCkv\nVdTwt4+V09zexZ1/dRJXnNpnqopQCHaUBYlg41NQsyE4XrgIzv6HIBFMWqC7ABE5RDSTwlSgqs9+\nNXB6vzJzAMzsTwRNTLe7+9NRjGnU6+wO8e/PVnDvqneZUzCeh69bypyCzGC1qK1/DO4GNj4VzPBp\niVC8DEo/C3PPg+zpsQ5fREa4WLcZJAElwFlAEfCSmZ3k7vV9C5nZ9cD1ANOnx2/FVrWvhZsfKaO8\nqp5PnT6dr59bROrW5+CPv4VNzwadxMnpMPscmHdhMOdPem6swxaRUSSaSWE7MK3PflH4WF/VwGvu\n3gm8Z2YVBEliTd9C7n4fcB9AaWnpCFg0dfj99s2d3PaLN8mzelaesZOTmx6E764KOorT82DBxUEi\nOOEsTVEsIscsmklhDVBiZjMJksGVQP8ni34FfBJ40MwmEjQnbYliTKPSD1a+RP1rD/PztHLmdm3A\nyjyYGuK06w90FA/lwigiEreilhTcvcvMbgKeIegveMDd15nZHcBad18ZPvcRM1sPdAN/7+610Ypp\n1GnZx/rHvsFn33uEccmd+MRF2Hx1FItI9Jj76GqNKS0t9bVr18Y6jOhqb4ZXv0/Xy3djHftZPf5c\nll5zJ0n5s2IdmYiMUmb2uruXHqlcrDuapa+udlj7ILz0f6FlLy/6afw86xruuvEKksbpf5WIRJ9q\nmpEg1A1v/gxe+A40VNIxbRl/0/l3vG0l/Opzy8hQQhCRYaLaJpbcgwFmf/h2MMCscDFt59/Fx58Z\nx5b2Fh77wqkUZulJIhEZPlGYYF8ismUV3H8O/OzTwZ3C5T+m+/MvcPNrOazf2cR/fHIJC6dmxTpK\nEYkzulMYbtvfgN/fAVteCNYUvvg/YdEnITGJf35yPc+t3803Lz6Rc+YXxDpSEYlDSgrDpaYC/vAt\neGdlsArZR/4JTv1871rDP169lf95+T2uXVbMZ95fHNNQRSR+KSlEW30VrLoTyh8OpqBYfhuccSOk\nTugt8sKGPdy+ch0fnj+Jr12wIIbBiki8U1KIlv174Y93wZr7AYfT/xo+8GXIOHhpzHU7Grjp4TeY\nXziBu69cQuKRFsYREYkiJYWh1t4Eq++BV/4TOvfDok/BWV8ZcIbSXQ1tfO6Ha5mQlswD15yqR09F\nJOZUCw2VzjZY+wD88d+gpRbmXwQf+j/B8pUD2N/exed+tIamtk5+/tfvp2BC6jAHLCJyKCWFodCy\nL3i8dN8WmLkczvkGFJ0yaPHukHPLI2W8s7OR/7nmVBZMmTBoWRGR4aSkcLzc4ckvBh3Kn/o5zPnI\nET/yrSfX8/sNe/jWpQs5e65WIBWRkUOD147XWz+H9b8OlrmMICE8+Kf3+OErW/ncmTO5eumMYQhQ\nRCRySgrHo2E7/PbvgvUMlt16xOK/f2c333pyPecuKOAfzp8/DAGKiBydiJKCmT1hZheYmZJIj1AI\nfn0DhLrgsnuPuMjN29sbuPmRMk6cksXdVy7Wo6ciMiJFWsl/n2DVtE1mdqeZDfxITTxZcz9seRE+\n+k+Qe8Jhi+5saOVzP1pDdloy//OZUtJT1JUjIiNTREnB3Z9396uA9wFbgefN7BUzu9bMkqMZ4Ii0\ndxM893Uo+Qiccs1hiza3d/HZH65lf3s3D1x7KpP06KmIjGARNweZWR5wDfB5oAy4myBJPBeVyEaq\n7i544vpgzqKL/+OwS2J2dYe4+eE3qNjdxD1XvY95k/XoqYiMbBG1Y5jZL4G5wE+Ai9x9Z/jUz8xs\njK+N2c8f/x12vAGf+CFkTh60mLtzx5PreWFjDf902UKWz8kfvhhFRI5RpI3b33P3FwY6Ecman2PG\n9jfgpX+Fky6HEy87bNEH/7SVH6/exvUfPIGrTtejpyIyOkTafLTAzLJ7dswsx8xuiFJMI1NnK/zy\nC5AxCc7/18MWfXbdLr712/WsOHEyt62YN0wBiogcv0iTwnXuXt+z4+51wHXRCWmEev6bsLcCLr0H\n0nIGLfZWdQO3PlrOyVOz+O4Vi0nQo6ciMopEmhQSzQ70qJpZIpASnZBGoC2r4LUfwGnXw6wPDVps\ne30rn/3RGnIzUvjvz5SSlnL4sQsiIiNNpH0KTxN0Kv9XeP8L4WNjX2s9/OoGyJsNH/7moMWa2jr5\n3A/X0NbRzUM3nM6kTD16KiKjT6RJ4SsEieBvwvvPAfdHJaKR5unboGknfO45SEkfsIh7MOvppj3N\n/PDaU5lTkDnMQYqIDI2IkoK7h4AfhF/xY/1K+MsjsPwrh50Ke1ttCy9srOHvPzqXD5To0VMRGb0i\nHadQAnwHWAD0tou4++HndxjNmnbDb26FwsXwwb8/bNGyqjoAPjRP02CLyOgWaUfzgwR3CV3A2cCP\ngZ9GK6iYc4ff3AKdLfBX90Hi4WfyKK+sJz0lUc1GIjLqRZoU0tz994C5+zZ3vx24IHphxVjZT6Di\nafjw7YMup3lQ8ap6FhVla+ZTERn1Ik0K7eFpszeZ2U1mdhkw/kgfMrMVZrbRzDab2W0DnL/GzGrM\nrDz8+vxRxj/09r0HT38VZn4QTvvCEYu3dXazfkcji6dnH7GsiMhIF+nTR7cC6cAtwLcImpA+c7gP\nhMcy3AOcC1QDa8xspbuv71f0Z+5+01FFHS2h7uDxU0uAS74PCUfOmet2NNAVcpZMU1IQkdHviEkh\nXLlf4e5/BzQD10b43acBm919S/h7HgUuAfonhZFj9X9C5Stw6b2QPS2ij5RVBgO9dacgImPBES+F\n3b0bOPMYvnsqUNVnvzp8rL+PmdmbZva4mUVWE0fD7nXwh2/D/Itg0ZURf6ysqp6p2WkarCYiY0Kk\nzUdlZrYS+Dmwv+eguz9xnD//N8Aj7t5uZl8AfgQcMo+EmV0PXA8wffr04/yRA+hqhye+AKnZcOH/\nO+waCf2VV9azRHcJIjJGRNrRnArUElTYF4VfFx7hM9uBvlf+ReFjvdy91t3bw7v3AwOOEHP3+9y9\n1N1L8/OjMDjsxe/A7rfg4u9BxsSIP7ansY3t9a0sVn+CiIwRkY5ojrQfoa81QImZzSRIBlcSrPPc\ny8wK+yzYczHwzjH8nONT+Sr86W5YcjXMPe+oPlpWFfQnLJk++KypIiKjSaQjmh8EvP9xd//sYJ9x\n9y4zuwl4BkgEHnD3dWZ2B7DW3VcCt5jZxQSD4vYRLPc5fNqbgzUSsopgxXeO+uNllfUkJxonTtEy\nmyIyNkTap/Bkn+1U4DJgx5E+5O5PAU/1O/b1PttfBb4aYQxD79mvQd02uPYpGHf0o5HLKutYUDiB\n1GRNkS0iY0OkzUe/6LtvZo8AL0clouFS8Sy8/iC8/xaY8f6j/nhXd4i3tjdweWnsHpgSERlqkXY0\n91cCjN7Z31r2wcqbYNICOPsfj+krKnY309LRrU5mERlTIu1TaOLgPoVdBGssjD7u8OSXgsRw1eOQ\nfGzjC8p7O5mVFERk7Ii0+WjsTP/51uOw/ldwzteh8ORj/pqyyjpyM1KYnjvwwjsiIqNRRM1HZnaZ\nmWX12c82s0ujF1aUNGyHp74M006HZV88rq8qq6pn8bRs7CgGuomIjHSR9il8w90benbcvR74RnRC\nipJQCH59A3R3wWX3QsKxPzHU0NrJ5j3NmgRPRMacSB9JHSh5RPrZkWHN/bDlRbjwu5B7fAvGvVmt\nSfBEZGyK9E5hrZndZWazwq+7gNejGdiQm/F+WHoDnHIsg7MPVl5Zjxks0p2CiIwxkSaFm4EO4GfA\no0AbcGO0goqKyQuDUctD0AdQVlXP7PzxTEg9/DKdIiKjTaRPH+0HDlk5LR65O2WVdXx4fkGsQxER\nGXKRPn30nJll99nPMbNnohfWyFW5r4W6lk5NgiciY1KkzUcTw08cAeDudYzmEc3HoXelNfUniMgY\nFGlSCJlZ7+o2ZlbMALOmxoPyqnrSUxKZUzA+1qGIiAy5SB8r/UfgZTNbBRjwAcIrocWbsso6Ti7K\nIinxWKeNEhEZuSKq2dz9aaAU2Ag8AnwZaI1iXCNSW2c363c2snia+hNEZGyKdEK8zwO3EiypWQ4s\nBVYzwHrKY9m6HY10drsmwRORMSvSNpBbgVOBbe5+NrAEqD/8R8aesso6AE1vISJjVqRJoc3d2wDM\nbJy7bwDmRi+skamsqp6p2WlMmnBs022LiIx0kXY0V4fHKfwKeM7M6oBt0QtrZCqvrNd8RyIypkU6\novmy8ObtZvYCkAU8HbWoRqA9TW1sr2/l2mXFsQ5FRCRqjnqmU3dfFY1ARrrySq20JiJjnx62j1BZ\nVT3JicaJU7KOXFhEZJRSUohQWWUd8wsnkJp87IvziIiMdEoKEegOOW9WN+hRVBEZ85QUIlCxu4mW\njm49eSQEGxt0AAAL6ElEQVQiY56SQgTKq8KdzJreQkTGOCWFCJRV1pGTnsyMvPRYhyIiElVKChEo\nq6xn8bRsbAiW8hQRGcmUFI6gsa2TzTXNWmlNROJCVJOCma0ws41mttnMBl3j2cw+ZmZuZqXRjOdY\nvFnVgLsGrYlIfIhaUjCzROAe4DxgAfBJM1swQLlMgllYX4tWLMejZ2bUk4uUFERk7IvmncJpwGZ3\n3+LuHcCjwCUDlPsW8C9AWxRjOWblVfXMnjSerLTkWIciIhJ10UwKU4GqPvvV4WO9zOx9wDR3/20U\n4zhm7k5ZVdDJLCISD2LW0WxmCcBdBEt7Hqns9Wa21szW1tTURD+4sKp9rezb36H+BBGJG9FMCtuB\naX32i8LHemQCC4EXzWwrwRKfKwfqbHb3+9y91N1L8/PzoxjywcqqelZa05NHIhIfopkU1gAlZjbT\nzFKAK4GVPSfdvcHdJ7p7sbsXA68CF7v72ijGdFTKKutJS05kTsH4WIciIjIsopYU3L0LuAl4BngH\neMzd15nZHWZ2cbR+7lAqq6rn5KIskhI1nENE4sNRL7JzNNz9KeCpfse+PkjZs6IZy9Fq6+xm/Y4G\nPnvmzFiHIiIybHQJPIj1Oxvp7Hb1J4hIXFFSGESZlt8UkTikpDCIsso6pmSlUjAhNdahiIgMGyWF\nQZRX1WsSPBGJO0oKA6hpaqe6rlVNRyISd5QUBtCz0pqmtxCReKOkMICyyjqSEoyFU7NiHYqIyLBS\nUhhAWWU98wsnkJqcGOtQRESGlZJCP90h583qevUniEhcUlLoZ9OeJvZ3dCspiEhcUlLop2fQ2mKN\nZBaROKSk0E95ZT3Z6ckU56XHOhQRkWGnpNBPWVUdi6dlY2axDkVEZNgpKfTR1NbJpj3NmgRPROKW\nkkIfb1Y34K5J8EQkfikp9FFWGSy/uUgjmUUkTikp9FFeVc+s/Ayy0pJjHYqISEwoKYS5O2WVmhlV\nROKbkkJYdV0rtfs7NAmeiMQ1JYWwN8L9CepkFpF4pqQQVlZZT1pyInMLMmMdiohIzCgphJVX1XNS\nURZJifpPIiLxSzUg0N7VzfodjWo6EpG4p6QArNvRSEd3iCXqZBaROKekQDAJHqDHUUUk7ikpAGVV\n9RRmpVIwITXWoYiIxJSSAlBeVaf+BBERlBSoaWqnal+rZkYVEUFJgfKq8EprulMQEVFSKK+qIynB\nWDglK9ahiIjEXFSTgpmtMLONZrbZzG4b4Pxfm9lbZlZuZi+b2YJoxjOQssp65hdOIC0lcbh/tIjI\niBO1pGBmicA9wHnAAuCTA1T6D7v7Se6+GPhX4K5oxTOQ7pDzZnWDJsETEQmL5p3CacBmd9/i7h3A\no8AlfQu4e2Of3QzAoxjPITbvaaa5vUtPHomIhCVF8bunAlV99quB0/sXMrMbgb8FUoAPDfRFZnY9\ncD3A9OnThyzAnpXWdKcgIhKIeUezu9/j7rOArwBfG6TMfe5e6u6l+fn5Q/azy6vqyUpLZubEjCH7\nThGR0SyaSWE7MK3PflH42GAeBS6NYjyHCFZay8bMhvPHioiMWNFMCmuAEjObaWYpwJXAyr4FzKyk\nz+4FwKYoxnOQ5vYuKvY0qelIRKSPqPUpuHuXmd0EPAMkAg+4+zozuwNY6+4rgZvM7MNAJ1AHfCZa\n8fT3ZlU97poET0Skr2h2NOPuTwFP9Tv29T7bt0bz5x9OWc9I5iLdKYiI9Ih5R3OslFXWc0J+Blnp\nybEORURkxIjLpODuwcyomgRPROQgcZkUquta2dvcoUnwRET6icuk0NOfoOU3RUQOFp9JobKO1OQE\n5k3OjHUoIiIjSlwmhfKqek6emk1SYlz++iIig4q7WrG9q5t12xs1CZ6IyADiLims39FIR3dII5lF\nRAYQd0mhZ/lNjWQWETlU3CWFssp6CrNSmZyVGutQRERGnLhLCuVV9Wo6EhEZRFwlhb3N7VTua1En\ns4jIIOIqKZRXhifB0/QWIiIDiq+kUFVPYoJx0tSsWIciIjIixVVSKKuqY35hJmkpibEORURkRIqb\npNAdcv5S1aBOZhGRw4ibpPBuTTPN7V2aLltE5DDiJimUVdYB6MkjEZHDiJukkJsxjnMXFDBzYkas\nQxERGbGiukbzSHLuggLOXVAQ6zBEREa0uLlTEBGRI1NSEBGRXkoKIiLSS0lBRER6KSmIiEgvJQUR\nEemlpCAiIr2UFEREpJe5e6xjOCpmVgNsO8aPTwT2DmE4w0mxx4ZiH36jNW4Y2bHPcPf8IxUadUnh\neJjZWncvjXUcx0Kxx4ZiH36jNW4Y3bH3UPORiIj0UlIQEZFe8ZYU7ot1AMdBsceGYh9+ozVuGN2x\nA3HWpyAiIocXb3cKIiJyGHGTFMxshZltNLPNZnZbrOOJlJlNM7MXzGy9ma0zs1tjHdPRMLNEMysz\nsydjHcvRMLNsM3vczDaY2TtmdkasY4qUmX0p/G/lbTN7xMxSYx3TYMzsATPbY2Zv9zmWa2bPmdmm\n8PuIXEN3kNj/b/jfzJtm9kszG3VLPcZFUjCzROAe4DxgAfBJM1sQ26gi1gV82d0XAEuBG0dR7AC3\nAu/EOohjcDfwtLvPAxYxSn4HM5sK3AKUuvtCIBG4MrZRHdYPgRX9jt0G/N7dS4Dfh/dHoh9yaOzP\nAQvd/WSgAvjqcAd1vOIiKQCnAZvdfYu7dwCPApfEOKaIuPtOd38jvN1EUDlNjW1UkTGzIuAC4P5Y\nx3I0zCwL+CDwPwDu3uHu9bGN6qgkAWlmlgSkAztiHM+g3P0lYF+/w5cAPwpv/wi4dFiDitBAsbv7\ns+7eFd59FSga9sCOU7wkhalAVZ/9akZJxdqXmRUDS4DXYhtJxP4f8L+BUKwDOUozgRrgwXDT1/1m\nNioW93b37cC/AZXATqDB3Z+NbVRHrcDdd4a3dwGjdR3dzwK/i3UQRyteksKoZ2bjgV8AX3T3xljH\ncyRmdiGwx91fj3UsxyAJeB/wA3dfAuxn5DZhHCTc/n4JQWKbAmSY2adjG9Wx8+DxyFH3iKSZ/SNB\n0+9DsY7laMVLUtgOTOuzXxQ+NiqYWTJBQnjI3Z+IdTwRWgZcbGZbCZrrPmRmP41tSBGrBqrdveeO\n7HGCJDEafBh4z91r3L0TeAJ4f4xjOlq7zawQIPy+J8bxHBUzuwa4ELjKR+Ez//GSFNYAJWY208xS\nCDreVsY4poiYmRG0bb/j7nfFOp5IuftX3b3I3YsJ/nv/wd1HxRWru+8CqsxsbvjQOcD6GIZ0NCqB\npWaWHv63cw6jpJO8j5XAZ8LbnwF+HcNYjoqZrSBoMr3Y3VtiHc+xiIukEO74uQl4huAP5DF3Xxfb\nqCK2DLia4Eq7PPw6P9ZBxYGbgYfM7E1gMfDPMY4nIuG7m8eBN4C3CP7GR+woWzN7BFgNzDWzajP7\nHHAncK6ZbSK487kzljEOZpDY/xPIBJ4L/63eG9Mgj4FGNIuISK+4uFMQEZHIKCmIiEgvJQUREeml\npCAiIr2UFEREpJeSgsgwMrOzRtuMsRJflBRERKSXkoLIAMzs02b25/AApP8KrwvRbGbfDa9V8Hsz\nyw+XXWxmr/aZQz8nfHy2mT1vZn8xszfMbFb468f3WavhofDIY5ERQUlBpB8zmw9cASxz98VAN3AV\nkAGsdfcTgVXAN8If+THwlfAc+m/1Of4QcI+7LyKYf6hn5s8lwBcJ1vY4gWDUusiIkBTrAERGoHOA\nU4A14Yv4NIJJ2ULAz8Jlfgo8EV57IdvdV4WP/wj4uZllAlPd/ZcA7t4GEP6+P7t7dXi/HCgGXo7+\nryVyZEoKIocy4EfuftCqWWb2f/qVO9Y5Ytr7bHejv0MZQdR8JHKo3wMfN7NJ0Ltm8AyCv5ePh8t8\nCnjZ3RuAOjP7QPj41cCq8Cp51WZ2afg7xplZ+rD+FiLHQFcoIv24+3oz+xrwrJklAJ3AjQSL7ZwW\nPreHoN8Bgumd7w1X+luAa8PHrwb+y8zuCH/HJ4bx1xA5JpolVSRCZtbs7uNjHYdINKn5SEREeulO\nQUREeulOQUREeikpiIhILyUFERHppaQgIiK9lBRERKSXkoKIiPT6/5bZXsMwTPyHAAAAAElFTkSu\nQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe7dfed7c18>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(model.history.history['acc'], label='train');\n",
    "plt.plot(model.history.history['val_acc'], label='val');\n",
    "plt.legend();\n",
    "plt.xlabel('epoch');\n",
    "plt.ylabel('accuracy');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEKCAYAAAAW8vJGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xt43GWd9/H3N5PzsWlOLU3b9MypUmw5KB4QxS26HBSh\nHFd0lWcXWcXD48KzrnjYXXku91of3cVFdFlRKYggUt0igiKsa1FaREtLkh5oaQo5NG2TSZrzfJ8/\nfpN2kqbNpGQyk5nP67rmmt95vkmb+zu/+75/923ujoiIyLCsZAcgIiKpRYlBRERGUGIQEZERlBhE\nRGQEJQYRERlBiUFEREZQYhARkRGUGEREZAQlBhERGSE72QGciMrKSq+rq0t2GCIi08qmTZv2uXvV\neMdNy8RQV1fHxo0bkx2GiMi0Yma74zlOVUkiIjKCEoOIiIygxCAiIiNMyzaGsQwMDNDU1ERvb2+y\nQ0mo/Px8amtrycnJSXYoIpKm0iYxNDU1UVJSQl1dHWaW7HASwt1pb2+nqamJBQsWJDscEUlTaVOV\n1NvbS0VFRdomBQAzo6KiIu3vikQkudImMQBpnRSGZcLPKCLJlTZVSSIik8XdGRhyegeH6O0foncg\nEiwPRJcHguWegSH6Ru0bHIqQlWVkmRHKMswgZMF6sJ0Ry6P3BedYdDsx1wq+GJ5+UhlVJXkJ/fmV\nGCbJwYMHWbt2LTfddNOEznvPe97D2rVrmTFjRoIiE8lsA0MRdrd3s62li+2tXexo66KzdzCmcI/Q\nF13uHTxS6Ec82ZGP7VvXr+TPTpuV0M9QYpgkBw8e5Jvf/OZRiWFwcJDs7GP/mtevX5/o0EQyQu/A\nEDvagsJ/+LWttYtd+7oZjCnl58wooLwoh/zsEEV52cwsCpGfk0V+TvQ9O3RkOScU8zrWvpjl7Cyy\nQ1m4O0MRJ+IQcY++YCjiY+4LthPdPvL4iDuRSHDskDsLK4sS/rtUYpgkt956Kzt27GDFihXk5OSQ\nn59PeXk59fX1NDY2ctlll7Fnzx56e3v5xCc+wY033ggcGd6jq6uLiy66iLe85S389re/Zc6cOTz6\n6KMUFBQk+ScTSS1dfYNBod8SZntbF9tbutje1sUr+w/h0fI/y6CuoohF1cW8+9QaltQUs7iqhEXV\nRRTmJr7YMzOyQ9O3PTAtE8MXf7qFra92Tuo1Tz2plNsvPu2Y+++44w5efPFFXnjhBX7961/z3ve+\nlxdffPFwt9J77rmHmTNn0tPTw1lnncXll19ORUXFiGts27aN+++/n29/+9tceeWVPPzww1x33XWT\n+nOITBcHuvvZ3tbFtpYutrWGD98FvNZxpFdebiiLBZVFnD6njMtWzGFJTTFLqkuoqywkLzuUxOin\nt7RMDKng7LPPHvGswTe+8Q0eeeQRAPbs2cO2bduOSgwLFixgxYoVAKxcuZJdu3ZNWbwir1f/YISe\n/iG6+wc51D/EoZj37r6ho/bFbhv5PsSB7n7au/sPX7sgJ8Ti6mLOXVjB4upiFlcXs6S6mHkzC8kO\npVXnypSQ8MRgZquBrwMh4Dvufseo/fOBe4AqYD9wnbs3vZ7PPN43+6lSVHSkHvDXv/41Tz75JBs2\nbKCwsJDzzz9/zGcR8vKO9DQIhUL09PRMSawiY+nuG6Sls5fmzl5aOntp6eyjuWN4uZeOnoFoIR8U\n9AND8bfWhrKMwtwQhbkhinKzKcwLUZiTTXlRLnPKQ5QV5LCwspjFNcUsripmzowCsrKmb9XMdJPQ\nxGBmIeBO4EKgCXjOzNa5+9aYw/4Z+J6732tmFwBfAa5PZFyJUFJSQjgcHnNfR0cH5eXlFBYWUl9f\nz7PPPjvF0YkcMTgUoa2r76iCPjYBtHT0Eu4bPOrc4rxsakrzqCnNZ3ZZQVCw52VTkBuiKDdEQW42\nRbkhCvOyKcwJBQX+GNtyQ1l6JieFJfqO4Wxgu7vvBDCzB4BLgdjEcCrwqejyU8BPEhxTQlRUVHDe\needx+umnU1BQQE1NzeF9q1ev5q677uKUU05h2bJlnHvuuUmMVNKRu3Oof4j90SqY/d197OvqpzVa\n4Dd39NEa7qW5o5d9XX1HdcXMzjKqS/KoKctnSXUxb1lcSU1pPrPK8qgpyaemLJ+a0nyK81T7nAkS\n/a88B9gTs94EnDPqmD8C7yeobnofUGJmFe7enuDYJt3atWvH3J6Xl8djjz025r7hdoTKykpefPHF\nw9s/85nPTHp8Mn24O939Q+zv6qe9u4/2rv4RhX57d//hbcH2PnoHImNeq7wwh5rSoGA/ZVZp8I2/\nLJ9Z0W01pflUFOWqqkYOS4X0/xng38zsBuAZYC8wNPogM7sRuBFg3rx5UxmfyKSJRJyWcC9NB3rY\ns/8QzZ29RxX6+7v62dfdT//g2AV9fk4WFUV5VBTnUlGcy5KaYiqKcqkozmNmUS4VRbnMLMqlsjiP\nqpI88nPUO0cmJtGJYS8wN2a9NrrtMHd/leCOATMrBi5394OjL+TudwN3A6xatSpFn0mUTOfu7O/u\nZ0+04N9z4NDhJNB0oIe9B3roHxpZ4BfkhIJCviiXquI8ltWUUlkcFO4zi3Kj+/IOL09FP3zJbIn+\nH/YcsMTMFhAkhKuAa2IPMLNKYL+7R4DbCHooiaSscO8Ae/b3sOfAocMF/uH3A4c41D/yhre8MIe5\nMws5dXYp7z6thrnlhdSWFzB3ZiGzy/JV0EvKSej/SHcfNLObgccJuqve4+5bzOxLwEZ3XwecD3zF\nzJygKuljiYxJJB5DEaehOcwLew6yq7378Lf/Pft76OgZGHFsUW6IuTMLmTuzkDcvrmBueWF0vYDa\n8kI12Mq0k/D/se6+Hlg/atvnY5YfAh5KdBwix9PVN8gLrxxk4+79bNp9gD+8cpCuaHfN3Oys4Bt+\neSFn1M4ICv3yoOCfW17IjMIcdb2UtKKvMpJx3J29B3vYtPsAG3cdYNPuA9Q3dxJxMINlNSVcuuIk\nVtWV88Z55cwtL1SPHckoSgxJUlxcTFdXV7LDyAgDQxG2vtrJpt1BEti4ez8tnX0AFOaGOHPeDG6+\nYAmr5pezYt4MSvM1n7ZkNiUGSTsdhwZ4/pUgAWzcdYA/Nh083Md/zowCzllQcfhu4ORZJRprR2QU\nJYZJcuuttzJ37lw+9rGg7fwLX/gC2dnZPPXUUxw4cICBgQH+4R/+gUsvvTTJkaaXSMTZvf8QG3ft\nD5LBrgNsaw3uxEJZxmknlXL12fNYOb+clfPLmV2mYcxFxpOeieGxW6F58+Rec9ZyuOiOY+5es2YN\nt9xyy+HE8OCDD/L444/z8Y9/nNLSUvbt28e5557LJZdcoobKE9DTP8TOfV3saOtmZ9uR951t3fQM\nBN1DS/OzWTm/nEtXnMTK+TM5Y26ZuoKKnAD91UySM888k9bWVl599VXa2tooLy9n1qxZfPKTn+SZ\nZ54hKyuLvXv30tLSwqxZiZ2Wb7pyd1o6+6IFf1D474gW/nsPHhlp1gxqywtYVBUMw7ykupiV88tZ\nVFWsRmKRSZCeieE43+wT6YorruChhx6iubmZNWvWcN9999HW1samTZvIycmhrq5uzOG2M03vwBC7\n2rvZ2dbNjugcvDv3BcvdMQ+HFeWGWFhVzFl15VxVNZeFVcUsqi6irqJIwzyIJFB6JoYkWbNmDR/9\n6EfZt28fTz/9NA8++CDV1dXk5OTw1FNPsXv37mSHOOXawn0809jG1tc6D3/733PgyBSMEDQIL6wq\n4opVc1lUVRQkgKpiakrzVO0mkgRKDJPotNNOIxwOM2fOHGbPns21117LxRdfzPLly1m1ahUnn3xy\nskNMuEjE2by3g1/Vt/Lrhlb+2NQBBAO/Laws5g21ZbzvzDksqi5mYWURC6umZg5eEYmf/iIn2ebN\nRxq9Kysr2bBhw5jHpdMzDB09A/z3tjaeqm/j6cZW9nX1YwZnzp3BZ969lPOXVXPq7FLV/8v0FYmA\nD4FHIBJ996Hoso/cZwbZ+ZCdF7xnTb9qTyUGmTB3Z1trF7+qb+Wp+lY27j7AUMQpK8jh7UuruODk\nat62tIqZRbnJDlUykTv0dkDnXujYC51N0fe90NEE4ddgoCemkB8u1COjCvyYZPB6ZGUfSRShvCMJ\n47jvx9m38B1QPn9yflfHoMQgcenpH2LDzn3RZNB2uJfQKbNL+au3L+Qdy6pZMXeGHhaTxOvrOlLI\nd+6FzlePLA8ngP5Rd+QWgpLZUDYn6HqeUwRZWWBZwb6s0JFls+h6dFvsvthzDu8bdY5HYKgfBnth\nsG+M9zG29XYE70Nj7I+MmmL1qrVKDBPh7mnfWOk+dVNR7Nl/KEgEDa1s2NFO32CEwtwQ5y2u5OYL\nFnP+sio9MJaO3KEvDD0HglfvwejywZHrkaGgMMzKCb4Vh3Ki69kx27Kj6zGvUM4xtsWcGxkMCvwR\n3/aj3/57O0YFbFBcDaVzoGopLLogSAClc6CsNngvrglimY6GBmMSRh/klyX8I6fpb+po+fn5tLe3\nU1FRkbbJwd1pb28nPz8/IdfvH4ywcfd+nqpv5Vf1rexo6wZgQWUR154zn3ecXMXZC2aSlz396kzT\nljtHungNL/uReu/ejqML9OH1Yxb6B4NzjyWUC/kzgvfIQFCIRwaDAmx4+XjnT1RhRVC4l8+H+W8a\nWeCXzYGSkyA7jastQ9EEm1s0ZR+ZNomhtraWpqYm2trakh1KQuXn51NbWztp13N3Nuxo5we/280z\njfvo6hskN5TFOQtnRpNBNQsqp+4/ZNJFhuDgK7CvMXi174CBQ8H24QIvMjTO+mBQXz1ifehIffaI\n9QhHFehHvTP2vtfFgm+eBTOgoDx4zZgXXY5uy58x9npOQVB1ctzfY+TIzzoUkzwOJ5GBI7+zw8ll\n6MixlgWlJwWvHN2VTrW0SQw5OTksWLAg2WFMG+7Or+pb+bentvOHVw5SWZzLxWfM5h3LqjlvcSVF\n6T65TP8haN9+JAHsa4S2xmDbUN+R4wpmQl7JkWoOG67uyDpSDTJcR52dO/Yxh9dDR+qhh9ctC7Aj\nBa1ZzHrM9qO2jfE+4nyCz8kvHbuQzysL4kuUrCwgK6giUsE+7aT5X7+MNhRxHnvxNe58agcvvdbJ\nnBkFfPmy07liZW36PU3sDt37Rhb+w6+DezjyrduCaorKZbDoHVC1DCqXBq/Cmcn8CUSSQokhQwwM\nRfjJH/by70/vYGdbNwurivjnK87g0hUnkTPdexJFhuDg7uAb/75G2NcA+7YFyz0HjhyXXQCVi6H2\nbFhxHVQuCZLAzEWQk5h2G5HpSIkhzfUODPGjjXu46+md7D3YwymzS7nzmjey+vRZhEY/cOYerfcd\nCOp6hwaOLB+1bTDokje8HBkI1ofriIei64frk8e45ojrD446fwLX6gsH+4cVVQXf9k+9NLgLqFwa\n9FYprU1s9YlImlBiSFPdfYPc97vdfPu/X6Yt3Mcb583gy5edxjuWVQe9trrbYc+z8Er01bwZBnvG\nv/BkyMoJ6p5DOWMvj96WWwihGdHt0e6Modwjy3nFULEkWv2zRNU/Iq+TEkOa6Tg0wHd/u4v//O3L\nHDw0wHmLK/j6mjN4U3kn9sqTsG4D7PldUM0CQcE6542w8oaggA3lxvQrH6cAH1GYZ0cL65g+7aHc\nowvzrND4PVpEJKmUGNLEvq4+vvPfL/ODZ3fT29fLhxZ28sHaZmrDf4RHnoXuaDfe/Bkw9xw442qY\n9yY46UzVr4vICEoM09yrB3u496nNbH/+V6zwl3i4ZDdLcurJerUHXgVmzIdF74R55wSJoHKZ6tlF\n5LiUGKajjibatjzNto1PUt6+ic/yCqGQ45aFzVgO8z4I886FuedC6exkRysi04wSw3TQ2wEv/Qx2\nPsXAy78lp2svVUCh59FcupzuUz9A6dK3YrWrgoexREReByWGVDXYB9t+AZt/BA0/h6E+DoZm8pv+\nJfzJLmT28vN577svZFFZcbIjFZE0o8SQSiJDsOs3QTLYug76OqCoipalV3HLlsVsZRk3vG0BN51X\nx4zCNB40TESSSokh2dzhtRdg80Pw4sPBJCK5xXDKxbD8A2zw5Xz4e39g9ox8Hv/IucwqUw8iEUks\nJYZkad8RJIPNP4L2bUE//yXvhuUfgKWrIbeQ/97Wxke/t5G55YXc99FzqC5RUhCRxFNimErhFtjy\n4yAZ7N0EGNS9Bd58M5xyyYgndp+qb+V//WATCyuLuO8j51BRnJe8uEUkoygxJFpvJ7z00yAZvPx0\nMP7+rDfAhV+G0y8PJhoZ5RdbmvnY2udZNquE73/4HMo1d7KITCElhkQY7INtT8DmBw/3KKK8Dt76\naVh+RTCi5zGs3/waH7//D5w2p4zvffhsygpypi5uERGmIDGY2Wrg60AI+I673zFq/zzgXmBG9Jhb\n3X19ouNKiN0b4IX7RvQoYuUN8IYrYc7KcccIevSFvXzqwT+yYu4MvvuhsyjJV1IQkamX0MRgZiHg\nTuBCoAl4zszWufvWmMM+Bzzo7v9uZqcC64G6RMaVEC/cDz/5qxE9ilhwftwTkD+0qYnPPvRHzqqb\nyT03nJX+M6iJSMpKdOlzNrDd3XcCmNkDwKVAbGJwoDS6XEYwws/0snsD/PTjsODtcPUDwTDRE/DA\n71/htkc2c96iSr79F6soyE2zmdREZFpJdGKYA+yJWW8Czhl1zBeAX5jZ3wBFwLsSHNPkOrAbfnht\nMJH6lfdOOCl8f8Mu/v7RLbx9aRXfun5l+k2vKSLTTioMs3k18F13rwXeA3zfzI6Ky8xuNLONZrax\nra1tyoMcU28nrF0TPLF8zYPBJOsT8B+/eZm/f3QL7zqlmrv/QklBRFJDohPDXmBuzHptdFusvwQe\nBHD3DUA+UDn6Qu5+t7uvcvdVVVVVCQp3AiJD8PBfBg+nXfk9qFg0odPvenoHX/7ZVi46fRbfvHYl\nedlKCiKSGhKdGJ4DlpjZAjPLBa4C1o065hXgnQBmdgpBYkiRW4Lj+MXfB4PcveersPDtEzr1X3+5\njTseq+fiM07iX68+k9zsVLhxExEJJLSNwd0Hzexm4HGCrqj3uPsWM/sSsNHd1wGfBr5tZp8kaIi+\nwd09kXG9bpu+C8/eCef8Naz6cNynuTtfe6KRb/xqO+8/cw5fveIMQlma5lJEUkvC+0RGn0lYP2rb\n52OWtwLnJTqOSfPyM/Bfn4bFF8Kf/WPcp7k7//fnDdz19A7WrJrLP71/uZKCiKQkdZafiPYd8MPr\noWIxfOCeYGL7OLg7X/7ZS9zzPy9z7Tnz+PKlp5OlpCAiKUqJIV49B2DtlUEyuOaHkF86/jlAJOLc\nvm4L3392Nze8uY7bLz4VG+cJaBGRZFJiiMfQADz4weCZhQ/+NBj3KA6RiPN3P9nM/b/fw41vW8ht\nF52spCAiKU+JYTzu8Nhng5FRL/t3mP+muE4bijiffehPPPx8Eze/YzGffvdSJQURmRaUGMbz+7th\n4z1w3i2w4pq4ThkcivDpH/2RR194lU++aymfeNeSBAcpIjJ5lBiOZ9uT8PNb4eQ/h3feHtcpA0MR\nbnngBf5r82t8dvUybjp/cYKDFBGZXEoMx9JaDw99CKpPg/d9C7LGfwitb3CIm9f+gSe2tvC5957C\nR966cAoCFRGZXEoMY+luh/vXQE4BXPMA5BWPe0ok4tz0g+f5ZX0rX7zkND745rrExykikgBKDKMN\n9sEPr4NwM9ywHspq4zpty6ud/LK+lc+uXqakICLTmhJDLHf42afgld/C5f8BtSvjPrW+uROA1afN\nSlR0IiJTIq7R28xseaIDSQm//Qa88AN4+63BDGwT0NgSJi87i/kVRQkKTkRkasQ7rOc3zez3ZnaT\nmZUlNKJkqV8PT9wOp70P3v63Ez+9OcySmmKNfyQi015cicHd3wpcSzC3wiYzW2tmFyY0sqnUvBke\n/gicdGbwEFscPZBGa2wJs6wmvmEyRERSWdwloLtvAz4H/C3wduAbZlZvZu9PVHBTItwCa6+C/DK4\n+v6gJ9IEHTzUT0tnH8tmjd97SUQk1cXbxvAGM/sa8BJwAXCxu58SXf5aAuNLrIEeeOAa6NkfdEst\nObGG44bmMADLZumOQUSmv3h7Jf0r8B3g/7h7z/BGd3/VzD6XkMgSzR0evRn2boQ1P4DZZ5zwpRpa\noomhpmSyohMRSZp4E8N7gR53HwIwsywg390Pufv3ExZdIj3zVXjxIXjn5+GUi1/XpRqaw5QV5FBT\nmjdJwYmIJE+8bQxPArGV74XRbdPTiz+Gp/4Rzrga3vKp1325huYwy2pKNHqqiKSFeBNDvrt3Da9E\nlwsTE1KC7d0EP/lrmHsuXPx1eJ2FubvT0BJm2SxVI4lIeog3MXSb2RuHV8xsJdBznONTU8deuP8a\nKK4O2hWyX3/Vz2sdvYR7B1mqxCAiaSLeNoZbgB+Z2auAAbOANQmLKlF+/RXo74brH4Hiqkm55HDD\n88lKDCKSJuJKDO7+nJmdDCyLbmpw94HEhZUg7/kqrPoQ1Jw6aZcc7qq6tFqJQUTSw0QG0VsGnArk\nA280M9z9e4kJK0FyCmBO/APjxaOxOczssnzKCnMm9boiIskSV2Iws9uB8wkSw3rgIuA3wPRKDAlQ\n3xxmqZ5fEJE0Em/j8weAdwLN7v4h4AwgPQfTm4DBoQjb27rUviAiaSXexNDj7hFg0MxKgVaCAfUy\n2q72Q/QPRnTHICJpJd42ho1mNgP4NrAJ6AI2JCyqaaJxeCgM3TGISBoZNzFY8DjvV9z9IHCXmf0c\nKHX3PyU8uhRX3xwmy2BxtUZVFZH0MW5icHc3s/XA8uj6rkQHNV00NoepqywiPyeU7FBERCZNvG0M\nz5vZWQmNZBpqaAlrRFURSTvxJoZzgA1mtsPM/mRmm80so6uSegeG2NXerfYFEUk78TY+/1lCo5iG\ntrV04a45GEQk/cR7x+DHeI3LzFabWYOZbTezW8fY/zUzeyH6ajSzg/EGn0wN6pEkImkq3juG/yJI\nBEYwJMYCoAE47XgnmVkIuBO4EGgCnjOzde6+dfgYd/9kzPF/A5w5kR8gWRqaO8nNzmJ+RVGyQxER\nmVTxDqK3PHY9OgT3TXGcejaw3d13Rs97ALgU2HqM468Gbo8npmRraOliSXUxoSxNziMi6SXeqqQR\n3P15ggbp8cwB9sSsN0W3HcXM5hPcifzqRGKaag3NnapGEpG0FO8gerHzX2YBbwReneRYrgIeGp5X\neowYbgRuBJg3b94kf/TEHDzUT0tnnxqeRSQtxXvHUBLzyiNoc7g0jvP2MnJMpdrotrFcBdx/rAu5\n+93uvsrdV1VVTc4kOydqeA4G3TGISDqKt43hiyd4/eeAJWa2gCAhXAVcM/qg6CRA5UyT8Zc0RpKI\npLO47hjM7InoIHrD6+Vm9vh457n7IHAz8DjwEvCgu28xsy+Z2SUxh14FPODucXWBTbb65jCl+dnM\nKs1PdigiIpMu3u6qVdFB9ABw9wNmVh3Pie6+nmByn9htnx+1/oU440gJjS1hls0qIRhfUEQkvcTb\nxjBkZodbfKM9iKbFt/vJ5u7UN4dVjSQiaSveO4a/A35jZk8TPOT2VqI9hDJNc2cv4d5B9UgSkbQV\nb+Pzz6MPtZ0b3XSLu+9LXFipq/5wj6TSJEciIpIY8TY+vw8YcPefufvPCKb4vCyxoaWmxuHEoDsG\nEUlT8bYx3O7uHcMr0YboaTF0xWRraA4zqzSfssKcZIciIpIQ8SaGsY6Lt30irTS0hFmqhmcRSWPx\nJoaNZvYvZrYo+voXYFMiA0tFg0MRtrV2cbISg4iksXgTw98A/cAPo68+4GOJCipV7d5/iP7BCEvV\nviAiaSzeXkndwFGT7GSa4TGSdMcgIuks3tFVq4DPEkzMc3gcCHe/IEFxpaSG5jBZBouri5MdiohI\nwsRblXQfUE8wX8IXgV0EA+RllIbmMHUVReTnhJIdiohIwsSbGCrc/T8InmV42t0/DGTU3QIEYySp\nfUFE0l28iWEg+v6amb3XzM4EZiYoppTUOzDErvZujZEkImkv3mcR/sHMyoBPA/8KlAKfTFhUKWh7\naxcR1xwMIpL+4u2V9LPoYgfwjtH7zew2d//KZAaWauo1a5uIZIh4q5LGc8UkXSdlNbaEyc3OYv7M\nwmSHIiKSUJOVGNJ+xpr65jBLqovJDk3Wr0xEJDVNVimX9pP2NDaHNaKqiGQE3THEoePQAM2dvWpf\nEJGMMFmJ4UeTdJ2U1NASNDxrVFURyQTHTQxmVjlq/Toz+4aZ3Whmh+8S3P2fEhVgKmho7gQ0RpKI\nZIbx7hh+MbxgZp8DricYbvtC4F8SGFdKaWgJU5KfzazS/PEPFhGZ5sZ7jiG27eD9wFvdvdvM1gLP\nJy6s1NLQHObkWSXE3CSJiKSt8e4YCszsTDNbCYSiw2/j7gPAUMKjSwHuTkOzxkgSkcwx3h3Daxyp\nMtpvZrPd/TUzqwAGExtaamju7KWzd1DtCyKSMY6bGNz9qOEvog4Ab5v8cFLP8OQ8umMQkUwR7yB6\nmNn7gbcQPMz2G3d/JGFRpZAGjZEkIhkmrucYzOybwF8Bm4EXgf9lZncmMrBU0dASpqY0jxmFuckO\nRURkSsR7x3ABcIq7O4CZ3QtsSVhUKaShOcyyWaXJDkNEZMrE++TzdmBezPrc6La0NhRxtrV2saxG\nczyLSOaI946hBHjJzH4fXT8L2Ghm6wDc/ZJEBJdsu9q76R+M6I5BRDJKvInh8wmNIkU1Djc8q0eS\niGSQuKqS3P1poJ7gzqEEeMndnx5+He9cM1ttZg1mtt3Mbj3GMVea2VYz2xJ9qjol1DeHMYMlqkoS\nkQwSb6+kK4HfE8zUdiXwOzP7QBznhYA7gYuAU4GrzezUUccsAW4DznP304BbJvQTJFBjS5i6iiLy\nc0LJDkVEZMrEW5X0d8BZ7t4KYGZVwJPAQ+Ocdzaw3d13Rs97ALgU2BpzzEeBO939AMDwZ6SCBk3O\nIyIZKN5eSVmjCuz2OM+dA+yJWW+Kbou1FFhqZv9jZs+a2eo4Y0qo3oEhdrV3aw4GEck48d4xPGZm\njwP3R9czkj7BAAALY0lEQVTXAOsnMYYlwPlALfCMmS1394OxB5nZjcCNAPPmzRt9jUm3vbWLiGsO\nBhHJPPHeMTjwLeAN0dfdcZ63l+CZh2G10W2xmoB17j7g7i8DjQSJYmQA7ne7+yp3X1VVVRXnx584\njZEkIpkq3sRwobv/2N0/FX09QtCgPJ7ngCVmtsDMcoGrgHWjjvkJwd3C8IxxS4GdccaVMA0tYXKz\ns6irKEx2KCIiU+q4VUlm9tfATcBCM/tTzK4S4H/Gu7i7D5rZzcDjQAi4x923mNmXgI3uvi66791m\ntpVgjof/7e7tJ/bjTJ6G5jCLq4rJDk3WtNgiItPDeG0Ma4HHgK8Asc8ghN19fzwf4O7rGdUe4e6f\nj1l24FPRV8poaA7z5kUVyQ5DRGTKjTcfQwfQAVw9NeGkho5DAzR39qpHkohkJNWTjKGhRXMwiEjm\nUmIYw+HEoB5JIpKBlBjG0NDcSUl+NrPL8pMdiojIlFNiGENjcxfLakows2SHIiIy5ZQYRnF36ps7\n1b4gIhlLiWGUls4+OnsHlRhEJGMpMYxS39wJqOFZRDKXEsMojeqqKiIZTolhlPrmMDWlecwozE12\nKCIiSaHEMEpjS1gjqopIRlNiiDEUcba1dGkOBhHJaEoMMXa3d9M3GNEdg4hkNCWGGMOT85w8qzTJ\nkYiIJI8SQ4yGljBmsLi6ONmhiIgkjRJDjIbmMHUVRRTkhpIdiohI0igxxGhoCbO0RncLIpLZlBii\negeG2LWvm2VqXxCRDKfEELW9tYuIaygMERElhqjhHkkaCkNEMp0SQ1RjS5jc7CzqKgqTHYqISFIp\nMUTVN4dZXFVMdki/EhHJbCoFoxpbwqpGEhFBiQGAjkMDvNbRq8QgIoISAwCNrdGGZ/VIEhFRYoCg\nfQHUI0lEBJQYAGhsDlOSn83ssvxkhyIiknRKDATPMCyrKcHMkh2KiEjSZXxicPdgjCRVI4mIAEoM\ntHT20dEzoFnbRESiMj4xNLQEDc+atU1EJKDE0NwJqKuqiMiwhCcGM1ttZg1mtt3Mbh1j/w1m1mZm\nL0RfH0l0TLEamruoLsmjvCh3Kj9WRCRlZSfy4mYWAu4ELgSagOfMbJ27bx116A/d/eZExnIsDS2d\nen5BRCRGou8Yzga2u/tOd+8HHgAuTfBnxm0o4mxr6VI1kohIjEQnhjnAnpj1pui20S43sz+Z2UNm\nNjfBMR22u72bvsGI7hhERGKkQuPzT4E6d38D8ARw71gHmdmNZrbRzDa2tbVNygc3tmgoDBGR0RKd\nGPYCsXcAtdFth7l7u7v3RVe/A6wc60Lufre7r3L3VVVVVZMSXH1zGDNYUq3EICIyLNGJ4TlgiZkt\nMLNc4CpgXewBZjY7ZvUS4KUEx3RYY0uY+TMLKcgNTdVHioikvIT2SnL3QTO7GXgcCAH3uPsWM/sS\nsNHd1wEfN7NLgEFgP3BDImOKVd+syXlEREZLaGIAcPf1wPpR2z4fs3wbcFui4xitd2CIXfu6+fPl\ns8c/WEQkg6RC43NSbG/tIuKwbFZpskMREUkpGZsYjvRIKk5yJCIiqSVjE0NDc5jcUBZ1FUXJDkVE\nJKVkbmJoCbOoupjsUMb+CkRExpSxpWJDc1hzMIiIjCEjE0NHzwCvdfRqDgYRkTFkZGIYbnjWHYOI\nyNEyMjE0NEdnbVNiEBE5SsYmhpK8bE4qy092KCIiKSczE0NLmKWzSjCzZIciIpJyMi4xuDsNGiNJ\nROSYMi4xtIb76OgZ0KxtIiLHkHGJob5Zk/OIiBxPxiWGxuHEoDsGEZExZVxiqG8OU12SR3lRbrJD\nERFJSRmXGBpb1PAsInI8GZUYhiIeJAZVI4mIHFNGJYZX9h+ibzCiJ55FRI4joxJDQ3MnoDGSRESO\nJ8MSQxdmsKRaiUFE5FgyKjHUlhfwvjPnUJAbSnYoIiIpKzvZAUyly1fWcvnK2mSHISKS0jLqjkFE\nRManxCAiIiMoMYiIyAhKDCIiMoISg4iIjKDEICIiIygxiIjICEoMIiIygrl7smOYMDNrA3af4OmV\nwL5JDGcqKfbkUOxTb7rGDakd+3x3rxrvoGmZGF4PM9vo7quSHceJUOzJodin3nSNG6Z37MNUlSQi\nIiMoMYiIyAiZmBjuTnYAr4NiTw7FPvWma9wwvWMHMrCNQUREji8T7xhEROQ4MioxmNlqM2sws+1m\ndmuy44mXmc01s6fMbKuZbTGzTyQ7pokws5CZ/cHMfpbsWCbCzGaY2UNmVm9mL5nZm5IdU7zM7JPR\n/ysvmtn9Zpaf7JiOxczuMbNWM3sxZttMM3vCzLZF38uTGeOxHCP2r0b/z/zJzB4xsxnJjPFEZExi\nMLMQcCdwEXAqcLWZnZrcqOI2CHza3U8FzgU+No1iB/gE8FKygzgBXwd+7u4nA2cwTX4GM5sDfBxY\n5e6nAyHgquRGdVzfBVaP2nYr8Et3XwL8Mrqeir7L0bE/AZzu7m8AGoHbpjqo1ytjEgNwNrDd3Xe6\nez/wAHBpkmOKi7u/5u7PR5fDBAXUnORGFR8zqwXeC3wn2bFMhJmVAW8D/gPA3fvd/WByo5qQbKDA\nzLKBQuDVJMdzTO7+DLB/1OZLgXujy/cCl01pUHEaK3Z3/4W7D0ZXnwWm3bSRmZQY5gB7YtabmCaF\naywzqwPOBH6X3Eji9v+AzwKRZAcyQQuANuA/o9Vg3zGzomQHFQ933wv8M/AK8BrQ4e6/SG5UE1bj\n7q9Fl5uBmmQG8zp8GHgs2UFMVCYlhmnPzIqBh4Fb3L0z2fGMx8z+HGh1903JjuUEZANvBP7d3c8E\nuknd6owRovXxlxIkt5OAIjO7LrlRnTgPuk5Ou+6TZvZ3BNXA9yU7lonKpMSwF5gbs14b3TYtmFkO\nQVK4z91/nOx44nQecImZ7SKourvAzH6Q3JDi1gQ0ufvwndlDBIliOngX8LK7t7n7APBj4M1Jjmmi\nWsxsNkD0vTXJ8UyImd0A/DlwrU/DZwIyKTE8BywxswVmlkvQGLcuyTHFxcyMoK77JXf/l2THEy93\nv83da929juD3/St3nxbfXN29GdhjZsuim94JbE1iSBPxCnCumRVG/++8k2nScB5jHfDB6PIHgUeT\nGMuEmNlqgurTS9z9ULLjOREZkxiijUE3A48T/JE86O5bkhtV3M4Drif4xv1C9PWeZAeVAf4GuM/M\n/gSsAP4pyfHEJXqX8xDwPLCZ4O88ZZ/GNbP7gQ3AMjNrMrO/BO4ALjSzbQR3QHckM8ZjOUbs/waU\nAE9E/1bvSmqQJ0BPPouIyAgZc8cgIiLxUWIQEZERlBhERGQEJQYRERlBiUFEREZQYhCZYmZ2/nQb\naVYyixKDiIiMoMQgcgxmdp2Z/T76kNK3ovNKdJnZ16JzHfzSzKqix64ws2djxuAvj25fbGZPmtkf\nzex5M1sUvXxxzFwP90WfUBZJCUoMImMws1OANcB57r4CGAKuBYqAje5+GvA0cHv0lO8Bfxsdg39z\nzPb7gDvd/QyC8YqGRww9E7iFYG6QhQRPt4ukhOxkByCSot4JrASei36ZLyAYyC0C/DB6zA+AH0fn\nbpjh7k9Ht98L/MjMSoA57v4IgLv3AkSv93t3b4quvwDUAb9J/I8lMj4lBpGxGXCvu4+YfcvM/n7U\ncSc6pkxfzPIQ+luUFKKqJJGx/RL4gJlVw+E5iOcT/M18IHrMNcBv3L0DOGBmb41uvx54OjrbXpOZ\nXRa9Rp6ZFU7pTyFyAvQtRWQM7r7VzD4H/MLMsoAB4GMEE/acHd3XStAOAcHQ0HdFC/6dwIei268H\nvmVmX4pe44op/DFETohGVxWZADPrcvfiZMchkkiqShIRkRF0xyAiIiPojkFEREZQYhARkRGUGERE\nZAQlBhERGUGJQURERlBiEBGREf4/LQh7dywQRgkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe7dff484e0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(model.history.history['top_k_categorical_accuracy'], label='train');\n",
    "plt.plot(model.history.history['val_top_k_categorical_accuracy'], label='val');\n",
    "plt.legend();\n",
    "plt.xlabel('epoch');\n",
    "plt.ylabel('top5_accuracy');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1.4546428885310889, 0.646484375, 0.859375]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate_generator(val_generator, 80)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}