7.PRETRAIN_METHOD/7.2.3.bert_finetune_NER.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import json\n",
    "import copy\n",
    "\n",
    "import tensorflow as tf\n",
    "from transformers import *\n",
    "\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "\n",
    "\n",
    "from seqeval.metrics import precision_score, recall_score, f1_score, classification_report\n",
    "\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 시각화\n",
    "\n",
    "def plot_graphs(history, string):\n",
    "    plt.plot(history.history[string])\n",
    "    plt.xlabel(\"Epochs\")\n",
    "    plt.ylabel(string)\n",
    "    plt.legend([string])\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#random seed 고정\n",
    "tf.random.set_seed(1234)\n",
    "np.random.seed(1234)\n",
    "\n",
    "BATCH_SIZE = 32\n",
    "NUM_EPOCHS = 3\n",
    "MAX_LEN = 111 # EDA에서 추출된 Max Length\n",
    "DATA_IN_PATH = 'data_in/KOR'\n",
    "DATA_OUT_PATH = \"data_out/KOR\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "개체명 인식 학습 데이터 개수: 81000\n",
      "개체명 인식 테스트 데이터 개수: 9000\n"
     ]
    }
   ],
   "source": [
    "# 데이터 전처리 준비\n",
    "DATA_TRAIN_PATH = os.path.join(DATA_IN_PATH, \"NER\", \"train.tsv\")\n",
    "DATA_LABEL_PATH = os.path.join(DATA_IN_PATH, \"NER\", \"label.txt\")\n",
    "DATA_TEST_PATH = os.path.join(DATA_IN_PATH, \"NER\", \"test.tsv\")\n",
    "\n",
    "def read_file(input_path):\n",
    "    \"\"\"Read tsv file, and return words and label as list\"\"\"\n",
    "    with open(input_path, \"r\", encoding=\"utf-8\") as f:\n",
    "        sentences = []\n",
    "        labels = []\n",
    "        for line in f:\n",
    "            split_line = line.strip().split(\"\\t\")\n",
    "            sentences.append(split_line[0])\n",
    "            labels.append(split_line[1])\n",
    "        return sentences, labels\n",
    "\n",
    "train_sentences, train_labels = read_file(DATA_TRAIN_PATH)\n",
    "\n",
    "train_ner_dict = {\"sentence\": train_sentences, \"label\": train_labels}\n",
    "train_ner_df = pd.DataFrame(train_ner_dict)\n",
    "\n",
    "test_sentences, test_labels = read_file(DATA_TEST_PATH)\n",
    "test_ner_dict = {\"sentence\": test_sentences, \"label\": test_labels}\n",
    "test_ner_df = pd.DataFrame(test_ner_dict)\n",
    "\n",
    "print(\"개체명 인식 학습 데이터 개수: {}\".format(len(train_ner_df)))\n",
    "print(\"개체명 인식 테스트 데이터 개수: {}\".format(len(test_ner_df)))\n",
    "\n",
    "# 개체명 인식 학습 데이터 개수: 81000\n",
    "# 개체명 인식 테스트 데이터 개수: 9000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "개체명 인식 레이블 개수: 30\n"
     ]
    }
   ],
   "source": [
    "# Label 불러오기\n",
    "\n",
    "def get_labels(label_path):\n",
    "    return [label.strip() for label in open(os.path.join(label_path), 'r', encoding='utf-8')]\n",
    "\n",
    "ner_labels = get_labels(DATA_LABEL_PATH)\n",
    "\n",
    "print(\"개체명 인식 레이블 개수: {}\".format(len(ner_labels)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 버트 토크나이저 설정\n",
    "\n",
    "tokenizer = BertTokenizer.from_pretrained(\"bert-base-multilingual-cased\", cache_dir='bert_ckpt')\n",
    "\n",
    "pad_token_id = tokenizer.pad_token_id # 0\n",
    "pad_token_label_id = 0\n",
    "cls_token_label_id = 0\n",
    "sep_token_label_id = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bert_tokenizer(sent, MAX_LEN):\n",
    "    \n",
    "    encoded_dict = tokenizer.encode_plus(\n",
    "        text = sent,\n",
    "        truncation=True,\n",
    "        add_special_tokens = True, #'[CLS]'와 '[SEP]' 추가\n",
    "        max_length = MAX_LEN,           # 문장 패딩 및 자르기 진행\n",
    "        pad_to_max_length = True,\n",
    "        return_attention_mask = True   # 어탠션 마스크 생성\n",
    "    )\n",
    "    \n",
    "    input_id = encoded_dict['input_ids']\n",
    "    attention_mask = encoded_dict['attention_mask'] \n",
    "    token_type_id = encoded_dict['token_type_ids']\n",
    "    \n",
    "    return input_id, attention_mask, token_type_id\n",
    "\n",
    "def convert_label(words, labels_idx, ner_begin_label, max_seq_len):\n",
    "            \n",
    "    tokens = []\n",
    "    label_ids = []\n",
    "\n",
    "    for word, slot_label in zip(words, labels_idx):\n",
    "\n",
    "        word_tokens = tokenizer.tokenize(word)\n",
    "        if not word_tokens:\n",
    "            word_tokens = [unk_token]\n",
    "        tokens.extend(word_tokens)\n",
    "        \n",
    "        # 슬롯 레이블 값이 Begin이면 I로 추가\n",
    "        if int(slot_label) in ner_begin_label:\n",
    "            label_ids.extend([int(slot_label)] + [int(slot_label) + 1] * (len(word_tokens) - 1))\n",
    "        else:\n",
    "            label_ids.extend([int(slot_label)] * len(word_tokens))\n",
    "  \n",
    "    # [CLS] and [SEP] 설정\n",
    "    special_tokens_count = 2\n",
    "    if len(label_ids) > max_seq_len - special_tokens_count:\n",
    "        label_ids = label_ids[: (max_seq_len - special_tokens_count)]\n",
    "\n",
    "    # [SEP] 토큰 추가\n",
    "    label_ids += [sep_token_label_id]\n",
    "\n",
    "    # [CLS] 토큰 추가\n",
    "    label_ids = [cls_token_label_id] + label_ids\n",
    "    \n",
    "    padding_length = max_seq_len - len(label_ids)\n",
    "    label_ids = label_ids + ([pad_token_label_id] * padding_length)\n",
    "    \n",
    "    return label_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 테스트용\n",
    "ner_begin_label = [ner_labels.index(begin_label) for begin_label in ner_labels if \"B\" in begin_label]\n",
    "ner_begin_label_string = [ner_labels[label_index] for label_index in ner_begin_label]\n",
    "\n",
    "print(ner_begin_label)\n",
    "print(ner_begin_label_string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "ner_begin_label = [ner_labels.index(begin_label) for begin_label in ner_labels if \"B\" in begin_label]\n",
    "\n",
    "def create_inputs_targets(df):\n",
    "    input_ids = []\n",
    "    attention_masks = []\n",
    "    token_type_ids = []\n",
    "    label_list = []\n",
    "\n",
    "    for i, data in enumerate(df[['sentence', 'label']].values):\n",
    "        sentence, labels = data\n",
    "        words = sentence.split()\n",
    "        labels = labels.split()\n",
    "        labels_idx = []\n",
    "        \n",
    "        for label in labels:\n",
    "            labels_idx.append(ner_labels.index(label) if label in ner_labels else ner_labels.index(\"UNK\"))\n",
    "\n",
    "        assert len(words) == len(labels_idx)\n",
    "\n",
    "        input_id, attention_mask, token_type_id = bert_tokenizer(sentence, MAX_LEN)\n",
    "\n",
    "        convert_label_id = convert_label(words, labels_idx, ner_begin_label, MAX_LEN)\n",
    "\n",
    "        input_ids.append(input_id)\n",
    "        attention_masks.append(attention_mask)\n",
    "        token_type_ids.append(token_type_id)\n",
    "        label_list.append(convert_label_id)\n",
    "\n",
    "    input_ids = np.array(input_ids, dtype=int)\n",
    "    attention_masks = np.array(attention_masks, dtype=int)\n",
    "    token_type_ids = np.array(token_type_ids, dtype=int)\n",
    "    label_list = np.asarray(label_list, dtype=int) #레이블 토크나이징 리스트\n",
    "    inputs = (input_ids, attention_masks, token_type_ids)\n",
    "    \n",
    "    return inputs, label_list\n",
    "\n",
    "train_inputs, train_labels = create_inputs_targets(train_ner_df)\n",
    "test_inputs, test_labels = create_inputs_targets(test_ner_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TFBertNERClassifier(tf.keras.Model):\n",
    "    def __init__(self, model_name, dir_path, num_class):\n",
    "        super(TFBertNERClassifier, self).__init__()\n",
    "\n",
    "        self.bert = TFBertModel.from_pretrained(model_name, cache_dir=dir_path)\n",
    "        self.dropout = tf.keras.layers.Dropout(self.bert.config.hidden_dropout_prob)\n",
    "        self.classifier = tf.keras.layers.Dense(num_class, \n",
    "                                                kernel_initializer=tf.keras.initializers.TruncatedNormal(self.bert.config.initializer_range),\n",
    "                                                name=\"ner_classifier\")\n",
    "\n",
    "    def call(self, inputs, attention_mask=None, token_type_ids=None, training=False):\n",
    "\n",
    "        #outputs 값: # sequence_output, pooled_output, (hidden_states), (attentions)\n",
    "        outputs = self.bert(inputs, attention_mask=attention_mask, token_type_ids=token_type_ids)\n",
    "        sequence_output = outputs[0]\n",
    "                \n",
    "        sequence_output = self.dropout(sequence_output, training=training)\n",
    "        logits = self.classifier(sequence_output)\n",
    "        \n",
    "\n",
    "        return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-multilingual-cased were not used when initializing TFBertModel: ['nsp___cls', 'mlm___cls']\n",
      "- This IS expected if you are initializing TFBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
      "- This IS NOT expected if you are initializing TFBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the weights of TFBertModel were initialized from the model checkpoint at bert-base-multilingual-cased.\n",
      "If your task is similar to the task the model of the ckeckpoint was trained on, you can already use TFBertModel for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "ner_model = TFBertNERClassifier(model_name='bert-base-multilingual-cased',\n",
    "                                  dir_path='bert_ckpt',\n",
    "                                  num_class=len(ner_labels))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_loss(labels, logits):\n",
    "    loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(\n",
    "        from_logits=True, reduction=tf.keras.losses.Reduction.NONE\n",
    "    )\n",
    "\n",
    "    # 0의 레이블 값은 손실 값을 계산할 때 제외\n",
    "    active_loss = tf.reshape(labels, (-1,)) != 0\n",
    "        \n",
    "    reduced_logits = tf.boolean_mask(tf.reshape(logits, (-1, shape_list(logits)[2])), active_loss)\n",
    "        \n",
    "    labels = tf.boolean_mask(tf.reshape(labels, (-1,)), active_loss)\n",
    "    \n",
    "    return loss_fn(labels, reduced_logits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "class F1Metrics(tf.keras.callbacks.Callback):\n",
    "    def __init__(self, x_eval, y_eval):\n",
    "        self.x_eval = x_eval\n",
    "        self.y_eval = y_eval\n",
    "\n",
    "    def compute_f1_pre_rec(self, labels, preds):\n",
    "\n",
    "        return {\n",
    "            \"precision\": precision_score(labels, preds, suffix=True),\n",
    "            \"recall\": recall_score(labels, preds, suffix=True),\n",
    "            \"f1\": f1_score(labels, preds, suffix=True)\n",
    "        }\n",
    "\n",
    "\n",
    "    def show_report(self, labels, preds):\n",
    "        return classification_report(labels, preds, suffix=True)\n",
    "        \n",
    "    def on_epoch_end(self, epoch, logs=None):\n",
    "\n",
    "        results = {}\n",
    "        \n",
    "        pred = self.model.predict(self.x_eval)\n",
    "        label = self.y_eval\n",
    "        pred_argmax = np.argmax(pred, axis = 2)\n",
    "\n",
    "        slot_label_map = {i: label for i, label in enumerate(ner_labels)}\n",
    "\n",
    "        out_label_list = [[] for _ in range(label.shape[0])]\n",
    "        preds_list = [[] for _ in range(label.shape[0])]\n",
    "\n",
    "        for i in range(label.shape[0]):\n",
    "            for j in range(label.shape[1]):\n",
    "                if label[i, j] != 0:\n",
    "                    out_label_list[i].append(slot_label_map[label[i][j]])\n",
    "                    preds_list[i].append(slot_label_map[pred_argmax[i][j]])\n",
    "                    \n",
    "        result = self.compute_f1_pre_rec(out_label_list, preds_list)\n",
    "        results.update(result)\n",
    "\n",
    "        print(\"********\")\n",
    "        print(\"F1 Score\")\n",
    "        for key in sorted(results.keys()):\n",
    "            print(\"{}, {:.4f}\".format(key, results[key]))\n",
    "        print(\"\\n\" + self.show_report(out_label_list, preds_list))\n",
    "        print(\"********\")\n",
    "\n",
    "f1_score_callback = F1Metrics(test_inputs, test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare training: Compile tf.keras model with optimizer, loss and learning rate schedule\n",
    "optimizer = tf.keras.optimizers.Adam(3e-5)\n",
    "# ner_model.compile(optimizer=optimizer, loss=compute_loss, run_eagerly=True)\n",
    "ner_model.compile(optimizer=optimizer, loss=compute_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"tf2_bert_ner\"\n",
    "\n",
    "checkpoint_path = os.path.join(DATA_OUT_PATH, model_name, 'weights.h5')\n",
    "checkpoint_dir = os.path.dirname(checkpoint_path)\n",
    "\n",
    "# Create path if exists\n",
    "if os.path.exists(checkpoint_dir):\n",
    "    print(\"{} -- Folder already exists \\n\".format(checkpoint_dir))\n",
    "else:\n",
    "    os.makedirs(checkpoint_dir, exist_ok=True)\n",
    "    print(\"{} -- Folder create complete \\n\".format(checkpoint_dir))\n",
    "    \n",
    "cp_callback = ModelCheckpoint(\n",
    "    checkpoint_path, verbose=1, save_best_only=True, save_weights_only=True)\n",
    "\n",
    "history = ner_model.fit(train_inputs, train_labels, batch_size=BATCH_SIZE, epochs=NUM_EPOCHS,\n",
    "                        callbacks=[cp_callback, f1_score_callback])\n",
    "\n",
    "print(history.history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8VPW9//HXJyEQgbAnoAKyg+xgWLQWgWqltpVaWwW0slNbtVpvvdqfXW1vq/W2trbcWkDcAe167bVqrYBbDRCQRZCwqgTRJOzInnx+f5wTHGMgA+TMJJP38/GYBzNnmfnkMOTNd76fOcfcHRERkRNJS3YBIiJS8yksRESkSgoLERGpksJCRESqpLAQEZEqKSxERKRKCgsREamSwkJERKqksBARkSrVS3YB1aVVq1beoUOHZJchIlKrLF26tMTds6vaLmXCokOHDuTn5ye7DBGRWsXM3olnO30MJSIiVVJYiIhIlRQWIiJSpZSZsxARqQ5HjhyhsLCQgwcPJruUapWZmUnbtm3JyMg4pf0VFiIiMQoLC8nKyqJDhw6YWbLLqRbuzvbt2yksLKRjx46n9Bz6GEpEJMbBgwdp2bJlygQFgJnRsmXL0xotKSxERCpIpaAod7o/U50Pi9Iy52f/eIvCnfuTXYqISI1V58Pi3R37mbf4XcbOzOO9XQeSXY6ICI0bN052CZ9Q58OiY6tGPDZ5CLv2H2HMjDy27VZgiIhUVOfDAqBfu2Y8NnkIOz88rMAQkRrD3bntttvo3bs3ffr04cknnwRg27ZtDBs2jP79+9O7d29eeeUVSktLmTBhwrFt77vvvmqtRa2zof7tmvHo5MFc9+Bixs7IY96082nTNDPZZYlIEv3476tZ896ean3Onmc14Ydf7BXXtn/5y19Yvnw5K1asoKSkhEGDBjFs2DDmzJnDpZdeyp133klpaSn79+9n+fLlbN26lTfffBOAXbt2VWvdGlnEGNC+OY9MHkzJvsOMnZnHB3tS60s5IlK7vPrqq4wdO5b09HRat27NRRddxJIlSxg0aBAPPfQQP/rRj1i1ahVZWVl06tSJTZs2cdNNN/Hcc8/RpEmTaq1FI4sKBrZvziOTBnPdg4vCEcZQcppohCFSF8U7Aki0YcOG8fLLL/PMM88wYcIEbr31Vq677jpWrFjB888/zwMPPMBTTz3F7Nmzq+01NbKoxHnnBIHxwZ6DjJmZR9FejTBEJPE+/elP8+STT1JaWkpxcTEvv/wygwcP5p133qF169ZMnTqVKVOmsGzZMkpKSigrK+PKK6/kpz/9KcuWLavWWiINCzMbZWYFZrbBzO6oZP0EMys2s+XhbUrMuvFmtj68jY+yzsrkdmjBw5MG8/7ug4ydkUfx3kOJLkFE6rgrrriCvn370q9fP0aOHMkvfvEL2rRpw8KFC+nXrx8DBgzgySef5Oabb2br1q0MHz6c/v37c+211/Lzn/+8Wmsxd6/WJzz2xGbpwDrgEqAQWAKMdfc1MdtMAHLd/cYK+7YA8oFcwIGlwHnuvvN4r5ebm+tRXPxo8eYdTHhoMWc1O4O5U4eSndWg2l9DRGqOt956i3PPPTfZZUSisp/NzJa6e25V+0Y5shgMbHD3Te5+GJgHjI5z30uBF9x9RxgQLwCjIqrzhAZ3bMFDEwaxdecBxs3Mo2SfRhgiUvdEGRZnA1tiHheGyyq60sxWmtmfzKzdyexrZtPMLN/M8ouLi6ur7k8Y0qklD00cRGEYGNsVGCJSxyR7gvvvQAd370swenjkZHZ29xnunuvuudnZVV5v/LQM7dSSByfk8u6O/Vwza5ECQySFRfXxfDKd7s8UZVhsBdrFPG4bLjvG3be7e/lv3VnAefHumwwXdG7F7PGD2FzyIdfMWsSODw8nuyQRqWaZmZls3749pQKj/HoWmZmn/jWAKCe46xFMcH+G4Bf9EmCcu6+O2eZMd98W3r8CuN3dh4YT3EuBgeGmywgmuHcc7/WimuCuzKvrS5j8yBI6ZTdmzpQhNG9UPyGvKyLRq2tXyot3gjuyL+W5+1EzuxF4HkgHZrv7ajO7C8h396eBb5nZ5cBRYAcwIdx3h5n9hCBgAO46UVAk2oVdWzFrfC6TH8nnmlmLeEKBIZIyMjIyTvlqcqksspFFoiVyZFHupXXFTH00n645jXliyhCaNVRgiEjtUhNaZ1PeRd2ymfG181hftI9rH1zE7v1Hkl2SiEgkFBanaXj3HP7wtfNY934YGAcUGCKSehQW1WBEGBgF7+/lOgWGiKQghUU1GdEjh99fO5A12/Zw3ezF7DmowBCR1KGwqEafObc1v7/mPNa8t5vrHlRgiEjqUFhUs4t7tmb6uIG8uXU342cvZq8CQ0RSgMIiAp/t1Ybp1wxkVWEQGPsOHU12SSIip0VhEZFLe7Xhd+MGsrJwNxMUGCJSyyksIjSqdxt+O3YAb2zZxcSHFvOhAkNEaimFRcQ+1+dM7h8zgGXv7mLiQ0sUGCJSKyksEuDzfc/kN2P6s/TdnUx8eAn7DyswRKR2UVgkyBf6nsV9V/cn/+0dTFJgiEgto7BIoMv7BYGxePMOJj+cz4HDpckuSUQkLgqLBBvd/2zuu7o/izZvZ/IjSxQYIlIrKCySYHT/s/nlVf14fdN2pj6az8EjCgwRqdkUFklyxYC2/PdX+vHaxhIFhojUeAqLJLryvLbc+5V+vLqhhGmPLVVgiEiNpbBIsq+c15Z7ruzLK+uL+boCQ0RqKIVFDXBVbjvu+XJfXlpXzPWPL+XQUQWGiNQsCosa4qpB7bj7y31YWFDMNx5fpsAQkRpFYVGDjBncnp9d0Yf5a4v4pgJDRGoQhUUNM25Ie/7rit68uLaIG55YxuGjZckuSUREYVETXTPkHH7ypd78660ibpijwBCR5FNY1FBfG3oOd43uxQtrPuDGOcs4UqrAEJHkUVjUYNed34EfX96Lf675gJvmvKHAEJGkUVjUcOMv6MAPv9iT51a/z7fmKjBEJDkUFrXAxE915Ptf6Mmzb77PLfOWc1SBISIJVi/ZBUh8Jl/YEXfnp8+8BQa/ubo/9dKV9SKSGJH+tjGzUWZWYGYbzOyOE2x3pZm5meWGjzuY2QEzWx7eHoiyztpiyqc7cedl5/LMym18+6kVGmGISMJENrIws3RgOnAJUAgsMbOn3X1Nhe2ygJuBRRWeYqO794+qvtpq6rBOlLnz82fXYsCvruqnEYaIRC7K3zKDgQ3uvsndDwPzgNGVbPcT4B7gYIS1pJSvX9SZ20f14OkV7/GdP66gtMyTXZKIpLgow+JsYEvM48Jw2TFmNhBo5+7PVLJ/RzN7w8xeMrNPR1hnrfSN4Z257dLu/G35e9ymwBCRiCVtgtvM0oBfARMqWb0NaO/u283sPOBvZtbL3fdUeI5pwDSA9u3bR1xxzXPDiC4A3Pt8ARjc+5V+pKdZkqsSkVQUZVhsBdrFPG4bLiuXBfQGFpoZQBvgaTO73N3zgUMA7r7UzDYC3YD82Bdw9xnADIDc3Nw6+V/rG0Z0oazM+eUL60gz454r+yowRKTaRRkWS4CuZtaRICTGAOPKV7r7bqBV+WMzWwh8x93zzSwb2OHupWbWCegKbIqw1lrtps90pczhvn+tw4B7ruxLmgJDRKpRZGHh7kfN7EbgeSAdmO3uq83sLiDf3Z8+we7DgLvM7AhQBlzv7juiqjUV3HxxV8rc+c2L60kz4+df7qPAEJFqE+mchbv/A/hHhWU/OM62w2Pu/xn4c5S1paJbLu6Ku3P//A2Ywc+uUGCISPXQN7hTiJnx7Uu6UebwuwUbMDP+60u9FRgictoUFinGzPiPz3bDcaYv2EiawU+/1JuwiUBE5JQoLFKQmfGdz3anzOH3CzdiBj8ZrcAQkVOnsEhRZsZ/XtqdMnf+8NIm0sz48eW9FBgickoUFinMzLhjVA/cYcbLQWD88Is9FRgictIUFinOzPju53pQVubMenUzZvCDLygwROTkKCzqADPjzs+fS5nD7Nc2Yxjf/8K5CgwRiZvCoo4wCwLCcWa/tpk0gzs/r8AQkfgoLOoQM+MHX+iJO8x6dTNpacFHVAoMEamKwqKOsXCSu8ydGS9vwoA7FBgiUgWFRR1kYRttmTt/eHkTZsbto7orMETkuBQWdZSZcdflvXGHB14Kvul926UKDBGpnMKiDktLM34yujdlDv+zcCNp4alCFBgiUpHCoo5LSwtONgjO7xZsIC3NuPWSbskuS0RqGIWFhIHRh7IyuP/F9RjwbQWGiMRQWAgQBMbPv9zn2AWUzOCWixUYIhJQWMgxaWnBNbwd+PW/givufeszXZNdlojUAAoL+ZjywChz51cvrCPN4MaRCgyRuk5hIZ+Qnmbc+5V+uMN//3MdZsYNI7okuywRSSKFhVQqPc3476/2w9259/kC0sz4xvDOyS5LRJJEYSHHlZ5m/PKq/jhwz3NrMYPrL1JgiNRFCgs5ofQ045df7UeZw93PriXNYNowBYZIXaOwkCrVS0/jvqv6UebOz/6xljQzpny6U7LLEpEEUlhIXOqlp/Gbq/uDw0+feQszY/KFHZNdlogkiMJC4lYvPY1fj+lPmTs/+b81pBlM/JQCQ6QuSEt2AVK7ZKSncf/YAYzq1YYf/30ND7+2OdkliUgCKCzkpGWkp/HbcQO4tFdrfvT3NTz6+tvJLklEIqawkFOSkZ7Gb8cO5JKerfnB/67msdffTnZJIhIhhYWcsvr10pg+biAXn5vD9/93NY/nvZPskkQkIpGGhZmNMrMCM9tgZnecYLsrzczNLDdm2XfD/QrM7NIo65RTV79eGtOvGchneuTwvb+9yZxF7ya7JBGJQGRhYWbpwHTgc0BPYKyZ9axkuyzgZmBRzLKewBigFzAK+J/w+aQGalAvnf+5diAjumfz//66inmLFRgiqSbKkcVgYIO7b3L3w8A8YHQl2/0EuAc4GLNsNDDP3Q+5+2ZgQ/h8UkM1qJfO7689j+Hds7njL6t4cokCQySVRBkWZwNbYh4XhsuOMbOBQDt3f+Zk95WaJzMjnQeuPY+LugWB8dSSLVXvJCK1QtImuM0sDfgV8B+n8RzTzCzfzPKLi4urrzg5ZZkZ6fzha+dxYZdW3P6XlfwxX4EhkgqiDIutQLuYx23DZeWygN7AQjN7GxgKPB1Ocle1LwDuPsPdc909Nzs7u5rLl1OVmZHOzOtyubBLK/7zzyv589LCZJckIqcpyrBYAnQ1s45mVp9gwvrp8pXuvtvdW7l7B3fvAOQBl7t7frjdGDNrYGYdga7A4ghrlWpWHhif6tyK7/xpBX99Q4EhUptFFhbufhS4EXgeeAt4yt1Xm9ldZnZ5FfuuBp4C1gDPATe4e2lUtUo0ygPj/E4t+Y+nVvC3Nz4xOBSRWsLcPdk1VIvc3FzPz89PdhlSiQOHS5n08BIWbd7OfVf3Z3R/9SqI1BRmttTdc6vaTt/glsidUT+dByfkMrhjC7795HKeXvFesksSkZMUV1iY2c1m1sQCD5rZMjP7bNTFSepoWL8esycMIrdDC26Z9wZ/V2CI1Crxjiwmufse4LNAc+BrwN2RVSUpqWH9ejw0YRC557TglieX88zKbckuSUTiFG9YWPjnZcBj4QS0nWB7kUo1alCPhyYOYmD7Znxr3hs8u0qBIVIbxBsWS83snwRh8Xx4Pqey6MqSVBYExmD6t2vGTXPf4Lk3FRgiNV28YTEZuAMY5O77gQxgYmRVScpr3KAeD08cRN+2Tblxzhs89+b7yS5JRE4g3rA4Hyhw911mdi3wPWB3dGVJXZCVmcEjkwbTp21TbpyzjH+uVmCI1FTxhsXvgf1m1o/gXE4bgUcjq0rqjPLA6H12U26Ys4wX1nyQ7JJEpBLxhsVRD769Nxr4nbtPJzi3k8hpa5KZwaOTB9PzrKZ884mlvPiWAkOkpok3LPaa2XcJWmafCc8YmxFdWVLXNMnM4NFJgzn3zCZ84/FlzF+rwBCpSeINi6uBQwTft3if4Cyw90ZWldRJTc/I4LFJQ+jeJovrH1vGgoKiZJckIqG4wiIMiCeApmb2BeCgu2vOQqpd04YZPD55CN3aNObrjy1loQJDpEaI93QfVxGcIvyrwFXAIjP7SpSFSd1VHhhdcxoz7bGlvLROF7YSSbZ4P4a6k+A7FuPd/TqC62F/P7qypK5r1rA+T0wZQpfsxkx9NJ+XFRgiSRVvWKS5e+znAdtPYl+RU1IeGJ3DwHh1fUmySxKps+L9hf+cmT1vZhPMbALwDPCP6MoSCTRvFARGx1aNmPzIEl7boMAQSYZ4J7hvA2YAfcPbDHe/PcrCRMq1CAOjQ8sgMP6twBBJuLg/SnL3P7v7reHtr1EWJVJRy8YNeGLqENq3aMikR5bw+sbtyS5JpE45YViY2V4z21PJba+Z7UlUkSIArRo3YM7UobRr3pBJDy8hb5MCQyRRThgW7p7l7k0quWW5e5NEFSlSrjwwzm5+BhMfWsIiBYZIQqijSWqd7KwGzJk6hLOaZTLx4SUseXtHsksSSXkKC6mVcrIymTt1KG2aZjJh9mLyFRgikVJYSK2V0ySTeVOH0rpJJuNnL2bpOwoMkagoLKRWy2mSydxpQ8lpksn42UtY+s7OZJckkpIUFlLrtW4SfCTVqnF9xs9ezLJ3FRgi1U1hISmhTdNghNGycX3GP7iY5Vt2JbskkZSisJCUcWbTM5g7dSjNG9Xnaw8uYoUCQ6TaKCwkpZzV7AzmThtKs4YZXPvgIlYWKjBEqkOkYWFmo8yswMw2mNkdlay/3sxWmdlyM3vVzHqGyzuY2YFw+XIzeyDKOiW1nN0sGGE0PSODa2ctYlXh7mSXJFLrRRYWZpYOTAc+B/QExpaHQYw57t7H3fsDvwB+FbNuo7v3D2/XR1WnpKa2zRsyd+pQsjKDEcabWxUYIqcjypHFYGCDu29y98PAPGB07AbuHnt+qUaAR1iP1DHtWjRk3rShNG5Qj2tmKTBETkeUYXE2sCXmcWG47GPM7AYz20gwsvhWzKqOZvaGmb1kZp+OsE5JYe1aBCOMRvXTufbBRax5T+e/FDkVSZ/gdvfp7t4ZuB34Xrh4G9De3QcAtwJzzOwTJy40s2lmlm9m+cXFuuymVK59y4bMm3Y+Z2Skc82sPN7apsAQOVlRhsVWoF3M47bhsuOZB3wJwN0Pufv28P5SYCPQreIO7j7D3XPdPTc7O7vaCpfUEwTGUBrUS+eaWYtY+74CQ+RkRBkWS4CuZtbRzOoDY4CnYzcws64xDz8PrA+XZ4cT5JhZJ6ArsCnCWqUOOKdlI+ZNG0pGujFu5iIK3t+b7JJEao3IwsLdjwI3As8DbwFPuftqM7vLzC4PN7vRzFab2XKCj5vGh8uHASvD5X8Crnd3nSVOTluHVo2YN+186qUZ42bmse4DBYZIPMw9NRqQcnNzPT8/P9llSC2xsXgfY2fkUebO3KlD6do6K9kliSSFmS1199yqtkv6BLdIMnTObszcaUMxM8bOXMSGIo0wRE5EYSF1VufsxsydOhSAMTMWsaFoX5IrEqm5FBZSp3XJacy8aUMAZ+zMPDYWKzBEKqOwkDqvS04Wc6cOpazMGTsjj00KDJFPUFiIAF1bZzFn6lBKy4IRxuaSD5NdkkiNorAQCXVvk8UTU4dwpDQYYbytwBA5RmEhEqNHmyY8MWUIh46WMnZmHu9sV2CIgMJC5BPOPbMJT0wZysEjpYydkce72/cnuySRpFNYiFSi51lNeHzKEPYfCUYYW3YoMKRuU1iIHEevs5ry+OQh7Dt0lDEzFBhStyksRE6g99lNeWLKEPYePMKYGXkU7lRgSN2ksBCpQhAYQ48FxtZdB5JdkkjCKSxE4tCnbVMenzKE3QeOMGbG67ynwJA6RmEhEqe+bZvx+OQh7PowGGFs263AkLpDYSFyEvq1a8ajkwez88PDCgypUxQWIidpQPvmPDJ5MNv3HWbsjDze330w2SWJRE5hIXIKBrZvziOTBlO89xBjZ+bxwR4FhqQ2hYXIKTrvnCAwivYcZOyMPIoUGJLCFBYipyG3QwsenjSY9/ccZMzMPIr2KjAkNSksRE7ToA4teHjiYN7fHYwwivceSnZJItVOYSFSDQZ3bMFDEwbx3q6DjJ2pwJDUo7AQqSZDOrXkoYmD2LrzAONm5lGyT4EhqUNhIVKNhnZqyYMTctmycz/jZuaxXYEhKUJhIVLNLujcitnjB/HO9v1cM2uRAkNSgsJCJAIXdGnFg+MHsbnkQ66ZtUhf3JNaT2EhEpELu7Zi1vhcNpd8yPl3v8iXpr/G/S+uZ1XhbsrKPNnliZwUc0+NN21ubq7n5+cnuwyRT9hQtI9nVm5jfkERKwt34Q7ZWQ0Y0T2bEd1zuLBrK7IyM5JdptRRZrbU3XOr3E5hIZI4JfsO8VJBMfMLinh5XTF7Dx4lI90Y1KEFI7rnMKJHDp2zG2FmyS5V6giFhUgNd6S0jGXv7GR+QREL1hax7oN9ALRv0TAYdfTIYWinlmRmpCe5UkllNSIszGwU8BsgHZjl7ndXWH89cANQCuwDprn7mnDdd4HJ4bpvufvzJ3othYXUdoU797OgoJgFa4v498YSDh4p44yMdD7VpSXDw1HH2c3OSHaZkmKSHhZmlg6sAy4BCoElwNjyMAi3aeLue8L7lwPfdPdRZtYTmAsMBs4C/gV0c/fS472ewkJSycEjpby+aTsL1hYxf20RhTuD62b0aJPF8O45jOyRw8D2zaiXrh4VOT3xhkW9CGsYDGxw901hQfOA0cCxsCgPilAjoDy5RgPz3P0QsNnMNoTP93qE9YrUGJkZ6cEcRvccfny5s7F4H/PXFrFgbTGzXtnEAy9tpElmPYZ1y2Zkjxwu6pZNy8YNkl22pLAow+JsYEvM40JgSMWNzOwG4FagPjAyZt+8CvueHU2ZIjWbmdElJ4suOVlMG9aZPQeP8Or6EhasLWJBQTH/t3IbZtCvbTNG9ghGHT3PbEJamibJpfpEGRZxcffpwHQzGwd8Dxgf775mNg2YBtC+fftoChSpYZpkZnBZnzO5rM+ZlJU5b763Oxh1FBRz37/W8asX1h1rzR3ZI4dPdVFrrpy+KMNiK9Au5nHbcNnxzAN+fzL7uvsMYAYEcxanU6xIbZSWZvRt24y+bZtxy8XdKNl3iIUFxSwoKOLZN9/nqfzCY625I3vkMLy7WnPl1EQ5wV2PYIL7MwS/6JcA49x9dcw2Xd19fXj/i8AP3T3XzHoBc/hogvtFoKsmuEXid6S0jKXv7GRBJa25QXBkqzVXkt8NFRZxGfBrgtbZ2e7+X2Z2F5Dv7k+b2W+Ai4EjwE7gxvIwMbM7gUnAUeAWd3/2RK+lsBA5sS079rNwXeWtuSN6BJPpZ6k1t86pEWGRSAoLkfidqDW3PDjUmls3KCxEJC7uzoaifSwoCIIj/+2dHC1zmp6RwbBu2Yzonq3W3BSmsBCRU1Lemjt/bRELC4op2XcIM+jfrhkjwi8E9jqriSbJU4TCQkRO28dac9cWsaJwNwA5WQ0YHrbmXtg1m8YNkt6FL6dIYSEi1a547yFeCifJX17/8bPmjuwRnL+qUyu15tYmCgsRidSx1ty1RSwo+Kg195yWDY+dbn1IxxZqza3hFBYiklBbduxnYUHwTfLXNpRw6Khac2sDhYWIJM3BI6W8vnH7sQ6riq25I3vkMKCdWnNrAoWFiNQIVbXmjuyRzUXdcmjRqH6yS62TFBYiUiPtOXiEV9aVsKCgiIUFRZTsO3ysNXdkONeh1tzEUViISI1XVuas2rr72PmrYltzg0nybLXmRkxhISK1zsdac9cVs/dQ0Jo7uGOLYx1Was2tXgoLEanVYltz568tYn2RWnOjoLAQkZRS3po7f20R/964PaY1txUjemSrNfcUKSxEJGUdOFxK3qbtzA9HHVt3qTX3VCksRKROKG/NnR9+k1ytuSdHYSEidVJ5a+78tUW8tE6tuVVRWIhInVfemhucbr2y1twcLuzaqk635iosREQqKN57iIUFwXU6KmvNHdkjh451rDVXYSEicgJHSsvIf3vnsQ6rutqaq7AQETkJJ2rNDa7Vkc2ZTVOvNVdhISJyik7Umlt+kadUac1VWIiIVIPY1tz5a4vIf2cnpWFr7kXdshlRy1tzFRYiIhHYfeAIr67/ZGvugHbNjs111KbWXIWFiEjEYltzFxQUsbIWtuYqLEREEqy8NXdBQRGvrCs51po7pGNLhnfPZmSPHDplN052mR+jsBARSaLy1tzyKwRuCFtzO7RsyPDwOx1DOrWgQb3ktuYqLEREapAtO/Yfu8hTeWtuw/rpXNA5ua25CgsRkRrqwOFSXt8UTJIvWFv8idbckT1y6J+g1lyFhYhILeDurC8/a25Ma26zhhkM6xrMcwzrlh1Za26NCAszGwX8BkgHZrn73RXW3wpMAY4CxcAkd38nXFcKrAo3fdfdLz/RayksRCQV7D5whFfWF7NgbfGx1ty08rPmhl8I7Hlm9bXmJj0szCwdWAdcAhQCS4Cx7r4mZpsRwCJ3329m3wCGu/vV4bp97h5324DCQkRSTVmZs3LrbhZUaM1t3eSj1txPdTm91tx4wyLK5t/BwAZ33xQWNA8YDRwLC3dfELN9HnBthPWIiNQqaWlG/3bN6N+uGd++pBtFew/yUkExCwqKeGblNuYt2UJGunFprzb8btzASGuJMizOBrbEPC4Ehpxg+8nAszGPM80sn+Ajqrvd/W8VdzCzacA0gPbt2592wSIiNVlOViZfzW3HV3Pbfaw1NyM9+m+L14ivFZrZtUAucFHM4nPcfauZdQLmm9kqd98Yu5+7zwBmQPAxVMIKFhFJsoz0NM7v3JLzO7dMyOtF2Ze1FWgX87htuOxjzOxi4E7gcnc/VL7c3beGf24CFgIDIqxVREROIMqwWAJ0NbOOZlYfGAM8HbuBmQ0A/kAQFEUxy5ubWYPwfivgU8TMdYiISGJF9jGUux81sxuB5wlaZ2e7+2ozuwvId/engXuBxsAfwzaw8hbZc4E/mFkZQaDdHdtFJSIiiaUv5YmI1GHxts7W/ss8iYhI5BQWIiJSJYWFiIhUSWEhIiKauA0aAAAHA0lEQVRVSpkJbjMrBt45jadoBZRUUznVSXWdHNV1clTXyUnFus5x9+yqNkqZsDhdZpYfT0dAoqmuk6O6To7qOjl1uS59DCUiIlVSWIiISJUUFh+ZkewCjkN1nRzVdXJU18mps3VpzkJERKqkkYWIiFQp5cPCzEaZWYGZbTCzOypZ38DMngzXLzKzDjHrvhsuLzCzSxNc161mtsbMVprZi2Z2Tsy6UjNbHt6errhvxHVNMLPimNefErNuvJmtD2/jE1zXfTE1rTOzXTHrojxes82syMzePM56M7P7w7pXmtnAmHVRHq+q6romrGeVmf3bzPrFrHs7XL48vABZIusabma7Y/6+fhCz7oTvgYjrui2mpjfD91SLcF2Ux6udmS0IfxesNrObK9kmMe8xd0/ZG8HZbjcCnYD6wAqgZ4Vtvgk8EN4fAzwZ3u8Zbt8A6Bg+T3oC6xoBNAzvf6O8rvDxviQerwnA7yrZtwWwKfyzeXi/eaLqqrD9TQRnOY70eIXPPQwYCLx5nPWXEVwB0oChBNecj/R4xVnXBeWvB3yuvK7w8dtAqyQdr+HA/53ue6C666qw7ReB+Qk6XmcCA8P7WcC6Sv5NJuQ9luoji2PXAXf3w0D5dcBjjQYeCe//CfiMmVm4fJ67H3L3zcCG8PkSUpe7L3D3/eHDPIKLR0UtnuN1PJcCL7j7DnffCbwAjEpSXWOBudX02ifk7i8DO06wyWjgUQ/kAc3M7EyiPV5V1uXu/w5fFxL3/orneB3P6bw3q7uuRL6/trn7svD+XuAtgktWx0rIeyzVw6Ky64BXPNDHtnH3o8BuoGWc+0ZZV6xKr09uZnlm9qVqqulk6royHO7+yczKr4ZYI45X+HFdR2B+zOKojlc8jld7lMfrZFV8fznwTzNbasF17hPtfDNbYWbPmlmvcFmNOF5m1pDgF+6fYxYn5HhZ8BH5AGBRhVUJeY/ViGtwy/HZKV6fPEJ/B+a6+yEz+zrBqGxkgl47HmOAP7l7acyyZB6vGs3MRhCExYUxiy8Mj1cO8IKZrQ3/550Iywj+vvaZ2WXA34CuCXrteHwReM3dY0chkR8vM2tMEFC3uPue6nzueKX6yCKe64Af28bM6gFNge1x7htlXcm4PnmVdbn79phaZgHnxbtvlHXFGEOFjwgiPF7xOF7tUR6vuJhZX4K/w9Huvr18eczxKgL+SvV9/Fold9/j7vvC+/8AMiy4tHLSj1foRO+vSI6XmWUQBMUT7v6XSjZJzHssikmZmnIjGDltIvhYonxSrFeFbW7g4xPcT4X3e/HxCe5NVN8Edzx1DSCY0OtaYXlzoEF4vxWwnmqa6IuzrjNj7l8B5PlHk2mbw/qah/dbJKqucLseBJONlojjFfMaHTj+hO3n+fjk4+Koj1ecdbUnmIe7oMLyRkBWzP1/A6MSWFeb8r8/gl+674bHLq73QFR1heubEsxrNErU8Qp/9keBX59gm4S8x6rtQNfUG0GnwDqCX7x3hsvuIvjfOkAm8MfwH85ioFPMvneG+xUAn0twXf8CPgCWh7enw+UXAKvCfyyrgMkJruvnwOrw9RcAPWL2nRQexw3AxETWFT7+EcH12mP3i/p4zQW2AUcIPhOeDFwPXB+uN2B6WPcqIDdBx6uqumYBO2PeX/nh8k7hsVoR/j3fmeC6box5f+URE2aVvQcSVVe4zQSCppfY/aI+XhcSzImsjPm7uiwZ7zF9g1tERKqU6nMWIiJSDRQWIiJSJYWFiIhUSWEhIiJVUliIiEiVFBYiVahw1trl1XnGUzPrcLwznYrUJDrdh0jVDrh7/2QXIZJMGlmInKLwOga/CK9lsNjMuoTLO5jZfPvoWiTtw+Wtzeyv4UnyVpjZBeFTpZvZzPB6Bf80szPC7b9lH13TZF6SfkwRQGEhEo8zKnwMdXXMut3u3gf4HfDrcNlvgUfcvS/wBHB/uPx+4CV370dw7YTV4fKuwHR37wXsAq4Ml98BDAif5/qofjiReOgb3CJVMLN97t64kuVvAyPdfVN4srf33b2lmZUQnEPrSLh8m7u3MrNioK3HnBQyPO30C+7eNXx8O5Dh7j81s+eAfQRnXv2bhyfYE0kGjSxETo8f5/7JOBRzv5SP5hI/T3DOn4HAkvCsyCJJobAQOT1Xx/z5enj/3wRnMAa4BnglvP8iwSVyMbN0M2t6vCc1szSgnbsvAG4nOOPpJ0Y3Iomi/6mIVO0MM1se8/g5dy9vn21uZisJRgdjw2U3AQ+Z2W1AMTAxXH4zMMPMJhOMIL5BcKbTyqQDj4eBYsD97r6r2n4ikZOkOQuRUxTOWeS6e0myaxGJmj6GEhGRKmlkISIiVdLIQkREqqSwEBGRKiksRESkSgoLERGpksJCRESqpLAQEZEq/X+M0S9LXqzPFwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_graphs(history, 'loss')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}