code_Custom_NER_with_Deep_learning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "code - Custom NER with Deep learning.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPXkn5rZGzbH39SxUu2pZNS",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/anshupandey/Natural_language_Processing/blob/master/code_Custom_NER_with_Deep_learning.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kLNoDhxIezAZ"
      },
      "source": [
        "!wget -q https://raw.githubusercontent.com/anshupandey/Data-Science-Resources/master/ner_dataset.csv"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 203
        },
        "id": "cVia1NxEfDTa",
        "outputId": "9a590163-85ba-4253-cf7a-8fe1f3c9324c"
      },
      "source": [
        "import pandas as pd\n",
        "data = pd.read_csv('ner_dataset.csv', encoding= 'unicode_escape')\n",
        "data.head()"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Sentence #</th>\n",
              "      <th>Word</th>\n",
              "      <th>POS</th>\n",
              "      <th>Tag</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>Thousands</td>\n",
              "      <td>NNS</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>NaN</td>\n",
              "      <td>of</td>\n",
              "      <td>IN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>NaN</td>\n",
              "      <td>demonstrators</td>\n",
              "      <td>NNS</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>NaN</td>\n",
              "      <td>have</td>\n",
              "      <td>VBP</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>NaN</td>\n",
              "      <td>marched</td>\n",
              "      <td>VBN</td>\n",
              "      <td>O</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    Sentence #           Word  POS Tag\n",
              "0  Sentence: 1      Thousands  NNS   O\n",
              "1          NaN             of   IN   O\n",
              "2          NaN  demonstrators  NNS   O\n",
              "3          NaN           have  VBP   O\n",
              "4          NaN        marched  VBN   O"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BDfyuwWff5tv",
        "outputId": "56d693d8-8297-403a-f196-790d1d353154"
      },
      "source": [
        "data.shape"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1048575, 4)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k0zY50q1fhd0"
      },
      "source": [
        "from itertools import chain\n",
        "def get_dict_map(data, token_or_tag):\n",
        "    tok2idx = {}\n",
        "    idx2tok = {}\n",
        "    \n",
        "    if token_or_tag == 'token':\n",
        "        vocab = list(set(data['Word'].to_list()))\n",
        "    else:\n",
        "        vocab = list(set(data['Tag'].to_list()))\n",
        "    \n",
        "    idx2tok = {idx:tok for  idx, tok in enumerate(vocab)}\n",
        "    tok2idx = {tok:idx for  idx, tok in enumerate(vocab)}\n",
        "    return tok2idx, idx2tok\n",
        "\n",
        "\n",
        "token2idx, idx2token = get_dict_map(data, 'token')\n",
        "tag2idx, idx2tag = get_dict_map(data, 'tag')"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lQt4ATzxfqIF",
        "outputId": "fd1e210b-8754-4593-9926-987867604e80"
      },
      "source": [
        "idx2tag"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{0: 'I-nat',\n",
              " 1: 'B-nat',\n",
              " 2: 'B-per',\n",
              " 3: 'B-org',\n",
              " 4: 'I-tim',\n",
              " 5: 'B-art',\n",
              " 6: 'I-art',\n",
              " 7: 'I-org',\n",
              " 8: 'B-eve',\n",
              " 9: 'I-gpe',\n",
              " 10: 'I-geo',\n",
              " 11: 'I-eve',\n",
              " 12: 'O',\n",
              " 13: 'B-geo',\n",
              " 14: 'B-gpe',\n",
              " 15: 'I-per',\n",
              " 16: 'B-tim'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 203
        },
        "id": "2NKOHbFWfscY",
        "outputId": "0ac05b35-5973-4a28-9faa-07fe2f576e20"
      },
      "source": [
        "data['Word_idx'] = data['Word'].map(token2idx)\n",
        "data['Tag_idx'] = data['Tag'].map(tag2idx)\n",
        "data.head()"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Sentence #</th>\n",
              "      <th>Word</th>\n",
              "      <th>POS</th>\n",
              "      <th>Tag</th>\n",
              "      <th>Word_idx</th>\n",
              "      <th>Tag_idx</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>Thousands</td>\n",
              "      <td>NNS</td>\n",
              "      <td>O</td>\n",
              "      <td>19718</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>NaN</td>\n",
              "      <td>of</td>\n",
              "      <td>IN</td>\n",
              "      <td>O</td>\n",
              "      <td>9634</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>NaN</td>\n",
              "      <td>demonstrators</td>\n",
              "      <td>NNS</td>\n",
              "      <td>O</td>\n",
              "      <td>1001</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>NaN</td>\n",
              "      <td>have</td>\n",
              "      <td>VBP</td>\n",
              "      <td>O</td>\n",
              "      <td>19330</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>NaN</td>\n",
              "      <td>marched</td>\n",
              "      <td>VBN</td>\n",
              "      <td>O</td>\n",
              "      <td>20934</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    Sentence #           Word  POS Tag  Word_idx  Tag_idx\n",
              "0  Sentence: 1      Thousands  NNS   O     19718       12\n",
              "1          NaN             of   IN   O      9634       12\n",
              "2          NaN  demonstrators  NNS   O      1001       12\n",
              "3          NaN           have  VBP   O     19330       12\n",
              "4          NaN        marched  VBN   O     20934       12"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 341
        },
        "id": "Qm-nMMhSgDjB",
        "outputId": "91d5b0d0-7ed3-4748-9c0a-f299b04e2e7f"
      },
      "source": [
        "# Fill na\n",
        "data_fillna = data.fillna(method='ffill', axis=0)\n",
        "# Groupby and collect columns\n",
        "data_group = data_fillna.groupby(\n",
        "['Sentence #'],as_index=False\n",
        ")['Word', 'POS', 'Tag', 'Word_idx', 'Tag_idx'].agg(lambda x: list(x))\n",
        "# Visualise data\n",
        "data_group.head()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:6: FutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.\n",
            "  \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Sentence #</th>\n",
              "      <th>Word</th>\n",
              "      <th>POS</th>\n",
              "      <th>Tag</th>\n",
              "      <th>Word_idx</th>\n",
              "      <th>Tag_idx</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Sentence: 1</td>\n",
              "      <td>[Thousands, of, demonstrators, have, marched, ...</td>\n",
              "      <td>[NNS, IN, NNS, VBP, VBN, IN, NNP, TO, VB, DT, ...</td>\n",
              "      <td>[O, O, O, O, O, O, B-geo, O, O, O, O, O, B-geo...</td>\n",
              "      <td>[19718, 9634, 1001, 19330, 20934, 12228, 8948,...</td>\n",
              "      <td>[12, 12, 12, 12, 12, 12, 13, 12, 12, 12, 12, 1...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Sentence: 10</td>\n",
              "      <td>[Iranian, officials, say, they, expect, to, ge...</td>\n",
              "      <td>[JJ, NNS, VBP, PRP, VBP, TO, VB, NN, TO, JJ, J...</td>\n",
              "      <td>[B-gpe, O, O, O, O, O, O, O, O, O, O, O, O, O,...</td>\n",
              "      <td>[23311, 30622, 28420, 34752, 8167, 9548, 23146...</td>\n",
              "      <td>[14, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 1...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Sentence: 100</td>\n",
              "      <td>[Helicopter, gunships, Saturday, pounded, mili...</td>\n",
              "      <td>[NN, NNS, NNP, VBD, JJ, NNS, IN, DT, NNP, JJ, ...</td>\n",
              "      <td>[O, O, B-tim, O, O, O, O, O, B-geo, O, O, O, O...</td>\n",
              "      <td>[23965, 19289, 9328, 20072, 733, 8082, 12777, ...</td>\n",
              "      <td>[12, 12, 16, 12, 12, 12, 12, 12, 13, 12, 12, 1...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Sentence: 1000</td>\n",
              "      <td>[They, left, after, a, tense, hour-long, stand...</td>\n",
              "      <td>[PRP, VBD, IN, DT, NN, JJ, NN, IN, NN, NNS, .]</td>\n",
              "      <td>[O, O, O, O, O, O, O, O, O, O, O]</td>\n",
              "      <td>[32960, 8323, 31790, 20800, 22954, 26504, 3209...</td>\n",
              "      <td>[12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Sentence: 10000</td>\n",
              "      <td>[U.N., relief, coordinator, Jan, Egeland, said...</td>\n",
              "      <td>[NNP, NN, NN, NNP, NNP, VBD, NNP, ,, NNP, ,, J...</td>\n",
              "      <td>[B-geo, O, O, B-per, I-per, O, B-tim, O, B-geo...</td>\n",
              "      <td>[28965, 1861, 23900, 21271, 28088, 4017, 23951...</td>\n",
              "      <td>[13, 12, 12, 2, 15, 12, 16, 12, 13, 12, 14, 12...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        Sentence #  ...                                            Tag_idx\n",
              "0      Sentence: 1  ...  [12, 12, 12, 12, 12, 12, 13, 12, 12, 12, 12, 1...\n",
              "1     Sentence: 10  ...  [14, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 1...\n",
              "2    Sentence: 100  ...  [12, 12, 16, 12, 12, 12, 12, 12, 13, 12, 12, 1...\n",
              "3   Sentence: 1000  ...       [12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12]\n",
              "4  Sentence: 10000  ...  [13, 12, 12, 2, 15, 12, 16, 12, 13, 12, 14, 12...\n",
              "\n",
              "[5 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0Le_w79lgGFy"
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from tensorflow.keras.utils import to_categorical"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iLLDLQCtgQvI",
        "outputId": "ebe3b2a2-7918-44b4-cb85-03e94450dc00"
      },
      "source": [
        "def get_pad_train_test_val(data_group, data):\n",
        "\n",
        "    #get max token and tag length\n",
        "    n_token = len(list(set(data['Word'].to_list())))\n",
        "    n_tag = len(list(set(data['Tag'].to_list())))\n",
        "\n",
        "    #Pad tokens (X var)    \n",
        "    tokens = data_group['Word_idx'].tolist()\n",
        "    maxlen = max([len(s) for s in tokens])\n",
        "    pad_tokens = pad_sequences(tokens, maxlen=maxlen, dtype='int32', padding='post', value= n_token - 1)\n",
        "\n",
        "    #Pad Tags (y var) and convert it into one hot encoding\n",
        "    tags = data_group['Tag_idx'].tolist()\n",
        "    pad_tags = pad_sequences(tags, maxlen=maxlen, dtype='int32', padding='post', value= tag2idx[\"O\"])\n",
        "    n_tags = len(tag2idx)\n",
        "    pad_tags = [to_categorical(i, num_classes=n_tags) for i in pad_tags]\n",
        "    \n",
        "    #Split train, test and validation set\n",
        "    tokens_, test_tokens, tags_, test_tags = train_test_split(pad_tokens, pad_tags, test_size=0.1, train_size=0.9, random_state=2020)\n",
        "    train_tokens, val_tokens, train_tags, val_tags = train_test_split(tokens_,tags_,test_size = 0.25,train_size =0.75, random_state=2020)\n",
        "\n",
        "    print(\n",
        "        'train_tokens length:', len(train_tokens),\n",
        "        '\\ntrain_tokens length:', len(train_tokens),\n",
        "        '\\ntest_tokens length:', len(test_tokens),\n",
        "        '\\ntest_tags:', len(test_tags),\n",
        "        '\\nval_tokens:', len(val_tokens),\n",
        "        '\\nval_tags:', len(val_tags),\n",
        "    )\n",
        "    \n",
        "    return train_tokens, val_tokens, test_tokens, train_tags, val_tags, test_tags\n",
        "\n",
        "train_tokens, val_tokens, test_tokens, train_tags, val_tags, test_tags = get_pad_train_test_val(data_group, data)\n",
        "\n"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "train_tokens length: 32372 \n",
            "train_tokens length: 32372 \n",
            "test_tokens length: 4796 \n",
            "test_tags: 4796 \n",
            "val_tokens: 10791 \n",
            "val_tags: 10791\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pDmcDCFhga63"
      },
      "source": [
        "import numpy as np\n",
        "import tensorflow\n",
        "from tensorflow.keras import Sequential, Model, Input\n",
        "from tensorflow.keras import layers\n",
        "from tensorflow.keras.utils import plot_model\n"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XxgtoOOPgdPh"
      },
      "source": [
        "from numpy.random import seed\n",
        "seed(1)\n",
        "tensorflow.random.set_seed(2)"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 229
        },
        "id": "YRldeOVLgfda",
        "outputId": "cebfc88b-2881-4e8e-eef9-bc13d1405f67"
      },
      "source": [
        "input_dim = len(list(set(data['Word'].to_list())))+1\n",
        "output_dim = 64\n",
        "input_length = max([len(s) for s in data_group['Word_idx'].tolist()])\n",
        "n_tags = len(tag2idx)\n",
        "print('input_dim: ', input_dim, '\\noutput_dim: ', output_dim, '\\ninput_length: ', input_length, '\\nn_tags: ', n_tags)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-5-775112a3b80b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0minput_dim\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Word'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_list\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0moutput_dim\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m64\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0minput_length\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0ms\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdata_group\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Word_idx'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtolist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mn_tags\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtag2idx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'input_dim: '\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_dim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'\\noutput_dim: '\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_dim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'\\ninput_length: '\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_length\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'\\nn_tags: '\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_tags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'data' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 229
        },
        "id": "J-NNN_ofgixJ",
        "outputId": "90ccb7df-20db-4972-9334-9a88f94aa2ee"
      },
      "source": [
        "\n",
        "input_layer = layers.Input(shape=(input_length,))\n",
        "\n",
        "# Add Embedding layer\n",
        "em = layers.Embedding(input_dim=input_dim, output_dim=output_dim)(input_layer)\n",
        "\n",
        "# Add bidirectional LSTM\n",
        "bilstm = layers.Bidirectional(LSTM(units=output_dim, return_sequences=True, dropout=0.2, recurrent_dropout=0.2), merge_mode = 'concat')(em)\n",
        "\n",
        "# Add LSTM\n",
        "lstm = layers.LSTM(units=output_dim, return_sequences=True, dropout=0.5, recurrent_dropout=0.5)(bilstm)\n",
        "\n",
        "# Add timeDistributed Layer\n",
        "op = layers.TimeDistributed(layers.Dense(n_tags, activation=\"relu\"))(lstm)\n",
        "\n",
        "model = models.Model(input_layer,op)\n",
        "# Compile model\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "model.summary()\n"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-6-97eca80e57af>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m# Add Embedding layer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mem\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlayers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mEmbedding\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_dim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minput_dim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_dim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_dim\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_layer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;31m# Add bidirectional LSTM\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'input_dim' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5N5OsLRMglml"
      },
      "source": [
        "def train_model(X, y, model):\n",
        "    loss = list()\n",
        "    for i in range(10):\n",
        "        # fit model for one epoch on this sequence\n",
        "        hist = model.fit(X, y, batch_size=1000, verbose=1, epochs=1, validation_split=0.2)\n",
        "        loss.append(hist.history['loss'][0])\n",
        "    return loss"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 673
        },
        "id": "8cO6NDE-goW0",
        "outputId": "4a48348e-9990-407e-d71f-428afb02a3f7"
      },
      "source": [
        "results = pd.DataFrame()\n",
        "plot_model(model)\n",
        "results['with_add_lstm'] = train_model(train_tokens, np.array(train_tags), model)"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding (Embedding)        (None, 104, 64)           2251456   \n",
            "_________________________________________________________________\n",
            "bidirectional (Bidirectional (None, 104, 128)          66048     \n",
            "_________________________________________________________________\n",
            "lstm_1 (LSTM)                (None, 104, 64)           49408     \n",
            "_________________________________________________________________\n",
            "time_distributed (TimeDistri (None, 104, 17)           1105      \n",
            "=================================================================\n",
            "Total params: 2,368,017\n",
            "Trainable params: 2,368,017\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "26/26 [==============================] - 151s 5s/step - loss: 0.7062 - accuracy: 0.9295 - val_loss: 0.3099 - val_accuracy: 0.9681\n",
            "26/26 [==============================] - 140s 5s/step - loss: 0.2992 - accuracy: 0.9677 - val_loss: 0.2453 - val_accuracy: 0.9681\n",
            "26/26 [==============================] - 170s 7s/step - loss: 0.4305 - accuracy: 0.9356 - val_loss: 0.3108 - val_accuracy: 0.9511\n",
            " 6/26 [=====>........................] - ETA: 2:05 - loss: 0.3076 - accuracy: 0.9505"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-22-8f959d0b9b18>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mmodel_bilstm_lstm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_bilstm_lstm_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mplot_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_bilstm_lstm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mresults\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'with_add_lstm'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_tokens\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_tags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel_bilstm_lstm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-21-bd4c1cb08fe3>\u001b[0m in \u001b[0;36mtrain_model\u001b[0;34m(X, y, model)\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m25\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m         \u001b[0;31m# fit model for one epoch on this sequence\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m         \u001b[0mhist\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1000\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_split\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'loss'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1181\u001b[0m                 _r=1):\n\u001b[1;32m   1182\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1183\u001b[0;31m               \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1184\u001b[0m               \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1185\u001b[0m                 \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    887\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    888\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mOptionalXlaContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jit_compile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 889\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    890\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    891\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    915\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    916\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 917\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    918\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    919\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   3022\u001b[0m        filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[1;32m   3023\u001b[0m     return graph_function._call_flat(\n\u001b[0;32m-> 3024\u001b[0;31m         filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access\n\u001b[0m\u001b[1;32m   3025\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3026\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1959\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1960\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1961\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1962\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1963\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    594\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    595\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 596\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    597\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    598\u001b[0m           outputs = execute.execute_with_cancellation(\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 60\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     61\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2wyzojzIgqzd"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}