Part I - spaCy NER and TextCat.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v7_kL3-qRVH3"
      },
      "source": [
        "# Data Preparation\n",
        "Here, we clean up our data, the AG News Dataset from Kaggle, for use in NER and Text Classification\n",
        "To install AG Dataset and NER Annotations, run following lines in Terminal:\n",
        "```\n",
        "aws s3 cp s3://applied-nlp-book/data/ data --recursive --no-sign-request\n",
        "aws s3 cp s3://applied-nlp-book/models/ag_dataset/ models/ag_dataset --recursive --no-sign-request\n",
        "```"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Install aws cli and get the data for this course and this task overall\n",
        "!pip install awscli\n",
        "!aws s3 cp s3://applied-nlp-book/data/ data --recursive --no-sign-request\n",
        "!aws s3 cp s3://applied-nlp-book/models/ag_dataset/ models/ag_dataset --recursive --no-sign-request"
      ],
      "metadata": {
        "id": "pnHryEIHRW-T"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "sMkCJqlMRVH_",
        "outputId": "b9bd741d-87ca-441a-8401-3afc8bf43f42"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   class_index                                              title  \\\n",
              "0            3  Wall St. Bears Claw Back Into the Black (Reuters)   \n",
              "1            3  Carlyle Looks Toward Commercial Aerospace (Reu...   \n",
              "2            3    Oil and Economy Cloud Stocks' Outlook (Reuters)   \n",
              "3            3  Iraq Halts Oil Exports from Main Southern Pipe...   \n",
              "4            3  Oil prices soar to all-time record, posing new...   \n",
              "\n",
              "                                         description class_name  \n",
              "0  Reuters - Short-sellers, Wall Street's dwindli...   Business  \n",
              "1  Reuters - Private investment firm Carlyle Grou...   Business  \n",
              "2  Reuters - Soaring crude prices plus worries ab...   Business  \n",
              "3  Reuters - Authorities have halted oil export f...   Business  \n",
              "4  AFP - Tearaway world oil prices, toppling reco...   Business  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-63aaa60f-9e0a-4db9-b78a-40192c729439\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>class_index</th>\n",
              "      <th>title</th>\n",
              "      <th>description</th>\n",
              "      <th>class_name</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>3</td>\n",
              "      <td>Wall St. Bears Claw Back Into the Black (Reuters)</td>\n",
              "      <td>Reuters - Short-sellers, Wall Street's dwindli...</td>\n",
              "      <td>Business</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3</td>\n",
              "      <td>Carlyle Looks Toward Commercial Aerospace (Reu...</td>\n",
              "      <td>Reuters - Private investment firm Carlyle Grou...</td>\n",
              "      <td>Business</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>Oil and Economy Cloud Stocks' Outlook (Reuters)</td>\n",
              "      <td>Reuters - Soaring crude prices plus worries ab...</td>\n",
              "      <td>Business</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>Iraq Halts Oil Exports from Main Southern Pipe...</td>\n",
              "      <td>Reuters - Authorities have halted oil export f...</td>\n",
              "      <td>Business</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3</td>\n",
              "      <td>Oil prices soar to all-time record, posing new...</td>\n",
              "      <td>AFP - Tearaway world oil prices, toppling reco...</td>\n",
              "      <td>Business</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-63aaa60f-9e0a-4db9-b78a-40192c729439')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-63aaa60f-9e0a-4db9-b78a-40192c729439 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-63aaa60f-9e0a-4db9-b78a-40192c729439');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ],
      "source": [
        "# Import libraries\n",
        "import pandas as pd\n",
        "import os\n",
        "\n",
        "# Get CWD\n",
        "cwd = os.getcwd()\n",
        "\n",
        "# Read dataset, replace spaces with underscores, and create a new column in data which maps categories to their names\n",
        "data = pd.read_csv(cwd + '/data/ag_dataset/train.csv')\n",
        "data = pd.DataFrame(data=data)\n",
        "data.columns = data.columns.str.replace(\" \", \"_\")\n",
        "data.columns = data.columns.str.lower()\n",
        "data[\"class_name\"] = data[\"class_index\"].map({1:\"world\", 2:\"Sports\", 3:\"Business\", 4:\"Sci_Tech\"})\n",
        "\n",
        "# Clean up data a little more\n",
        "cols = [\"title\", \"description\"]\n",
        "data[cols] = data[cols].applymap(lambda x: x.replace(\"\\\\\",\" \"))\n",
        "data[cols] = data[cols].applymap(lambda x: x.replace(\"#36\",\"$\"))\n",
        "data[cols] = data[cols].applymap(lambda x: x.replace(\"  \",\" \"))\n",
        "data[cols] = data[cols].applymap(lambda x: x.strip())\n",
        "\n",
        "# Grab data head\n",
        "data.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KyuI5A8vRVIB"
      },
      "source": [
        "# Install spaCy"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zYBDTAc-RVIC",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9491b430-09aa-4fc0-c125-94c92ffbf23d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: spacy[cuda112,lookups,transformers] in /usr/local/lib/python3.8/dist-packages (3.4.3)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.8/dist-packages (from spacy[cuda112,lookups,transformers]) (2.0.7)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.8/dist-packages (from spacy[cuda112,lookups,transformers]) (1.0.3)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from spacy[cuda112,lookups,transformers]) (3.0.8)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.8/dist-packages (from spacy[cuda112,lookups,transformers]) (21.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.8/dist-packages (from spacy[cuda112,lookups,transformers]) (57.4.0)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.8/dist-packages (from spacy[cuda112,lookups,transformers]) (2.0.8)\n",
            "Requirement already satisfied: thinc<8.2.0,>=8.1.0 in /usr/local/lib/python3.8/dist-packages (from spacy[cuda112,lookups,transformers]) (8.1.5)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4 in /usr/local/lib/python3.8/dist-packages (from spacy[cuda112,lookups,transformers]) (1.10.2)\n",
            "Requirement already satisfied: pathy>=0.3.5 in /usr/local/lib/python3.8/dist-packages (from spacy[cuda112,lookups,transformers]) (0.10.0)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.8/dist-packages (from spacy[cuda112,lookups,transformers]) (1.0.9)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.8/dist-packages (from spacy[cuda112,lookups,transformers]) (4.64.1)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.8/dist-packages (from spacy[cuda112,lookups,transformers]) (2.23.0)\n",
            "Requirement already satisfied: typer<0.8.0,>=0.3.0 in /usr/local/lib/python3.8/dist-packages (from spacy[cuda112,lookups,transformers]) (0.7.0)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.9.1 in /usr/local/lib/python3.8/dist-packages (from spacy[cuda112,lookups,transformers]) (0.10.1)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.10 in /usr/local/lib/python3.8/dist-packages (from spacy[cuda112,lookups,transformers]) (3.0.10)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.8/dist-packages (from spacy[cuda112,lookups,transformers]) (2.4.5)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.8/dist-packages (from spacy[cuda112,lookups,transformers]) (1.21.6)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.8/dist-packages (from spacy[cuda112,lookups,transformers]) (3.3.0)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.8/dist-packages (from spacy[cuda112,lookups,transformers]) (2.11.3)\n",
            "Collecting spacy-transformers<1.2.0,>=1.1.2\n",
            "  Downloading spacy_transformers-1.1.8-py2.py3-none-any.whl (53 kB)\n",
            "\u001b[K     |████████████████████████████████| 53 kB 1.8 MB/s \n",
            "\u001b[?25hCollecting cupy-cuda112<12.0.0,>=5.0.0b4\n",
            "  Downloading cupy_cuda112-10.6.0-cp38-cp38-manylinux1_x86_64.whl (80.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 80.8 MB 93 kB/s \n",
            "\u001b[?25hCollecting spacy-lookups-data<1.1.0,>=1.0.3\n",
            "  Downloading spacy_lookups_data-1.0.3-py2.py3-none-any.whl (98.5 MB)\n",
            "\u001b[K     |████████████████████████████████| 98.5 MB 1.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: fastrlock>=0.5 in /usr/local/lib/python3.8/dist-packages (from cupy-cuda112<12.0.0,>=5.0.0b4->spacy[cuda112,lookups,transformers]) (0.8.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.8/dist-packages (from packaging>=20.0->spacy[cuda112,lookups,transformers]) (3.0.9)\n",
            "Requirement already satisfied: smart-open<6.0.0,>=5.2.1 in /usr/local/lib/python3.8/dist-packages (from pathy>=0.3.5->spacy[cuda112,lookups,transformers]) (5.2.1)\n",
            "Requirement already satisfied: typing-extensions>=4.1.0 in /usr/local/lib/python3.8/dist-packages (from pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4->spacy[cuda112,lookups,transformers]) (4.4.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests<3.0.0,>=2.13.0->spacy[cuda112,lookups,transformers]) (2022.9.24)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests<3.0.0,>=2.13.0->spacy[cuda112,lookups,transformers]) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests<3.0.0,>=2.13.0->spacy[cuda112,lookups,transformers]) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests<3.0.0,>=2.13.0->spacy[cuda112,lookups,transformers]) (3.0.4)\n",
            "Requirement already satisfied: torch>=1.6.0 in /usr/local/lib/python3.8/dist-packages (from spacy-transformers<1.2.0,>=1.1.2->spacy[cuda112,lookups,transformers]) (1.13.0+cu116)\n",
            "Collecting transformers<4.22.0,>=3.4.0\n",
            "  Downloading transformers-4.21.3-py3-none-any.whl (4.7 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.7 MB 67.3 MB/s \n",
            "\u001b[?25hCollecting spacy-alignments<1.0.0,>=0.7.2\n",
            "  Downloading spacy_alignments-0.8.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1 MB 60.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.8/dist-packages (from thinc<8.2.0,>=8.1.0->spacy[cuda112,lookups,transformers]) (0.0.3)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.8/dist-packages (from thinc<8.2.0,>=8.1.0->spacy[cuda112,lookups,transformers]) (0.7.9)\n",
            "Collecting huggingface-hub<1.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.11.1-py3-none-any.whl (182 kB)\n",
            "\u001b[K     |████████████████████████████████| 182 kB 78.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.8/dist-packages (from transformers<4.22.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->spacy[cuda112,lookups,transformers]) (3.8.0)\n",
            "Collecting tokenizers!=0.11.3,<0.13,>=0.11.1\n",
            "  Downloading tokenizers-0.12.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.6 MB 57.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.8/dist-packages (from transformers<4.22.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->spacy[cuda112,lookups,transformers]) (2022.6.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.8/dist-packages (from transformers<4.22.0,>=3.4.0->spacy-transformers<1.2.0,>=1.1.2->spacy[cuda112,lookups,transformers]) (6.0)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.8/dist-packages (from typer<0.8.0,>=0.3.0->spacy[cuda112,lookups,transformers]) (7.1.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.8/dist-packages (from jinja2->spacy[cuda112,lookups,transformers]) (2.0.1)\n",
            "Installing collected packages: tokenizers, huggingface-hub, transformers, spacy-alignments, spacy-transformers, spacy-lookups-data, cupy-cuda112\n",
            "Successfully installed cupy-cuda112-10.6.0 huggingface-hub-0.11.1 spacy-alignments-0.8.6 spacy-lookups-data-1.0.3 spacy-transformers-1.1.8 tokenizers-0.12.1 transformers-4.21.3\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting spacy-lookups-data==1.0.0\n",
            "  Downloading spacy_lookups_data-1.0.0-py2.py3-none-any.whl (93.4 MB)\n",
            "\u001b[K     |████████████████████████████████| 93.4 MB 60 kB/s \n",
            "\u001b[?25hRequirement already satisfied: setuptools in /usr/local/lib/python3.8/dist-packages (from spacy-lookups-data==1.0.0) (57.4.0)\n",
            "Installing collected packages: spacy-lookups-data\n",
            "  Attempting uninstall: spacy-lookups-data\n",
            "    Found existing installation: spacy-lookups-data 1.0.3\n",
            "    Uninstalling spacy-lookups-data-1.0.3:\n",
            "      Successfully uninstalled spacy-lookups-data-1.0.3\n",
            "Successfully installed spacy-lookups-data-1.0.0\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: cupy-cuda112 in /usr/local/lib/python3.8/dist-packages (10.6.0)\n",
            "Requirement already satisfied: numpy<1.25,>=1.18 in /usr/local/lib/python3.8/dist-packages (from cupy-cuda112) (1.21.6)\n",
            "Requirement already satisfied: fastrlock>=0.5 in /usr/local/lib/python3.8/dist-packages (from cupy-cuda112) (0.8.1)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting en-core-web-lg==3.4.1\n",
            "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.4.1/en_core_web_lg-3.4.1-py3-none-any.whl (587.7 MB)\n",
            "\u001b[K     |████████████████████████████████| 587.7 MB 17 kB/s \n",
            "\u001b[?25hRequirement already satisfied: spacy<3.5.0,>=3.4.0 in /usr/local/lib/python3.8/dist-packages (from en-core-web-lg==3.4.1) (3.4.3)\n",
            "Requirement already satisfied: typer<0.8.0,>=0.3.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (0.7.0)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (1.21.6)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (2.0.7)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (2.23.0)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (1.0.3)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (2.0.8)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (4.64.1)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.10 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (3.0.10)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (2.11.3)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (1.10.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (21.3)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (1.0.9)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.9.1 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (0.10.1)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (3.3.0)\n",
            "Requirement already satisfied: pathy>=0.3.5 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (0.10.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (57.4.0)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (3.0.8)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (2.4.5)\n",
            "Requirement already satisfied: thinc<8.2.0,>=8.1.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (8.1.5)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.8/dist-packages (from packaging>=20.0->spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (3.0.9)\n",
            "Requirement already satisfied: smart-open<6.0.0,>=5.2.1 in /usr/local/lib/python3.8/dist-packages (from pathy>=0.3.5->spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (5.2.1)\n",
            "Requirement already satisfied: typing-extensions>=4.1.0 in /usr/local/lib/python3.8/dist-packages (from pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4->spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (4.4.0)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (2022.9.24)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.8/dist-packages (from thinc<8.2.0,>=8.1.0->spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (0.7.9)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.8/dist-packages (from thinc<8.2.0,>=8.1.0->spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (0.0.3)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.8/dist-packages (from typer<0.8.0,>=0.3.0->spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (7.1.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.8/dist-packages (from jinja2->spacy<3.5.0,>=3.4.0->en-core-web-lg==3.4.1) (2.0.1)\n",
            "Installing collected packages: en-core-web-lg\n",
            "Successfully installed en-core-web-lg-3.4.1\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the package via spacy.load('en_core_web_lg')\n"
          ]
        }
      ],
      "source": [
        "# NOTE: Here, we put cuda112 since we are using cuda 11.2 in Jupyter notebooks (as shown by a simple nvcc --version call)\n",
        "!pip install -U spacy[cuda112,transformers,lookups]\n",
        "!pip install -U spacy-lookups-data==1.0.0 \n",
        "!pip install -U cupy-cuda112\n",
        "!python -m spacy download en_core_web_lg"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TpPfDofdRVIE"
      },
      "source": [
        "# Train NER model\n",
        "\n",
        "To train our spaCy model, we have to start by generating a config file for our model based off the template we have and using spaCy's `init fill-config` command."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KfxMiN9GRVIF"
      },
      "outputs": [],
      "source": [
        "# Set cfg file parameters for file path of various things:\n",
        "ner_path = \"/data/ag_dataset/ner/\"                                                  # Providing path to NER annotation directory\n",
        "config_file_path_input = cwd + ner_path + \"config_spacy_template_gpu_blank.cfg\"     # Path to downloaded spaCy cfg template\n",
        "config_file_path_output = cwd + ner_path + \"config_final_gpu_blank.cfg\"             # File path for filled config to be saved at"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JVMKQ8_fRVIF",
        "outputId": "8075fe2d-9f3f-40e2-af09-fdc619f5e92d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[38;5;2m✔ Auto-filled config with all values\u001b[0m\n",
            "\u001b[38;5;2m✔ Saved config\u001b[0m\n",
            "/content/data/ag_dataset/ner/config_final_gpu_blank.cfg\n",
            "You can now add your data and train your pipeline:\n",
            "python -m spacy train config_final_gpu_blank.cfg --paths.train ./train.spacy --paths.dev ./dev.spacy\n"
          ]
        }
      ],
      "source": [
        "# Generate config file, taking in the template as input file path and giving a path for the config to be saved\n",
        "!python -m spacy init fill-config \"$config_file_path_input\" \"$config_file_path_output\""
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now that we've generated a config file for the model to be trained off, we can start training:"
      ],
      "metadata": {
        "id": "mvQ8s5YbUU0e"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "12mSsoOvRVIG"
      },
      "outputs": [],
      "source": [
        "# Set file parameters for training\n",
        "output_path = cwd + \"/models/ag_dataset/ner/ner-gpu-blank\"                          # File path for model to be saved at\n",
        "train_path = cwd + \"/data/ag_dataset/ner/annotations/binary/train\"                  # File path for training data annotations\n",
        "dev_path = cwd + \"/data/ag_dataset/ner/annotations/binary/eval\"                     # File path for evaluation data annotations"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ArnYyljqRVIG"
      },
      "outputs": [],
      "source": [
        "# Do the training, passing in config file path, output path, training path, and eval path (called dev path here)\n",
        "!python -m spacy train \"$config_file_path_output\" \\\n",
        " --output \"$output_path\" --paths.train \"$train_path\" --paths.dev \"$dev_path\" \\\n",
        "  --training.max_epochs 30 --gpu-id 0 --verbose\n",
        "\n",
        "# Output removed because it was too large, but it essentially saved our model at the output_path for\n",
        "#   later use."
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Run NER Model:"
      ],
      "metadata": {
        "id": "2Vhriy9FcOfU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import spacy\n",
        "from spacy import displacy\n",
        "import random\n",
        "\n",
        "spacy.require_gpu()\n",
        "custom_ner_model = spacy.load(cwd + \\\n",
        "    '/models/ag_dataset/ner/ner-gpu-blank/model-last')\n",
        "options = {\"ents\": [\"ORG\",\"PERSON\",\"GPE\",\"TICKER\"]}\n",
        "\n",
        "for j in range(3):\n",
        "    i = random.randint(0, len(data))\n",
        "    print(\"Article\",i)\n",
        "    doc_custom = custom_ner_model(data.loc[i,\"description\"])\n",
        "    print(\"Custom Model NER:\")\n",
        "    displacy.render(doc_custom, style=\"ent\", options=options, jupyter=True)\n",
        "    print(\"\\n\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 525
        },
        "id": "WH9y7yoSV9e0",
        "outputId": "c554c545-2fa8-4986-f1c9-cb7629a71f16"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.8/dist-packages/spacy/util.py:877: UserWarning: [W095] Model 'en_pipeline' (0.0.0) was trained with spaCy v3.0 and may not be 100% compatible with the current version (3.4.3). If you see errors or degraded performance, download a newer compatible model or retrain your custom model with the current spaCy version. For more details and available updates, run: python -m spacy validate\n",
            "  warnings.warn(warn_msg)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Article 104315\n",
            "Custom Model NER:\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Reuters\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              " - Five-time American League batting champion \n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Wade Boggs\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              ", two-time NL batting champ \n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Willie McGee\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              " and former Cy Young Award winner \n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Jack McDowell\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              " are among 12 first-year candidates on the 2005 Baseball Hall of Fame ballot.</div></span>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "Article 72745\n",
            "Custom Model NER:\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
              "<mark class=\"entity\" style=\"background: #feca74; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Mexico\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">GPE</span>\n",
              "</mark>\n",
              " has some of the highest and the lowest standards of living in the world, such is the extent of the gap between rich and poor, a \n",
              "<mark class=\"entity\" style=\"background: #7aecec; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    UN\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">ORG</span>\n",
              "</mark>\n",
              " report says.</div></span>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "Article 116312\n",
            "Custom Model NER:\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.8/dist-packages/spacy/displacy/__init__.py:206: UserWarning: [W006] No entities to visualize found in Doc object. If this is surprising to you, make sure the Doc was processed using a model that supports named entity recognition, and check the `doc.ents` property manually if necessary.\n",
            "  warnings.warn(Warnings.W006)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">The City Council narrowly approved a taxpayer-financed plan late Tuesday night to build a new riverfront baseball stadium. But it was not clear whether the amended measure</div></span>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "prediction = custom_ner_model(\"Bolsonaro supporters try to storm police HQ in 'January 6-style' rampage\")\n",
        "print(\"Custom Model NER:\")\n",
        "displacy.render(prediction, style=\"ent\", options=options, jupyter=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "id": "nvjkI_5hXYKF",
        "outputId": "8e2cf62b-9c3b-4385-87f4-7fc54a19b595"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Custom Model NER:\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Bolsonaro\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              " supporters try to storm police HQ in 'January 6-style' rampage</div></span>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Train Text Classification Model\n",
        "The process of training a text classificaiton model is similar to that of training an NER model with spaCy. We do this by generating a config file for training and then undergoing training using annotated data."
      ],
      "metadata": {
        "id": "109RWF3m9ACi"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "config_file_path_output = cwd + \"/data/ag_dataset/textcat/config_final.cfg\"     # Config file parameter\n",
        "\n",
        "# Create config from scratch. This creation is a little more complicated, taking in language as a parameter as well as\n",
        "#   GPU and pipeline. Textcat means text categoriztion, and multilabel means that a single training example\n",
        "#   might have multiple labels due to conflicting annotations.\n",
        "!python -m spacy init config \"$config_file_path_output\" --lang en \\\n",
        "--pipeline textcat_multilabel --optimize efficiency --gpu --force"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SZ8j9xDa-Sot",
        "outputId": "303aae28-6c5a-4182-f625-64ba8d3240d0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[38;5;4mℹ Generated config template specific for your use case\u001b[0m\n",
            "- Language: en\n",
            "- Pipeline: textcat_multilabel\n",
            "- Optimize for: efficiency\n",
            "- Hardware: GPU\n",
            "- Transformer: roberta-base\n",
            "\u001b[38;5;2m✔ Auto-filled config with all values\u001b[0m\n",
            "\u001b[38;5;2m✔ Saved config\u001b[0m\n",
            "/content/data/ag_dataset/textcat/config_final.cfg\n",
            "You can now add your data and train your pipeline:\n",
            "python -m spacy train config_final.cfg --paths.train ./train.spacy --paths.dev ./dev.spacy\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now that we've created our config file from scratch, we can train our model:"
      ],
      "metadata": {
        "id": "A-2HdqeAAx8M"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import spacy\n",
        "annots_path = \"/data/ag_dataset/textcat/annotations/binary/\"\n",
        "output_path = cwd + \"/models/ag_dataset/textcat/full_labels\"\n",
        "train_path = cwd + annots_path + \"train_full_labels\"\n",
        "dev_path = cwd + annots_path + \"eval\""
      ],
      "metadata": {
        "id": "LxGdtz8V_GVw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!python -m spacy train \"$config_file_path_output\" \\\n",
        "--output \"$output_path\" --paths.train \"$train_path\" \\\n",
        "--paths.dev \"$dev_path\" --gpu-id 0 --training.max_epochs 1 --verbose"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "92uP6Hf0CW5O",
        "outputId": "f3347b40-1545-475e-e6d8-a81057f19f39"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[2022-12-14 05:54:50,119] [DEBUG] Config overrides from CLI: ['paths.train', 'paths.dev', 'training.max_epochs']\n",
            "DEBUG:spacy:Config overrides from CLI: ['paths.train', 'paths.dev', 'training.max_epochs']\n",
            "\u001b[38;5;4mℹ Saving to output directory:\n",
            "/content/models/ag_dataset/textcat/full_labels\u001b[0m\n",
            "\u001b[38;5;4mℹ Using GPU: 0\u001b[0m\n",
            "\u001b[1m\n",
            "=========================== Initializing pipeline ===========================\u001b[0m\n",
            "[2022-12-14 05:54:51,963] [INFO] Set up nlp object from config\n",
            "INFO:spacy:Set up nlp object from config\n",
            "[2022-12-14 05:54:51,972] [DEBUG] Loading corpus from path: /content/data/ag_dataset/textcat/annotations/binary/eval\n",
            "DEBUG:spacy:Loading corpus from path: /content/data/ag_dataset/textcat/annotations/binary/eval\n",
            "[2022-12-14 05:54:51,974] [DEBUG] Loading corpus from path: /content/data/ag_dataset/textcat/annotations/binary/train_full_labels\n",
            "DEBUG:spacy:Loading corpus from path: /content/data/ag_dataset/textcat/annotations/binary/train_full_labels\n",
            "[2022-12-14 05:54:51,974] [INFO] Pipeline: ['textcat_multilabel']\n",
            "INFO:spacy:Pipeline: ['textcat_multilabel']\n",
            "[2022-12-14 05:54:51,977] [INFO] Created vocabulary\n",
            "INFO:spacy:Created vocabulary\n",
            "[2022-12-14 05:54:51,978] [INFO] Finished initializing nlp object\n",
            "INFO:spacy:Finished initializing nlp object\n",
            "[2022-12-14 05:56:06,897] [INFO] Initialized pipeline components: ['textcat_multilabel']\n",
            "INFO:spacy:Initialized pipeline components: ['textcat_multilabel']\n",
            "\u001b[38;5;2m✔ Initialized pipeline\u001b[0m\n",
            "\u001b[1m\n",
            "============================= Training pipeline =============================\u001b[0m\n",
            "[2022-12-14 05:56:06,908] [DEBUG] Loading corpus from path: /content/data/ag_dataset/textcat/annotations/binary/eval\n",
            "DEBUG:spacy:Loading corpus from path: /content/data/ag_dataset/textcat/annotations/binary/eval\n",
            "[2022-12-14 05:56:06,909] [DEBUG] Loading corpus from path: /content/data/ag_dataset/textcat/annotations/binary/train_full_labels\n",
            "DEBUG:spacy:Loading corpus from path: /content/data/ag_dataset/textcat/annotations/binary/train_full_labels\n",
            "[2022-12-14 05:56:06,911] [DEBUG] Removed existing output directory: /content/models/ag_dataset/textcat/full_labels/model-best\n",
            "DEBUG:spacy:Removed existing output directory: /content/models/ag_dataset/textcat/full_labels/model-best\n",
            "[2022-12-14 05:56:06,913] [DEBUG] Removed existing output directory: /content/models/ag_dataset/textcat/full_labels/model-last\n",
            "DEBUG:spacy:Removed existing output directory: /content/models/ag_dataset/textcat/full_labels/model-last\n",
            "\u001b[38;5;4mℹ Pipeline: ['textcat_multilabel']\u001b[0m\n",
            "\u001b[38;5;4mℹ Initial learn rate: 0.0\u001b[0m\n",
            "E    #       LOSS TEXTC...  CATS_SCORE  SCORE \n",
            "---  ------  -------------  ----------  ------\n",
            "  0       0           1.00       50.00    0.50\n",
            "  0     200         169.87       77.15    0.77\n",
            "  0     400         146.31       81.50    0.81\n",
            "  0     600         127.47       85.49    0.85\n",
            "  0     800         116.73       88.69    0.89\n",
            "  0    1000         108.84       90.93    0.91\n",
            "  0    1200         103.31       92.48    0.92\n",
            "  0    1400          95.60       93.52    0.94\n",
            "  0    1600          92.38       94.24    0.94\n",
            "  0    1800          87.69       94.76    0.95\n",
            "  0    2000          82.77       95.14    0.95\n",
            "  0    2200          80.19       95.42    0.95\n",
            "\u001b[38;5;2m✔ Saved pipeline to output directory\u001b[0m\n",
            "/content/models/ag_dataset/textcat/full_labels/model-last\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Run Text Classification Model"
      ],
      "metadata": {
        "id": "wYb-XxtuEmtR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import spacy\n",
        "\n",
        "spacy.require_gpu()\n",
        "custom_text_class_model = spacy.load(cwd + \\\n",
        "    '/models/ag_dataset/textcat/full_labels/model-best')\n",
        "options = {\"ents\": [\"ORG\",\"PERSON\",\"GPE\",\"TICKER\"]}\n",
        "\n",
        "# To do prediction, simply load the model and use it as a function, giving in the string. Access predictions\n",
        "#   using .cats and print them out to find prediction values\n",
        "prediction = custom_text_class_model(\"2023 NFL draft QB Hot Board: Ranking top 17 quarterbacks, risers\")\n",
        "print(f\"Custom Model classifications:\")\n",
        "print(prediction.cats)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6Y4srPESCbWA",
        "outputId": "c620fdaa-c7c5-4734-e278-e442c47c0e33"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Custom Model classifications:\n",
            "{'World': 0.394867479801178, 'Sci_Tech': 0.42078670859336853, 'Business': 0.44176068902015686, 'Sports': 0.518925130367279}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!zip -r \"text-categorization-model.zip\" \"/content/models/ag_dataset/textcat/full_labels/model-best\""
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CnJZPM2gPnO1",
        "outputId": "6c2f1b39-0f83-444f-fa92-18a326ec4fd6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  adding: content/models/ag_dataset/textcat/full_labels/model-best/ (stored 0%)\n",
            "  adding: content/models/ag_dataset/textcat/full_labels/model-best/tokenizer (deflated 81%)\n",
            "  adding: content/models/ag_dataset/textcat/full_labels/model-best/meta.json (deflated 62%)\n",
            "  adding: content/models/ag_dataset/textcat/full_labels/model-best/config.cfg (deflated 61%)\n",
            "  adding: content/models/ag_dataset/textcat/full_labels/model-best/vocab/ (stored 0%)\n",
            "  adding: content/models/ag_dataset/textcat/full_labels/model-best/vocab/vectors (deflated 45%)\n",
            "  adding: content/models/ag_dataset/textcat/full_labels/model-best/vocab/lookups.bin (stored 0%)\n",
            "  adding: content/models/ag_dataset/textcat/full_labels/model-best/vocab/key2row (stored 0%)\n",
            "  adding: content/models/ag_dataset/textcat/full_labels/model-best/vocab/strings.json (deflated 75%)\n",
            "  adding: content/models/ag_dataset/textcat/full_labels/model-best/vocab/vectors.cfg (stored 0%)\n",
            "  adding: content/models/ag_dataset/textcat/full_labels/model-best/textcat_multilabel/ (stored 0%)\n",
            "  adding: content/models/ag_dataset/textcat/full_labels/model-best/textcat_multilabel/cfg (deflated 24%)\n",
            "  adding: content/models/ag_dataset/textcat/full_labels/model-best/textcat_multilabel/model (deflated 78%)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "9NyDM5JLR6Il"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Running Saved Models\n"
      ],
      "metadata": {
        "id": "NkJTZZfCNXlz"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "First, we reinstall spaCy and mount our google drive:"
      ],
      "metadata": {
        "id": "ax6pssB2jxET"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -U spacy[cuda112,transformers,lookups]\n",
        "!pip install -U spacy-lookups-data==1.0.0 \n",
        "!pip install -U cupy-cuda112\n",
        "!python -m spacy download en_core_web_lg"
      ],
      "metadata": {
        "id": "2UPZ2qEAj1Zj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Fgu_j8T6Dhcw",
        "outputId": "a0decb6d-e579-458f-dea0-7225fdae9f48"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now, we can load our spaCy models from the google drive:"
      ],
      "metadata": {
        "id": "kQiTWl2uPROZ"
      }
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "RFC5MAgbRka_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "j-ZowC2FRkRo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "01eyZrptRkDf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "Mj91Mcu7Rj4u"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "2rO7JBsZRjqt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# NER Demo"
      ],
      "metadata": {
        "id": "fnNFAsEyRYlQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Import necessary libraries\n",
        "import spacy\n",
        "from spacy import displacy\n",
        "import random"
      ],
      "metadata": {
        "id": "Wjc0fBlGZTeK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Load in our NER Model\n",
        "spacy.require_gpu()\n",
        "custom_ner_model = spacy.load('/content/drive/MyDrive/NLP Book Notes/ner-last-model/models/ag_dataset/ner/ner-gpu-blank/model-last')\n",
        "options = {\"ents\": [\"ORG\",\"PERSON\",\"GPE\",\"TICKER\"]}\n",
        "\n",
        "# Generate Results\n",
        "headline = input(\"Input a headline to predict on: \\n\\t\")\n",
        "print()\n",
        "doc_custom = custom_ner_model(headline)\n",
        "print(\"Custom Model NER Output:\")\n",
        "displacy.render(doc_custom, style=\"ent\", options=options, jupyter=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 125
        },
        "id": "wm_3IlJyOFK2",
        "outputId": "8b62b5c4-faec-4b82-c9c5-531f08595298"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Input a headline to predict on: \n",
            "\tUkraine's Zelenskiy opposes idea of Russian athletes at Olympics under neutral banner\n",
            "\n",
            "Custom Model NER Output:\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "<span class=\"tex2jax_ignore\"><div class=\"entities\" style=\"line-height: 2.5; direction: ltr\">\n",
              "<mark class=\"entity\" style=\"background: #feca74; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Ukraine\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">GPE</span>\n",
              "</mark>\n",
              "'s \n",
              "<mark class=\"entity\" style=\"background: #aa9cfc; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Zelenskiy\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">PERSON</span>\n",
              "</mark>\n",
              " opposes idea of \n",
              "<mark class=\"entity\" style=\"background: #feca74; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;\">\n",
              "    Russian\n",
              "    <span style=\"font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; vertical-align: middle; margin-left: 0.5rem\">GPE</span>\n",
              "</mark>\n",
              " athletes at Olympics under neutral banner</div></span>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "E3XamVvcRnot"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "Fg33bZc3Rnmg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "EXf4kT_QRnkq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "Nm4lTJV-Rnic"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "mH01vtKIRng4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "JXET6OxSRnfG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "t5hgphBDRna1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "bvoblGRpRnYb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "RSiQbN4bRnWG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "k-eJu9SkRnTl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "-s_8hAbqRqYI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "inxu5cWdRqWO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "NduQ2tslRqT7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "cKfVs92dRqRp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "s8kYIRf7RqOa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Text Categorization Demo"
      ],
      "metadata": {
        "id": "qajZNFtaRd1h"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import spacy\n",
        "\n",
        "spacy.require_gpu()\n",
        "custom_text_class_model = spacy.load('/content/drive/MyDrive/NLP Book Notes/text-categorization-model/content/models/ag_dataset/textcat/full_labels/model-best')\n",
        "options = {\"ents\": [\"ORG\",\"PERSON\",\"GPE\",\"TICKER\"]}\n",
        "\n",
        "# To do prediction, simply load the model and use it as a function, giving in the string. Access predictions\n",
        "#   using .cats and print them out to find prediction values\n",
        "headline = input(\"Input a headline to predict on: \\n\\t\")\n",
        "print()\n",
        "probabilities = custom_text_class_model(headline).cats\n",
        "print(f\"Model's probabilities of each category are:\")\n",
        "print(probabilities)"
      ],
      "metadata": {
        "id": "76M5yvGHOgSe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "GHL1KIhhMc87"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.4"
    },
    "orig_nbformat": 4,
    "vscode": {
      "interpreter": {
        "hash": "3ad933181bd8a04b432d3370b9dc3b0662ad032c4dfaa4e4f1596c548f763858"
      }
    },
    "colab": {
      "provenance": []
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}