7.PRETRAIN_METHOD/7.2.4.bert_finetune_KorSTS.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import tensorflow as tf\n",
    "from transformers import BertTokenizer, TFBertModel\n",
    "\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 시각화\n",
    "def plot_graphs(history, string):\n",
    "    plt.plot(history.history[string])\n",
    "    plt.plot(history.history['val_'+string], '')\n",
    "    plt.xlabel(\"Epochs\")\n",
    "    plt.ylabel(string)\n",
    "    plt.legend([string, 'val_'+string])\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SEED_NUM = 1234\n",
    "tf.random.set_seed(SEED_NUM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained(\"bert-base-multilingual-cased\",\n",
    "                                          cache_dir='bert_ckpt',\n",
    "                                          do_lower_case=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#random seed 고정\n",
    "\n",
    "tf.random.set_seed(0)\n",
    "np.random.seed(0)\n",
    "\n",
    "# BASE PARAM\n",
    "\n",
    "BATCH_SIZE = 32\n",
    "NUM_EPOCHS = 3\n",
    "VALID_SPLIT = 0.2\n",
    "MAX_LEN = 28 * 2 \n",
    "\n",
    "DATA_IN_PATH = 'data_in/KOR'\n",
    "DATA_OUT_PATH = \"data_out/KOR\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Special Tokens\n",
    "print(tokenizer.all_special_tokens, \"\\n\", tokenizer.all_special_ids)\n",
    "\n",
    "# Test Tokenizers\n",
    "kor_encode = tokenizer.encode(\"안녕하세요, 반갑습니다\")\n",
    "eng_encode = tokenizer.encode(\"Hello world\")\n",
    "\n",
    "kor_decode = tokenizer.decode(kor_encode)\n",
    "eng_decode = tokenizer.decode(eng_encode)\n",
    "\n",
    "print(kor_encode)\n",
    "print(eng_encode)\n",
    "print(kor_decode)\n",
    "print(eng_decode)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# KorSTS Dataset\n",
    "\n",
    "Data from Kakaobrain:  https://github.com/kakaobrain/KorNLUDatasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Train dataset\n",
    "\n",
    "TRAIN_STS_DF = os.path.join(DATA_IN_PATH, 'KorSTS', 'sts-train.tsv')\n",
    "DEV_STS_DF = os.path.join(DATA_IN_PATH, 'KorSTS', 'sts-dev.tsv')\n",
    "\n",
    "train_data = pd.read_csv(TRAIN_STS_DF, header=0, delimiter = '\\t', quoting = 3)\n",
    "dev_data = pd.read_csv(DEV_STS_DF, header=0, delimiter = '\\t', quoting = 3)\n",
    "\n",
    "print(\"Total # dataset: train - {}, dev - {}\".format(len(train_data), len(dev_data)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Bert Tokenizer\n",
    "\n",
    "# 참조: https://huggingface.co/transformers/main_classes/tokenizer.html?highlight=encode_plus#transformers.PreTrainedTokenizer.encode_plus\n",
    "\n",
    "def bert_tokenizer_v2(sent1, sent2, MAX_LEN):\n",
    "    \n",
    "    # For Two setenece input\n",
    "    \n",
    "    encoded_dict = tokenizer.encode_plus(\n",
    "        text = sent1,\n",
    "        text_pair = sent2,\n",
    "        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
    "        max_length = MAX_LEN,           # Pad & truncate all sentences.\n",
    "        pad_to_max_length = True,\n",
    "        return_attention_mask = True   # Construct attn. masks.\n",
    "        \n",
    "    )\n",
    "    \n",
    "    input_id = encoded_dict['input_ids']\n",
    "    attention_mask = encoded_dict['attention_mask'] # And its attention mask (simply differentiates padding from non-padding).\n",
    "    token_type_id = encoded_dict['token_type_ids'] # differentiate two sentences\n",
    "    \n",
    "    return input_id, attention_mask, token_type_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 토크나이저를 제외하고는 5장에서 처리한 방식과 유사하게 접근\n",
    "def clean_text(sent):\n",
    "    sent_clean = re.sub(\"[^a-zA-Z0-9ㄱ-ㅣ가-힣\\\\s]\", \" \", sent)\n",
    "    return sent_clean\n",
    "\n",
    "input_ids = []\n",
    "attention_masks = []\n",
    "token_type_ids = []\n",
    "data_labels = []\n",
    "\n",
    "\n",
    "for sent1, sent2, score in train_data[['sentence1', 'sentence2', 'score']].values:\n",
    "    try:\n",
    "        input_id, attention_mask, token_type_id = bert_tokenizer_v2(clean_text(sent1), clean_text(sent2), MAX_LEN)\n",
    "        input_ids.append(input_id)\n",
    "        attention_masks.append(attention_mask)\n",
    "        token_type_ids.append(token_type_id)\n",
    "        data_labels.append(score)\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        print(sent1, sent2)\n",
    "        pass\n",
    "    \n",
    "train_input_ids = np.array(input_ids, dtype=int)\n",
    "train_attention_masks = np.array(attention_masks, dtype=int)\n",
    "train_type_ids = np.array(token_type_ids, dtype=int)\n",
    "train_inputs = (train_input_ids, train_attention_masks, train_type_ids)\n",
    "train_data_labels = np.array(data_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DEV SET Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 토크나이저를 제외하고는 5장에서 처리한 방식과 유사하게 접근\n",
    "input_ids = []\n",
    "attention_masks = []\n",
    "token_type_ids = []\n",
    "data_labels = []\n",
    "\n",
    "for sent1, sent2, score in dev_data[['sentence1', 'sentence2', 'score']].values:\n",
    "    try:\n",
    "        input_id, attention_mask, token_type_id = bert_tokenizer_v2(clean_text(sent1), clean_text(sent2), MAX_LEN)\n",
    "        input_ids.append(input_id)\n",
    "        attention_masks.append(attention_mask)\n",
    "        token_type_ids.append(token_type_id)\n",
    "        data_labels.append(score)\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        print(sent1, sent2)\n",
    "        pass\n",
    "    \n",
    "dev_input_ids = np.array(input_ids, dtype=int)\n",
    "dev_attention_masks = np.array(attention_masks, dtype=int)\n",
    "dev_type_ids = np.array(token_type_ids, dtype=int)\n",
    "dev_inputs = (dev_input_ids, dev_attention_masks, dev_type_ids)\n",
    "dev_data_labels = np.array(data_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"# train labels: {}, #dev labels: {}\".format(len(train_data_labels), len(dev_data_labels)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TFBertRegressor(tf.keras.Model):\n",
    "    def __init__(self, model_name, dir_path, num_class):\n",
    "        super(TFBertRegressor, self).__init__()\n",
    "        \n",
    "        self.bert = TFBertModel.from_pretrained(model_name, cache_dir=dir_path)\n",
    "        self.num_class = num_class\n",
    "        self.dropout = tf.keras.layers.Dropout(self.bert.config.hidden_dropout_prob)\n",
    "        self.regressor = tf.keras.layers.Dense(self.num_class, \n",
    "                                                kernel_initializer=tf.keras.initializers.TruncatedNormal(self.bert.config.initializer_range), \n",
    "                                                name=\"regressor\")\n",
    "        \n",
    "        \n",
    "    def call(self, inputs, attention_mask=None, token_type_ids=None, training=False):\n",
    "        \n",
    "        #outputs 값: # sequence_output, pooled_output, (hidden_states), (attentions)\n",
    "        outputs = self.bert(inputs, attention_mask=attention_mask, token_type_ids=token_type_ids)\n",
    "        pooled_output = outputs[1]\n",
    "        pooled_output = self.dropout(pooled_output, training=training)\n",
    "        logits = self.regressor(pooled_output)\n",
    "\n",
    "        return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "regression_model = TFBertRegressor(model_name='bert-base-multilingual-cased',\n",
    "                                  dir_path='bert_ckpt',\n",
    "                                  num_class=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PearsonCorrelationMetric(tf.keras.metrics.Metric):\n",
    "    def __init__(self, name=\"pearson_correlation\", **kwargs):\n",
    "        super(PearsonCorrelationMetric, self).__init__(name=name, **kwargs)\n",
    "        self.y_true_list = []\n",
    "        self.y_pred_list = []\n",
    "\n",
    "    def update_state(self, y_true, y_pred, sample_weight=None):\n",
    "        y_true = tf.reshape(y_true, shape=[-1])\n",
    "        y_pred = tf.reshape(y_pred, shape=[-1])\n",
    "        self.y_true_list.append(y_true)\n",
    "        self.y_pred_list.append(y_pred)\n",
    "\n",
    "    def result(self):\n",
    "        y_true = tf.concat(self.y_true_list, -1)\n",
    "        y_pred = tf.concat(self.y_pred_list, -1)\n",
    "        pearson_correlation = self.pearson(y_true, y_pred)\n",
    "        \n",
    "        return pearson_correlation\n",
    "\n",
    "    def reset_states(self):\n",
    "        self.y_true_list = []\n",
    "        self.y_pred_list = []\n",
    "        \n",
    "\n",
    "    def pearson(self, true, pred):\n",
    "        m_true = tf.reduce_mean(true)\n",
    "        m_pred = tf.reduce_mean(pred)\n",
    "        m_true, m_pred = true-m_true, pred-m_pred\n",
    "        num = tf.reduce_sum(tf.multiply(m_true, m_pred))\n",
    "        den = tf.sqrt(tf.multiply(tf.reduce_sum(tf.square(m_true)), tf.reduce_sum(tf.square(m_pred)))) + 1e-12\n",
    "        return num / den"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.keras.optimizers.Adam(3e-5)\n",
    "loss = tf.keras.losses.MeanSquaredError()\n",
    "metric = PearsonCorrelationMetric()\n",
    "regression_model.compile(optimizer=optimizer, loss=loss, metrics=[metric], run_eagerly=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#학습 진행하기\n",
    "model_name = \"tf2_BERT_KorSTS\"\n",
    "\n",
    "# overfitting을 막기 위한 ealrystop 추가\n",
    "earlystop_callback = EarlyStopping(monitor='val_pearson_correlation', min_delta=0.0001,patience=2,mode='max')\n",
    "# min_delta: the threshold that triggers the termination (acc should at least improve 0.0001)\n",
    "# patience: no improvment epochs (patience = 1, 1번 이상 상승이 없으면 종료)\\\n",
    "\n",
    "checkpoint_path = os.path.join(DATA_OUT_PATH, model_name, 'weights.h5')\n",
    "checkpoint_dir = os.path.dirname(checkpoint_path)\n",
    "\n",
    "# Create path if exists\n",
    "if os.path.exists(checkpoint_dir):\n",
    "    print(\"{} -- Folder already exists \\n\".format(checkpoint_dir))\n",
    "else:\n",
    "    os.makedirs(checkpoint_dir, exist_ok=True)\n",
    "    print(\"{} -- Folder create complete \\n\".format(checkpoint_dir))\n",
    "    \n",
    "cp_callback = ModelCheckpoint(\n",
    "    checkpoint_path, monitor='val_pearson_correlation', verbose=1, save_best_only=True, save_weights_only=True,mode='max')\n",
    "\n",
    "# 학습과 eval 시작\n",
    "history = regression_model.fit(train_inputs, train_data_labels, epochs=NUM_EPOCHS,\n",
    "            validation_data = (dev_inputs, dev_data_labels),\n",
    "            batch_size=BATCH_SIZE, callbacks=[earlystop_callback, cp_callback])\n",
    "\n",
    "#steps_for_epoch\n",
    "print(history.history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_graphs(history, 'pearson_correlation')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_graphs(history, 'loss')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# KorSTS Test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Test dataset\n",
    "TEST_STS_DF = os.path.join(DATA_IN_PATH, 'KorSTS', 'sts-test.tsv')\n",
    "\n",
    "test_data = pd.read_csv(TEST_STS_DF, header=0, delimiter = '\\t', quoting = 3)\n",
    "test_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test set도 똑같은 방법으로 구성한다.\n",
    "input_ids = []\n",
    "attention_masks = []\n",
    "token_type_ids = []\n",
    "data_labels = []\n",
    "\n",
    "for sent1, sent2, score in test_data[['sentence1', 'sentence2', 'score']].values:\n",
    "    try:\n",
    "        input_id, attention_mask, token_type_id = bert_tokenizer_v2(clean_text(sent1), clean_text(sent2), MAX_LEN)\n",
    "        input_ids.append(input_id)\n",
    "        attention_masks.append(attention_mask)\n",
    "        token_type_ids.append(token_type_id)\n",
    "        data_labels.append(score)\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        print(sent1, sent2)\n",
    "        pass\n",
    "    \n",
    "test_input_ids = np.array(input_ids, dtype=int)\n",
    "test_attention_masks = np.array(attention_masks, dtype=int)\n",
    "test_type_ids = np.array(token_type_ids, dtype=int)\n",
    "test_inputs = (test_input_ids, test_attention_masks, test_type_ids)\n",
    "test_data_labels = np.array(data_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"# sents: {}, # labels: {}\".format(len(test_input_ids), len(test_data_labels)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "regression_model.load_weights(checkpoint_path)\n",
    "\n",
    "results = regression_model.evaluate(test_inputs, test_data_labels, batch_size=512)\n",
    "print(\"test loss, test pearson correlation: \", results)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}