7. bert_scores_multilingual_layer_selection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import pandas as pd\n",
    "from scipy.stats import pearsonr \n",
    "import matplotlib.pyplot as plt\n",
    "plt.style.use('seaborn-whitegrid')\n",
    "import numpy as np\n",
    "\n",
    "pretrained = 'bert-base-multilingual-uncased'\n",
    "#pretrained = 'bert-base-multilingual-cased' \n",
    "#pretrained = 'xlm-roberta-base'\n",
    "#pretrained = 'xlm-roberta-large'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_array(arr):\n",
    "    arr = arr.replace('[','').replace(']','')\n",
    "    num = arr.split(', ')\n",
    "    num = [float(a) for a in num]\n",
    "    return num\n",
    "\n",
    "def read_array2(arr):\n",
    "    arr = arr.replace('[','').replace(']','')\n",
    "    num = arr.split(', ')\n",
    "    num = [a.replace('\\'','') for a in num]\n",
    "    return num\n",
    "\n",
    "def match(keys, values):\n",
    "    res = {}\n",
    "    for idx, key in enumerate(keys):\n",
    "        res[key] = values[idx]\n",
    "    return res\n",
    "        \n",
    "def correlation(human_metric, id2score):\n",
    "    filtered_human = []\n",
    "    filtered_metric = []\n",
    "    \n",
    "    for key in id2score.keys():\n",
    "        if len(human_metric[human_metric['id']==int(key)]) == 0:\n",
    "            continue\n",
    "        human_score = human_metric[human_metric['id'] == int(key)]['score'].tolist()[0]\n",
    "        filtered_human.append(human_score)\n",
    "        filtered_metric.append(id2score[key])\n",
    "    return pearsonr(filtered_human, filtered_metric)[0], len(filtered_human)\n",
    "\n",
    "def get_max(arr):\n",
    "    max_id = np.argmax(arr)\n",
    "    print(max_id+1, '--', arr[max_id])\n",
    "    min_id = np.argmin(arr)\n",
    "    print(min_id+1, '--', arr[min_id])\n",
    "\n",
    "def get_best_rank(corr_arrs):\n",
    "    score = 0\n",
    "    rank = -1\n",
    "    \n",
    "    for idx in range(len(corr_arrs[0])):\n",
    "        tmp = 0\n",
    "        for corr_arr in corr_arrs:\n",
    "            tmp += corr_arr[idx]\n",
    "        tmp /= len(corr_arrs)\n",
    "        if tmp > score:\n",
    "            rank = idx\n",
    "            score = tmp        \n",
    "    print('Best layer:', rank+1)\n",
    "    return rank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_human_annotation(lang, model, types='focus'):\n",
    "    if types=='focus':\n",
    "        path_human = f'mturk/annotation_result/{lang}/human_focus_final.csv'\n",
    "    else:\n",
    "        assert types=='coverage'\n",
    "        path_human = f'mturk/annotation_result/{lang}/human_coverage_final.csv'\n",
    "    human = pd.read_csv(path_human)\n",
    "    return human[human['model']==model]\n",
    "\n",
    "def read(lang, model, human_score, prec_or_rec):\n",
    "    path = f'bert_score/{lang}--{model}--{pretrained}.csv'\n",
    "    df = pd.read_csv(path)\n",
    "    xlabel = []; cors = []\n",
    "    for idx, row in df.iterrows():\n",
    "        cur_layer = row['layer']\n",
    "        score = read_array(row[prec_or_rec])\n",
    "        doc_id = read_array2(row['fnames'])\n",
    "        id2score = match(doc_id, score)\n",
    "        \n",
    "        xlabel.append(cur_layer)\n",
    "        cors.append(correlation(human_score, id2score)[0])\n",
    "    return xlabel, cors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Focus of bert-base-multilingual-uncased\n",
      "Best layer: 12\n",
      "Individual model performance:\n",
      "('EN', 'PG') 0.6366980318400038\n",
      "('EN', 'BERT') 0.6100619544153179\n",
      "('ID', 'PG') 0.6826102703970842\n",
      "('ID', 'BERT') 0.7183780273572422\n",
      "('FR', 'PG') 0.6720144746324448\n",
      "('FR', 'BERT') 0.7330420943115018\n",
      "('TR', 'PG') 0.860145104227336\n",
      "('TR', 'BERT') 0.8092996194988722\n",
      "('ZH', 'PG') 0.7944995696980006\n",
      "('ZH', 'BERT') 0.7744011505746847\n",
      "('RU', 'PG') 0.4806242731518413\n",
      "('RU', 'BERT') 0.6933440880722743\n",
      "('DE', 'PG') 0.888640529382837\n",
      "('DE', 'BERT') 0.8994458047094434\n",
      "('ES', 'PG') 0.7015689507007323\n",
      "('ES', 'BERT') 0.570212706066844\n"
     ]
    }
   ],
   "source": [
    "# Focus\n",
    "cors = {}\n",
    "for lang in LANGS:\n",
    "    for model in ['PG', 'BERT']:\n",
    "        human_score = read_human_annotation(lang, model, 'focus')\n",
    "        _, cor = read(lang, model, human_score, 'precision')\n",
    "        cors[(lang, model)] = cor\n",
    "\n",
    "print('Focus of', pretrained)\n",
    "layer = get_best_rank(list(cors.values()))\n",
    "print('Individual model performance:')\n",
    "for key in cors:\n",
    "    print(key, cors[key][layer])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coverage of bert-base-multilingual-uncased\n",
      "Best layer: 6\n",
      "Individual model performance:\n",
      "('EN', 'PG') 0.6337745838190866\n",
      "('EN', 'BERT') 0.6432568225528881\n",
      "('ID', 'PG') 0.7539321018441252\n",
      "('ID', 'BERT') 0.7291109531081518\n",
      "('FR', 'PG') 0.6966728453074331\n",
      "('FR', 'BERT') 0.7584867035912555\n",
      "('TR', 'PG') 0.8497323827865577\n",
      "('TR', 'BERT') 0.8906809021022299\n",
      "('ZH', 'PG') 0.810508927683462\n",
      "('ZH', 'BERT') 0.7653215496265204\n",
      "('RU', 'PG') 0.6744441649350216\n",
      "('RU', 'BERT') 0.6811374244786897\n",
      "('DE', 'PG') 0.9068873906259574\n",
      "('DE', 'BERT') 0.9033131846111686\n",
      "('ES', 'PG') 0.7123259131508018\n",
      "('ES', 'BERT') 0.7331705718492405\n"
     ]
    }
   ],
   "source": [
    "# Coverage\n",
    "cors = {}\n",
    "for lang in LANGS:\n",
    "    for model in ['PG', 'BERT']:\n",
    "        human_score = read_human_annotation(lang, model, 'coverage')\n",
    "        _, cor = read(lang, model, human_score, 'recall')\n",
    "        cors[(lang, model)] = cor\n",
    "\n",
    "print('Coverage of', pretrained)\n",
    "layer = get_best_rank(list(cors.values()))\n",
    "print('Individual model performance:')\n",
    "for key in cors:\n",
    "    print(key, cors[key][layer])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}