5. transform_RU_and_ZH.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Russian"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json, pickle, argparse\n",
    "import glob, os\n",
    "\n",
    "\n",
    "def write(fname, string):\n",
    "    file = open(fname,'w')\n",
    "    file.write(string.lower())\n",
    "    file.close()\n",
    "\n",
    "def get_dic(str1, str2, str3):\n",
    "    dic = {}\n",
    "    id=0\n",
    "    for word in (' '.join(str1)).split(' ') + (' '.join(str2)).split(' ') + (' '.join(str3)).split(' '):\n",
    "        word = word.strip()\n",
    "        if dic.get(word,-1)==-1:\n",
    "            dic[word] = id\n",
    "            id+=1\n",
    "    return dic\n",
    "\n",
    "def convert(dic, list_str):\n",
    "    new_list = []\n",
    "    for string in list_str:\n",
    "        new_str = []\n",
    "        for word in string.split(' '):\n",
    "            new_str.append(f'w{dic[word.strip()]}')\n",
    "        new_list.append(' '.join(new_str))\n",
    "    return new_list\n",
    "\n",
    "def fix(sents):\n",
    "    return [s.replace(' [UNK] ', '') for s in sents]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "source = 'resulting_data/RU'\n",
    "target = 'resulting_data/RU_latin'\n",
    "os.makedirs(target+'/gold', exist_ok=True)\n",
    "os.makedirs(target+'/pred_BERT', exist_ok=True)\n",
    "os.makedirs(target+'/pred_PG', exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "for file in glob.glob(source+'/gold/*'):\n",
    "    fname = file.split('/')[-1]\n",
    "    gold = open(file).readlines()\n",
    "    sys_BERT = fix(open(source+'/pred_BERT/'+fname).readlines())\n",
    "    sys_PG = fix(open(source+'/pred_PG/'+fname).readlines())\n",
    "    \n",
    "    dic = get_dic(gold, sys_BERT, sys_PG)\n",
    "    \n",
    "    gold = convert(dic, gold)\n",
    "    sys_BERT = convert(dic, sys_BERT)\n",
    "    sys_PG = convert(dic, sys_PG)\n",
    "    \n",
    "    write(target+'/gold/'+fname, '\\n'.join(gold))\n",
    "    write(target+'/pred_BERT/'+fname, '\\n'.join(sys_BERT))\n",
    "    write(target+'/pred_PG/'+fname, '\\n'.join(sys_PG))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Chinese"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def write(fname, string):\n",
    "    file = open(fname,'w')\n",
    "    file.write(string.lower())\n",
    "    file.close()\n",
    "\n",
    "def get_dic(str1, str2, str3):\n",
    "    dic = {}\n",
    "    id=0\n",
    "    for word in (' '.join(str1)).split(' ') + (' '.join(str2)).split(' ') + (' '.join(str3)).split(' '):\n",
    "        word = word.strip()\n",
    "        if dic.get(word,-1)==-1:\n",
    "            dic[word] = id\n",
    "            id+=1\n",
    "    return dic\n",
    "\n",
    "def convert(dic, list_str):\n",
    "    new_list = []\n",
    "    for string in list_str:\n",
    "        new_str = []\n",
    "        for word in string.split(' '):\n",
    "            new_str.append(f'w{dic[word.strip()]}')\n",
    "        new_list.append(' '.join(new_str))\n",
    "    return new_list\n",
    "\n",
    "def fix(sents):\n",
    "    temp = ''.join([s.replace('[UNK]', '') for s in sents])\n",
    "    return temp.replace(' ','')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "source = 'resulting_data/ZH'\n",
    "target = 'resulting_data/ZH_latin'\n",
    "os.makedirs(target+'/gold', exist_ok=True)\n",
    "os.makedirs(target+'/pred_BERT', exist_ok=True)\n",
    "os.makedirs(target+'/pred_PG', exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "for file in glob.glob(source+'/gold/*'):\n",
    "    fname = file.split('/')[-1]\n",
    "    gold = ''.join(open(file).readlines()).replace('\\n','')\n",
    "    sys_BERT = fix(open(source+'/pred_BERT/'+fname).readlines())\n",
    "    sys_PG = fix(open(source+'/pred_PG/'+fname).readlines())\n",
    "    \n",
    "    dic = get_dic(gold, sys_BERT, sys_PG)\n",
    "    \n",
    "    gold = convert(dic, gold)\n",
    "    sys_BERT = convert(dic, sys_BERT)\n",
    "    sys_PG = convert(dic, sys_PG)\n",
    "    \n",
    "    write(target+'/gold/'+fname, ' '.join(gold))\n",
    "    write(target+'/pred_BERT/'+fname, ' '.join(sys_BERT))\n",
    "    write(target+'/pred_PG/'+fname, ' '.join(sys_PG))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}