check_start_text_eos.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "en finished...\n",
      "zh finished...\n",
      "9698532\n",
      "9698532\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 9698532/9698532 [10:47<00:00, 14968.70it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9698304 9698532\n"
     ]
    }
   ],
   "source": [
    "# 将en和zh都dump到同一个h5py文件，valid.h5py, train.h5py\n",
    "import json\n",
    "import h5py\n",
    "from collections import defaultdict\n",
    "from tqdm import *\n",
    "import random\n",
    "\n",
    "_en = json.load(open('/data/xuwenshen/ai_challenge/data/train/train/en_data-50-60.json'))\n",
    "en_voc = json.load(open('/data/xuwenshen/ai_challenge/data/train/train/en_voc.json'))\n",
    "print ('en finished...')\n",
    "\n",
    "_zh = json.load(open('/data/xuwenshen/ai_challenge/data/train/train/zh_data-50-60.json'))\n",
    "zh_voc = json.load(open('/data/xuwenshen/ai_challenge/data/train/train/zh_voc.json'))\n",
    "print ('zh finished...')\n",
    "\n",
    "print (len(_en))\n",
    "print (len(_zh))\n",
    "\n",
    "data = []\n",
    "\n",
    "for i in range(len(_en)):\n",
    "    data.append([_en[i], _zh[i]])\n",
    "\n",
    "# shuffle isf train data\n",
    "random.shuffle(data)\n",
    "\n",
    "f = h5py.File(\"/data/xuwenshen/ai_challenge/data/train/train/ibm_train-50-60.h5py1\", \"a\")\n",
    "\n",
    "\n",
    "en_set = f.create_dataset(\"en\", (len(data), 50), dtype='i8')\n",
    "en_length_set = f.create_dataset('en_len', (len(data), ), dtype='i8')\n",
    "\n",
    "\n",
    "label_set = f.create_dataset(\"label\", (len(data), 61), dtype='i8')\n",
    "zh_length_set = f.create_dataset('zh_len', (len(data), ), dtype='i8')\n",
    "gtruth_set = f.create_dataset('ground_truth', (len(data), 62), dtype='i8')\n",
    "\n",
    "zh_str_set = f.create_dataset('zh_str', (len(data), ), dtype='S400')\n",
    "en_str_set = f.create_dataset('en_str', (len(data), ), dtype='S400')\n",
    "\n",
    "\n",
    "en = []\n",
    "en_len = []\n",
    "en_str = []\n",
    "\n",
    "labels = []\n",
    "zh_len = []\n",
    "gtruths = []\n",
    "zh_str = []\n",
    "\n",
    "for i in tqdm(range(len(data))):\n",
    "    \n",
    "    #en data\n",
    "    tmp = [en_voc['pad#'] for j in range(50)]\n",
    "    if len(data[i][0]) == 0 or len(data[i][0]) > 50:\n",
    "        print ('en ', len(data[i][0]))\n",
    "        data[i][0] = data[i][0][:50]\n",
    "        continue\n",
    "    if len(data[i][1]) == 0 or len(data[i][1]) > 60:\n",
    "        print ('zh ', len(data[i][1]))\n",
    "        data[i][1] = data[i][1][:60]\n",
    "        continue\n",
    "    \n",
    "    for j in range(len(data[i][0])):\n",
    "        if data[i][0][j].lower() not in en_voc:\n",
    "            tmp[j] = en_voc['nuk#']\n",
    "        else:\n",
    "            tmp[j] = en_voc[data[i][0][j].lower()]\n",
    "    en.append(tmp)\n",
    "    en_len.append(len(data[i][0]))\n",
    "    en_str.append(' '.join(data[i][0][:min(400, len(data[i][0]))]).encode('utf8'))\n",
    "    \n",
    "    # zh data\n",
    "    tlabel = [zh_voc['pad#'] for j in range(61)]\n",
    "    tgtruth = [zh_voc['pad#'] for j in range(62)]\n",
    "    for j in range(len(data[i][1])):\n",
    "        if data[i][1][j] not in zh_voc:\n",
    "            tlabel[j] = zh_voc['nuk#']\n",
    "            tgtruth[j+1] = zh_voc['nuk#']\n",
    "        else:\n",
    "            tlabel[j] = zh_voc[data[i][1][j]]\n",
    "            tgtruth[j+1] = zh_voc[data[i][1][j]]\n",
    "            \n",
    "    tlabel[len(data[i][1])] = zh_voc['eos#']\n",
    "    tgtruth[0] = zh_voc['go#']\n",
    "    tgtruth[len(data[i][1])] = zh_voc['eos#']\n",
    "    \n",
    "    \n",
    "    labels.append(tlabel)\n",
    "    gtruths.append(tgtruth)\n",
    "    zh_len.append(len(data[i][1])+1)    \n",
    "    zh_str.append(''.join(data[i][1][:min(200, len(data[i][1]))]).encode('gb2312', 'ignore'))\n",
    "    \n",
    "    if len(en) == 256:\n",
    "        start = i+1-256\n",
    "        end = i+1\n",
    "        # en data\n",
    "        en_set[start : end] = en\n",
    "        en_length_set[start : end] = en_len\n",
    "        en_str_set[start : end] =  en_str\n",
    "        en = []\n",
    "        en_len = []\n",
    "        en_str = []\n",
    "        \n",
    "        #zh data\n",
    "        label_set[start : end] = labels\n",
    "        zh_length_set[start : end] = zh_len\n",
    "        gtruth_set[start : end] = gtruths      \n",
    "        zh_str_set[start:end] = zh_str\n",
    "        labels = []\n",
    "        zh_len = []\n",
    "        gtruths = []\n",
    "        zh_str = []\n",
    "\n",
    "        \n",
    "end = len(data)\n",
    "start = end-len(en)\n",
    "print (start, end)\n",
    "en_set[start : end] = en\n",
    "en_length_set[start : end] = en_len\n",
    "en_str_set[start : end] =  en_str\n",
    "\n",
    "label_set[start : end] = labels\n",
    "zh_length_set[start : end] = zh_len\n",
    "gtruth_set[start : end] = gtruths\n",
    "zh_str_set[start : end] =  zh_str\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|███▎      | 2605/8000 [00:00<00:00, 26036.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8000/8000 [00:00<00:00, 24286.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7936 8000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# 将test en dump到h5py文件，test.h5py\n",
    "import json\n",
    "import h5py\n",
    "from collections import defaultdict\n",
    "from tqdm import *\n",
    "\n",
    "_en = json.load(open('/data/xuwenshen/ai_challenge/data/test/en_b.json'))\n",
    "en_voc = json.load(open('/data/xuwenshen/ai_challenge/data/train/train/en_voc.json'))\n",
    "\n",
    "\n",
    "print (len(_en))\n",
    "\n",
    "\n",
    "f = h5py.File(\"/data/xuwenshen/ai_challenge/data/test/ibm_test-30.h5py\", \"a\")\n",
    "\n",
    "\n",
    "en_set = f.create_dataset(\"en\", (len(_en), 30), dtype='i8')\n",
    "en_length_set = f.create_dataset('en_len', (len(_en), ), dtype='i8')\n",
    "\n",
    "en_str_set = f.create_dataset('en_str', (len(_en), ), dtype='S400')\n",
    "\n",
    "\n",
    "en = []\n",
    "en_len = []\n",
    "en_str = []\n",
    "\n",
    "\n",
    "for i in tqdm(range(len(_en))):\n",
    "    \n",
    "    #en data\n",
    "    tmp = [en_voc['pad#'] for j in range(30)]\n",
    "    if len(_en[i]) == 0 or len(_en[i]) > 30:\n",
    "        print ('en ', len(_en[i]))\n",
    "        _en[i] = _en[i][:30]\n",
    "        continue\n",
    "    \n",
    "    for j in range(len(_en[i])):\n",
    "        if _en[i][j].lower() not in en_voc:\n",
    "            tmp[j] = en_voc['nuk#']\n",
    "        else:\n",
    "            tmp[j] = en_voc[_en[i][j].lower()]\n",
    "    en.append(tmp)\n",
    "    en_len.append(len(_en[i]))\n",
    "    en_str.append(' '.join(_en[i][:min(400, len(_en[i]))]).encode('utf8'))\n",
    "     \n",
    "    if len(en) == 256:\n",
    "        start = i+1-256\n",
    "        end = i+1\n",
    "        # en data\n",
    "        en_set[start : end] = en\n",
    "        en_length_set[start : end] = en_len\n",
    "        en_str_set[start : end] =  en_str\n",
    "        en = []\n",
    "        en_len = []\n",
    "        en_str = []\n",
    "        \n",
    "        \n",
    "end = len(_en)\n",
    "start = end-len(en)\n",
    "print (start, end)\n",
    "en_set[start : end] = en\n",
    "en_length_set[start : end] = en_len\n",
    "en_str_set[start : end] =  en_str\n",
    "\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 将en都dump到同一个h5py文件，valid.h5py\n",
    "import json\n",
    "import h5py\n",
    "from collections import defaultdict\n",
    "from tqdm import *\n",
    "\n",
    "_en = json.load(open('/data/xuwenshen/ai_challenge/data/valid/valid/en.json'))\n",
    "en_voc = json.load(open('/data/xuwenshen/ai_challenge/data/train/train/en_voc.json'))\n",
    "\n",
    "\n",
    "print (len(_en))\n",
    "print (len(_zh))\n",
    "\n",
    "f = h5py.File(\"/data/xuwenshen/ai_challenge/data/valid/valid/ibm_valid-50-60.h5py1\", \"a\")\n",
    "\n",
    "\n",
    "en_set = f.create_dataset(\"en\", (len(_en), 50), dtype='i8')\n",
    "en_length_set = f.create_dataset('en_len', (len(_en), ), dtype='i8')\n",
    "\n",
    "\n",
    "label_set = f.create_dataset(\"label\", (len(_zh), 61), dtype='i8')\n",
    "zh_length_set = f.create_dataset('zh_len', (len(_zh), ), dtype='i8')\n",
    "gtruth_set = f.create_dataset('ground_truth', (len(_zh), 62), dtype='i8')\n",
    "\n",
    "zh_str_set = f.create_dataset('zh_str', (len(_zh), ), dtype='S400')\n",
    "en_str_set = f.create_dataset('en_str', (len(_en), ), dtype='S400')\n",
    "\n",
    "\n",
    "en = []\n",
    "en_len = []\n",
    "en_str = []\n",
    "\n",
    "labels = []\n",
    "zh_len = []\n",
    "gtruths = []\n",
    "zh_str = []\n",
    "\n",
    "for i in tqdm(range(len(_en))):\n",
    "    \n",
    "    #en data\n",
    "    tmp = [en_voc['pad#'] for j in range(50)]\n",
    "    if len(_en[i]) == 0 or len(_en[i]) > 50:\n",
    "        print ('en ', len(_en[i]))\n",
    "        _en[i] = _en[i][:50]\n",
    "        continue\n",
    "    if len(_zh[i]) == 0 or len(_zh[i]) > 60:\n",
    "        print ('zh ', len(_zh[i]))\n",
    "        _zh[i] = _zh[i][:60]\n",
    "        continue\n",
    "    \n",
    "    for j in range(len(_en[i])):\n",
    "        if _en[i][j].lower() not in en_voc:\n",
    "            tmp[j] = en_voc['nuk#']\n",
    "        else:\n",
    "            tmp[j] = en_voc[_en[i][j].lower()]\n",
    "    en.append(tmp)\n",
    "    en_len.append(len(_en[i]))\n",
    "    en_str.append(' '.join(_en[i][:min(400, len(_en[i]))]).encode('utf8'))\n",
    "    \n",
    "    # zh data\n",
    "    tlabel = [zh_voc['pad#'] for j in range(61)]\n",
    "    tgtruth = [zh_voc['pad#'] for j in range(62)]\n",
    "    for j in range(len(_zh[i])):\n",
    "        if _zh[i][j] not in zh_voc:\n",
    "            tlabel[j] = zh_voc['nuk#']\n",
    "            tgtruth[j+1] = zh_voc['nuk#']\n",
    "        else:\n",
    "            tlabel[j] = zh_voc[_zh[i][j]]\n",
    "            tgtruth[j+1] = zh_voc[_zh[i][j]]\n",
    "            \n",
    "    tlabel[len(_zh[i])] = zh_voc['eos#']\n",
    "    tgtruth[0] = zh_voc['go#']\n",
    "    tgtruth[len(_zh[i])] = zh_voc['eos#']\n",
    "    \n",
    "    \n",
    "    labels.append(tlabel)\n",
    "    gtruths.append(tgtruth)\n",
    "    zh_len.append(len(_zh[i])+1)    \n",
    "    zh_str.append(''.join(_zh[i][:min(200, len(_zh[i]))]).encode('gb2312', 'ignore'))\n",
    "    \n",
    "    if len(en) == 256:\n",
    "        start = i+1-256\n",
    "        end = i+1\n",
    "        # en data\n",
    "        en_set[start : end] = en\n",
    "        en_length_set[start : end] = en_len\n",
    "        en_str_set[start : end] =  en_str\n",
    "        en = []\n",
    "        en_len = []\n",
    "        en_str = []\n",
    "        \n",
    "        #zh data\n",
    "        label_set[start : end] = labels\n",
    "        zh_length_set[start : end] = zh_len\n",
    "        gtruth_set[start : end] = gtruths      \n",
    "        zh_str_set[start:end] = zh_str\n",
    "        labels = []\n",
    "        zh_len = []\n",
    "        gtruths = []\n",
    "        zh_str = []\n",
    "\n",
    "        \n",
    "end = len(_en)\n",
    "start = end-len(en)\n",
    "print (start, end)\n",
    "en_set[start : end] = en\n",
    "en_length_set[start : end] = en_len\n",
    "en_str_set[start : end] =  en_str\n",
    "\n",
    "label_set[start : end] = labels\n",
    "zh_length_set[start : end] = zh_len\n",
    "gtruth_set[start : end] = gtruths\n",
    "zh_str_set[start : end] =  zh_str\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 ground_truth  go#但今天根本没有关于这方面的报道…\n",
      "0 label  但今天根本没有关于这方面的报道…eos#\n",
      "0 zh_str  但今天根本没有关于这方面的报道…\n",
      "1 ground_truth  go#希望这些可以弥补那些弄丢了的货。 \n",
      "1 label  希望这些可以弥补那些弄丢了的货。 eos#\n",
      "1 zh_str  希望这些可以弥补那些弄丢了的货。 \n",
      "2 ground_truth  go#其他人可以醉但是我… \n",
      "2 label  其他人可以醉但是我… eos#\n",
      "2 zh_str  其他人可以醉但是我… \n",
      "3 ground_truth  go#在这儿见她真是大错特错。要是她们起了疑心，那就完了。 \n",
      "3 label  在这儿见她真是大错特错。要是她们起了疑心，那就完了。 eos#\n",
      "3 zh_str  在这儿见她真是大错特错。要是她们起了疑心，那就完了。 \n",
      "4 ground_truth  go#本论文分析，能使杭州映山红纺织印染厂提高生产物流的效率，提升其在纺织印染行业的生产竞争力。\n",
      "4 label  本论文分析，能使杭州映山红纺织印染厂提高生产物流的效率，提升其在纺织印染行业的生产竞争力。eos#\n",
      "4 zh_str  本论文分析，能使杭州映山红纺织印染厂提高生产物流的效率，提升其在纺织印染行业的生产竞争力。\n",
      "5 ground_truth  go#希望我进去时，她已睡着。\n",
      "5 label  希望我进去时，她已睡着。eos#\n",
      "5 zh_str  希望我进去时，她已睡着。\n",
      "6 ground_truth  go#对于室井的事情总有个说法吧。 \n",
      "6 label  对于室井的事情总有个说法吧。 eos#\n",
      "6 zh_str  对于室井的事情总有个说法吧。 \n",
      "7 ground_truth  go#我还讨厌运河两岸房子上的乱涂乱画。讨厌那些紧挨着这些房子的面目可憎的新楼房。还有价高质差的食物。\n",
      "7 label  我还讨厌运河两岸房子上的乱涂乱画。讨厌那些紧挨着这些房子的面目可憎的新楼房。还有价高质差的食物。eos#\n",
      "7 zh_str  我还讨厌运河两岸房子上的乱涂乱画。讨厌那些紧挨着这些房子的面目可憎的新楼房。还有价高质差的食物。\n",
      "8 ground_truth  go#但能不能让他自己做决定？\n",
      "8 label  但能不能让他自己做决定？eos#\n",
      "8 zh_str  但能不能让他自己做决定？\n",
      "9 ground_truth  go#好怀念过去。 \n",
      "9 label  好怀念过去。 eos#\n",
      "9 zh_str  好怀念过去。 \n",
      "10 ground_truth  go#志愿参军，在看清事实后要求退出。 \n",
      "10 label  志愿参军，在看清事实后要求退出。 eos#\n",
      "10 zh_str  志愿参军，在看清事实后要求退出。 \n",
      "11 ground_truth  go#我知道老大相信我你看他死了事情就对你很不利了。 \n",
      "11 label  我知道老大相信我你看他死了事情就对你很不利了。 eos#\n",
      "11 zh_str  我知道老大相信我你看他死了事情就对你很不利了。 \n",
      "12 ground_truth  go#你儿子都明白值得为此牺牲。 \n",
      "12 label  你儿子都明白值得为此牺牲。 eos#\n",
      "12 zh_str  你儿子都明白值得为此牺牲。 \n",
      "13 ground_truth  go#已经结束了，劳伦。\n",
      "13 label  已经结束了，劳伦。eos#\n",
      "13 zh_str  已经结束了，劳伦。\n",
      "14 ground_truth  go#那应该改变。你喜欢红葡萄酒吗？\n",
      "14 label  那应该改变。你喜欢红葡萄酒吗？eos#\n",
      "14 zh_str  那应该改变。你喜欢红葡萄酒吗？\n",
      "15 ground_truth  go#当你和他一起回来，\n",
      "15 label  当你和他一起回来，eos#\n",
      "15 zh_str  当你和他一起回来，\n",
      "16 ground_truth  go#但仍然很美。\n",
      "16 label  但仍然很美。eos#\n",
      "16 zh_str  但仍然很美。\n",
      "17 ground_truth  go#为了以示对我的朋友和同事布伦特兰博士的尊重,我就不在这次研讨会上播放那些幻灯片。\n",
      "17 label  为了以示对我的朋友和同事布伦特兰博士的尊重,我就不在这次研讨会上播放那些幻灯片。eos#\n",
      "17 zh_str  为了以示对我的朋友和同事布伦特兰博士的尊重,我就不在这次研讨会上播放那些幻灯片。\n",
      "18 ground_truth  go#右边再低点。\n",
      "18 label  右边再低点。eos#\n",
      "18 zh_str  右边再低点。\n",
      "19 ground_truth  go#她不会听的。她根本不在乎自己的生活一塌糊涂。\n",
      "19 label  她不会听的。她根本不在乎自己的生活一塌糊涂。eos#\n",
      "19 zh_str  她不会听的。她根本不在乎自己的生活一塌糊涂。\n",
      "20 ground_truth  go#国内两大反对党都同意上述观点。\n",
      "20 label  国内两大反对党都同意上述观点。eos#\n",
      "20 zh_str  国内两大反对党都同意上述观点。\n",
      "21 ground_truth  go#实在不行就偷点。 \n",
      "21 label  实在不行就偷点。 eos#\n",
      "21 zh_str  实在不行就偷点。 \n",
      "22 ground_truth  go#我来查明红队其他队员的身份。\n",
      "22 label  我来查明红队其他队员的身份。eos#\n",
      "22 zh_str  我来查明红队其他队员的身份。\n",
      "23 ground_truth  go#很抱歉我也配不上你。 \n",
      "23 label  很抱歉我也配不上你。 eos#\n",
      "23 zh_str  很抱歉我也配不上你。 \n",
      "24 ground_truth  go#我吃了药的。一直得上厕所。 \n",
      "24 label  我吃了药的。一直得上厕所。 eos#\n",
      "24 zh_str  我吃了药的。一直得上厕所。 \n",
      "25 ground_truth  go#你喜欢选？- 我想.... \n",
      "25 label  你喜欢选？- 我想.... eos#\n",
      "25 zh_str  你喜欢选？- 我想.... \n",
      "26 ground_truth  go#如果你好好干这里一切都是你的。 \n",
      "26 label  如果你好好干这里一切都是你的。 eos#\n",
      "26 zh_str  如果你好好干这里一切都是你的。 \n",
      "27 ground_truth  go#他不想跟我唠叨什么海明威。 \n",
      "27 label  他不想跟我唠叨什么海明威。 eos#\n",
      "27 zh_str  他不想跟我唠叨什么海明威。 \n",
      "28 ground_truth  go#当家庭主男不会有损你的男子气概\n",
      "28 label  当家庭主男不会有损你的男子气概eos#\n",
      "28 zh_str  当家庭主男不会有损你的男子气概\n",
      "29 ground_truth  go#柯纳马拉松与半马拉松暨家庭趣味赛跑。\n",
      "29 label  柯纳马拉松与半马拉松暨家庭趣味赛跑。eos#\n",
      "29 zh_str  柯纳马拉松与半马拉松暨家庭趣味赛跑。\n",
      "30 ground_truth  go#它的皮肤溃烂发脓有着长长的指甲。 \n",
      "30 label  它的皮肤溃烂发脓有着长长的指甲。 eos#\n",
      "30 zh_str  它的皮肤溃烂发脓有着长长的指甲。 \n",
      "31 ground_truth  go#好的，当然。我要好的。\n",
      "31 label  好的，当然。我要好的。eos#\n",
      "31 zh_str  好的，当然。我要好的。\n",
      "32 ground_truth  go#我宁愿先弄好再生火。 \n",
      "32 label  我宁愿先弄好再生火。 eos#\n",
      "32 zh_str  我宁愿先弄好再生火。 \n",
      "33 ground_truth  go#关于驻马店地区农业生产形势的调查报告。\n",
      "33 label  关于驻马店地区农业生产形势的调查报告。eos#\n",
      "33 zh_str  关于驻马店地区农业生产形势的调查报告。\n",
      "34 ground_truth  go#因此他们有更好的理由，相信他们所付出的。\n",
      "34 label  因此他们有更好的理由，相信他们所付出的。eos#\n",
      "34 zh_str  因此他们有更好的理由，相信他们所付出的。\n",
      "35 ground_truth  go#不用担心那些蛇。\n",
      "35 label  不用担心那些蛇。eos#\n",
      "35 zh_str  不用担心那些蛇。\n",
      "36 ground_truth  go#好吧。我再说一次，个人经验之谈。\n",
      "36 label  好吧。我再说一次，个人经验之谈。eos#\n",
      "36 zh_str  好吧。我再说一次，个人经验之谈。\n",
      "37 ground_truth  go#偷窥狂很容易从性骚扰转变成谋杀犯。\n",
      "37 label  偷窥狂很容易从性骚扰转变成谋杀犯。eos#\n",
      "37 zh_str  偷窥狂很容易从性骚扰转变成谋杀犯。\n",
      "38 ground_truth  go#所以明年会是不安定的一年，到那个时候，俄罗斯将实行笨拙的双头政治。\n",
      "38 label  所以明年会是不安定的一年，到那个时候，俄罗斯将实行笨拙的双头政治。eos#\n",
      "38 zh_str  所以明年会是不安定的一年，到那个时候，俄罗斯将实行笨拙的双头政治。\n",
      "39 ground_truth  go#好像防刮花防偷窥， \n",
      "39 label  好像防刮花防偷窥， eos#\n",
      "39 zh_str  好像防刮花防偷窥， \n",
      "40 ground_truth  go#在吉尼米上，一点点的改变。\n",
      "40 label  在吉尼米上，一点点的改变。eos#\n",
      "40 zh_str  在吉尼米上，一点点的改变。\n",
      "41 ground_truth  go#是吗？看我都被划了。\n",
      "41 label  是吗？看我都被划了。eos#\n",
      "41 zh_str  是吗？看我都被划了。\n",
      "42 ground_truth  go#喂，罗伦佐，你好吗？ \n",
      "42 label  喂，罗伦佐，你好吗？ eos#\n",
      "42 zh_str  喂，罗伦佐，你好吗？ \n",
      "43 ground_truth  go#在时空里的运动。 \n",
      "43 label  在时空里的运动。 eos#\n",
      "43 zh_str  在时空里的运动。 \n",
      "44 ground_truth  go#你好？是长湾旅馆吗？ \n",
      "44 label  你好？是长湾旅馆吗？ eos#\n",
      "44 zh_str  你好？是长湾旅馆吗？ \n",
      "45 ground_truth  go#当了6年会计了。我要去。\n",
      "45 label  当了6年会计了。我要去。eos#\n",
      "45 zh_str  当了6年会计了。我要去。\n",
      "46 ground_truth  go#wesley clark jim webb和chuck hagel以前都是越战英雄。\n",
      "46 label  wesley clark jim webb和chuck hagel以前都是越战英雄。eos#\n",
      "46 zh_str  wesley clark jim webb和chuck hagel以前都是越战英雄。\n",
      "47 ground_truth  go#十友为善，难抵一敌为恶。\n",
      "47 label  十友为善，难抵一敌为恶。eos#\n",
      "47 zh_str  十友为善，难抵一敌为恶。\n",
      "48 ground_truth  go#什么呀？你是在勾引我吗？ \n",
      "48 label  什么呀？你是在勾引我吗？ eos#\n",
      "48 zh_str  什么呀？你是在勾引我吗？ \n",
      "49 ground_truth  go#快走，你得感谢神，是神让我放了你。\n",
      "49 label  快走，你得感谢神，是神让我放了你。eos#\n",
      "49 zh_str  快走，你得感谢神，是神让我放了你。\n",
      "50 ground_truth  go#大连新世纪连发贸易有限公司。\n",
      "50 label  大连新世纪连发贸易有限公司。eos#\n",
      "50 zh_str  大连新世纪连发贸易有限公司。\n",
      "51 ground_truth  go#把我干掉，你下部戏上哪弄去。 \n",
      "51 label  把我干掉，你下部戏上哪弄去。 eos#\n",
      "51 zh_str  把我干掉，你下部戏上哪弄去。 \n",
      "52 ground_truth  go#你妈妈和我都不希望你喝酒。 \n",
      "52 label  你妈妈和我都不希望你喝酒。 eos#\n",
      "52 zh_str  你妈妈和我都不希望你喝酒。 \n",
      "53 ground_truth  go#差不多两万元吧。\n",
      "53 label  差不多两万元吧。eos#\n",
      "53 zh_str  差不多两万元吧。\n",
      "54 ground_truth  go#有一天，一位音乐家问贝多芬，这部安魂曲是否真的是苏士梅耶尔而不是莫扎特所作的。\n",
      "54 label  有一天，一位音乐家问贝多芬，这部安魂曲是否真的是苏士梅耶尔而不是莫扎特所作的。eos#\n",
      "54 zh_str  有一天，一位音乐家问贝多芬，这部安魂曲是否真的是苏士梅耶尔而不是莫扎特所作的。\n",
      "55 ground_truth  go#我不知道你怎么发现钱的事情的…。\n",
      "55 label  我不知道你怎么发现钱的事情的…。eos#\n",
      "55 zh_str  我不知道你怎么发现钱的事情的…。\n",
      "56 ground_truth  go#她当时并没有戴戒指。 \n",
      "56 label  她当时并没有戴戒指。 eos#\n",
      "56 zh_str  她当时并没有戴戒指。 \n",
      "57 ground_truth  go#我是成岛亮。 \n",
      "57 label  我是成岛亮。 eos#\n",
      "57 zh_str  我是成岛亮。 \n",
      "58 ground_truth  go#却唯独留下这些花瓣？ \n",
      "58 label  却唯独留下这些花瓣？ eos#\n",
      "58 zh_str  却唯独留下这些花瓣？ \n",
      "59 ground_truth  go#你这么老，还偷窥。 \n",
      "59 label  你这么老，还偷窥。 eos#\n",
      "59 zh_str  你这么老，还偷窥。 \n",
      "60 ground_truth  go#我是上海大学的一名研究生，希望在上海找个语言伙伴。\n",
      "60 label  我是上海大学的一名研究生，希望在上海找个语言伙伴。eos#\n",
      "60 zh_str  我是上海大学的一名研究生，希望在上海找个语言伙伴。\n",
      "61 ground_truth  go#你委托人在关禁闭。 \n",
      "61 label  你委托人在关禁闭。 eos#\n",
      "61 zh_str  你委托人在关禁闭。 \n",
      "62 ground_truth  go#所以他很讨厌酒。 \n",
      "62 label  所以他很讨厌酒。 eos#\n",
      "62 zh_str  所以他很讨厌酒。 \n",
      "63 ground_truth  go#你知道吗，要啤酒吗？我要喝点。 \n",
      "63 label  你知道吗，要啤酒吗？我要喝点。 eos#\n",
      "63 zh_str  你知道吗，要啤酒吗？我要喝点。 \n",
      "64 ground_truth  go#如果不能信任你，我可不得不毁了你。 \n",
      "64 label  如果不能信任你，我可不得不毁了你。 eos#\n",
      "64 zh_str  如果不能信任你，我可不得不毁了你。 \n",
      "65 ground_truth  go#我知道发生的一切都不是你的错。 \n",
      "65 label  我知道发生的一切都不是你的错。 eos#\n",
      "65 zh_str  我知道发生的一切都不是你的错。 \n",
      "66 ground_truth  go#我们的特大床吸烟房提供MP3单套闹钟、微波炉、冰箱/冰柜和一个单杯咖啡机。 \n",
      "66 label  我们的特大床吸烟房提供MP3单套闹钟、微波炉、冰箱/冰柜和一个单杯咖啡机。 eos#\n",
      "66 zh_str  我们的特大床吸烟房提供MP3单套闹钟、微波炉、冰箱/冰柜和一个单杯咖啡机。 \n",
      "67 ground_truth  go#你能给我提点建议吗？\n",
      "67 label  你能给我提点建议吗？eos#\n",
      "67 zh_str  你能给我提点建议吗？\n",
      "68 ground_truth  go#你能让我上飞机吗，长官？\n",
      "68 label  你能让我上飞机吗，长官？eos#\n",
      "68 zh_str  你能让我上飞机吗，长官？\n",
      "69 ground_truth  go#你要是回不来，我就给你带一盘。 \n",
      "69 label  你要是回不来，我就给你带一盘。 eos#\n",
      "69 zh_str  你要是回不来，我就给你带一盘。 \n",
      "70 ground_truth  go#我们要面试实习生。你有纸和笔吗？\n",
      "70 label  我们要面试实习生。你有纸和笔吗？eos#\n",
      "70 zh_str  我们要面试实习生。你有纸和笔吗？\n",
      "71 ground_truth  go#噢，只是挂在墙上的东西而已。 \n",
      "71 label  噢，只是挂在墙上的东西而已。 eos#\n",
      "71 zh_str  噢，只是挂在墙上的东西而已。 \n",
      "72 ground_truth  go#就在去年，两国在伊拉克战争问题上产生严重分歧。英国首相托尼。布莱尔支持对伊开战，而法国总统雅克。希拉克持反对态度。\n",
      "72 label  就在去年，两国在伊拉克战争问题上产生严重分歧。英国首相托尼。布莱尔支持对伊开战，而法国总统雅克。希拉克持反对态度。eos#\n",
      "72 zh_str  就在去年，两国在伊拉克战争问题上产生严重分歧。英国首相托尼。布莱尔支持对伊开战，而法国总统雅克。希拉克持反对态度。\n",
      "73 ground_truth  go#告诉我真话：这趟出访你有何希望？\n",
      "73 label  告诉我真话：这趟出访你有何希望？eos#\n",
      "73 zh_str  告诉我真话：这趟出访你有何希望？\n",
      "74 ground_truth  go#这家豪华的捷克布尔诺酒店位于市中心的优越地理位置，为商务旅行人士和休闲旅游人士提供优质便利设施。 \n",
      "74 label  这家豪华的捷克布尔诺酒店位于市中心的优越地理位置，为商务旅行人士和休闲旅游人士提供优质便利设施。 eos#\n",
      "74 zh_str  这家豪华的捷克布尔诺酒店位于市中心的优越地理位置，为商务旅行人士和休闲旅游人士提供优质便利设施。 \n",
      "75 ground_truth  go#我需要有人上来。\n",
      "75 label  我需要有人上来。eos#\n",
      "75 zh_str  我需要有人上来。\n",
      "76 ground_truth  go#管你是我还是你！\n",
      "76 label  管你是我还是你！eos#\n",
      "76 zh_str  管你是我还是你！\n",
      "77 ground_truth  go#我知道如何吸引男人的注意力。 \n",
      "77 label  我知道如何吸引男人的注意力。 eos#\n",
      "77 zh_str  我知道如何吸引男人的注意力。 \n",
      "78 ground_truth  go#客房内配有无线高速互联网连接、咖啡机及熨斗和熨衣板、吹风机。 \n",
      "78 label  客房内配有无线高速互联网连接、咖啡机及熨斗和熨衣板、吹风机。 eos#\n",
      "78 zh_str  客房内配有无线高速互联网连接、咖啡机及熨斗和熨衣板、吹风机。 \n",
      "79 ground_truth  go#好啊。- 意式烘焙咖啡？\n",
      "79 label  好啊。- 意式烘焙咖啡？eos#\n",
      "79 zh_str  好啊。- 意式烘焙咖啡？\n",
      "80 ground_truth  go#所以你没有听到我的消息。 \n",
      "80 label  所以你没有听到我的消息。 eos#\n",
      "80 zh_str  所以你没有听到我的消息。 \n",
      "81 ground_truth  go#军队必须配备适当的医药用品。\n",
      "81 label  军队必须配备适当的医药用品。eos#\n",
      "81 zh_str  军队必须配备适当的医药用品。\n",
      "82 ground_truth  go#你的热牛奶和蜂蜜，\n",
      "82 label  你的热牛奶和蜂蜜，eos#\n",
      "82 zh_str  你的热牛奶和蜂蜜，\n",
      "83 ground_truth  go#我把它加在“混杂物”。目录下\n",
      "83 label  我把它加在“混杂物”。目录下eos#\n",
      "83 zh_str  我把它加在“混杂物”。目录下\n",
      "84 ground_truth  go#在我自己的家里告诉我 \n",
      "84 label  在我自己的家里告诉我 eos#\n",
      "84 zh_str  在我自己的家里告诉我 \n",
      "85 ground_truth  go#我不能承认， \n",
      "85 label  我不能承认， eos#\n",
      "85 zh_str  我不能承认， \n",
      "86 ground_truth  go#他们明天会去那里签署协定。\n",
      "86 label  他们明天会去那里签署协定。eos#\n",
      "86 zh_str  他们明天会去那里签署协定。\n",
      "87 ground_truth  go#看不看得到？\n",
      "87 label  看不看得到？eos#\n",
      "87 zh_str  看不看得到？\n",
      "88 ground_truth  go#你们不是真的朋友吧？\n",
      "88 label  你们不是真的朋友吧？eos#\n",
      "88 zh_str  你们不是真的朋友吧？\n",
      "89 ground_truth  go#如果超过时间，红灯亮了， \n",
      "89 label  如果超过时间，红灯亮了， eos#\n",
      "89 zh_str  如果超过时间，红灯亮了， \n",
      "90 ground_truth  go#我老婆昨天自杀了 \n",
      "90 label  我老婆昨天自杀了 eos#\n",
      "90 zh_str  我老婆昨天自杀了 \n",
      "91 ground_truth  go#你就别在添乱了。 \n",
      "91 label  你就别在添乱了。 eos#\n",
      "91 zh_str  你就别在添乱了。 \n",
      "92 ground_truth  go#就好像他在掩护什么人似的。- 所以我们需要跟那些执法警再谈谈。\n",
      "92 label  就好像他在掩护什么人似的。- 所以我们需要跟那些执法警再谈谈。eos#\n",
      "92 zh_str  就好像他在掩护什么人似的。- 所以我们需要跟那些执法警再谈谈。\n",
      "93 ground_truth  go#听着，那个女孩不适合你。\n",
      "93 label  听着，那个女孩不适合你。eos#\n",
      "93 zh_str  听着，那个女孩不适合你。\n",
      "94 ground_truth  go#为什么他不住在酒店或者公寓里 \n",
      "94 label  为什么他不住在酒店或者公寓里 eos#\n",
      "94 zh_str  为什么他不住在酒店或者公寓里 \n",
      "95 ground_truth  go#去年有个混蛋把比目鱼都吃光了。 \n",
      "95 label  去年有个混蛋把比目鱼都吃光了。 eos#\n",
      "95 zh_str  去年有个混蛋把比目鱼都吃光了。 \n",
      "96 ground_truth  go#我绝对是心理失常了？ \n",
      "96 label  我绝对是心理失常了？ eos#\n",
      "96 zh_str  我绝对是心理失常了？ \n",
      "97 ground_truth  go#我想改变自己。\n",
      "97 label  我想改变自己。eos#\n",
      "97 zh_str  我想改变自己。\n",
      "98 ground_truth  go#对，看得出来你好。\n",
      "98 label  对，看得出来你好。eos#\n",
      "98 zh_str  对，看得出来你好。\n",
      "99 ground_truth  go#不管怎样，医生和房间都准备好了。我们该做什么呢？ \n",
      "99 label  不管怎样，医生和房间都准备好了。我们该做什么呢？ eos#\n",
      "99 zh_str  不管怎样，医生和房间都准备好了。我们该做什么呢？ \n"
     ]
    }
   ],
   "source": [
    "import h5py\n",
    "import json\n",
    "from collections import defaultdict\n",
    "\n",
    "zh_voc = json.load(open('/data/xuwenshen/ai_challenge/data/train/train/zh_voc.json'))\n",
    "\n",
    "h5 = h5py.File('/data/xuwenshen/ai_challenge/data/train/train/ibm_train-50-60.h5py', 'r')\n",
    "\n",
    "rvoc = defaultdict()\n",
    "for (k,v) in zh_voc.items():\n",
    "    rvoc[v] = k\n",
    "rvoc = dict(rvoc)\n",
    "\n",
    "f1 = h5['ground_truth']\n",
    "f2 = h5['label']\n",
    "f3 = h5['zh_str']\n",
    "for i in range(100):\n",
    "    text1 = ''\n",
    "    text2 = ''\n",
    "    for j in range(62):\n",
    "        if f1[i][j] == zh_voc['pad#']: \n",
    "            break\n",
    "        text1 += rvoc[f1[i][j]]\n",
    "    for j in range(61):\n",
    "        if f2[i][j] == zh_voc['pad#']:\n",
    "            break\n",
    "        text2 += rvoc[f2[i][j]]\n",
    "    \n",
    "    \n",
    "    print (i, 'ground_truth ', text1)\n",
    "    print (i, 'label ', text2)\n",
    "    print (i, 'zh_str ', f3[i].decode('gb2312'))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0  i knew this would happen for you . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "0 I knew this would happen for you .\n",
      "1  that 's fine . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "1 That 's fine .\n",
      "2  wildflowers are nuk# domestic flowers pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "2 Wildflowers are betterthan domestic flowers\n",
      "3  but there is a richie nuk# in philadelphia . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "3 But there is a richie cusack in philadelphia .\n",
      "4  ah , lucky you , i 'm still poor . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "4 Ah , lucky you , I 'm still poor .\n",
      "5  since when do i do what i 'm told ? pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "5 Since when do I do what I 'm told ?\n",
      "6  he 's called me a few times , but i 'm not calling him back . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "6 He 's called me a few times , but I 'm not calling him back .\n",
      "7  though `` under the weather '' really does n't quite cover it , does it ? pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "7 Though `` under the weather '' really does n't quite cover it , does it ?\n",
      "8  a sex ... i ... i did not have one of them damn things . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "8 A sex ... I ... I did not have one of them damn things .\n",
      "9  and they have saved a life . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "9 And they have saved a life .\n",
      "10  well , they are worth every penny , of course . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "10 Well , they are worth every penny , of course .\n",
      "11  and my shit looked like a fucking stuffed animal ! pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "11 And my shit looked like a fucking stuffed animal !\n",
      "12  all right . and you told him ... pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "12 All right . And you told him ...\n",
      "13  i just ca n't quite remember . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "13 I just ca n't quite remember .\n",
      "14  she was vehemently against the whole concept of marriage . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "14 She was vehemently against the whole concept of marriage .\n",
      "15  you seem to be doing a pretty good job of that yourself . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "15 You seem to be doing a pretty good job of that yourself .\n",
      "16  nuk# i like cheesecake , but i kind of love ... pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "16 I- I like cheesecake , but I kind of love ...\n",
      "17  or ... happiness . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "17 Or ... Happiness .\n",
      "18  oooh look ! is n't this adorable , ross let ben address mine ! pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "18 Oooh look ! Is n't this adorable , ross let ben address mine !\n",
      "19  but it 's my fault , you know ? i ... i 'm an idiot , and i started it . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "19 But it 's my fault , you know ? I ... I 'm an idiot , and I started it .\n",
      "20  when i sing , it is an nuk# me . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "20 When I sing , it is an escapefor me .\n",
      "21  probably because i had too much champagne . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "21 Probably because I had too much champagne .\n",
      "22  do you wan na consider it ? no , sir . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "22 Do you wan na consider it ? No , sir .\n",
      "23  i promised your maester i 'd get you to castle black pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "23 I promised your Maester I 'd get you to castle black\n",
      "24  all right , now i 'm officially terrified . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "24 All right , now I 'm officially terrified .\n",
      "25  okay , so , uh , this is where it happened . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "25 Okay , so , uh , this is where it happened .\n",
      "26  i do n't know . you 'd have to ask her . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "26 I do n't know . You 'd have to ask her .\n",
      "27  or i 'm gon na take it , and you wo n't like how . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "27 Or I 'm gon na take it , and you wo n't like how .\n",
      "28  we help with that , we 're golden . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "28 We help with that , we 're golden .\n",
      "29  there 's something seriously wrong with him . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "29 There 's something seriously wrong with him .\n",
      "30  it 's just ever since my dad left , i 've felt responsible for her . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "30 It 's just ever since my dad left , I 've felt responsible for her .\n",
      "31  okay , mr. man . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "31 Okay , Mr. Man .\n",
      "32  currently , cds on 10-year u.s. treasuries are trading at nuk# , which is well off its highs for the year . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "32 Currently , cds on 10-year u.s. treasuries are trading at 58bps , which is well off its highs for the year .\n",
      "33  so chances are that if you 're watching this , i 'll already be dead . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "33 So chances are that if you 're watching this , I 'll already be dead .\n",
      "34  the dials are set for ... pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "34 The dials are set for ...\n",
      "35  you may want to take your leave , ma 'am . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "35 You may want to take your leave , ma 'am .\n",
      "36  are they ok ? pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "36 Are they OK ?\n",
      "37  i do n't think there 's anything . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "37 I do n't think there 's anything .\n",
      "38  looking for the main idea . even when you do n't grasp everything , you can still find the main point . think : what is the author really trying to say ? pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "38 Looking for the main idea . Even when you do n't grasp everything , you can still find the main point . Think : what is the author really trying to say ?\n",
      "39  so this could do for the brain what colonoscopy did for the gut ! pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "39 So this could do for the brain what colonoscopy did for the gut !\n",
      "40  and we 're here once again with the incomparable simone pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "40 And we 're here once again with the incomparable Simone\n",
      "41  no , it 's not . you want to have lunch ? pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "41 No , it 's not . You want to have lunch ?\n",
      "42  deep - seated problems . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "42 Deep - seated problems .\n",
      "43  you 've got to be careful . i do n't know if you hit him . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "43 You 've got to be careful . I do n't know if you hit him .\n",
      "44  practice for installation of thermoplastic pipe and corrugated pipe in septic tank leach fields . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "44 Practice for installation of thermoplastic pipe and corrugated pipe in septic tank leach fields .\n",
      "45  and i 'm pretty used to getting yelled at , so i was doing okay . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "45 And I 'm pretty used to getting yelled at , so I was doing okay .\n",
      "46  fine . living with it . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "46 Fine . Living with it .\n",
      "47  retinal tissue contains nuk# cells that absorb wavelengths in the visible spectrum and convert them into electrical signals . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "47 Retinal tissue contains light-sensitive cells that absorb wavelengths in the visible spectrum and convert them into electrical signals .\n",
      "48  i did n't feel like ratting him out . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "48 I did n't feel like ratting him out .\n",
      "49  if it 's not going anywhere , what does it hurt to look around ? pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "49 If it 's not going anywhere , what does it hurt to look around ?\n",
      "50  collection itself is not nuk# . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "50 Collection itself is not modifiable .\n",
      "51  course , i asked for talent . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "51 Course , I asked for talent .\n",
      "52  vanilla ? what did you say ? pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "52 Vanilla ? What did you say ?\n",
      "53  the prefrontal cortex is key to complex thoughts and behaviors , including language , social behavior and decision-making . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "53 The prefrontal cortex is key to complex thoughts and behaviors , including language , social behavior and decision-making .\n",
      "54  somebody put it out before the alarms go off ! pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "54 Somebody put it out before the alarms go off !\n",
      "55  yeah , you do n't have any game . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "55 Yeah , you do n't have any game .\n",
      "56  rather , you can arrange your own nuk# dinner at home . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "56 Rather , you can arrange your own candle-lit dinner at home .\n",
      "57  the doorway is open . what ? pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "57 The doorway is open . What ?\n",
      "58  throughout smoke and mirrors , nuk# alternates between describing chinese people , places , and events , and nuk# on their indian counterparts . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "58 Throughout smoke and mirrors , aiyar alternates between describing chinese people , places , and events , and ruminating on their indian counterparts .\n",
      "59  simulation study on the nuk# bombing nuk# principle in invisible area . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "59 Simulation study on the air-to-ground bombing fire-control principle in invisible area .\n",
      "60  i like my new nuk# . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "60 I like my new deskmate .\n",
      "61  if you love that girl , leave me , pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "61 If you love that girl , leave me ,\n",
      "62  that is , if he wants you as his old lady . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "62 That is , if he wants you as his old lady .\n",
      "63  jacob shall again find rest , shall be tranquil and undisturbed . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "63 Jacob shall again find rest , shall be tranquil and undisturbed .\n",
      "64  i plan to pension off the secret police , restore free elections , pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "64 I plan to pension off the secret police , restore free elections ,\n",
      "65  get your butts in the saddle . we 're riding . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "65 Get your butts in the saddle . We 're riding .\n",
      "66  i wear pleated pants because i have to , pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "66 I wear pleated pants because I have to ,\n",
      "67  sorry about that ! i 'll keep this sucker closed . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "67 Sorry about that ! I 'll keep this sucker closed .\n",
      "68  mostly out of morbid curiosity . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "68 Mostly out of morbid curiosity .\n",
      "69  studying is hard and boring . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "69 Studying is hard and boring .\n",
      "70  defrost the fridge regularly so that it works at maximum efficiency . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "70 Defrost the fridge regularly so that it works at maximum efficiency .\n",
      "71  this `` nuk# '' could reflect market forces , not official policies . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "71 This `` co-movement '' could reflect market forces , not official policies .\n",
      "72  i 'm not an ostrich ! pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "72 I 'm not an ostrich !\n",
      "73  you know , i 'm surprised you 're still on the job . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "73 You know , I 'm surprised you 're still on the job .\n",
      "74  bingo . look at that . current went through the phone . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "74 Bingo . Look at that . Current went through the phone .\n",
      "75  were you okay out here last night ? pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "75 Were you okay out here last night ?\n",
      "76  uh , that 's a great idea . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "76 Uh , that 's a great idea .\n",
      "77  let me take your bag . we 've been really looking forward to this . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "77 Let me take your bag . We 've been really looking forward to this .\n",
      "78  so i found myself a boyfriend . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "78 So I found myself a boyfriend .\n",
      "79  we 're paying you . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "79 We 're paying you .\n",
      "80  well , she had a yeast infection , and then she had her period right afterwards . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "80 Well , she had a yeast infection , and then she had her period right afterwards .\n",
      "81  nuk# nobel is multinational company that serves customers around the world with healthcare products , coatings and chemicals . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "81 Akzo nobel is multinational company that serves customers around the world with healthcare products , coatings and chemicals .\n",
      "82  i wo n't go back on my word pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "82 I wo n't go back on my word\n",
      "83  i think that 's the wrong sauce . do you ? pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "83 I think that 's the wrong sauce . Do you ?\n",
      "84  so would cutting the high taxes and welfare payments that contribute to what investors call nuk# brasil the cost of doing business in the country . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "84 So would cutting the high taxes and welfare payments that contribute to what investors call custo brasil the cost of doing business in the country .\n",
      "85  how would you respond to mr. nuk# 's arguments ? pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "85 How would you respond to mr. richman 's arguments ?\n",
      "86  i find it interesting that you find it interesting . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "86 I find it interesting that you find it interesting .\n",
      "87  hey , l love you to death . you too , man . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "87 Hey , l love you to death . You too , man .\n",
      "88  do n't ask me cause i have n't got a clue . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "88 do n't ask me cause I Have n't got a clue .\n",
      "89  sh ... sh ... sh ... she wo n't ... she wo n't suffer , will she ? pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "89 SH ... SH ... SH ... She wo n't ... She wo n't suffer , will she ?\n",
      "90  could you help a friend with a big problem ? pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "90 Could you help a friend with a big problem ?\n",
      "91  okay , i 'm a little ... pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "91 Okay , I 'm a little ...\n",
      "92  so far , nuk# joint venture tnk-bp still faces an uncertain future . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "92 So far , russian-british joint venture tnk-bp still faces an uncertain future .\n",
      "93  recently he found one dead , with its horns cut off . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "93 Recently he found one dead , with its horns cut off .\n",
      "94  did he have his name on it ? and his room number . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "94 Did he have his name on it ? And his room number .\n",
      "95  i do n't know ! bullshit ! pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "95 I do n't know ! Bullshit !\n",
      "96  the tree is a universal metaphor for the self , pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "96 The tree is a universal metaphor for the self ,\n",
      "97  but i 'd have to do the surgery , so i need to decide what i can live with . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "97 But I 'd have to do the surgery , so I need to decide what I can live with .\n",
      "98  you 're all really in trouble now . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "98 You 're all really in trouble now .\n",
      "99  tarzan , the king of the jungle . pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad# pad#\n",
      "99 Tarzan , the king of the jungle .\n"
     ]
    }
   ],
   "source": [
    "import h5py\n",
    "import json\n",
    "from collections import defaultdict\n",
    "\n",
    "en_voc = json.load(open('/data/xuwenshen/ai_challenge/data/train/train/en_voc.json'))\n",
    "\n",
    "h5 = h5py.File('/data/xuwenshen/ai_challenge/data/train/train/ibm_train-50-60.h5py', 'r')\n",
    "\n",
    "rvoc = defaultdict()\n",
    "for (k,v) in en_voc.items():\n",
    "    rvoc[v] = k\n",
    "rvoc = dict(rvoc)\n",
    "\n",
    "f1 = h5['en_len']\n",
    "f2 = h5['en']\n",
    "f3 = h5['en_str']\n",
    "for i in range(100):\n",
    "    text = ''\n",
    "    \n",
    "    for j in range(len(f2[i])):\n",
    "        text += ' ' + rvoc[f2[i][j]]\n",
    "    print (i, text)\n",
    "    print (i, f3[i].decode('utf-8'))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}