REDONE_LSTM.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function, division\n",
    "\n",
    "from sklearn import metrics\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.optim import lr_scheduler\n",
    "import numpy as np\n",
    "import torchvision\n",
    "from torchvision import datasets, models, transforms\n",
    "import torch.utils.data as data\n",
    "from torchvision.datasets.folder import *\n",
    "import PIL\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "import os\n",
    "import copy\n",
    "from itertools import chain\n",
    "import pretrainedmodels.utils as utils\n",
    "import pretrainedmodels\n",
    "\n",
    "from torchviz import make_dot, make_dot_from_trace\n",
    "\n",
    "%matplotlib inline\n",
    "plt.ion()   # interactive mode\n",
    "BATCH = 2\n",
    "N_layers = 2 # LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "code_folding": [
     1,
     23
    ]
   },
   "outputs": [],
   "source": [
    "# Transforms\n",
    "data_transforms = {\n",
    "    'train': transforms.Compose([\n",
    "        transforms.RandomResizedCrop(224),\n",
    "        transforms.RandomHorizontalFlip(),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ]),\n",
    "    'val': transforms.Compose([\n",
    "        transforms.Resize(256),\n",
    "        transforms.CenterCrop(224),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ]),\n",
    "    'test': transforms.Compose([\n",
    "        transforms.Resize(256),\n",
    "        transforms.CenterCrop(224),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ]),\n",
    "}\n",
    "\n",
    "# Special class\n",
    "class MultiImageFolder(data.Dataset):\n",
    "    \"\"\"A generic data loader where the images are arranged in this way: ::\n",
    "        root/dog/xxx.png\n",
    "        root/dog/xxy.png\n",
    "        root/dog/xxz.png\n",
    "        root/cat/123.png\n",
    "        root/cat/nsdf3.png\n",
    "        root/cat/asd932_.png\n",
    "    Args:\n",
    "        root (string): Root directory path.\n",
    "        transform (callable, optional): A function/transform that  takes in an PIL image\n",
    "            and returns a transformed version. E.g, ``transforms.RandomCrop``\n",
    "        target_transform (callable, optional): A function/transform that takes in the\n",
    "            target and transforms it.\n",
    "        loader (callable, optional): A function to load an image given its path.\n",
    "     Attributes:\n",
    "        classes (list): List of the class names.\n",
    "        class_to_idx (dict): Dict with items (class_name, class_index).\n",
    "        imgs (list): List of (image path, class_index) tuples\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, root, transform=None, target_transform=None,\n",
    "                 loader=default_loader, len_seq=4):\n",
    "        classes, class_to_idx = find_classes(root)\n",
    "        imgs = make_dataset(root, class_to_idx, [\".jpg\",\".jpeg\",\".png\"])\n",
    "        if len(imgs) == 0:\n",
    "            raise(RuntimeError(\"Found 0 images in subfolders of: \" + root + \"\\n\"\n",
    "                               \"Supported image extensions are: \" + \",\".join(IMG_EXTENSIONS)))\n",
    "\n",
    "        self.root = root\n",
    "        self.imgs = imgs\n",
    "        self.classes = classes\n",
    "        self.class_to_idx = class_to_idx\n",
    "        self.transform = transform\n",
    "        self.target_transform = target_transform\n",
    "        self.loader = loader\n",
    "        self.len_seq =len_seq\n",
    "\n",
    "    def __len__(self): \n",
    "        return len(self.imgs)//self.len_seq\n",
    "        \n",
    "    def __getitem__(self, index):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            index (int): Index\n",
    "        Returns:\n",
    "            tuple: (image, target) where target is class_index of the target class.\n",
    "        \"\"\"\n",
    "        img_l = []\n",
    "        target_l = []\n",
    "        for i in range(self.len_seq):\n",
    "            path, target = self.imgs[self.len_seq*index+i]\n",
    "            img = self.loader(path)\n",
    "            if self.transform is not None:\n",
    "                img = self.transform(img)\n",
    "            if self.target_transform is not None:\n",
    "                target = self.target_transform(target)\n",
    "            if target_l and target != target_l[0]: break\n",
    "            img_l += [img]\n",
    "            target_l += [target]\n",
    "\n",
    "        return img_l, target\n",
    "\n",
    "# Datasets\n",
    "data_dir = './LittlePlaNet/data/cities_L/'\n",
    "image_datasets = {x: MultiImageFolder(os.path.join(data_dir, x),\n",
    "                                           data_transforms[x], len_seq=8)\n",
    "                   for x in ['train', 'val', 'test']}\n",
    "dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=BATCH,\n",
    "                                           shuffle=True, num_workers=8, drop_last=True)\n",
    "                   for x in ['train', 'val', 'test']}\n",
    "dataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'val', 'test']}\n",
    "class_names = image_datasets['train'].classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "def imshow(inp, title=None):\n",
    "    \"\"\"Imshow for Tensor.\"\"\"\n",
    "    inp = inp.numpy().transpose((1, 2, 0))\n",
    "    mean = np.array([0.5, 0.5, 0.5])\n",
    "    std = np.array([0.229, 0.224, 0.225])\n",
    "    inp = std * inp + mean\n",
    "    inp = np.clip(inp, 0, 1)\n",
    "    plt.imshow(inp)\n",
    "    if title is not None:\n",
    "        plt.title(title)\n",
    "    plt.pause(0.001)  # pause a bit so that plots are updated\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "device = torch.device(\"cuda\")\n",
    "m = torch.load(\"models/resnet50_trained\")\n",
    "\n",
    "num_ftrs = m.fc.in_features\n",
    "hidden = 80\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "?nn.LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "class SeqModel(nn.Module):\n",
    "    def __init__(self, num_ftrs, hidden, transf, num_layers=N_layers):\n",
    "        super(SeqModel, self).__init__()        \n",
    "        self.hidden = hidden\n",
    "        self.num_ftrs = num_ftrs\n",
    "        \n",
    "        self.emb = transf\n",
    "        self.L = nn.LSTM(num_ftrs, hidden, num_layers=num_layers, bidirectional = True).to(device)\n",
    "        self.h2la = nn.Linear(2*hidden, len(class_names)).to(device)\n",
    "        self.drop = nn.Dropout(0.3)\n",
    "        self.h = (torch.zeros(2*N_layers, BATCH ,hidden).to(device),\n",
    "                  torch.zeros(2*N_layers,BATCH ,hidden).to(device))\n",
    "\n",
    "    def init_hidden(self):\n",
    "        self.h = (torch.randn(self.num_layers, BATCH ,self.hidden).to(device),\n",
    "                  torch.randn(self.num_layers, BATCH ,self.hidden).to(device))\n",
    "        \n",
    "    def forward(self,inp):        \n",
    "        X = torch.stack([self.emb(inp[i].to(device)) for i in range(len(inp))]).squeeze() \n",
    "#         print(X.shape)\n",
    "        o,h = self.L(X.view(len(inp), -1 , self.num_ftrs),self.h)\n",
    "        res = self.h2la(nn.Softmax(dim=1)(self.drop(o).mean(0)))\n",
    "        return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "x,y = next(iter(nt_dataloader))\n",
    "y = y.to(device)\n",
    "sm = SeqModel(num_ftrs, hidden, nn.Sequential(*list(m.children())[:-1]).to(device))\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "# Observe that all parameters are being optimized\n",
    "optimizer_ft = optim.Adam(chain(sm.L.parameters(),sm.h2la.parameters()) , lr=0.01)\n",
    "\n",
    "# Decay LR by a factor of 0.1 every 7 epochs\n",
    "exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=5, gamma=0.1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "\n",
    "# inputs, classes = next(iter(dataloaders['train']))\n",
    "# inputs = torch.stack( inputs )\n",
    "# # Make a grid from batch\n",
    "# out = torchvision.utils.make_grid(inputs[:,1,:,:,:])\n",
    "\n",
    "# imshow(out, title=class_names[classes[1]])\n",
    "\n",
    "# Reset optimizer\n",
    "optimizer_ft = optim.Adam(chain(sm.L.parameters(),sm.h2la.parameters()) , lr=0.01)\n",
    "\n",
    "# Decay LR by a factor of 0.1 every 7 epochs\n",
    "exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=5, gamma=0.1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "def train_model(model, criterion, optimizer, scheduler, num_epochs=25):\n",
    "    since = time.time()\n",
    "\n",
    "    hist = {\"train\":[], \"val\":[]}\n",
    "        \n",
    "    best_model_wts = copy.deepcopy(model.state_dict())\n",
    "    best_acc = 0.0\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n",
    "        print('-' * 10)\n",
    "\n",
    "        # Each epoch has a training and validation phase\n",
    "        for phase in ['train', 'val']:\n",
    "            if phase == 'train':            \n",
    "                scheduler.step()\n",
    "                model.train()  # Set model to training mode\n",
    "                model.emb.eval()\n",
    "            else:\n",
    "                model.eval()   # Set model to evaluate mode\n",
    "\n",
    "            running_loss = 0.0\n",
    "            running_corrects = 0\n",
    "\n",
    "            # Iterate over data.\n",
    "            for inputs, labels in dataloaders[phase]:\n",
    "#                 inputs = inputs.to(device)\n",
    "                labels = labels.to(device)\n",
    "\n",
    "                # zero the parameter gradients\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "                # forward\n",
    "                # track history if only in train\n",
    "                with torch.set_grad_enabled(phase == 'train'):\n",
    "                                            # last batch classes\n",
    "                    outputs = model(inputs)                                                \n",
    "                    _, preds = torch.max(outputs, 1)\n",
    "                    labels,\n",
    "                    loss = criterion(outputs, labels)\n",
    "\n",
    "                    # backward + optimize only if in training phase\n",
    "                    if phase == 'train':\n",
    "                        loss.backward()\n",
    "                        optimizer.step()\n",
    "\n",
    "                # statistics\n",
    "                running_loss += loss.item() * BATCH\n",
    "                running_corrects += torch.sum(preds == labels.data)\n",
    "            \n",
    "            epoch_loss = running_loss / dataset_sizes[phase]\n",
    "            epoch_acc = running_corrects.double() / dataset_sizes[phase]\n",
    "            \n",
    "            print('{} Loss: {:.4f} Acc: {:.4f}'.format(\n",
    "                phase, epoch_loss, epoch_acc))\n",
    "            hist[phase] += [[epoch_loss, epoch_acc]]\n",
    "            \n",
    "            # deep copy the model\n",
    "            if phase == 'val' and epoch_acc > best_acc:\n",
    "                best_acc = epoch_acc\n",
    "                best_model_wts = copy.deepcopy(model.state_dict())\n",
    "\n",
    "        print()\n",
    "\n",
    "    time_elapsed = time.time() - since\n",
    "    print('Training complete in {:.0f}m {:.0f}s'.format(\n",
    "        time_elapsed // 60, time_elapsed % 60))\n",
    "    print('Best val Acc: {:4f}'.format(best_acc))\n",
    "\n",
    "    # load best model weights\n",
    "    model.load_state_dict(best_model_wts)\n",
    "    return model, hist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0/24\n",
      "----------\n",
      "train Loss: 2.3155 Acc: 0.0888\n",
      "val Loss: 2.3213 Acc: 0.0000\n",
      "\n",
      "Epoch 1/24\n",
      "----------\n",
      "train Loss: 2.3133 Acc: 0.0970\n",
      "val Loss: 2.3503 Acc: 0.0147\n",
      "\n",
      "Epoch 2/24\n",
      "----------\n",
      "train Loss: 2.3117 Acc: 0.0916\n",
      "val Loss: 2.3262 Acc: 0.0147\n",
      "\n",
      "Epoch 3/24\n",
      "----------\n",
      "train Loss: 2.3124 Acc: 0.0898\n",
      "val Loss: 2.2863 Acc: 0.2206\n",
      "\n",
      "Epoch 4/24\n",
      "----------\n",
      "train Loss: 2.3164 Acc: 0.0898\n",
      "val Loss: 2.2917 Acc: 0.2206\n",
      "\n",
      "Epoch 5/24\n",
      "----------\n",
      "train Loss: 2.3025 Acc: 0.0934\n",
      "val Loss: 2.3014 Acc: 0.0147\n",
      "\n",
      "Epoch 6/24\n",
      "----------\n",
      "train Loss: 2.3020 Acc: 0.0852\n",
      "val Loss: 2.3086 Acc: 0.0147\n",
      "\n",
      "Epoch 7/24\n",
      "----------\n",
      "train Loss: 2.3016 Acc: 0.1034\n",
      "val Loss: 2.3133 Acc: 0.0147\n",
      "\n",
      "Epoch 8/24\n",
      "----------\n",
      "train Loss: 2.3016 Acc: 0.1006\n",
      "val Loss: 2.3157 Acc: 0.0147\n",
      "\n",
      "Epoch 9/24\n",
      "----------\n",
      "train Loss: 2.3014 Acc: 0.0888\n",
      "val Loss: 2.3132 Acc: 0.0147\n",
      "\n",
      "Epoch 10/24\n",
      "----------\n",
      "train Loss: 2.2989 Acc: 0.1052\n",
      "val Loss: 2.3137 Acc: 0.0147\n",
      "\n",
      "Epoch 11/24\n",
      "----------\n",
      "train Loss: 2.2989 Acc: 0.1061\n",
      "val Loss: 2.3140 Acc: 0.0147\n",
      "\n",
      "Epoch 12/24\n",
      "----------\n",
      "train Loss: 2.2987 Acc: 0.1052\n",
      "val Loss: 2.3137 Acc: 0.0147\n",
      "\n",
      "Epoch 13/24\n",
      "----------\n",
      "train Loss: 2.2985 Acc: 0.1006\n",
      "val Loss: 2.3138 Acc: 0.0147\n",
      "\n",
      "Epoch 14/24\n",
      "----------\n",
      "train Loss: 2.2984 Acc: 0.1070\n",
      "val Loss: 2.3137 Acc: 0.0147\n",
      "\n",
      "Epoch 15/24\n",
      "----------\n",
      "train Loss: 2.2982 Acc: 0.1052\n",
      "val Loss: 2.3138 Acc: 0.0147\n",
      "\n",
      "Epoch 16/24\n",
      "----------\n",
      "train Loss: 2.2978 Acc: 0.1052\n",
      "val Loss: 2.3138 Acc: 0.0147\n",
      "\n",
      "Epoch 17/24\n",
      "----------\n",
      "train Loss: 2.2980 Acc: 0.1052\n",
      "val Loss: 2.3138 Acc: 0.0147\n",
      "\n",
      "Epoch 18/24\n",
      "----------\n",
      "train Loss: 2.2980 Acc: 0.1052\n",
      "val Loss: 2.3138 Acc: 0.0147\n",
      "\n",
      "Epoch 19/24\n",
      "----------\n",
      "train Loss: 2.2979 Acc: 0.1052\n",
      "val Loss: 2.3138 Acc: 0.0147\n",
      "\n",
      "Epoch 20/24\n",
      "----------\n",
      "train Loss: 2.2979 Acc: 0.1043\n",
      "val Loss: 2.3137 Acc: 0.0147\n",
      "\n",
      "Epoch 21/24\n",
      "----------\n",
      "train Loss: 2.2978 Acc: 0.1043\n",
      "val Loss: 2.3137 Acc: 0.0147\n",
      "\n",
      "Epoch 22/24\n",
      "----------\n",
      "train Loss: 2.2979 Acc: 0.1052\n",
      "val Loss: 2.3138 Acc: 0.0147\n",
      "\n",
      "Epoch 23/24\n",
      "----------\n",
      "train Loss: 2.2978 Acc: 0.1052\n",
      "val Loss: 2.3138 Acc: 0.0147\n",
      "\n",
      "Epoch 24/24\n",
      "----------\n",
      "train Loss: 2.2979 Acc: 0.1061\n",
      "val Loss: 2.3138 Acc: 0.0147\n",
      "\n",
      "Training complete in 147m 35s\n",
      "Best val Acc: 0.220588\n"
     ]
    }
   ],
   "source": [
    "sm, hist = train_model(sm, criterion, optimizer_ft, exp_lr_scheduler, num_epochs=25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "def avgmodel(m, inp):\n",
    "    mn = torch.stack([m(inp[i].to(device)) for i in range(len(inp))]).squeeze().mean(0)\n",
    "    if len(mn.shape) == 1: return mn.max(0)[1]\n",
    "    return mn.max(1)[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 0.2475, -0.4040, -0.5777, -0.0768,  0.0328,  0.4112,  0.7051,\n",
      "          0.2025,  1.3419, -0.9156],\n",
      "        [ 0.1969,  0.5837, -0.0008,  0.5255,  0.1755, -0.2451,  1.1130,\n",
      "         -0.7582,  0.0414, -0.3068]], device='cuda:0')\n",
      "torch.Size([2, 10])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([ 8,  6], device='cuda:0')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x, y = next(iter(dataloaders['train']))\n",
    "mn = torch.stack([m(x[i].to(device)) for i in range(len(x))]).squeeze().mean(0)\n",
    "print(mn)\n",
    "print(mn.shape)\n",
    "mn.max(1)[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "newtestds = MultiImageFolder(\"data/cities/\",data_transforms[\"test\"],len_seq=2)\n",
    "nt_dataloader = torch.utils.data.DataLoader(\n",
    "    newtestds,batch_size=BATCH, shuffle=True, num_workers=4, drop_last=True)\n",
    "class_names = newtestds.classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.652\n",
      "0.6519748352746799\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f46f75831d0>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAECCAYAAADesWqHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAC+VJREFUeJzt3V2MXXUVhvH3nTOdtlNpGcAYaYmFxACKmpLRVDCYUC9UCPgVgwlEuemN8mGIBPTCS2+MkURD0qBcaCMmBRNDjPiBXBhNdWhJtAxEQCht+ahQC8KUaWeWFzM1CLVnT9zr7Dms55eQ0LJZXZnM031mus+/jggBqGWk6wUADB7hAwURPlAQ4QMFET5QEOEDBXUWvu2P237U9mO2b+lqj6Zsn2X7d7anbe+xfUPXOzVhu2d7t+17u96lCdun2t5h+5HFj/WHu96pH9tfXfyc+Kvtn9he1fVO/XQSvu2epO9L+oSk90j6gu33dLHLEhyTdFNEnC9ps6QvD8HOknSDpOmul1iC2yT9MiLOk/QBLfPdba+XdL2kyYi4QFJP0lXdbtVfV3f8D0l6LCKeiIhZSXdJurKjXRqJiGciYtfiv7+shU/I9d1udXK2N0i6TNIdXe/ShO21ki6R9ANJiojZiPhnt1s1Mippte1RSeOSDnS8T19dhb9e0tOv+/E+LfOIXs/2RkmbJO3sdpO+vivpZknzXS/S0DmSDkq6c/HLkztsr+l6qZOJiP2Svi1pr6RnJB2OiF91u1V/XYXvE/zcUDw7bPttku6WdGNEvNT1Pv+L7cslPR8RD3a9yxKMSrpQ0u0RsUnSK5KW9fd/bE9o4dXq2ZLOlLTG9tXdbtVfV+Hvk3TW6368QUPw8sj2Ci1Evz0i7ul6nz4ulnSF7Se18KXUpbZ/3O1Kfe2TtC8ijr+S2qGF3wiWs49J+ntEHIyIo5LukXRRxzv11VX4f5b0bttn2x7TwjdDft7RLo3Ytha+9pyOiO90vU8/EXFrRGyIiI1a+PjeHxHL+k4UEc9Ketr2uYs/tUXSwx2u1MReSZttjy9+jmzRMv+GpLTw0mrgIuKY7a9Iuk8L3wX9YUTs6WKXJbhY0jWS/mL7ocWf+3pE/KLDnd6KrpO0ffGG8ISkazve56QiYqftHZJ2aeFPfnZL2tbtVv2Zt+UC9fDkHlAQ4QMFET5QEOEDBRE+UFDn4dve2vUOSzFs+0rsPAjDtm/n4Usaqg+Yhm9fiZ0HYaj2XQ7hAxiwlAd4Vqwbj5XvWNvo2qOHZ7Ri3epG1/aePtF7e1py9Fijy2bjiMaWyzkL883edDer1zSmlc3njiTdD0Z7jS+dnXtVY73x5rPnkx5EO9bw82KpH2NJGm3/wdmZYy9pdn6mbygpj+yufMdave97X2x97rqbVrQ+8z8OPJ83O0nMzKTM9epmvxEvee7EupS5kqTXZlPGzr/wYspcSRo5/bTWZ/7hubua/dqt/8oAlj3CBwoifKAgwgcKInygoEbhD9sZ+ABOrm/4Q3oGPoCTaHLHH7oz8AGcXJPwh/oMfABv1iT8Rmfg295qe8r21NHDOU+UAWhHk/AbnYEfEdsiYjIiJps+ew+gG03CH7oz8AGcXN836QzpGfgATqLRu/MW/9II/uII4C2CJ/eAgggfKIjwgYIIHyiI8IGCUs7c6z12VGs/9aZnfP5vn939ZOszj7v7cx9NmTu359GUuZLUm5hImevxnAew5g88mzJXyjsncP7IkZS5kuRXE55wbXgAK3d8oCDCBwoifKAgwgcKInygIMIHCiJ8oCDCBwoifKAgwgcKInygIMIHCiJ8oCDCBwoifKAgwgcKInygIMIHCiJ8oCDCBwoifKAgwgcKSjleO8vdF52fNvsXe36aMvcT52xOmStJMZNwPHOirCOwpeH7WEjS3KFDrc+MmGt0HXd8oCDCBwoifKAgwgcKInygIMIHCiJ8oKC+4ds+y/bvbE/b3mP7hkEsBiBPkwd4jkm6KSJ22T5F0oO2fx0RDyfvBiBJ3zt+RDwTEbsW//1lSdOS1mcvBiDPkr7Gt71R0iZJOzOWATAYjZ/Vt/02SXdLujEiXjrBf98qaaskrfKa1hYE0L5Gd3zbK7QQ/faIuOdE10TEtoiYjIjJMa1sc0cALWvyXX1L+oGk6Yj4Tv5KALI1ueNfLOkaSZfafmjxn08m7wUgUd+v8SPi95I8gF0ADAhP7gEFET5QEOEDBRE+UBDhAwXlnLI7MpJ6omqGyz6Y8yeUe7dPpMyVpI1feyVlbhw6nDL3hcvPS5krSaf+6I9ps9+KuOMDBRE+UBDhAwURPlAQ4QMFET5QEOEDBRE+UBDhAwURPlAQ4QMFET5QEOEDBRE+UBDhAwURPlAQ4QMFET5QEOEDBRE+UBDhAwURPlBQ3vHa4+0frx2vzrQ+M3v2xutTxkqSttz3SMrc+y5YmzL3jPufSpkrSTGRc4z53KFDKXMlqZewsw/3Gl3HHR8oiPCBgggfKIjwgYIIHyiI8IGCCB8oqHH4tnu2d9u+N3MhAPmWcse/QdJ01iIABqdR+LY3SLpM0h256wAYhKZ3/O9KulnSfOIuAAakb/i2L5f0fEQ82Oe6rbanbE/Nzuc9Uw/g/9fkjn+xpCtsPynpLkmX2v7xGy+KiG0RMRkRk2Mj7b9BB0B7+oYfEbdGxIaI2CjpKkn3R8TV6ZsBSMOf4wMFLen9+BHxgKQHUjYBMDDc8YGCCB8oiPCBgggfKIjwgYJSTtmNo0d1bP+B1ueOrj+z9ZnHxamn5Mw98HzKXEn67RXvT5l77tT+lLmPX5kyVlLeabgZJ+Eel7FzxFyj67jjAwURPlAQ4QMFET5QEOEDBRE+UBDhAwURPlAQ4QMFET5QEOEDBRE+UBDhAwURPlAQ4QMFET5QEOEDBRE+UBDhAwURPlAQ4QMFpZyy615PvXXtn06acXLvf+QcLKvee8/NGay8E3z/9pFeytxvTf8sZa4kfeMjn06Zm/o51yHu+EBBhA8URPhAQYQPFET4QEGEDxRE+EBBjcK3fartHbYfsT1t+8PZiwHI0/QBntsk/TIiPmd7TNJ44k4AkvUN3/ZaSZdI+pIkRcSspNnctQBkavJS/xxJByXdaXu37Ttsr0neC0CiJuGPSrpQ0u0RsUnSK5JueeNFtrfanrI9NRtHWl4TQJuahL9P0r6I2Ln44x1a+I3gv0TEtoiYjIjJMa9qc0cALesbfkQ8K+lp28ffZrZF0sOpWwFI1fS7+tdJ2r74Hf0nJF2btxKAbI3Cj4iHJE0m7wJgQHhyDyiI8IGCCB8oiPCBgggfKIjwgYIcEa0PXTdyemxe9cnW584f4VHg1xtZlfOE5Mjpp6XMzfSZ3+xKmbtj08aUuZI0d2H7R6//6aHb9dK/9rvfddzxgYIIHyiI8IGCCB8oiPCBgggfKIjwgYIIHyiI8IGCCB8oiPCBgggfKIjwgYIIHyiI8IGCCB8oiPCBgggfKIjwgYIIHyiI8IGCmv5tuUszMiKvXt362F7CzOPmDh1KmZt1Eq6klI+xJB05950pc0fvfzBlrpR3Gu67fz+XMleSHr/yYOszPXus0XXc8YGCCB8oiPCBgggfKIjwgYIIHyiI8IGCGoVv+6u299j+q+2f2M77w2kA6fqGb3u9pOslTUbEBZJ6kq7KXgxAnqYv9UclrbY9Kmlc0oG8lQBk6xt+ROyX9G1JeyU9I+lwRPwqezEAeZq81J+QdKWksyWdKWmN7atPcN1W21O2p2bjSPubAmhNk5f6H5P094g4GBFHJd0j6aI3XhQR2yJiMiImx/jeH7CsNQl/r6TNtsdtW9IWSdO5awHI1ORr/J2SdkjaJekvi//PtuS9ACRq9H78iPimpG8m7wJgQHhyDyiI8IGCCB8oiPCBgggfKIjwgYJyjteen1fMzLQ+Nus46Uwjp5+WNnv+hRdT5q587l8pczUxkTM30aOTOceuS9Lnp3e1PnP6s682uo47PlAQ4QMFET5QEOEDBRE+UBDhAwURPlAQ4QMFET5QEOEDBRE+UBDhAwURPlAQ4QMFET5QEOEDBRE+UBDhAwURPlAQ4QMFET5QkCOi/aH2QUlPNbz8DEn/aH2JPMO2r8TOg7Bc9n1XRLy930Up4S+F7amImOx0iSUYtn0ldh6EYduXl/pAQYQPFLQcwt/W9QJLNGz7Suw8CEO1b+df4wMYvOVwxwcwYIQPFET4QEGEDxRE+EBB/wbaKrjq+HrG8gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f4681d44550>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "preds = []\n",
    "Y = []\n",
    "for x,y in nt_dataloader:\n",
    "    prs = avgmodel(m,x)\n",
    "#     prs = sm(x)[len(x)-1,:,:].max(1)[1]\n",
    "    preds +=list(prs)\n",
    "    Y += list(y)\n",
    "scor = metrics.accuracy_score(Y, preds)\n",
    "print( scor )\n",
    "print(metrics.f1_score(Y,preds, average=\"weighted\"))\n",
    "plt.matshow(metrics.confusion_matrix(Y,preds).astype(np.double))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7f4680550048>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xd4VGXax/HvPZNJpYYqxEBEulI0wmLHihUFC6CAa8FVcVdXd1fdfa27a1nruu6urqugaBCQZhelufREWmiKSAkoJXRSZ+Z+/5iDjjEyATM5k8n9ua65Muec58z5JZBz57TnEVXFGGOMORSP2wGMMcbEPisWxhhjIrJiYYwxJiIrFsYYYyKyYmGMMSYiKxbGGGMismJhjDEmIisWxhhjIrJiYYwxJqIEtwNUl6ZNm2rbtm3djmGMMbVKXl7eDlVtFqld3BSLtm3bkpub63YMY4ypVURkQ1Xa2WkoY4wxEVmxMMYYE5EVC2OMMRHFzTWLypSXl1NQUEBJSYnbUaIiOTmZjIwMfD6f21GMMXEurotFQUEB9evXp23btoiI23GqlapSWFhIQUEBWVlZbscxxsS5qJ6GEpF+IrJGRNaKyD2VLG8jIp+KyDIRmSkiGWHLAiKyxHlNPZLtl5SU0KRJk7grFAAiQpMmTeL2qMkYE1uidmQhIl7gBeBcoABYJCJTVXVlWLMngddUdbSInAU8Cgx1lhWrao9qyPFzPyJmxfP3ZoyJLdE8sugFrFXVdapaBowF+ldo0wX41Hk/o5LlxhhjDmHxtDdYNPG5qG8nmsWiNbApbLrAmRduKTDQeX85UF9EmjjTySKSKyLzReSyyjYgIiOcNrnbt2+vzuzVbtKkSYgIq1ev/m7eF198wYUXXsixxx5L586dueqqq9i6dSsACxcu5PTTT6djx4506tSJG2+8kaKiIrfiG2NizL69u5j/3LX0nHMr9VfmEAwEorq9aBaLys6RaIXpu4EzRGQxcAawGfA7yzJVNRsYAjwrIu1+9GGqL6lqtqpmN2sW8Wl1V+Xk5HDqqacyduxYIHQ95aKLLuKWW25h7dq1rFq1iltuuYXt27ezdetWrrzySh5//HHWrFnDqlWr6NevH/v27XP5uzDGxILl8z9l9zN96LXzXRa0GsYxv5uJx+uN6jajeTdUAXB02HQGsCW8gapuAQYAiEg9YKCq7glbhqquE5GZQE/gqyjmjZr9+/czZ84cZsyYwaWXXsqDDz7Im2++SZ8+fbjkkku+a9e3b18A7r//foYPH06fPn2A0LWJK664wpXsxpjYUVJayoLR93HK5lfY4WnClxeOpXfvfjWy7WgWi0VAexHJInTEMIjQUcJ3RKQpsFNVg8C9wCvO/MZAkaqWOm1OAZ74OWEeemcFK7fs/Tkf8SNdWjXggUu6Rmw3efJk+vXrR4cOHUhPT+fzzz8nPz+fE088sdL2+fn5DB8+vFqzGmNqty9WLiXw9o2cEfiCJenn0+GX/6Zlg/Qa237UTkOpqh8YCXwErALGqeoKEXlYRC51mp0JrBGRL4AWwF+c+Z2BXBFZSujC92MV7qKqVXJychg0aBAAgwYNIicnx+VExpjawu8P8Okbf6P1W+fSOrCZFSc/S4/fjCO1BgsFRPmhPFV9H3i/wrz7w95PACZUst5c4PjqzFKVI4BoKCwsZPr06eTn5yMiBAIBRIQHHniAWbNmVbpO165dycvLo39/uznMmLps/cYNbB0zgrPL5rMm7QRaDn+Vri3aupLF+oaKsgkTJjBs2DA2bNjA+vXr2bRpE1lZWXTo0IG5c+fy3nvvfdf2ww8/ZPny5YwcOZLRo0ezYMGC75aNGTOGb7/91o1vwRhTw1SVT6aOIe2/p9GzLJf84/5Ax7s/paFLhQKsWERdTk4Ol19++Q/mDRw4kDfffJN3332X559/nvbt29OlSxdGjRpF8+bNadGiBWPHjuXuu++mY8eOdO7cmc8++4wGDRq49F0YY2rKNzsK+fSpoZzz+W0UJ6az99qPOe6K+8Dj7u5aVCvezVo7ZWdna8XBj1atWkXnzp1dSlQz6sL3aExdoKrMmjWNNjN/QxZbWNl2GJ2veQLxpUR1uyKS5zymcEhx3ZGgMcbUBrv2FTN79J+4cPur7PE24ptL3qJLz5q5JbaqrFgYY4yL5uXlkfzubfTXVXzZ/Fyyhr9IQr0mkVesYVYsjDHGBQdKynn/jWfot/FpPCIU9H2W9qdfBzHaQagVC2OMqWFL1nzF7nEjuTIwl40NetJ8+CgymrZ1O9YhWbEwxpgaUuoPMHnCGM5Y9QBdZR8bT/g9mRffA57o9utUHaxYGGNMDVi9aSurXr+bq8umsjW5DWWD3yazbeVd/sQie84iyrxeLz169PjutX79egoLC+nbty/16tVj5MiRbkc0xkRRIKiMf+8DPC+fxeVlU9l47FBa3L2AtFpUKMCOLKIuJSWFJUuW/GDegQMHeOSRR8jPzyc/P9+lZMaYaNu4Yz8zR9/P1XtHU5LQgL2XjSXz+AvcjnVErFi4IC0tjVNPPZW1a9e6HcUYEwWqytTZC2k5/Q6GyUo2H3UOrYa+iKQ1dTvaEas7xeKDe+Db5dX7mS2PhwseO2ST4uJievQIDSWelZXFpEmTqjeDMSambNtXwqTRzzF4+7P4PMquc56h9cm/jNlbYquq7hQLl1R2GsoYE58+zltD4J07uZk5bG3UnXrDRpPSJMvtWNWi7hSLCEcAxhhzpPYUlzMm53Uu2/BnWsouCnv9jhbn3wPe+NnFxs93YowxLpizejNfj7+PW/xT2JNyNDrkLZpknuR2rGpnxcIlbdu2Ze/evZSVlTF58mQ+/vhjunTp4nYsY0wVFZcFeHXSe5y54k+c4tnAjs7X0nTAE5CY5na0qLBiEWX79++vdP769etrNogxptos2biTOW/8mRtLXsOfmEbZgByadrnQ7VhRZcXCGGOqqDwQ5NUP5tJl4R+4zZPPzoyzSB/8ItRr7na0qLNiYYwxVbB22z7Gv/YCt+z7OyneAEXnPUX6L26o9bfEVlXcFwtVReL0HzNeRjk0JpYFg8qY2fnUn34f93pmszu9G0nXvApNj3U7Wo2K62KRnJxMYWEhTZo0ibuCoaoUFhaSnJzsdhRj4tbm3cX85/U3uGH7Y7T2FHLgF3fR6Nx7wetzO1qNi+tikZGRQUFBAdu3b3c7SlQkJyeTkZHhdgxj4o6qMjH3awrffZj/YzLF9Vojgz4kLbO329FcE9fFwufzkZUVH09PGmNqRuH+Up576z2u3PAQx3vWs7/LEOr1fwKS6rsdzVVR7aJcRPqJyBoRWSsi91SyvI2IfCoiy0RkpohkhC0bLiJfOq/h0cxpjDEA01Z8y3+evo97N95M+6TdBK56nXpX/avOFwqI4pGFiHiBF4BzgQJgkYhMVdWVYc2eBF5T1dEichbwKDBURNKBB4BsQIE8Z91d0cprjKm79pWU88ykzzht5YPc413K/swzqXfVi1C/pdvRYkY0T0P1Ataq6joAERkL9AfCi0UX4E7n/QxgsvP+fGCaqu501p0G9ANyopjXGFMHzV9XyJSxL3F36Qs0SCjDf/4T1Os9os7cEltV0SwWrYFNYdMFQMWrQ0uBgcBzwOVAfRFp8hPrtq64AREZAYwAyMzMrLbgxpj4V1Ie4O8fLKbNokd41DuTA02PwzfoFWjW0e1oMSma1ywqK8sVHwy4GzhDRBYDZwCbAX8V10VVX1LVbFXNbtas2c/Na4ypI/I37+H3z77MVblDuNI7i7I+d5B2ywwrFIcQzSOLAuDosOkMYEt4A1XdAgwAEJF6wEBV3SMiBcCZFdadGcWsxpg6wB8I8tKMNfhn/Y2nvZPw1z8Kz1Xvk9jmZLejxbxoFotFQHsRySJ0xDAIGBLeQESaAjtVNQjcC7ziLPoI+KuINHamz3OWG2PMEfl6xwH+9uZ7jNjxGD28X1HW9SqSL3kSkhu6Ha1WiFqxUFW/iIwktOP3Aq+o6goReRjIVdWphI4eHhURBWYDtznr7hSRRwgVHICHD17sNsaYw6GqjJm3ni8//CdPeUbjTUqG/q+SeNwAt6PVKhIv/QtlZ2drbm6u2zGMMTHk2z0lPPLWTC7b9ATnevMozTydpCtehAat3I4WM0QkT1WzI7WL6ye4jTF115Qlm/l48ms8pP+isa8YPfevJPW+BTxRfRY5blmxMMbEld1FZTw0MZcTVj/FCwmfUNa0M96r/gsturodrVazYmGMiRsz12zj1XETecD/LFkJ3xLsM5LEs++HhCS3o9V6ViyMMbXegVI/j723nAZ5/+S/vrfRes2RgVOQY85wO1rcsGJhjKnV8jbs5G9jP+LuA0+T7fuCQJcBJFzyNKQ0jryyqTIrFsaYWqnMH+TZaWvY/r9XecU3msQkH1zyMt5uV7odLS5ZsTDG1Dqrv93L/Tmf8cudz3CBbxH+o08mYeCL0Mj6iIsWKxbGmFojEFRe/mwd86eN54WEf9PEtx/OeoiEk28Hj9fteHHNioUxplbYtLOIe95ayDmb/8mrCR/hb9IRzxVT4ahubkerE6xYGGNimqry1qJNjH/3PR6Xf3BsQgHa+1cknPMg+FLcjldnWLEwxsSsbftKuG/CEtqtHcVbvvFIWhO4fCJy7NluR6tzrFgYY2LSB8u/4fmJ03kg+Dy9favQzpcilzwHqeluR6uTrFgYY2LKnuJyHpqST2DZOMYnjSIlUeCifyHdB9tQpy6yYmGMiQmrv93L2IWbmPb5au4J/IdLEucRzOiN5/IXIT3L7Xh1nhULY4xrissCvLtsCzkLNlBesJghCTP5g28eyVIKff8Pz6l32i2xMcKKhTGmxq3cspechRuZsXg15/hn8bek2bRLWo8mpCBd+kOfW+Go7m7HNGGsWBhjasSBUv93RxH1tsxhcMJMHvDmkuArR1v0gBOeRo6/woY5jVFWLIwxUZW/eQ85CzeyYMkyLvRP59+Js2mZuI1gciM83a6HE4YiLY93O6aJwIqFMaba7S/1887SLYxf8BUtv5nO4IRZPOJZhsenaNaZ0PNRPJ0uBl+y21FNFVmxMMZUm+UFe3hz4UZWLpnHpcHpvOr7Hw0T9xGs3wpPz99Bz2uQxm3djmmOgBULY8zPsq+knClLtjB5wWqO3fYxgxNm0t2zlmCCD+l0IfQchqddX7urqZazYmGMOWyqytKCPeTM30DBshlcpp8yJmEByb5SAk07wol/xdPtakhr6nZUU02sWBhjqmxvSTlTFm/m/fnL6LbjfW5OmMUx3i0EfGl4jrsaThiGNyPbnrSOQ1YsjDGHpKp8vnE3by1Yx97lH3A5M3jdu5gEXwB/Rm848Y94u1wGSfXcjmqiKKrFQkT6Ac8BXuBlVX2swvJMYDTQyGlzj6q+LyJtgVXAGqfpfFX9VTSzGmN+aE9ROZMWFzBr3gKyd7/HXd7PaOHdRXlyUxJOuBV6DiOhWQe3Y5oaErViISJe4AXgXKAAWCQiU1V1ZVizPwHjVPVfItIFeB9o6yz7SlV7RCufMebHVJW8DbsYP/9LgiumMIAZXOddSTDBQ7DdOZA9DF+HfuD1uR3V1LBoHln0Ataq6joAERkL9AfCi4UCDZz3DYEtUcxjjPkJu4vKmJhXQO78GfTZ8z5/9M6lgbeI0gZtIPv/8PQYgqdBK7djGhdFs1i0BjaFTRcAvSu0eRD4WERuB9KAc8KWZYnIYmAv8CdV/SyKWY2pc1SVRet3MWVePsmr3magzOB6zwb8iUnQ+RLIHk5Sm1PB43E7qokB0SwWld0OoRWmBwOjVPUpEekDvC4ixwHfAJmqWigiJwKTRaSrqu79wQZERgAjADIzM6v/OzAmDu08UMbEvI2snvcep+3/kPu9i0jyllPc9Hjo9SQJx18JKY3cjmliTDSLRQFwdNh0Bj8+zXQD0A9AVeeJSDLQVFW3AaXO/DwR+QroAOSGr6yqLwEvAWRnZ1csRMYYh6oyf91OPpiTS/qX4xkgM7nRs52y5AZ4ul0H2cNIOaqb2zFNDItmsVgEtBeRLGAzMAgYUqHNRuBsYJSIdAaSge0i0gzYqaoBETkGaA+si2JWY+JS4f5SJuV+zab5E+l74EMe8C7D61UOtD4Fej9KYueLwZfidkxTC0StWKiqX0RGAh8Rui32FVVdISIPA7mqOhW4C/iPiNxJ6BTVdaqqInI68LCI+IEA8CtV3RmtrMbEk2BQmbeukOmfzabVuvFc5vkfTWUvxWkt0BN/CycOJc1GnjOHSVTj4+xNdna25ubmRm5oTJzavq+UKQvWsHPhWM4p+YgTPGsJiJfitudRr8/1cOzZ1j+T+RERyVPV7Ejt7AluY2qxYFCZs3Y7C2Z/QJsNExnsmUealLKv4TGU93oYX88h1KvXzO2YJg5YsTCmFtq2r4R35y6jOHcM55dO427PFsp8KRR3HAAnX0/9jJOsfyZTraxYGFNLBIPKZ198y/KZE2m/ZRJD5XN8EmBn0x6U9/kDid0GkphU3+2YJk5ZsTAmxm3dW8KHs+ehi8dwvn86Z8hODiQ2pui4G2l48vWkN+/kdkRTB1ixMCYGBYLK/1Zu4ovZORz37WSGe1YSxMOOlqdSftqNpHW6ABIS3Y5p6hArFsbEkG/2FDNjxickLnuDcwOzOEOK2J3Sit09f0+jPtfRvGFrtyOaOsqKhTEu8weC/G/5WjZ/9hrdt7/DEM96yvCxI/N8Uk+/iUbtTrf+mYzrrFgY45LNuw4w95Mp1F+Vw5mBeSRLOVvrdWDXSX+h8S+uoVVKY7cjGvMdKxbG1CB/IMicxcvZ8b9XyN71PlfKNookjW3HXslRfUfQIqOn2xGNqZQVC2NqwKbte/j8kxyafvEWpwYX4xVlY8MTKexzP02yryDT+mcyMc6KhTE/RRX1l1JcfIDiAwcoKd5PSdEBykqLKC0uorz0AOUlRfhLiwiUFRMoKyFYVoT6S6C8GMpLkEAJXv8Bupfk0l/2ssvbhA2dbibzrBFkNmvn9ndoTJVZsTC1gyr4S8Ef2gnjd17lxeAvQcuL8ZcWUVpSRFlJ0Xc78dCOvIRgeRHBsmK0vCT0Gf5SPP4SPIESvIESvMFSEoJl+IKl+LSMJEpJ1HI8oqQCqUcQuYQkyiWRck8ShU1OgNNuoGm3C2nstV87U/vY/1pz+FTDdtYlYTvw0E744A78h1+/39EHnR27v6yYQFkxWlZMsKz4+/X9JWE78lISgqV4tQzPj8bO+p4APudVUVCFEhIpwUcpiZSoj1JJokQSKZckAt5UAp50golJBLzJaEIyJCQjvmTEl4onMRlPYgoJiSkkJKWSkJRKYnIqiclpJKWkkZScSkpqGknJaXgSU0JdfnsTSRYh2cmQHoV/BmNqkhULc3gOFFL2n3NI3H1kw4uE77hLSKREEyk9uBN3pktoRCm+0DJJJOBNIuhJJpiQDAlJkJAS2pEnpuDxpeBNDL18yWkkJKWQlJxGYnIKySn1SExJJTU5ldTkBNISE0hL8tLM5yXBa7eiGnM4qlQsRORyYLqq7nGmGwFnqurkaIYzMUaVPRNGkrJrA0/7r2A/KZSQSKmGdvwH/yLHl4LHl4w3KbQTT0gM7cR9yakkJyWTmuQjLclLamICqYleUhO9pCWF3qcnHZwX+pqU4EGsQzxjXFfVI4sHVHXSwQlV3S0iDwBWLOqQsqXjafj1+7zgvYazbvwrjVJ8pCZ5SUtMIMXnxeOxnbox8aqqxaKyY3Y7hVWX7PuWwDt3sSTYjq5X/4keRzdyO5ExpgZV9cRtrog8LSLtROQYEXkGyItmMBNDVNn91i2Iv5jpnR7mzM6t3E5kjKlhVS0WtwNlwFvAeKAEuC1aoUxsKc19nUYF03kpcSgjBvZzO44xxgVVOpWkqgeAe6KcxcSi3ZvQD+5hQbATvQfdR70kO/toTF10yN98EXlWVe8QkXfgxze5q+qlUUtm3KfKrrE3kxjws6j7nxnZzsZyNqauivRn4uvO1yejHcTEnuK5L9H42zk8k3Irt1x6lttxjDEuOmSxUNU8EfECN6nqtTWUycSCnevwfHI/s4PdOPua35Ps87qdyBjjoogXuFU1ADQTERvDsa4IBtj5xo2UBj2s6fUXuh1t4yoYU9dV9W6o9cAcEfk/EfntwVeklUSkn4isEZG1IvKjC+QikikiM0RksYgsE5ELw5bd66y3RkTOr/J3ZH62/bOfJ70wj5fr38x1F5zqdhxjTAyo6q0tW5yXB6jvzPvpXt0A5/TVC8C5QAGwSESmqurKsGZ/Asap6r9EpAvwPtDWeT8I6Aq0Aj4RkQ7OUY6JIt22mqRZf+aT4IlcfO1v8VkfSsYYql4sVqrq+PAZInJlhHV6AWtVdZ3TfizQHwgvFgo0cN43JFSQcNqNVdVS4GsRWet83rwq5jVHIuBn15s3QDCJb057lHNaNoi8jjGmTqjqn433VnFeuNbAprDpAmdeuAeBa0WkgNBRxe2Hsa6pZns//Rvpu/MZ1fjXDDm7l9txjDExJNJzFhcAFwKtReTvYYsaAP4In11Zr3IVT10NBkap6lMi0gd4XUSOq+K6iMgIYARAZmZmhDjmUIJblpI690ne1z4MHDoSr3UKaIwJE+nIYguQS6h7j7yw11Qg0kXnAuDosOkMvj/NdNANwDgAVZ0HJANNq7guqvqSqmaranazZvbA2BHzl7H7zRvZpWnsP/tx2jRJczuRMSbGHLJYqOpSVR0NHEtopz5fVUer6kRV3RXhsxcB7UUky7ntdhChIhNuI3A2gIh0JlQstjvtBolIkohkAe2BhYf5vZkq2v3hn0nf/wWvN7uLK0/r5nYcY0wMquo1i37AEuBDABHpISIVd/w/oKp+YCTwEbCK0F1PK0TkYRE52E3IXcBNIrIUyAGu05AVhIrTSmebt9mdUNER2LiI+rnPM5kzGTL0ZhtoyBhTKVE95B2woUYiecBZwExV7enMW6aqMfNnaHZ2tubm5rodo3YpL2bX07+gqGgfiy96n4t7dXI7kTGmholInqpmR2pX1SML/8EhVU382PnO/9G4eD0TWt/DRSd1dDuOMSaGVfU5i3wRGQJ4RaQ98GtgbvRimWgr/+ozGi17mfFyPtcOGW6nn4wxh3Q4gx91BUoJXVvYC9wRrVAmykr3c2DcCDYFm5F+2aM0qZfkdiJjTIyr6uBHRcAfnZep5XZM+gPpJd8wJuvvjOzezu04xphaINJDeZHueLLBj2qZsjXTaLp6DDkJlzBs8GC34xhjaolIRxZ9CHW7kQMsoPInq01tUbyb4gm3sjHYiraDHqNBss/tRMaYWiLSNYuWwH3AccBzhHqQ3aGqs1R1VrTDmeq1bcJvSSvbwfROD9GnU4bbcYwxtUikJ7gDqvqhqg4HfgGsBWaKyO2HWs/EnqLlU2n+1dvkJA7k2isGuB3HGFPLRLzALSJJwEWEOv1rC/wdmBjdWKZaFe0kMOU3rApm0nXwX0hNrOod08YYExLpAvdoQqegPgAeUtX8GkllqtXWsbfRuHwP87o/x/XHtHA7jjGmFor0J+ZQ4ADQAfh12INbAqiq2ug4MW5/3jhabHyfUSnXck3/i9yOY4yppQ5ZLFTVxtSszfZthffuYmmwHb2ufYSkBK/biYwxtZQVg3ilyrdv/IqEQDH5vR+nS0a624mMMbWYFYs4tXf+a7T8djpv1hvO1f3OdjuOMaaWs2IRh3T3JhI+vpdF2okzht1Pgtf+mY0xP4/tReKNKlvH3ARBPxtO/RvtWjR0O5ExJg5YsYgzO2f/m5Y75pHT6CYGnH2a23GMMXHCikUcCRZ+TerMB5mr3Th/+H14PNaVlzGmelixiBfBINtev56yoFB49lNkpKe5ncgYE0esWMSJ7Z8+S8vdnzOh6UguPu0kt+MYY+KMFYs44N+6mgZzHmUWJ3Lx8LtsiFRjTLWzHuVqu4CfHWOuJ0kTKb/oWZo3SHE7kTEmDtmRRS239YPHablvBZNb/ZZzenVzO44xJk5ZsajFyjYvIz33aabJyVw+1IYYMcZET1SLhYj0E5E1IrJWRO6pZPkzIrLEeX0hIrvDlgXClh1yLPA6yV/GrjeuZ7emknLZMzRKTXQ7kTEmjkXtmoWIeIEXCA3FWgAsEpGpqrryYBtVvTOs/e1Az7CPKFbVHtHKV9ttmfoQrYq+5LW2jzKseye34xhj4lw0jyx6AWtVdZ2qlgFjgf6HaD8YyIlinrhR/PVCmi/7Jx94+zJgyAi34xhj6oBoFovWwKaw6QJn3o+ISBsgC5geNjtZRHJFZL6IXBa9mLVMeTH7xt7Edm1EsyufoV6S3dBmjIm+aO5pKrvZX3+i7SBggqoGwuZlquoWETkGmC4iy1X1qx9sQGQEMAIgMzOzOjLHvE1v/5GjS9fzZodnGdIpy+04xpg6IppHFgXA0WHTGcCWn2g7iAqnoFR1i/N1HTCTH17PONjmJVXNVtXsZs2aVUfmmLb/y89ovfoV3vH1Y8BVw9yOY4ypQ6JZLBYB7UUkS0QSCRWEH93VJCIdgcbAvLB5jUUkyXnfFDgFWFlx3TqldD8l40ZQoM3IGvI0yT4bItUYU3OiVixU1Q+MBD4CVgHjVHWFiDwsIpeGNR0MjFXV8FNUnYFcEVkKzAAeC7+Lqi7aMO53pJd9w/zjH+a4rEov/RhjTNTID/fRtVd2drbm5ua6HSMq9qz4mIbjr2Rycn8u+t0ofDbynTGmmohInqpmR2pne50Yp8W7CUy8jXXaiq5Dn7RCYYxxhe15YtyGN++goX87S7MfpX3r5m7HMcbUUVYsYljh51Nou2kSU+pdxaUXHep5RmOMiS4rFjFKDxTiffcO1mgmJw5/DK8NkWqMcZEVixi1/vVbSQvsYe0pT9KmebrbcYwxdZwVixi0bf5Ysr79kCmNhnLhuee5HccYY6xYxJrA3q0kf/Q78mnHqdf9xYZINcbEBCsWsUSVja/dTFKwmK19n6Fl43puJzLGGMCKRUzZ8tkosnbM4J2mN3DW6ae7HccYY75jxSJGlO3cRIMZf2QJneg7/AE7/WSMiSlWLGKBKptfuxFP0M++C/5OkwapbicyxpgfsGIsCCtXAAAO3UlEQVQRAzZ98i+yds/ng6Nu5bTevd2OY4wxP2LFwmWl29fRdM5DLJJunDv8XrfjGGNMpaxYuCkY5JvR1+NXgf4v0CAlye1ExhhTKSsWLvr6/adpu38xn7S5k5N6dHM7jjHG/CQrFi7Zv3klrXIfZ643m/Ov+a3bcYwx5pCsWLgh4KdwzI0Uq4+0K/5JapLP7UTGGHNIVixc8NWUv9KmeAWz299D984d3Y5jjDERWbGoYXvWLyFz2XN85juF86++1e04xhhTJQluB6hT/GXszbmRck2l2aB/kOSzH78xpnawI4satGbCAxxd+iULut5Pp3bHuB3HGGOqzIpFDdn5xXzarf43M5LO4vyBN7gdxxhjDoudB6kBWl5M8fgRlGkj2l77PAleq9HGmNrF9lo1YHXOvbQu38DSE/5M1tEZbscxxpjDFtViISL9RGSNiKwVkXsqWf6MiCxxXl+IyO6wZcNF5EvnNTyaOaNpa/5MOn41ik9SL+TcS4a4HccYY45I1E5DiYgXeAE4FygAFonIVFVdebCNqt4Z1v52oKfzPh14AMgGFMhz1t0VrbzRECzZT3DSLWyhKZ2HP4fHY2NUGGNqp2geWfQC1qrqOlUtA8YC/Q/RfjCQ47w/H5imqjudAjEN6BfFrFGxasxdHBXYwpd9nqB1i+ZuxzHGmCMWzWLRGtgUNl3gzPsREWkDZAHTD3fdWLX58w/oWjCWafUHcOb5l7sdxxhjfpZoFovKzrnoT7QdBExQ1cDhrCsiI0QkV0Ryt2/ffoQxq5+/aDe+d29nPUfR/ZdP2RCpxphaL5rFogA4Omw6A9jyE20H8f0pqCqvq6ovqWq2qmY3a9bsZ8atPmtG/5omgR1sPvMZmqenux3HGGN+tmgWi0VAexHJEpFEQgVhasVGItIRaAzMC5v9EXCeiDQWkcbAec68mLd+3kS6bp3Cp00Gc8qZF7gdxxhjqkXU7oZSVb+IjCS0k/cCr6jqChF5GMhV1YOFYzAwVlU1bN2dIvIIoYID8LCq7oxW1upSum8H9T++i7Vk0uu6J9yOY4wx1UbC9tG1WnZ2tubm5rqaYcXzV9Fhxycs6fc2J/Xp62oWY4ypChHJU9XsSO3sCe5q8tWsN+ha+BEzWw63QmGMiTtWLKpB0a5vaDLjHlZLO/oM/6vbcYwxptpZsfi5VFk/agQpWkTpJS9QLzXF7UTGGFPtrFj8TKun/Zcue2bz2dE30/2EPm7HMcaYqLBi8TPs3baRVnPvJ9/biVOHPuh2HGOMiRorFkdKlc2jb8Cn5SQM+DfJSYluJzLGmKixYnGE8t/9B50PLGReu9/QqWtPt+MYY0xUWbE4Ajs3f0lW3p9ZktCN04b8aJgOY4yJO1YsDpMGA2x//QZUhfpXv4QvwUamNcbEPysWh2npxCfpWLKU3E6/o137zm7HMcaYGmHF4jBsW59Px/wn+TzpJE676s7IKxhjTJywYlFFGvCz582bKFMfzYe8iNdrPzpjTN1he7wq+vytR2hftpLl3f9ERpt2bscxxpgaZcWiCjavyeP4Nf9gYcqpnHLZr9yOY4wxNc6KRQSB8jJKx9/EPkmlzbB/Ix77kRlj6h7b80Ww+I0/cYz/K9ae9DAtjjo68grGGBOHrFgcwvrlc+j+9cvMr3cOvS68zu04xhjjGisWP6GspBgm/4pd0pAO1/0TEXE7kjHGuMaKxU9Y8vrvaRvYyKZTHyO9aQu34xhjjKusWFTiy9xPOLHgdeY1upgTz7na7TjGGOM6KxYVlBTtI+W929kmTely3fNuxzHGmJhgxaKCpaN+S4ZuYcfZz9CwUbrbcYwxJiZYsQizcu679N42jnnNruT40y5xO44xxsQMKxaO/Xt30njaHWySVnS/7mm34xhjTEyxYuFY8eqvaR7cwYEL/k5qWgO34xhjTEyJarEQkX4iskZE1opIpUPKichVIrJSRFaIyJth8wMissR5TY1mziXTx9F71zssbDWUTr3OjeamjDGmVoraMG8i4gVeAM4FCoBFIjJVVVeGtWkP3Aucoqq7RKR52EcUq2qPaOU7aHfhNlrN/gPrPZmcMPzxaG/OGGNqpWgeWfQC1qrqOlUtA8YC/Su0uQl4QVV3AajqtijmqVTQX8o3qZ0I9P8nScmpNb15Y4ypFaI5gHRrYFPYdAHQu0KbDgAiMgfwAg+q6ofOsmQRyQX8wGOqOrniBkRkBDACIDMz84hCprc4mvTff3BE6xpjTF0RzWJRWWdKWsn22wNnAhnAZyJynKruBjJVdYuIHANMF5HlqvrVDz5M9SXgJYDs7OyKn22MMaaaRPM0VAEQ3qd3BrClkjZTVLVcVb8G1hAqHqjqFufrOmAm0DOKWY0xxhxCNIvFIqC9iGSJSCIwCKh4V9NkoC+AiDQldFpqnYg0FpGksPmnACsxxhjjiqidhlJVv4iMBD4idD3iFVVdISIPA7mqOtVZdp6IrAQCwO9UtVBETgZeFJEgoYL2WPhdVMYYY2qWqMbHqf7s7GzNzc11O4YxxtQqIpKnqtmR2tkT3MYYYyKyYmGMMSYiKxbGGGMiiptrFiKyHdjwMz6iKbCjmuJUJ8t1eCzX4bFchycec7VR1WaRGsVNsfi5RCS3Khd5aprlOjyW6/BYrsNTl3PZaShjjDERWbEwxhgTkRWL773kdoCfYLkOj+U6PJbr8NTZXHbNwhhjTER2ZGGMMSaiOl0sRORoEZkhIqucYV1/43YmABFJFpGFIrLUyfWQ25nCiYhXRBaLyLtuZzlIRNaLyHJnGN6Y6fdFRBqJyAQRWe38P+vjdiYAEekYNmzxEhHZKyJ3xECuO53/8/kikiMiyW5nAhCR3ziZVrj9cxKRV0Rkm4jkh81LF5FpIvKl87VxdW+3ThcLQgMr3aWqnYFfALeJSBeXMwGUAmepanegB9BPRH7hcqZwvwFWuR2iEn1VtUeM3dr4HPChqnYCuhMjPzdVXeP8rHoAJwJFwCQ3M4lIa+DXQLaqHkeoA9JBbmYCEJHjCI3q2YvQv+HFzpDQbhkF9Ksw7x7gU1VtD3zqTFerOl0sVPUbVf3ceb+P0C9ya3dTgYbsdyZ9zismLi6JSAZwEfCy21linYg0AE4H/gugqmXOwF6x5mzgK1X9OQ+1VpcEIEVEEoBUfjwGjhs6A/NVtUhV/cAs4HK3wqjqbGBnhdn9gdHO+9HAZdW93TpdLMKJSFtCAywtcDdJiHOqZwmwDZimqjGRC3gW+D0QdDtIBQp8LCJ5znC7seAYYDvwqnPa7mURSXM7VCUGATluh1DVzcCTwEbgG2CPqn7sbioA8oHTRaSJiKQCF/LDgd1iQQtV/QZCfwQDzat7A1YsABGpB7wN3KGqe93OA6CqAecUQQbQyzkUdpWIXAxsU9U8t7NU4hRVPQG4gNDpxNPdDkTor+QTgH+pak/gAFE4PfBzOAOTXQqMj4EsjQn9hZwFtALSRORad1OBqq4CHgemAR8CSwmdwq5T6nyxEBEfoULxhqpOdDtPRc5pi5n8+BylG04BLhWR9cBY4CwRGeNupJCwYXi3ETr33svdREBo2OCCsKPCCYSKRyy5APhcVbe6HQQ4B/haVberajkwETjZ5UwAqOp/VfUEVT2d0CmgL93OVMFWETkKwPm6rbo3UKeLhYgIofPJq1T1abfzHCQizUSkkfM+hdAv0Wp3U4Gq3quqGaraltCpi+mq6vpffiKSJiL1D74HziN06sBVqvotsElEOjqzzib2hgceTAycgnJsBH4hIqnO7+bZxMgNASLS3PmaCQwgdn5mB00FhjvvhwNTqnsDURtWtZY4BRgKLHeuDwDcp6rvu5gJ4ChgtIh4CRX0caoaM7epxqAWwKTQ/oUE4E1V/dDdSN+5HXjDOd2zDvily3m+45x/Pxe42e0sAKq6QEQmAJ8TOs2zmNh5YvptEWkClAO3qeout4KISA5wJtBURAqAB4DHgHEicgOhontltW/XnuA2xhgTSZ0+DWWMMaZqrFgYY4yJyIqFMcaYiKxYGGOMiciKhTHGmIisWJi4IyIqIk+FTd8tIg+6GCmqROQ6EfmH2zlMfLNiYeJRKTBARJq6HcSYeGHFwsQjP6GHue6suEBERonIFWHT+52vZ4rILBEZJyJfiMhjInKNM67IchFpV8lnpTljCyxyOgrs78z/rYi84rw/3hkHIVVEeonIXKft3INPdjtHBpNF5B0R+VpERjqfsVhE5otIutNupog866ybLyI/6tLEefr/bSfTIhE5xZl/hnw/dsXig0+8G1NVVixMvHoBuEZEGh7GOt0JjdVxPKEn+zuoai9C3bHfXkn7PxLq8uQkoC/wN6e7kWeBY0XkcuBV4GZVLSLUZcvpTqeC9wN/Dfus44AhhPq0+gtQ5LSbBwwLa5emqicDtwKvVJLpOeAZJ9NAvu9K/m5CTx73AE4Diqv+YzHGuvswcUpV94rIa4QG06nqjnHRwW6eReQr4GD32MsJFYOKziPUseLdznQykKmqq0TkOmAZ8KKqznGWNyTUjUt7Ql2q+8I+a4Yzpso+EdkDvBO27W5h7XKc72+2iDQ42IdYmHOALk7XJwANnKOIOcDTIvIGMFFVC6rw8zDmO1YsTDx7llA/Q6+GzfPjHFE7ndUlhi0rDXsfDJsOUvnvigADVXVNJcvaA/sJdbV90COEisLlzvgpM49g2xX756k47QH6qGrFAvmYiLxHaCyG+SJyjqq63jmlqT3sNJSJW6q6ExgH3BA2ez2hYUQhNHaCjyP3EXC7U3QQkZ7O14aETgedDjQJu0bSENjsvL/uCLd5tbONUwkNDrSnwvKPgZEHJ0Skh/O1naouV9XHgVyg0xFu39RRVixMvHsKCL8r6j/AGSKyEOhNaECiI/UIoWKzTETynWmAZ4B/quoXhArVY04X108Aj4rIHELjSx+JXSIyF/g3PyyCB/0ayBaRZSKyEviVM/8O56L4UkKn5T44wu2bOsp6nTWmlhCRmcDdqprrdhZT99iRhTHGmIjsyMIYY0xEdmRhjDEmIisWxhhjIrJiYYwxJiIrFsYYYyKyYmGMMSYiKxbGGGMi+n+xlnznyTBRngAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f4671bf1b00>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = [10,8,6,4, 2 \n",
    "]\n",
    "y1= [0.95,0.8709677419354839,0.8253012048192772,0.82, 0.652\n",
    "]\n",
    "y2=[0.95,0.8694682717292381,0.8250352445494066,0.8210904969040139, 0.6519748352746799\n",
    "]\n",
    "\n",
    "plt.plot(x,y1, label=\"ACC\")\n",
    "plt.plot(x,y2, label=\"F1\")\n",
    "plt.ylabel(\"Metric\")\n",
    "plt.xlabel(\"Num examples\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6\n",
      "0.5748962905000057\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f432017e0f0>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAECCAYAAADesWqHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAACyBJREFUeJzt3U2IXfUZx/Hfz5lJxiSVxNqNk9QxfbEVaY1ONZriIhH6EtEuurCgUDfZVI1BsFoKLktBrC6KMI11oalCYxbFFrUYXZTawckL1WQs2JjmVYyKRkfMzMSni7mCpuncc9vzv2dunu8HhMx48udhmG/OuXfO+Y8jQgByOavpAQB0H+EDCRE+kBDhAwkRPpAQ4QMJNRa+7e/a/oft12zf3dQcVdleYft52xO299je2PRMVdjus73L9lNNz1KF7aW2t9p+tfW1vqrpmdqxvan1PfGK7cdtDzY9UzuNhG+7T9KvJX1P0sWSfmT74iZm6cCMpDsj4uuSVkv6SQ/MLEkbJU00PUQHHpT0dER8TdI3Nc9ntz0k6XZJIxFxiaQ+STc2O1V7TZ3xr5D0WkTsi4gpSU9IuqGhWSqJiKMRsbP15/c1+w051OxUc7O9XNJ6SZubnqUK2+dIukbSw5IUEVMR8W6zU1XSL+ls2/2SFkk60vA8bTUV/pCkg5/6+JDmeUSfZntY0ipJY81O0tYDku6S9HHTg1S0UtIxSY+0Xp5str246aHmEhGHJd0n6YCko5Lei4hnm52qvabC92k+1xP3DtteIulJSXdExPGm5/lvbF8n6c2I2NH0LB3ol3SZpIciYpWkSUnz+v0f28s0e7V6oaTzJS22fVOzU7XXVPiHJK341MfL1QOXR7YHNBv9lojY1vQ8bayRdL3t/Zp9KbXW9mPNjtTWIUmHIuKTK6mtmv2HYD67VtLrEXEsIqYlbZN0dcMztdVU+C9J+ortC20v0OybIX9oaJZKbFuzrz0nIuL+pudpJyLuiYjlETGs2a/v9oiY12eiiHhD0kHbF7U+tU7S3gZHquKApNW2F7W+R9Zpnr8hKc1eWnVdRMzYvlXSM5p9F/S3EbGniVk6sEbSzZJetr279bmfRcSfGpzpTHSbpC2tE8I+Sbc0PM+cImLM9lZJOzX7k59dkkabnao981gukA937gEJET6QEOEDCRE+kBDhAwk1Hr7tDU3P0Ilem1di5m7otXkbD19ST33B1HvzSszcDT0173wIH0CXFbmBZ2DB4hgcXFbp2OnpSQ0MVHsAy+9/+P+MVYtpndCAFjY9Rkfmy8wnP1/9QbuZjybVP1j9+L63J/+Xkdo6saLaDCc/mFTfks4eJFx4sP6ZP9KkpuLE6R6C+4wit+wODi7TyBW31r5u//ZeetAMp3p3fbnNdJY++mKRdV+7c3WRdSXpy5v+VvuaY/FcpeO41AcSInwgIcIHEiJ8ICHCBxKqFH6v7YEPYG5tw+/RPfABzKHKGb/n9sAHMLcq4ff0HvgA/lOV8CvtgW97g+1x2+PT02VunwRQjyrhV9oDPyJGI2IkIkaq3nsPoBlVwu+5PfABzK3tQzo9ugc+gDlUejqv9Usj+MURwBmCO/eAhAgfSIjwgYQIH0iI8IGEiuy5d9bUjAb3v137uh+tvbz2NT/xwdCCIuuW2gtOkvpXDhdbu4Qlh6eaHqFjJfbFmw844wMJET6QEOEDCRE+kBDhAwkRPpAQ4QMJET6QEOEDCRE+kBDhAwkRPpAQ4QMJET6QEOEDCRE+kBDhAwkRPpAQ4QMJET6QEOEDCRE+kJAjovZFz/G5caXX1b5uSc8c2V1k3e+cf2mRdYHTGYvndDzecbvjOOMDCRE+kBDhAwkRPpAQ4QMJET6QEOEDCbUN3/YK28/bnrC9x/bGbgwGoJz+CsfMSLozInba/pykHbb/HBF7C88GoJC2Z/yIOBoRO1t/fl/ShKSh0oMBKKej1/i2hyWtkjRWYhgA3VHlUl+SZHuJpCcl3RERx0/z/zdI2iBJg1pU24AA6lfpjG97QLPRb4mIbac7JiJGI2IkIkYGtLDOGQHUrMq7+pb0sKSJiLi//EgASqtyxl8j6WZJa23vbv33/cJzASio7Wv8iPiLpLbP9wLoHdy5ByRE+EBChA8kRPhAQoQPJFT5zr0z3fpv/6DIuvt+t7TIupL0xc19Rdbt376jzLorh4usK0kz+/YXW/tMxBkfSIjwgYQIH0iI8IGECB9IiPCBhAgfSIjwgYQIH0iI8IGECB9IiPCBhAgfSIjwgYQIH0iI8IGECB9IiPCBhAgfSIjwgYQIH0iI8IGEimyv7YUL1L98uPZ1S26hXGrtr/58uMi6kjTzm5kyC28vsyzmD874QEKEDyRE+EBChA8kRPhAQoQPJET4QEKVw7fdZ3uX7adKDgSgvE7O+BslTZQaBED3VArf9nJJ6yVtLjsOgG6oesZ/QNJdkj4uOAuALmkbvu3rJL0ZETvaHLfB9rjt8amTH9Y2IID6VTnjr5F0ve39kp6QtNb2Y6ceFBGjETESESML+hbVPCaAOrUNPyLuiYjlETEs6UZJ2yPipuKTASiGn+MDCXX0PH5EvCDphSKTAOgazvhAQoQPJET4QEKEDyRE+EBCRXbZjRNTRXfERcu6Q0WWPfzTq4use8HvjxRZt6RSXwtJGvrlX4ut3Q5nfCAhwgcSInwgIcIHEiJ8ICHCBxIifCAhwgcSInwgIcIHEiJ8ICHCBxIifCAhwgcSInwgIcIHEiJ8ICHCBxIifCAhwgcSInwgIUdE7Yue43PjSq+rfd2S+lcONz1Cx3ptJ+Nv7T5ZbO2XLu0rtnYvGYvndDzecbvjOOMDCRE+kBDhAwkRPpAQ4QMJET6QEOEDCVUK3/ZS21ttv2p7wvZVpQcDUE7VX5P9oKSnI+KHthdIWlRwJgCFtQ3f9jmSrpH0Y0mKiClJU2XHAlBSlUv9lZKOSXrE9i7bm20vLjwXgIKqhN8v6TJJD0XEKkmTku4+9SDbG2yP2x6f1omaxwRQpyrhH5J0KCLGWh9v1ew/BJ8REaMRMRIRIwNaWOeMAGrWNvyIeEPSQdsXtT61TtLeolMBKKrqu/q3SdrSekd/n6Rbyo0EoLRK4UfEbkkjhWcB0CXcuQckRPhAQoQPJET4QEKEDyRE+EBCVX+Of8YrtVV1L27bXUrJLbDv+effi6z7iy99o8i6TeOMDyRE+EBChA8kRPhAQoQPJET4QEKEDyRE+EBChA8kRPhAQoQPJET4QEKEDyRE+EBChA8kRPhAQoQPJET4QEKEDyRE+EBChA8kxC67hZXavbekUjsDl/xalNoNt9TuvVKzO/hyxgcSInwgIcIHEiJ8ICHCBxIifCAhwgcSqhS+7U2299h+xfbjtgdLDwagnLbh2x6SdLukkYi4RFKfpBtLDwagnKqX+v2SzrbdL2mRpCPlRgJQWtvwI+KwpPskHZB0VNJ7EfFs6cEAlFPlUn+ZpBskXSjpfEmLbd90muM22B63PT6tE/VPCqA2VS71r5X0ekQci4hpSdskXX3qQRExGhEjETEyoIV1zwmgRlXCPyBpte1Fti1pnaSJsmMBKKnKa/wxSVsl7ZT0cuvvjBaeC0BBlZ7Hj4h7Jd1beBYAXcKde0BChA8kRPhAQoQPJET4QEKEDyTkiKh90cXnrYiL12+qfd2lj75Y+5ronpm1lxdbu3/7jiLrlpz5g6EFta+594+/0uRbB93uOM74QEKEDyRE+EBChA8kRPhAQoQPJET4QEKEDyRE+EBChA8kRPhAQoQPJET4QEKEDyRE+EBChA8kRPhAQoQPJET4QEKEDyRE+EBCRXbZtX1M0r8qHn6epLdqH6KcXptXYuZumC/zXhARX2h3UJHwO2F7PCJGGh2iA702r8TM3dBr83KpDyRE+EBC8yH80aYH6FCvzSsxczf01LyNv8YH0H3z4YwPoMsIH0iI8IGECB9IiPCBhP4NHeqK5hJ8QXMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f4321a1c908>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "scor = metrics.accuracy_score(Y, preds)\n",
    "print( scor )\n",
    "print(metrics.f1_score(Y,preds, average=\"weighted\"))\n",
    "plt.matshow(metrics.confusion_matrix(Y,preds).astype(np.double))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0.0395,  0.1186,  0.2006,  0.0897,  0.0829,  0.0907,  0.0668,\n",
       "           0.0928,  0.1574,  0.0610],\n",
       "         [ 0.0296,  0.1044,  0.4025,  0.0485,  0.0662,  0.0468,  0.0407,\n",
       "           0.0692,  0.1449,  0.0471]],\n",
       "\n",
       "        [[ 0.0172,  0.0475,  0.5648,  0.0503,  0.0398,  0.0293,  0.0723,\n",
       "           0.0562,  0.0728,  0.0498],\n",
       "         [ 0.0143,  0.0660,  0.3085,  0.0573,  0.0603,  0.0474,  0.0964,\n",
       "           0.0531,  0.1701,  0.1265]],\n",
       "\n",
       "        [[ 0.0193,  0.0432,  0.3705,  0.0484,  0.1606,  0.0406,  0.0742,\n",
       "           0.0473,  0.1235,  0.0724],\n",
       "         [ 0.0069,  0.0686,  0.5940,  0.0305,  0.0597,  0.0289,  0.0387,\n",
       "           0.0354,  0.1119,  0.0253]],\n",
       "\n",
       "        [[ 0.0192,  0.0687,  0.4567,  0.0727,  0.0475,  0.0289,  0.0297,\n",
       "           0.0693,  0.1643,  0.0430],\n",
       "         [ 0.0101,  0.0444,  0.6012,  0.0382,  0.0645,  0.0278,  0.0256,\n",
       "           0.0537,  0.0962,  0.0383]],\n",
       "\n",
       "        [[ 0.0402,  0.0892,  0.2531,  0.0645,  0.0485,  0.0333,  0.0593,\n",
       "           0.0697,  0.1940,  0.1483],\n",
       "         [ 0.0302,  0.0646,  0.2240,  0.0562,  0.1815,  0.0310,  0.0865,\n",
       "           0.0378,  0.1578,  0.1305]],\n",
       "\n",
       "        [[ 0.0183,  0.0693,  0.3903,  0.0238,  0.1662,  0.0367,  0.0556,\n",
       "           0.0421,  0.1378,  0.0598],\n",
       "         [ 0.0474,  0.0406,  0.3815,  0.0506,  0.1805,  0.0209,  0.0468,\n",
       "           0.0539,  0.1271,  0.0505]],\n",
       "\n",
       "        [[ 0.0163,  0.0572,  0.5570,  0.0442,  0.0403,  0.0563,  0.0291,\n",
       "           0.0754,  0.0710,  0.0531],\n",
       "         [ 0.0192,  0.0838,  0.2733,  0.1035,  0.0723,  0.0684,  0.0894,\n",
       "           0.0690,  0.1297,  0.0913]],\n",
       "\n",
       "        [[ 0.0150,  0.0399,  0.6344,  0.0413,  0.0446,  0.0195,  0.0223,\n",
       "           0.0503,  0.0857,  0.0470],\n",
       "         [ 0.0147,  0.0341,  0.5691,  0.0312,  0.0617,  0.0238,  0.0566,\n",
       "           0.0446,  0.0654,  0.0987]],\n",
       "\n",
       "        [[ 0.0229,  0.0479,  0.2622,  0.0625,  0.1311,  0.0307,  0.1312,\n",
       "           0.0503,  0.1526,  0.1085],\n",
       "         [ 0.0238,  0.0648,  0.3103,  0.0619,  0.1868,  0.0311,  0.1047,\n",
       "           0.0437,  0.1105,  0.0625]],\n",
       "\n",
       "        [[ 0.0211,  0.0502,  0.5113,  0.0336,  0.0720,  0.0219,  0.0385,\n",
       "           0.0537,  0.0786,  0.1189],\n",
       "         [ 0.0157,  0.0288,  0.5235,  0.0488,  0.0718,  0.0211,  0.0466,\n",
       "           0.0479,  0.1387,  0.0571]]], device='cuda:0')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sm(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f43201bbd30>]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAFoRJREFUeJzt3X+QXWd93/H3d3/cFXvXsnevlOL4BzLBtHEait1FToE4mQJGpqmdduxEpEyVBsahxW1TJjN1S2sYpzMtMEnbZJyCKZ4QCNgEQqK2YhyVuG1mUoNkYxtkYyyrDlbk2MIr+dda0u7q2z/uWXN9fVd7pf1xd+/zfs3s7L3nPOfer87e/eyj55zznMhMJEllGOh1AZKk1WPoS1JBDH1JKoihL0kFMfQlqSCGviQVxNCXpIIY+pJUEENfkgoy1OsC2m3atCm3bNnS6zIkaV255557vp+Zmxdrt+ZCf8uWLezdu7fXZUjSuhIRf95NO4d3JKkghr4kFcTQl6SCGPqSVBBDX5IKYuhLUkEMfUkqSN+E/rPHZviPu7/LfY8f7XUpkrRm9U3o50n4z197hL2PTfW6FElas/om9M/aMMTgQDD1wolelyJJa1bfhP7AQDA+WuPItKEvSQvpm9AHaNRrPP28oS9JC+mr0J+o1xzekaRTMPQlqSD9F/qO6UvSgvou9I9OzzA7d7LXpUjSmtRXod8YqwFwZHqmx5VI0trUV6E/PtoMfcf1Jamzvgr9Rt3Ql6RT6avQnxgz9CXpVPor9F/q6R/vcSWStDb1VejPj+k/bU9fkjrqq9AfHhxg44Yhjhj6ktRRV6EfEdsi4uGI2B8RN3ZY/8GIeDAiHoiIr0XEa1rW7YiIR6qvHctZfCeNsRF7+pK0gEVDPyIGgVuAq4BLgHdHxCVtzb4JTGbmG4AvAR+rtp0APgxcDmwFPhwR48tX/is5FYMkLaybnv5WYH9mHsjME8DtwDWtDTLzrsycrp7eDZxfPX4nsDszpzLzCLAb2LY8pXc2PmroS9JCugn984DHW54frJYt5L3AV89w2yVr2NOXpAUNddEmOizLjg0j3gNMAj91OttGxPXA9QAXXnhhFyUtbGKseSOVzCSi09tLUrm66ekfBC5oeX4+cKi9UUS8HfgQcHVmHj+dbTPz1syczMzJzZs3d1t7R416jZm55Nljs0t6HUnqR92E/h7g4oi4KCJqwHZgZ2uDiLgU+CTNwH+qZdWdwJURMV4dwL2yWrZinH9Hkha2aOhn5ixwA82wfgj4Ymbui4ibI+LqqtnHgTHg9yPivojYWW07BfwazT8ce4Cbq2UrxqkYJGlh3Yzpk5m7gF1ty25qefz2U2x7G3DbmRZ4upx0TZIW1ldX5ILz70jSqfRt6HtVriS9Ut+F/mhtiA3DA86/I0kd9F3oAzTqzr8jSZ30Zeg7/44kddaXoT9u6EtSR30Z+s6/I0md9WXoO7wjSZ31behPn5jj2Mxcr0uRpDWlb0MfPFdfktr1deh7rr4kvVxfhn7Dnr4kddSXoe/8O5LUWV+H/tPP29OXpFZ9GfobNwwzOBAcmTb0JalVX4b+wEAwPuq5+pLUri9DH5oHcx3ekaSX69vQH68P29OXpDZ9G/qN+ghTjulL0sv0beg7/44kvVJfh/7R6Rlm5072uhRJWjP6OvQBjkzP9LgSSVo7Cgh9h3gkaV7fhn7Dq3Il6RX6NvQnxubn3zH0JWle/4b+qJOuSVK7vg398Zdm2vRAriTN69vQHx4cYOOGIXv6ktSib0MfoDE24o1UJKlFX4f++Oiwp2xKUou+Dv2J+oinbEpSi74O/Ybz70jSy/R16E+M1TgyfYLM7HUpkrQm9Hfoj9aYmUueOz7b61IkaU3o79CfP1ffcX1JAvo99KupGDxtU5Kaugr9iNgWEQ9HxP6IuLHD+isi4t6ImI2Ia9vWzUXEfdXXzuUqvBuNuvPvSFKrocUaRMQgcAvwDuAgsCcidmbmgy3Nvgf8IvCrHV7ixcx84zLUetrGq/l3jhj6kgR0EfrAVmB/Zh4AiIjbgWuAl0I/Mx+r1q2p21Q1HN6RpJfpZnjnPODxlucHq2Xd2hAReyPi7oj42U4NIuL6qs3ew4cPn8ZLn9pobYgNwwPOvyNJlW5CPzosO50T3y/MzEngF4D/FBE/8ooXy7w1Myczc3Lz5s2n8dKLa9Sdf0eS5nUT+geBC1qenw8c6vYNMvNQ9f0A8L+AS0+jviUbrw87pi9JlW5Cfw9wcURcFBE1YDvQ1Vk4ETEeESPV403AW2g5FrAaJuojnr0jSZVFQz8zZ4EbgDuBh4AvZua+iLg5Iq4GiIg3RcRB4DrgkxGxr9r8R4G9EXE/cBfwH9rO+llxjXrN4R1JqnRz9g6ZuQvY1bbsppbHe2gO+7Rv92fAjy+xxiWZcNI1SXpJX1+RC83Qnz4xx7GZuV6XIkk9V0Tog1flShIY+pJUlL4P/fn5dzyYK0kFhP543fl3JGle34e+PX1J+oG+D/2NG4YZHAjn35EkCgj9gYFgfNRz9SUJCgh9gIn6sKEvSRQT+vb0JQkKCX2nV5akpiJC356+JDUVEfrj9RrPvDjD7NyaupujJK26IkK/Ua+RCUdfnOl1KZLUU0WEvvPvSFJTEaH/0lW5zxv6kspWROi/NP/OtKEvqWxFhL7z70hSUxGhP9/Tn3J4R1Lhigj94cEBNm4YctI1ScUrIvShukBr2lM2JZWtrNC3py+pcAWF/oinbEoqXjGh33D+HUkqJ/TH6zWOTJ8gM3tdiiT1TDGh36jXmJlLnjs+2+tSJKlnign9Cc/Vl6SCQn/Mq3IlqZzQH63m3zH0JRWsnNB3emVJKif0Gw7vSFI5oT9aG2LD8IDTK0sqWjGhD81xfa/KlVSyskJ/zPl3JJWtrNCvj3ggV1LRigr9Rr3GlGP6kgrWVehHxLaIeDgi9kfEjR3WXxER90bEbERc27ZuR0Q8Un3tWK7Cz8T4aM0rciUVbdHQj4hB4BbgKuAS4N0RcUlbs+8Bvwh8vm3bCeDDwOXAVuDDETG+9LLPTGOsxgsn5jg2M9erEiSpp7rp6W8F9mfmgcw8AdwOXNPaIDMfy8wHgJNt274T2J2ZU5l5BNgNbFuGus+IF2hJKl03oX8e8HjL84PVsm50tW1EXB8ReyNi7+HDh7t86dNn6EsqXTehHx2WdTspfVfbZuatmTmZmZObN2/u8qVPn6EvqXTdhP5B4IKW5+cDh7p8/aVsu+wMfUml6yb09wAXR8RFEVEDtgM7u3z9O4ErI2K8OoB7ZbWsJxp159+RVLZFQz8zZ4EbaIb1Q8AXM3NfRNwcEVcDRMSbIuIgcB3wyYjYV207BfwazT8ce4Cbq2U9sXHDMIMD4fTKkoo11E2jzNwF7GpbdlPL4z00h246bXsbcNsSalw2AwPB+OiwPX1JxSrqilxojus7/46kUhUa+vb0JZWpuNBvOOmapIIVF/rj9WFDX1Kxigv9ifoIR1+cYe5kt9eXSVL/KC70G/UamXjbRElFKi7056/K9Vx9SSUqNvQ9V19SiYoNfQ/mSipRcaHv/DuSSlZc6I87pi+pYMWF/vDgAGdtGHJ4R1KRigt9aA7xOLwjqURFhr6TrkkqVaGhP8LUCzO9LkOSVl2hoT9sT19SkQoN/eZMm5nOvyOpLEWGfqNeY2Yuee74bK9LkaRVVWToO/+OpFIVHfqetimpNEWH/tTzhr6kspQd+vb0JRWmyNBvjFWh741UJBWmyNB/1fAgI0MD9vQlFafI0I+I5vw7julLKkyRoQ8wMeb8O5LKU27o10eYmnb+HUllKTf0R51/R1J5yg39+ojn6UsqTrGh3xir8cKJOY7NzPW6FElaNcWG/kvz73iuvqSCFBv646PV/DsO8UgqSLGh/9JVuV6gJakgxYa+8+9IKlGxod8w9CUVqKvQj4htEfFwROyPiBs7rB+JiDuq9V+PiC3V8i0R8WJE3Fd9fWJ5yz9zGzcMMzgQhr6kogwt1iAiBoFbgHcAB4E9EbEzMx9safZe4Ehmvi4itgMfBX6+WvdoZr5xmetesoGBYHx02BupSCpKNz39rcD+zDyQmSeA24Fr2tpcA3ymevwl4G0REctX5sqYqNe8ZaKkonQT+ucBj7c8P1gt69gmM2eBZ4BGte6iiPhmRPzviPjJJda7rCbqNYd3JBWlm9Dv1GPPLts8AVyYmZcCHwQ+HxEbX/EGEddHxN6I2Hv48OEuSloeE/UaTzv/jqSCdBP6B4ELWp6fDxxaqE1EDAFnA1OZeTwznwbIzHuAR4HXt79BZt6amZOZObl58+bT/1ecIXv6kkrTTejvAS6OiIsiogZsB3a2tdkJ7KgeXwv8SWZmRGyuDgQTEa8FLgYOLE/pSzdRH+HoizPMnWz/j4sk9adFz97JzNmIuAG4ExgEbsvMfRFxM7A3M3cCnwY+GxH7gSmafxgArgBujohZYA54f2ZOrcQ/5Ew06jUy4ej0CRpjI70uR5JW3KKhD5CZu4Bdbctuanl8DLiuw3ZfBr68xBpXzHjLBVqGvqQSFHtFLvzgqlzP1ZdUiqJD/6XplQ19SYUoOvTt6UsqTdGhf86ok65JKkvRoV8bGuCsDUOGvqRiFB360BziMfQllaL40PeqXEklMfTrNQ/kSiqGoV+vMeWka5IKYejXRzjywgyZzr8jqf8VH/qNeo0Tcyd5/vhsr0uRpBVXfOiPe4N0SQUpPvS9KldSSYoPfeffkVQSQ9+evqSCGPqO6UsqSPGhP1obZGRowNCXVITiQz8inH9HUjGKD32AiTFDX1IZDH1gfNT5dySVwdBnfnpl59+R1P8MfX4w/44k9TtDH2iM1Xj++CzHZ+d6XYokrShDn+aYPniuvqT+Z+jTclXu84a+pP421OsC1oLGWDX/zvT6Dv3M5KS3BZDWtcGBWNHXN/Tpj+GdPY9NccPn7+XJZz0LSVqv3njBOfzhB96you9h6NMyvfI6Hd65/Rvf49/+0be5YHyUf/H21xAr21GQtEJevXHDir+HoQ+c/aphBgdi3fX0Z+dO8u/+x0P8zp89xhWv38xvvftSzn7VcK/LkrSGGfrAwEAwPjrM1Doa039meoYbvnAvf/rI93nfWy/ixqv+GkODHpeXdGqGfmV8tMbUOhne2f/Uc7zvM3s5dPQYH7v2Dfzc5AW9LknSOmHoVybWyUybd33nKf7ZF77JyPAAX7j+cv7mayZ6XZKkdcTQrzTGajz8l8/1uowFZSaf+tMD/PuvfocfffVGPrVjkvPOeVWvy5K0zhj6lYl6jSPTa3P+nWMzc/zrr3yLP7j3L/g7P34uH7/uDYzW/NFJOn0mR2VitMaR6RPMncwVvzjidDz17DF++XP38M3vHeWD73g9//Rvv47wnExJZ8jQr0zUa2TC0ekTNMZGel0OAA8cPMr1v3sPz7w4wyfecxnb/vq5vS5J0jrX1Tl+EbEtIh6OiP0RcWOH9SMRcUe1/usRsaVl3b+qlj8cEe9cvtKX10QV9GvlYO7O+w9x3Sf+L4MDwZf/8ZsNfEnLYtGefkQMArcA7wAOAnsiYmdmPtjS7L3Akcx8XURsBz4K/HxEXAJsB34M+GHgf0bE6zNzzc1hPH9Vbq9D/+TJ5Nd3P8wtdz3K1i0T/PZ7LmPTGvmfh6T1r5ue/lZgf2YeyMwTwO3ANW1trgE+Uz3+EvC2aA48XwPcnpnHM/P/Afur11tz1sL8O88fn+WXP3cPt9z1KNvfdAGfe9/lBr6kZdXNmP55wOMtzw8Cly/UJjNnI+IZoFEtv7tt2/POuNoVND/T5kf+2z5+Y/d3e1LDkekZjkyf4CN/9xJ2vHmLB2wlLbtuQr9T8rRP4LtQm262JSKuB64HuPDCC7soafn90FkjvO+tF3HomRd78v4AAxH8wtYLefPrNvWsBkn9rZvQPwi0Xud/PnBogTYHI2IIOBuY6nJbMvNW4FaAycnJnswIHxH8m5+5pBdvLUmrppsx/T3AxRFxUUTUaB6Y3dnWZiewo3p8LfAnmZnV8u3V2T0XARcD31ie0iVJp2vRnn41Rn8DcCcwCNyWmfsi4mZgb2buBD4NfDYi9tPs4W+vtt0XEV8EHgRmgQ+sxTN3JKkU0eyQrx2Tk5O5d+/eXpchSetKRNyTmZOLtXMCdkkqiKEvSQUx9CWpIIa+JBXE0Jekgqy5s3ci4jDw50t4iU3A95epnJVgfUtjfUtjfUuzlut7TWZuXqzRmgv9pYqIvd2cttQr1rc01rc01rc0a72+bji8I0kFMfQlqSD9GPq39rqARVjf0ljf0ljf0qz1+hbVd2P6kqSF9WNPX5K0gHUZ+ku5Ufsq1HZBRNwVEQ9FxL6I+Ocd2vx0RDwTEfdVXzetVn0tNTwWEd+q3v8VM9xF029W+/CBiLhsFWv7qy375r6IeDYifqWtzaruw4i4LSKeiohvtyybiIjdEfFI9X18gW13VG0eiYgdndqsUH0fj4jvVD+/r0TEOQtse8rPwgrW95GI+IuWn+G7Ftj2lL/vK1jfHS21PRYR9y2w7Yrvv2WVmevqi+b0zo8CrwVqwP3AJW1t/gnwierxduCOVazvXOCy6vFZwHc71PfTwH/v8X58DNh0ivXvAr5K8+5nPwF8vYc/77+keQ5yz/YhcAVwGfDtlmUfA26sHt8IfLTDdhPAger7ePV4fJXquxIYqh5/tFN93XwWVrC+jwC/2sXP/5S/7ytVX9v6Xwdu6tX+W86v9djTX8qN2ldcZj6RmfdWj58DHmKN3hd4EdcAv5tNdwPnRMS5PajjbcCjmbmUC/aWLDP/D817RbRq/Zx9BvjZDpu+E9idmVOZeQTYDWxbjfoy848zc7Z6ejfNO9f1xAL7rxvd/L4v2anqq7Lj54AvLPf79sJ6DP1ON2pvD9WX3agdmL9R+6qqhpUuBb7eYfXfioj7I+KrEfFjq1pYUwJ/HBH3VPcobtfNfl4N21n4l63X+/CvZOYT0PxjD/xQhzZrZT/+Es3/uXWy2GdhJd1QDT/dtsDw2FrYfz8JPJmZjyywvpf777Stx9Bfyo3aV01EjAFfBn4lM59tW30vzeGKvwH8FvCHq1lb5S2ZeRlwFfCBiLiibf1a2Ic14Grg9zusXgv7sBtrYT9+iOad635vgSaLfRZWyn8BfgR4I/AEzSGUdj3ff8C7OXUvv1f774ysx9A/nRu1Ey+/UfuqiIhhmoH/e5n5B+3rM/PZzHy+erwLGI6ITatVX/W+h6rvTwFfofnf6FZd3dR+hV0F3JuZT7avWAv7EHhyfsir+v5UhzY93Y/VgeOfAf5BVgPQ7br4LKyIzHwyM+cy8yTwqQXet9f7bwj4+8AdC7Xp1f47U+sx9Jdyo/YVV43/fRp4KDN/Y4E2r54/xhARW2n+HJ5ejfqq96xHxFnzj2ke8Pt2W7OdwD+szuL5CeCZ+aGMVbRgD6vX+7DS+jnbAfxRhzZ3AldGxHg1fHFltWzFRcQ24F8CV2fm9AJtuvksrFR9rceI/t4C79vN7/tKejvwncw82GllL/ffGev1keQz+aJ5Zsl3aR7V/1C17GaaH26ADTSHBPYD3wBeu4q1vZXmfz8fAO6rvt4FvB94f9XmBmAfzTMR7gbevMr777XVe99f1TG/D1trDOCWah9/C5hc5RpHaYb42S3LerYPaf7xeQKYodn7fC/N40RfAx6pvk9UbSeB/9qy7S9Vn8X9wD9axfr20xwPn/8czp/R9sPArlN9Flapvs9Wn60HaAb5ue31Vc9f8fu+GvVVy39n/jPX0nbV999yfnlFriQVZD0O70iSzpChL0kFMfQlqSCGviQVxNCXpIIY+pJUEENfkgpi6EtSQf4/GdBq/pas8W8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f432015d240>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot([b for a,b in hist[\"val\"]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}