prediction_explorer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image, ImageDraw, ImageFont\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "from concurrent.futures import ThreadPoolExecutor\n",
    "from itertools import repeat\n",
    "from functools import partial\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def expand_path(path):\n",
    "    if os.path.isfile(TRAIN + path):\n",
    "        return TRAIN + path\n",
    "    if os.path.isfile(TEST + path):\n",
    "        return TEST + path\n",
    "    return p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to write with black outline\n",
    "def write_on_image(results_image, text, x, y, x_shift=0, y_shift=0, font_cr='rgb(0,0,0)', FNT_SZ = 45, font='RobotoSlab-Regular.ttf'): \n",
    "    font = ImageFont.truetype(font, FNT_SZ)\n",
    "    x += x_shift\n",
    "    y += y_shift\n",
    "    results_image_draw = ImageDraw.Draw(results_image)\n",
    "    results_image_draw.text((x-1, y), text, font=font, fill='rgb(0,0,0)')\n",
    "    results_image_draw.text((x+1, y), text, font=font, fill='rgb(0,0,0)')\n",
    "    results_image_draw.text((x, y-1), text, font=font, fill='rgb(0,0,0)')\n",
    "    results_image_draw.text((x, y+1), text, font=font, fill='rgb(0,0,0)')\n",
    "    results_image_draw.text((x, y), text, fill=font_cr, font=font)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# when multiple neighbours are passed in the function to plot ascending scored images, a series will be returned in some\n",
    "# cases instead of string\n",
    "\n",
    "def extract_from_series(sr):\n",
    "    if type(sr) == pd.core.series.Series:\n",
    "        try: sr = sr[0]\n",
    "        except: pass\n",
    "    return sr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_box(results_image, x, y, size, enlarge=0, line_color= 'rgb(0,0,150)'):\n",
    "    area = (x-enlarge, y-enlarge, x+size[0]+enlarge, y+size[0]+enlarge)\n",
    "    results_image_draw = ImageDraw.Draw(results_image)\n",
    "    results_image_draw.rectangle(area, outline = 'rgb(0,0,0)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "def image_matrix_draw(val_image_df, train_df, TRAIN, TEST, trn_vs_val_df = None, topk = False, ascending = True, \n",
    "                      dcut = 0, dcut_label = '', SZ=512, MRGN=20, SMALL_MRGN=0, HDR=100,\n",
    "                      BKGR_COLOR=(245,245,245), Font='RobotoSlab-Regular.ttf', FNT_SZ=25):\n",
    "    \"\"\"\n",
    "    This is a function to explore the prediciton's of a classifier. It is comparing the image that has been classified from the validation set, \n",
    "    with the images that has the same predicted class from the train set, and with the images that has the same class from the ground truth label of the validation image in question.\n",
    "    It will draw 7 columns of images. Each column has 4 images, with size (sz).\n",
    "    \n",
    "    The 1st column is composed of 4 replicates of the validation image that we want to explore.\n",
    "    The 2nd to 4th column composed of images that has the same predicted class from the train set.\n",
    "    The 5th to 7th column composed of images that has the same class from the ground truth label of the validation image in question.\n",
    "    \n",
    "    # For the pred columns we have 3 states:\n",
    "    # 1. sort by topk. Req. : topk == True && hasattr(val_image_df, 'nbs_Image')\n",
    "    # 2. sort by class. Req.: topk == False (or hasattr(val_image_df, 'nbs_Image')=False) &&  trn_vs_val_df is not None\n",
    "    # 3. unsorted randomly picked from pred class.\n",
    "    \n",
    "    # For the ground truth columns:\n",
    "    # 1. sort by class (pred_sorted_by_class = True). Req.: trn_vs_val_df is not None\n",
    "    # 2. unsorted randomly picked from pred class (pred_sorted_by_class = False)\n",
    "    \n",
    "    Parameters:\n",
    "    val_image_df    :    pandas series that has the following information about the validation/test image:\n",
    "                            Image     : file name of the validation images\n",
    "                            Id        : ground truth label (if Id is not present, the function will regard the images to be analyzed as test images instead of validation images)\n",
    "                            nbs       : predicted labels (space separated if multiple labels provided for multiple topk=True\n",
    "                                         e.g., 100 of labels in one column otherwise only one).\n",
    "                            d         : score  (space separated if multiple labels provided for multiple topk=True\n",
    "                                         e.g., 100 of labels in one column otherwise only one)\n",
    "                            nbs_Image : file names of Images of the predicted labels (space separated in one column). If this column not provided, \n",
    "                                        randomly picked images will be provided of the predicted class instead of sorted from low to high score or in reverse (even if topk = True)\n",
    "\n",
    "    train_df        :    pandas dataframe of the whole dataset's labels (train without validation) contain the following columns:\n",
    "                            Image: file name\n",
    "                            Id   : label\n",
    "                            \n",
    "    trn_vs_val_df   :    pandas dataframe matrix of scores between all train images and validation images. \n",
    "                         If provided, the function will plot in the columns of the ground truth the images that belongs to the ground truth label\n",
    "                         sorted (either ascending or descending) by scores of the validation image in question vs train ground truth class images.\n",
    "                         Columns:\n",
    "                            Image     : file name of the train images\n",
    "                            Id        : ground truth label \n",
    "                            file names: multiple columns: each column for each validation file name\n",
    "    \n",
    "    topk            :    True         : Plotting the images of the top-1, top-2, top-3  ...  instead of analyzing only one of the top-k\n",
    "                         False        : Analyzing the top-1 score or top-2 or top-3 ...etc. Which top-k score is chosen is determined by the \n",
    "                                        val_image_df data of which top-k that is passed to this function\n",
    "                        \n",
    "    ascending       :    sorting scores from low to high if True (e.g., for sorting nearest negihbours), or sorting from High to low if Flase (e.g., sorting according to accuracy scores)                   \n",
    "    dcut            :    if score > dcut assign dcut_label to the predictions if ascending = True\n",
    "                         if score < dcut assign dcut_label to the predictions if ascending = False\n",
    "    SZ              :    Size of each image\n",
    "    MRGN            :    Size of the margin between the major 3 columns (validation column + predicted class images column, ground truth class images column)\n",
    "    SMALL_MRGN      :    Size of the margin between the each columns of the 7 columns\n",
    "    BKGR_COLOR      :    Background color \n",
    "    HDR             :    Height of the header of the total image\n",
    "    INPUT_PATH      :    Path of the whole dataset images\n",
    "    Font            :    Font of the written text \n",
    "    FNT_SZ          :    Font size\n",
    "    \n",
    "    TODO: ascending = False\n",
    "    dcut <> 0 and dcut_label <>'' else if dcut <>0 and dcut_label ='' raise error\n",
    "    \"\"\"\n",
    "\n",
    "    if  not hasattr(val_image_df, 'Id'): # if it is a test set without labels info\n",
    "        test_set = True\n",
    "    else:\n",
    "        test_set = False\n",
    "   \n",
    "    if topk == True and hasattr(val_image_df, 'nbs_Image'):\n",
    "        pred_sorted_by_topk = True\n",
    "    else:\n",
    "        pred_sorted_by_topk = False  # Without nb_Image file name info we cannot plot as topk so disable topk or if topk chosen as False\n",
    "        topk = False\n",
    "            \n",
    "    if trn_vs_val_df is not None:\n",
    "        y_sorted_by_class = True\n",
    "    else:\n",
    "        y_sorted_by_class = False # Cannot sort anything by class if trn_vs_val_df is not provided \n",
    "\n",
    "    if topk == False  and  trn_vs_val_df is not None:\n",
    "        pred_sorted_by_class = True\n",
    "    else:\n",
    "        pred_sorted_by_class = False # Cannot sort anything by class if trn_vs_val_df is not provided \n",
    "    \n",
    "    if not test_set:\n",
    "        if topk == False:\n",
    "            if len(val_image_df['nbs'].split(\" \")) == 1: # means only one neighbor passed\n",
    "                if val_image_df.Id == val_image_df.nbs: # if pred is correct then the pred columns will already shows the ground truth, so no meaning to repeat but better to continue\n",
    "                    pred_correct = 1\n",
    "                else:\n",
    "                    pred_correct = -1\n",
    "        else:\n",
    "            if len(val_image_df['nbs'].split(\" \")) != 1: # means more than one neighbor passed\n",
    "                if val_image_df['nbs'].split(\" \")[0] == val_image_df['Id']: # if pred is correct then the pred columns will already shows the ground truth, so no meaning to repeat but better to continue\n",
    "                    pred_correct = 2\n",
    "                else:\n",
    "                    pred_correct = -2\n",
    "    else:\n",
    "        pred_correct = 0\n",
    "        # To make this function works for test-set it is easier to append label data to the test set too\n",
    "        # So it will work like a validation-set. It will be colored like correct always\n",
    "        if len(val_image_df['nbs'].split(\" \")) == 1: # means only one neighbor passed\n",
    "            val_image_df.Id = val_image_df.nbs #  Label = prediciton\n",
    "        else:\n",
    "            val_image_df['Id'] = val_image_df['nbs'].split(\" \")[0]  #Label = top-1 prediction\n",
    "\n",
    "    if  test_set == True: # if it is a test set without labels info\n",
    "        y_cls_show = False  # y_class is not shown, so instead those columns will show continuation of the pred columns if enough extra pred images available\n",
    "    else:\n",
    "        if topk:\n",
    "            y_cls_show = True # if topk = True: always show labels (either sorted (y_sorted_by_class = True) or randomly) if it is not dataset (if it is dataset topk already disabled by the 1st if statement)\n",
    "        elif pred_correct == 1 or pred_correct == 2 or pred_correct == 0: # if pred is correct then the pred columns will already shows the ground truth, so no meaning to repeat but better to continue\n",
    "            y_cls_show = False\n",
    "        else:\n",
    "            y_cls_show = True    \n",
    "    \n",
    "    \n",
    "    # For the pred columns we have 3 states:\n",
    "    # 1. sort by topk (pred_sorted_by_topk = True)\n",
    "    # 2. sort by class (pred_sorted_by_class = True)\n",
    "    # 3. unsorted randomly picked from pred class (pred_sorted_by_topk = False & pred_sorted_by_class = False)\n",
    "    \n",
    "    # For the ground truth columns:\n",
    "    # 1. sort by class (pred_sorted_by_class = True)\n",
    "    # 2. unsorted randomly picked from pred class (pred_sorted_by_class = False)\n",
    "    # 3. If topk = False , pred_correct = 1 means that prediction is correct then gorund truth will not plotted but continue plotting pred\n",
    "    \n",
    "    \n",
    "    pred_scores   = []\n",
    "    pred_trn_imgs = []\n",
    "    pred_classes  = []\n",
    "    y_scores      = []\n",
    "    y_trn_imgs    = []\n",
    "    y_classes     = []\n",
    "    \n",
    "    \n",
    "    if pred_sorted_by_class == False and pred_sorted_by_topk == False: # pick randomly for preds columns\n",
    "        pred_trn_imgs = train_df.Image[train_df.Id == val_image_df.nbs]\n",
    "        y_trn_imgs = train_df.Image[train_df.Id == val_image_df.Id]\n",
    "\n",
    "        if not y_cls_show:\n",
    "            pred_trn_imgs = pd.DataFrame(pred_trn_imgs.sample(n=min(24, len(pred_trn_imgs)), replace = False))\n",
    "            y_trn_imgs = []\n",
    "        else:\n",
    "            pred_trn_imgs = pd.DataFrame(pred_trn_imgs.sample(n=min(12, len(pred_trn_imgs)), replace = False))\n",
    "            y_trn_imgs = pd.DataFrame(y_trn_imgs.sample(n=min(12, len(y_trn_imgs)), replace = False))\n",
    "            if trn_vs_val_df is not None:\n",
    "                y_scores = trn_vs_val_df[trn_vs_val_df.Image.isin(y_trn_imgs.Image)][[val_image_df.Image]]\n",
    "                y_trn_imgs =  trn_vs_val_df[trn_vs_val_df.Image.isin(y_trn_imgs.Image)][['Image']] # This is because the sorting of y is different. By this it will be the same\n",
    "                y_scores = y_scores.reset_index()\n",
    "            y_trn_imgs = y_trn_imgs.reset_index()\n",
    "           \n",
    "        if trn_vs_val_df is not None:\n",
    "            pred_scores = trn_vs_val_df[trn_vs_val_df.Image.isin(pred_trn_imgs.Image)][[val_image_df.Image]]\n",
    "            pred_trn_imgs = trn_vs_val_df[trn_vs_val_df.Image.isin(pred_trn_imgs.Image)][['Image']] # This is because the sorting of y is different. By this it will be the same\n",
    "            pred_scores = pred_scores.reset_index()\n",
    "        pred_trn_imgs = pred_trn_imgs.reset_index()\n",
    "        pred_top1_img = \"nothing\"\n",
    "        \n",
    "    elif pred_sorted_by_class == True and pred_sorted_by_topk == False: # sort images picked from pred class images\n",
    "        temp_df = trn_vs_val_df[['Image', 'Id',val_image_df.Image]]\n",
    "        pred_trn_imgs = temp_df[temp_df.Id.isin([val_image_df.nbs])].sort_values(by=[val_image_df.Image])\n",
    "        if not y_cls_show:\n",
    "            pred_trn_imgs, pred_scores = pred_trn_imgs.head(24)['Image'], pred_trn_imgs.head(24)[val_image_df.Image]\n",
    "            y_trn_imgs = []\n",
    "        else:\n",
    "            pred_trn_imgs, pred_scores = pred_trn_imgs.head(12)['Image'], pred_trn_imgs.head(12)[val_image_df.Image]\n",
    "            temp_df = trn_vs_val_df[['Image', 'Id', val_image_df.Image]]\n",
    "            y_trn_imgs = temp_df[temp_df.Id.isin([val_image_df.Id])].sort_values(by=[val_image_df.Image])\n",
    "            y_trn_imgs, y_scores = y_trn_imgs.head(12)['Image'], y_trn_imgs.head(12)[val_image_df.Image]\n",
    "            y_scores = y_scores.reset_index()\n",
    "            y_trn_imgs = y_trn_imgs.reset_index()\n",
    "        pred_scores = pred_scores.reset_index()\n",
    "        pred_trn_imgs = pred_trn_imgs.reset_index()\n",
    "        pred_top1_img = pred_trn_imgs.loc[0].Image\n",
    "    elif pred_sorted_by_topk == True: # sort images in the pred column by top-1, top-2,...\n",
    "        pred_trn_imgs = pd.DataFrame(val_image_df['nbs_Image'].split(\" \"), columns=['Image'])\n",
    "        pred_scores   = pd.DataFrame(val_image_df['d'].split(\" \"), columns=[[val_image_df.Image]])\n",
    "        pred_classes  = pd.DataFrame(val_image_df['nbs'].split(\" \"), columns=['nbs'])\n",
    "        pred_top1_img = \"nothing\"\n",
    "        \n",
    "        if not y_cls_show:\n",
    "            pred_trn_imgs = pred_trn_imgs.head(24)\n",
    "            pred_scores   = pred_scores.head(24)\n",
    "            pred_classes = pred_classes.head(24)\n",
    "            y_trn_imgs = []\n",
    "        else:\n",
    "            pred_trn_imgs = pred_trn_imgs.head(12)\n",
    "            pred_scores   = pred_scores.head(12)\n",
    "            pred_classes = pred_classes.head(12)\n",
    "            if y_sorted_by_class == True:\n",
    "                temp_df = trn_vs_val_df[['Image', 'Id', val_image_df.Image]]\n",
    "                y_trn_imgs = temp_df[temp_df.Id.isin([val_image_df.Id])].sort_values(by=[val_image_df.Image])\n",
    "                y_trn_imgs, y_scores = y_trn_imgs.head(12)['Image'], y_trn_imgs.head(12)[val_image_df.Image]\n",
    "                y_scores = y_scores.reset_index()\n",
    "                y_trn_imgs = y_trn_imgs.reset_index()\n",
    "            else:            \n",
    "                y_trn_imgs = train_df.Image[train_df.Id == val_image_df.Id]\n",
    "                y_trn_imgs = pd.DataFrame(y_trn_imgs.sample(n=min(12, len(y_trn_imgs)), replace = False))\n",
    "                if trn_vs_val_df is not None:\n",
    "                    y_scores = trn_vs_val_df[trn_vs_val_df.Image.isin(y_trn_imgs.Image)][[val_image_df.Image]]\n",
    "                    y_trn_imgs =  trn_vs_val_df[trn_vs_val_df.Image.isin(y_trn_imgs.Image)][['Image']] # This is because the sorting of y is different. By this it will be the same\n",
    "                    y_scores = y_scores.reset_index()\n",
    "                y_trn_imgs = y_trn_imgs.reset_index()\n",
    "        \n",
    "    font = ImageFont.truetype(Font, FNT_SZ)\n",
    "    results_image = Image.new('RGB', (SZ*7+MRGN*2+SMALL_MRGN*6, SZ*4+HDR), BKGR_COLOR)\n",
    "    \n",
    "    img = Image.open(expand_path(val_image_df.Image))\n",
    "    img.thumbnail((SZ,SZ),Image.LANCZOS)  # LANCZOS : means Anitalias\n",
    "    \n",
    "    # draw the validation image\n",
    "    for i in range(4):\n",
    "        results_image.paste(img, (0, i*SZ+HDR))\n",
    "\n",
    "    preds_imgs = []\n",
    "    preds_file_names = []\n",
    "    pred_scores_list = []\n",
    "    pred_class_list = []\n",
    "    scr = ''\n",
    "    cls = ''\n",
    "    for i in range(24):\n",
    "        if i < len(pred_trn_imgs):\n",
    "            name = extract_from_series(pred_trn_imgs.loc[i].Image)\n",
    "            img = Image.open(expand_path(name))\n",
    "            if len(pred_scores) > 0:\n",
    "                scr  = extract_from_series(pred_scores.loc[i][val_image_df.Image])\n",
    "            if len(pred_classes) > 0:\n",
    "                cls  = extract_from_series(pred_classes.loc[i]['nbs'])\n",
    "            else:\n",
    "                cls  = \"\"\n",
    "        else:\n",
    "            img = Image.new('RGB', (SZ, SZ), BKGR_COLOR)\n",
    "            name = \"\"\n",
    "            scr  = \"\"\n",
    "            cls  = \"\"\n",
    "        img.thumbnail((SZ,SZ),Image.LANCZOS) \n",
    "        preds_imgs.append(img)\n",
    "        preds_file_names.append(name)\n",
    "        pred_scores_list.append(scr)\n",
    "        pred_class_list.append(cls)\n",
    "\n",
    "    ground_truth_imgs = []\n",
    "    ground_truth_file_names = []\n",
    "    ground_truth_scores_list = []\n",
    "    ground_truth_class_list = []\n",
    "    scr = ''\n",
    "    cls = ''\n",
    "\n",
    "    if len(y_trn_imgs) > 0:\n",
    "        for i in range(12):\n",
    "            if i < len(y_trn_imgs):\n",
    "                name = extract_from_series(y_trn_imgs.loc[i].Image)\n",
    "                img = Image.open(expand_path(name))\n",
    "                if len(y_scores) > 0:\n",
    "                    scr  = extract_from_series(y_scores.loc[i][val_image_df.Image]) \n",
    "                if len(y_classes) > 0:\n",
    "                    cls  = extract_from_series(y_classes.loc[i]['nbs'])\n",
    "            else:\n",
    "                img = Image.new('RGB', (SZ, SZ), BKGR_COLOR)\n",
    "                name = \"\"\n",
    "                scr  = \"\"\n",
    "                cls  = \"\"               \n",
    "            img.thumbnail((SZ,SZ),Image.LANCZOS)\n",
    "            ground_truth_imgs.append(img) \n",
    "            ground_truth_file_names.append(name)\n",
    "            ground_truth_scores_list.append(scr)\n",
    "            ground_truth_class_list.append(cls)\n",
    "            \n",
    "    # Color the fonts with purple if the prediciton is wrong, if correct with green, if topk sorted blue\n",
    "    if pred_correct > 0:\n",
    "        font_color  = 'rgb(0,150,0)'\n",
    "    elif pred_correct < 0:\n",
    "        font_color  = 'rgb(148,0,211)' \n",
    "\n",
    "    if test_set:\n",
    "        font_color  = 'rgb(0,0,180)'\n",
    "\n",
    "\n",
    "    # Put at the header of the whole image:\n",
    "    # file name : (real class) \n",
    "    # top-x   score : (pred class)\n",
    "\n",
    "    if test_set:\n",
    "        write_on_image(results_image, f\"{val_image_df.Image}   (Test set)\",10 ,5 , font_cr=font_color, FNT_SZ=FNT_SZ)\n",
    "    else:\n",
    "        write_on_image(results_image, f\"{val_image_df.Image} : ({val_image_df.Id})   (Validation set)\",10 ,5 , font_cr=font_color, FNT_SZ=FNT_SZ)\n",
    "    if topk:\n",
    "        if y_cls_show:\n",
    "            if test_set:\n",
    "                # write_on_image(results_image, f\"test set top-k: [[top-1, top-2, top-3],[top-4..],[..top-12]]\",10, 45,  font_cr=font_color, FNT_SZ=FNT_SZ)\n",
    "                write_on_image(results_image, f\"top-k\",10, 40,  font_cr=font_color, FNT_SZ=FNT_SZ)\n",
    "            else:\n",
    "                # write_on_image(results_image, f\"validation set top-k: [[top-1, top-2, top-3],[top-4..],[..top-12]]\",10, 45,  font_cr=font_color, FNT_SZ=FNT_SZ)\n",
    "                write_on_image(results_image, f\"top-k\",10, 40,  font_cr=font_color, FNT_SZ=FNT_SZ)\n",
    "        else:\n",
    "            if test_set:\n",
    "                # write_on_image(results_image, f\"test set top-k: [[top-1, top-2, top-3],[top-4..],[..top-12]] + [[top-13, top-14, top-15],[top-16..],[..top-24]]\",10, 45,  font_cr=font_color, FNT_SZ=FNT_SZ)\n",
    "                write_on_image(results_image, f\"top-k\",10, 40,  font_cr=font_color, FNT_SZ=FNT_SZ)\n",
    "            else:\n",
    "                # write_on_image(results_image, f\"validation set top-k: [[top-1, top-2, top-3],[top-4..],[..top-12]] + [[top-13, top-14, top-15],[top-16..],[..top-24]]\",10, 45,  font_cr=font_color, FNT_SZ=FNT_SZ)\n",
    "                write_on_image(results_image, f\"top-k\",10, 40,  font_cr=font_color, FNT_SZ=FNT_SZ)\n",
    "    else:\n",
    "        write_on_image(results_image, f\"top-{K}    {float(val_image_df.d):.3f}   : ({val_image_df.nbs})\",10, 40,  font_cr=font_color, FNT_SZ=FNT_SZ)\n",
    "    if not pred_sorted_by_topk:\n",
    "        write_on_image(results_image, f\"predicted ({val_image_df.nbs})\",SZ*2.25+MRGN+SMALL_MRGN, 5,  font_cr=font_color, FNT_SZ=FNT_SZ)\n",
    "    else:\n",
    "        write_on_image(results_image, f\"predicted top-1 ({val_image_df.nbs.split(' ')[0]})\",SZ*2.25+MRGN+SMALL_MRGN, 5,  font_cr=font_color, FNT_SZ=FNT_SZ)\n",
    "    if y_cls_show:\n",
    "        if not pred_sorted_by_topk:\n",
    "            write_on_image(results_image, f\"ground truth ({val_image_df.Id})\",SZ*5.25+MRGN*2+SMALL_MRGN*3, 5, font_cr=font_color, FNT_SZ=FNT_SZ)\n",
    "        else:\n",
    "            write_on_image(results_image, f\"ground truth ({val_image_df.Id.split(' ')[0]})\",SZ*5.25+MRGN*2+SMALL_MRGN*3, 5, font_cr=font_color, FNT_SZ=FNT_SZ)\n",
    " \n",
    "    index = 0\n",
    "    for i in range(4):\n",
    "        for j in range(1,4):\n",
    "            x = j*(SZ+SMALL_MRGN)+int(j/4+1)*MRGN\n",
    "            y = i*SZ+HDR\n",
    "            results_image.paste(preds_imgs[index], (x, y))\n",
    "            if len(pred_scores_list)>0:            \n",
    "                write_on_image(results_image, preds_file_names[index], x, y, x_shift = 5, y_shift=-70, font_cr= font_color, FNT_SZ=FNT_SZ)\n",
    "                write_on_image(results_image, str(pred_scores_list[index]), x, y, x_shift = 5, y_shift=-35, font_cr= font_color, FNT_SZ=FNT_SZ)\n",
    "            else:\n",
    "                write_on_image(results_image, preds_file_names[index], x, y, x_shift = 5, y_shift=-35, font_cr= font_color, FNT_SZ=FNT_SZ)\n",
    "            if len(pred_class_list)>0: \n",
    "                write_on_image(results_image, str(pred_class_list[index]), x, y, x_shift = 5, y_shift=5, font_cr= font_color, FNT_SZ=FNT_SZ)\n",
    "            \n",
    "            if pred_top1_img == preds_file_names[index]:\n",
    "                draw_box(results_image, x, y, size=(SZ,SZ), enlarge=0, line_color= font_color)\n",
    "            index += 1\n",
    "            if len(preds_imgs)<=index:\n",
    "                break\n",
    "        if len(preds_imgs)<=index:\n",
    "            break\n",
    "\n",
    "            \n",
    "    # draw ground truth train images\n",
    "    if len(y_trn_imgs) > 0:\n",
    "        index = 0\n",
    "        for i in range(4):\n",
    "            for j in range(4,7):\n",
    "                x = j*(SZ+SMALL_MRGN)+int(j/4+1)*MRGN\n",
    "                y = i*SZ+HDR            \n",
    "                results_image.paste(ground_truth_imgs[index], (x, y))\n",
    "                if len(ground_truth_scores_list)>0:               \n",
    "                    write_on_image(results_image, ground_truth_file_names[index], x, y, x_shift = 5, y_shift=-70, \n",
    "                                   font_cr=font_color, FNT_SZ=FNT_SZ)\n",
    "                    write_on_image(results_image, str(ground_truth_scores_list[index]), x, y, x_shift = 5, y_shift=-35,\n",
    "                                   font_cr=font_color, FNT_SZ=FNT_SZ)\n",
    "                else:\n",
    "                    write_on_image(results_image, str(ground_truth_file_names[index]), x, y, x_shift = 5, y_shift=-35, \n",
    "                                       font_cr=font_color, FNT_SZ=FNT_SZ)\n",
    "                if len(ground_truth_class_list)>0:\n",
    "                    write_on_image(results_image, str(ground_truth_class_list[index]), x, y, x_shift = 5, y_shift=5, \n",
    "                                   font_cr=font_color, FNT_SZ=FNT_SZ)\n",
    "                index += 1\n",
    "                if len(ground_truth_imgs)<=index:\n",
    "                    break\n",
    "            if len(ground_truth_imgs)<=index:\n",
    "                break\n",
    "    elif len(preds_imgs) > 12:\n",
    "        index = 12\n",
    "        for i in range(4):\n",
    "            for j in range(4,7):\n",
    "                x = j*(SZ+SMALL_MRGN)+int(j/4+1)*MRGN\n",
    "                y = i*SZ+HDR\n",
    "                results_image.paste(preds_imgs[index], (x, y))\n",
    "                if len(str(pred_scores_list[12]))>0:\n",
    "                    write_on_image(results_image, preds_file_names[index], x, y, x_shift = 5, y_shift=-70, \n",
    "                                   font_cr=font_color, FNT_SZ=FNT_SZ)\n",
    "                    write_on_image(results_image, str(pred_scores_list[index]), x, y, x_shift = 5, y_shift=-35, \n",
    "                                   font_cr=font_color, FNT_SZ=FNT_SZ)\n",
    "                else:\n",
    "                    write_on_image(results_image, preds_file_names[index], x, y, x_shift = 5, y_shift=-35, \n",
    "                                   font_cr=font_color, FNT_SZ=FNT_SZ)\n",
    "                if len(str(pred_class_list[12]))>0:\n",
    "                    write_on_image(results_image, str(pred_class_list[index]), x, y, x_shift = 5, y_shift=5, \n",
    "                                   font_cr=font_color, FNT_SZ=FNT_SZ)\n",
    "\n",
    "                index += 1\n",
    "                if len(preds_imgs)<=index:\n",
    "                    break\n",
    "            if len(preds_imgs)<=index:\n",
    "                break\n",
    "    \n",
    "    return results_image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "def save_single_topk_images(val_image_df, train_df, TRAIN, TEST, CORRECT_PRED_PATH, INCORRECT_PRED_PATH, TOPK_CORRECT_PRED_PATH,\n",
    "                            TOPK_INCORRECT_PRED_PATH, TEST_SET_PRED_PATH, TEST_SET_TOPK_PRED_PATH, topk=False, **kwargs):\n",
    "    \n",
    "    if hasattr(val_image_df, 'Id'): # if it is a test set without labels info\n",
    "        test_set = False\n",
    "    else:\n",
    "        test_set = True    \n",
    "    \n",
    "    result = image_matrix_draw(val_image_df, train_df, TRAIN, TEST, topk=topk, **kwargs)\n",
    "    \n",
    "    leading_zeros = 5\n",
    "    \n",
    "    # file named as : score_predclass_realclass_validationfilename\n",
    "    \n",
    "    if topk == False:        \n",
    "        if len(val_image_df['nbs'].split(\" \")) != 1: \n",
    "            # means more than one neighbor passed\n",
    "            raise Exception('You should pass only one neighbor image data in the columns(nbs, d, nbs_Image)  of val_image_df  if you do not want to plot all topk: top-1, top2, top3 in the same result image')\n",
    "    else:\n",
    "        if len(val_image_df['nbs'].split(\" \")) == 1: \n",
    "            # means only one neighbor passed\n",
    "            raise Exception('if you want to plot all topk you should pass more than one neighbor image data in the columns(nbs, d, nbs_Image) of val_image_df  [space seperated]')\n",
    "\n",
    "    \n",
    "    \n",
    "    if not test_set:\n",
    "        if topk == False:        \n",
    "            if len(val_image_df['nbs'].split(\" \")) == 1: # means only one neighbor passed\n",
    "                if val_image_df.Id == val_image_df.nbs: # if pred is correct then the pred columns will already shows the ground truth, so no meaning to repeat but better to continue\n",
    "                    pred_correct = 1\n",
    "                else:\n",
    "                    pred_correct = -1\n",
    "        else:\n",
    "            if len(val_image_df['nbs'].split(\" \")) != 1: # means more than one neighbor passed\n",
    "                if val_image_df['nbs'].split(\" \")[0] == val_image_df['Id']: # if pred is correct then the pred columns will already shows the ground truth, so no meaning to repeat but better to continue\n",
    "                    pred_correct = 2\n",
    "                else:\n",
    "                    pred_correct = -2\n",
    "            \n",
    "        \n",
    "        if pred_correct == 1:\n",
    "            # saving correct prediction\n",
    "            result.save(f\"{CORRECT_PRED_PATH}{float(val_image_df.d):012.5f}_{val_image_df.nbs}_{val_image_df.Id}_{val_image_df.Image}\", format='JPEG')\n",
    "            \n",
    "        elif pred_correct == -1:\n",
    "            # saving incorrect prediction\n",
    "            result.save(f\"{INCORRECT_PRED_PATH}{float(val_image_df.d):012.5f}_{val_image_df.nbs}_{val_image_df.Id}_{val_image_df.Image}\", format='JPEG') \n",
    "        elif pred_correct == 2:\n",
    "            # saving correct topk\n",
    "            result.save(f\"{TOPK_CORRECT_PRED_PATH}{float(val_image_df.d.split(' ')[0]):012.5f}_{val_image_df.nbs.split(' ')[0]}_{val_image_df.Id.split(' ')[0]}_{val_image_df.Image}\", format='JPEG') \n",
    "        elif pred_correct == -2:\n",
    "            # saving incorrect topk\n",
    "            result.save(f\"{TOPK_INCORRECT_PRED_PATH}{float(val_image_df.d.split(' ')[0]):012.5f}_{val_image_df.nbs.split(' ')[0]}_{val_image_df.Id.split(' ')[0]}_{val_image_df.Image}\", format='JPEG') \n",
    "    else:\n",
    "        if topk == False:        \n",
    "            if len(val_image_df['nbs'].split(\" \")) != 1: # means not only one neighbor passed\n",
    "                raise Exception('You should pass only one neighbor image data in the columns(nbs, d, nbs_Image)  of val_image_df  if you do not want to plot all topk: top-1, top2, top3 in the same result image')\n",
    "            os.makedirs(TEST_SET_PRED_PATH, exist_ok = True)\n",
    "            # saving top-x prediction\n",
    "            result.save(f\"{TEST_SET_PRED_PATH}{float(val_image_df.d):012.5f}_{val_image_df.nbs}_{val_image_df.Image}\", format='JPEG')        \n",
    "        else:\n",
    "            os.makedirs(TEST_SET_TOPK_PRED_PATH, exist_ok = True)\n",
    "            # saving topk prediction\n",
    "            result.save(f\"{TEST_SET_TOPK_PRED_PATH}{float(val_image_df.d.split(' ')[0]):012.5f}_{val_image_df.nbs.split(' ')[0]}_{val_image_df.Image}\", format='JPEG') "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **Customize your options**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "## CHOOSE your config of the output\n",
    "SZ = 512\n",
    "MRGN = 20\n",
    "SMALL_MRGN = 0\n",
    "HDR = 100\n",
    "BKGR_COLOR = (0,0,0) \n",
    "FNT = 'RobotoSlab-Regular.ttf'\n",
    "FNT_SZ = 45"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "K = 2          # CHOOSE top-K nearest neighbours (1,2.3...)\n",
    "NW = 16        # number of workers\n",
    "\n",
    "TRAIN = '../input/humpback-whale-identification/train/' # for train set analysis\n",
    "TEST = '../input/humpback-whale-identification/test/'    # for test set analysis\n",
    "LABELS = '../input/humpback-whale-identification/train.csv'\n",
    "OUTPUT = '../output/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [],
   "source": [
    "### CHOOSE to load either validation set or test set to analyze ####\n",
    "\n",
    "### LOAD your dataset\n",
    "\n",
    "# You can skip loading 'trn_vs_val_df' scores matrix if you don't have it,\n",
    "# but analysis will be by random pick rather than sorted by scores\n",
    "\n",
    "# Load: Image, Id, nbs, d   (Image file name, label, nearest neighbours, distances)    \n",
    "# val_df = pd.read_csv('Iofoss-COMMIT7-baseline-sz512-val-v7-iafoss-sz512-dcut17.csv')    # for train set analysis\n",
    "# trn_vs_val_df = pd.read_csv('trn_vs_val-v7-iafoss-sz512-dcut17.csv')                    # for train set analysis\n",
    "val_df = pd.read_csv('Iofoss-COMMIT7-baseline-sz512-test-v7-iafoss-sz512-dcut17.csv')   # for test set analysis\n",
    "trn_vs_val_df = pd.read_csv('trn_vs_test-v7-iafoss-sz512-dcut17.csv')                   # for test set analysis\n",
    "\n",
    "# Load dataframe: Image, Id\n",
    "\n",
    "train_df = trn_vs_val_df[['Image','Id']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CHOOSE the K-th nearest neighbours to save all resulted images by using only them\n",
    "\n",
    "# In case we wanted to check other K-th nearest neighbours, we can run again with another K value\n",
    "# The following code will save them in a different folder for each K\n",
    "nearest_nb = pd.DataFrame(val_df['nbs'].apply(lambda x: str.split(x,\" \")[K-1]))\n",
    "nearest_nb_distance = pd.DataFrame(val_df['d'].apply(lambda x: str.split(x,\" \")[K-1]))\n",
    "if hasattr(val_df,'nbs_Image'):\n",
    "    nearest_nb_img = pd.DataFrame(val_df['nbs_Image'].apply(lambda x: str.split(x,\" \")[K-1]))\n",
    "\n",
    "if hasattr(val_df,'Id'):\n",
    "    data = val_df[['Image','Id']]\n",
    "else:\n",
    "    data = val_df[['Image']]    \n",
    "    \n",
    "data = data.join(nearest_nb)\n",
    "data = data.join(nearest_nb_distance)\n",
    "if hasattr(val_df,'nbs_Image'): data = data.join(nearest_nb_img)\n",
    "# data = val_df  # If you want to plot topk THEN you should do this, because we want all the info and not only the k-th neighbor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "## CHOOSE your ouput paths\n",
    "\n",
    "# saving all the resulted images in the following paths:\n",
    "CORRECT_PRED_PATH = OUTPUT+'top'+str(K)+'_correct/'\n",
    "INCORRECT_PRED_PATH = OUTPUT+'top'+str(K)+'_incorrect/'\n",
    "TOPK_CORRECT_PRED_PATH = OUTPUT+'topk_correct/'\n",
    "TOPK_INCORRECT_PRED_PATH = OUTPUT+'topk_incorrect/'\n",
    "TEST_SET_PRED_PATH = OUTPUT+'Test_top'+str(K)+'/'\n",
    "TEST_SET_TOPK_PRED_PATH = OUTPUT+'Test_topk/'\n",
    "\n",
    "os.makedirs(CORRECT_PRED_PATH, exist_ok = True) # set exist_ok = False to raise error so preventing accidental over writing\n",
    "os.makedirs(INCORRECT_PRED_PATH, exist_ok = True) \n",
    "os.makedirs(TOPK_CORRECT_PRED_PATH, exist_ok = True)\n",
    "os.makedirs(TOPK_INCORRECT_PRED_PATH, exist_ok = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "############# CHOOSE one of the following 2 cases  ########\n",
    "\n",
    "## in case you want top-1 or top-2 or ... according to you choice of K in previous constants cell\n",
    "func = partial(save_single_topk_images,\n",
    "               train_df=train_df,\n",
    "               TRAIN=TRAIN,\n",
    "               TEST=TEST,\n",
    "               CORRECT_PRED_PATH=CORRECT_PRED_PATH,\n",
    "               INCORRECT_PRED_PATH=INCORRECT_PRED_PATH,\n",
    "               TOPK_CORRECT_PRED_PATH=TOPK_CORRECT_PRED_PATH, \n",
    "               TOPK_INCORRECT_PRED_PATH=TOPK_INCORRECT_PRED_PATH,\n",
    "               TEST_SET_PRED_PATH=TEST_SET_PRED_PATH, \n",
    "               TEST_SET_TOPK_PRED_PATH=TEST_SET_PRED_PATH, \n",
    "               BKGR_COLOR='rgb(255,255,255)', \n",
    "               trn_vs_val_df=trn_vs_val_df, # pass None here if you dont have this matrix, but analysis of the ground truth images (if training set is analyzed) will be by random pick rather than sorted by scores\n",
    "               topk=False)\n",
    "\n",
    "### in case you want topk\n",
    "# data = val_df \n",
    "# func = partial(save_single_topk_images,\n",
    "#                train_df=train_df,\n",
    "#                TRAIN=TRAIN,\n",
    "#                TEST=TEST,\n",
    "#                CORRECT_PRED_PATH=CORRECT_PRED_PATH,\n",
    "#                INCORRECT_PRED_PATH=INCORRECT_PRED_PATH,\n",
    "#                TOPK_CORRECT_PRED_PATH=TOPK_CORRECT_PRED_PATH, \n",
    "#                TOPK_INCORRECT_PRED_PATH=TOPK_INCORRECT_PRED_PATH,\n",
    "#                TEST_SET_PRED_PATH=TEST_SET_PRED_PATH, \n",
    "#                TEST_SET_TOPK_PRED_PATH=TEST_SET_TOPK_PRED_PATH, \n",
    "#                BKGR_COLOR='rgb(255,255,255)', \n",
    "#                trn_vs_val_df=trn_vs_val_df, # pass None here if you dont have this matrix, but analysis of the ground truth images (if training set is analyzed) will be by random pick rather than sorted by scores\n",
    "#                topk=True)           "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Image        cfcc21b0e.jpg\n",
       "nbs              w_5665c80\n",
       "d                6.0546875\n",
       "nbs_Image    732b981cb.jpg\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.loc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ## run for 1 image\n",
    "func(data.loc[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving all images time:  18.66553550561269  min.\n"
     ]
    }
   ],
   "source": [
    "### RUN FOR THE WHOLE DATASET ###\n",
    "tic = time.time()\n",
    "\n",
    "with ThreadPoolExecutor(NW) as e: \n",
    "    # Saving only the 1st 20 validation images for testing everything is okay\n",
    "#     e.map(func, [val_image_df for (_, val_image_df) in data.head(20).iterrows()])\n",
    "    e.map(func, [val_image_df for (_, val_image_df) in data.iterrows()])\n",
    "\n",
    "toc = time.time()\n",
    "print(\"Saving all images time: \", (toc - tic) / 60.,' min.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from my_python_tricks import *\n",
    "notify_me('interpreter saving images finished')"
   ]
  }
 ],
 "metadata": {
  "jupytext": {
   "formats": "ipynb,py:percent",
   "text_representation": {
    "extension": ".py",
    "format_name": "percent",
    "format_version": "1.2",
    "jupytext_version": "0.8.6"
   }
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}