GraphSage-sample-tunable-products_gpu0.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using backend: pytorch\n"
     ]
    }
   ],
   "source": [
    "import dgl\n",
    "import time\n",
    "import torch as th\n",
    "import numpy as np\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from load_data import load_ogb\n",
    "from model import SAGE, GAT, get_eigen\n",
    "from train_eval import train_ft_embed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook shows the performance of GraphSage on the OGBN-products graph with trainable embeddings on the nodes. We use normal distribution to initialize the node embeddings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load ogbn-products\n",
      "finish loading ogbn-products\n",
      "finish constructing ogbn-products\n",
      "embedding is in  cuda:0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/numpy/core/fromnumeric.py:3373: RuntimeWarning: Mean of empty slice.\n",
      "  out=out, **kwargs)\n",
      "/usr/local/lib/python3.6/dist-packages/numpy/core/_methods.py:170: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00000 | Step 00000 | Loss 9.1122 | Train Acc 0.0137 | Speed (samples/sec) nan | GPU 4979.9 MB\n",
      "Epoch 00000 | Step 00020 | Loss 2.8422 | Train Acc 0.2695 | Speed (samples/sec) 349.5740 | GPU 5002.2 MB\n",
      "Epoch 00000 | Step 00040 | Loss 2.7205 | Train Acc 0.2871 | Speed (samples/sec) 356.9307 | GPU 5002.2 MB\n",
      "Epoch 00000 | Step 00060 | Loss 2.5919 | Train Acc 0.3086 | Speed (samples/sec) 351.5551 | GPU 5002.2 MB\n",
      "Epoch 00000 | Step 00080 | Loss 2.5316 | Train Acc 0.3096 | Speed (samples/sec) 343.6506 | GPU 5011.8 MB\n",
      "Epoch 00000 | Step 00100 | Loss 2.5456 | Train Acc 0.3096 | Speed (samples/sec) 339.3753 | GPU 5011.8 MB\n",
      "Epoch 00000 | Step 00120 | Loss 2.4427 | Train Acc 0.3125 | Speed (samples/sec) 336.6420 | GPU 5011.8 MB\n",
      "Epoch 00000 | Step 00140 | Loss 2.3357 | Train Acc 0.3174 | Speed (samples/sec) 336.5644 | GPU 5011.8 MB\n",
      "Epoch 00000 | Step 00160 | Loss 2.2046 | Train Acc 0.3076 | Speed (samples/sec) 335.0605 | GPU 5011.8 MB\n",
      "Epoch 00000 | Step 00180 | Loss 1.9670 | Train Acc 0.4043 | Speed (samples/sec) 333.7067 | GPU 5011.8 MB\n",
      "Epoch Time(s): 594.7645\n",
      "Epoch 00001 | Step 00000 | Loss 1.8988 | Train Acc 0.4219 | Speed (samples/sec) 331.3403 | GPU 5012.0 MB\n",
      "Epoch 00001 | Step 00020 | Loss 1.7638 | Train Acc 0.4678 | Speed (samples/sec) 330.6266 | GPU 5012.0 MB\n",
      "Epoch 00001 | Step 00040 | Loss 1.4588 | Train Acc 0.5459 | Speed (samples/sec) 333.2729 | GPU 5012.0 MB\n",
      "Epoch 00001 | Step 00060 | Loss 1.3273 | Train Acc 0.5869 | Speed (samples/sec) 332.6086 | GPU 5012.0 MB\n",
      "Epoch 00001 | Step 00080 | Loss 1.2189 | Train Acc 0.6279 | Speed (samples/sec) 331.4439 | GPU 5012.0 MB\n",
      "Epoch 00001 | Step 00100 | Loss 1.0794 | Train Acc 0.6748 | Speed (samples/sec) 330.2883 | GPU 5012.0 MB\n",
      "Epoch 00001 | Step 00120 | Loss 1.1239 | Train Acc 0.6758 | Speed (samples/sec) 329.9808 | GPU 5012.0 MB\n",
      "Epoch 00001 | Step 00140 | Loss 1.0720 | Train Acc 0.6982 | Speed (samples/sec) 328.9085 | GPU 5012.0 MB\n",
      "Epoch 00001 | Step 00160 | Loss 0.9674 | Train Acc 0.7324 | Speed (samples/sec) 328.3581 | GPU 5012.0 MB\n",
      "Epoch 00001 | Step 00180 | Loss 0.8720 | Train Acc 0.7588 | Speed (samples/sec) 328.3091 | GPU 5012.0 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 606.8168\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:39<00:00,  6.18it/s]\n",
      "100%|██████████| 245/245 [01:27<00:00,  2.81it/s]\n",
      "100%|██████████| 245/245 [01:21<00:00,  3.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8205, Test Acc: 0.5783\n",
      "Best val acc 0.8205, best test acc: 0.5783\n",
      "Epoch 00002 | Step 00000 | Loss 0.8105 | Train Acc 0.7520 | Speed (samples/sec) 327.8016 | GPU 5012.0 MB\n",
      "Epoch 00002 | Step 00020 | Loss 0.7621 | Train Acc 0.7803 | Speed (samples/sec) 330.4879 | GPU 5012.0 MB\n",
      "Epoch 00002 | Step 00040 | Loss 0.7675 | Train Acc 0.7842 | Speed (samples/sec) 332.5371 | GPU 5012.0 MB\n",
      "Epoch 00002 | Step 00060 | Loss 0.7653 | Train Acc 0.7871 | Speed (samples/sec) 334.1143 | GPU 5012.0 MB\n",
      "Epoch 00002 | Step 00080 | Loss 0.7824 | Train Acc 0.7998 | Speed (samples/sec) 335.5085 | GPU 5012.0 MB\n",
      "Epoch 00002 | Step 00100 | Loss 0.6361 | Train Acc 0.8320 | Speed (samples/sec) 337.0354 | GPU 5012.0 MB\n",
      "Epoch 00002 | Step 00120 | Loss 0.6843 | Train Acc 0.8252 | Speed (samples/sec) 338.7689 | GPU 5012.0 MB\n",
      "Epoch 00002 | Step 00140 | Loss 0.6214 | Train Acc 0.8477 | Speed (samples/sec) 339.8166 | GPU 5012.0 MB\n",
      "Epoch 00002 | Step 00160 | Loss 0.6745 | Train Acc 0.8281 | Speed (samples/sec) 340.6662 | GPU 5012.0 MB\n",
      "Epoch 00002 | Step 00180 | Loss 0.5339 | Train Acc 0.8672 | Speed (samples/sec) 341.2891 | GPU 5012.0 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 532.9240\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:34<00:00,  7.03it/s]\n",
      "100%|██████████| 245/245 [01:09<00:00,  3.52it/s]\n",
      "100%|██████████| 245/245 [01:11<00:00,  3.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8834, Test Acc: 0.6547\n",
      "Best val acc 0.8834, best test acc: 0.6547\n",
      "Epoch 00003 | Step 00000 | Loss 0.5463 | Train Acc 0.8623 | Speed (samples/sec) 341.5257 | GPU 5012.0 MB\n",
      "Epoch 00003 | Step 00020 | Loss 0.4993 | Train Acc 0.8701 | Speed (samples/sec) 342.0532 | GPU 5012.0 MB\n",
      "Epoch 00003 | Step 00040 | Loss 0.5271 | Train Acc 0.8789 | Speed (samples/sec) 343.0589 | GPU 5012.0 MB\n",
      "Epoch 00003 | Step 00060 | Loss 0.4912 | Train Acc 0.8750 | Speed (samples/sec) 344.4389 | GPU 5012.0 MB\n",
      "Epoch 00003 | Step 00080 | Loss 0.4553 | Train Acc 0.8789 | Speed (samples/sec) 344.7817 | GPU 5012.8 MB\n",
      "Epoch 00003 | Step 00100 | Loss 0.5229 | Train Acc 0.8662 | Speed (samples/sec) 345.2473 | GPU 5012.8 MB\n",
      "Epoch 00003 | Step 00120 | Loss 0.4778 | Train Acc 0.8838 | Speed (samples/sec) 345.9997 | GPU 5012.8 MB\n",
      "Epoch 00003 | Step 00140 | Loss 0.3903 | Train Acc 0.9004 | Speed (samples/sec) 346.7484 | GPU 5012.8 MB\n",
      "Epoch 00003 | Step 00160 | Loss 0.4412 | Train Acc 0.8906 | Speed (samples/sec) 347.2615 | GPU 5012.8 MB\n",
      "Epoch 00003 | Step 00180 | Loss 0.5022 | Train Acc 0.8818 | Speed (samples/sec) 347.4845 | GPU 5012.8 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 538.6316\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:35<00:00,  6.84it/s]\n",
      "100%|██████████| 245/245 [01:09<00:00,  3.53it/s]\n",
      "100%|██████████| 245/245 [01:12<00:00,  3.37it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8948, Test Acc: 0.6707\n",
      "Best val acc 0.8948, best test acc: 0.6707\n",
      "Epoch 00004 | Step 00000 | Loss 0.4292 | Train Acc 0.8916 | Speed (samples/sec) 347.2987 | GPU 5012.8 MB\n",
      "Epoch 00004 | Step 00020 | Loss 0.4027 | Train Acc 0.8975 | Speed (samples/sec) 347.8318 | GPU 5012.8 MB\n",
      "Epoch 00004 | Step 00040 | Loss 0.4261 | Train Acc 0.8945 | Speed (samples/sec) 348.1624 | GPU 5012.8 MB\n",
      "Epoch 00004 | Step 00060 | Loss 0.3729 | Train Acc 0.8965 | Speed (samples/sec) 348.4304 | GPU 5012.8 MB\n",
      "Epoch 00004 | Step 00080 | Loss 0.5057 | Train Acc 0.8740 | Speed (samples/sec) 349.0230 | GPU 5012.8 MB\n",
      "Epoch 00004 | Step 00100 | Loss 0.4483 | Train Acc 0.8896 | Speed (samples/sec) 349.6477 | GPU 5012.8 MB\n",
      "Epoch 00004 | Step 00120 | Loss 0.3828 | Train Acc 0.9062 | Speed (samples/sec) 349.7671 | GPU 5012.8 MB\n",
      "Epoch 00004 | Step 00140 | Loss 0.4335 | Train Acc 0.8877 | Speed (samples/sec) 349.7806 | GPU 5012.8 MB\n",
      "Epoch 00004 | Step 00160 | Loss 0.4219 | Train Acc 0.8955 | Speed (samples/sec) 349.9481 | GPU 5012.8 MB\n",
      "Epoch 00004 | Step 00180 | Loss 0.3872 | Train Acc 0.8975 | Speed (samples/sec) 350.4056 | GPU 5012.8 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 542.1306\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:38<00:00,  6.28it/s]\n",
      "100%|██████████| 245/245 [01:12<00:00,  3.37it/s]\n",
      "100%|██████████| 245/245 [01:13<00:00,  3.35it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9003, Test Acc: 0.6802\n",
      "Best val acc 0.9003, best test acc: 0.6802\n",
      "Epoch 00005 | Step 00000 | Loss 0.3678 | Train Acc 0.9043 | Speed (samples/sec) 350.2325 | GPU 5012.8 MB\n",
      "Epoch 00005 | Step 00020 | Loss 0.3625 | Train Acc 0.9082 | Speed (samples/sec) 350.7592 | GPU 5012.8 MB\n",
      "Epoch 00005 | Step 00040 | Loss 0.4018 | Train Acc 0.8955 | Speed (samples/sec) 351.0882 | GPU 5012.8 MB\n",
      "Epoch 00005 | Step 00060 | Loss 0.3684 | Train Acc 0.8994 | Speed (samples/sec) 351.4710 | GPU 5012.8 MB\n",
      "Epoch 00005 | Step 00080 | Loss 0.3737 | Train Acc 0.9072 | Speed (samples/sec) 351.7009 | GPU 5012.8 MB\n",
      "Epoch 00005 | Step 00100 | Loss 0.4246 | Train Acc 0.8906 | Speed (samples/sec) 352.2521 | GPU 5012.8 MB\n",
      "Epoch 00005 | Step 00120 | Loss 0.3289 | Train Acc 0.9131 | Speed (samples/sec) 352.5059 | GPU 5012.8 MB\n",
      "Epoch 00005 | Step 00140 | Loss 0.3866 | Train Acc 0.9043 | Speed (samples/sec) 352.5522 | GPU 5012.8 MB\n",
      "Epoch 00005 | Step 00160 | Loss 0.3904 | Train Acc 0.8984 | Speed (samples/sec) 352.6611 | GPU 5012.8 MB\n",
      "Epoch 00005 | Step 00180 | Loss 0.3454 | Train Acc 0.9102 | Speed (samples/sec) 353.0260 | GPU 5012.8 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 535.1405\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:35<00:00,  6.81it/s]\n",
      "100%|██████████| 245/245 [01:13<00:00,  3.33it/s]\n",
      "100%|██████████| 245/245 [01:12<00:00,  3.36it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9052, Test Acc: 0.6883\n",
      "Best val acc 0.9052, best test acc: 0.6883\n",
      "Epoch 00006 | Step 00000 | Loss 0.3295 | Train Acc 0.9150 | Speed (samples/sec) 353.0431 | GPU 5012.8 MB\n",
      "Epoch 00006 | Step 00020 | Loss 0.3409 | Train Acc 0.9082 | Speed (samples/sec) 353.5913 | GPU 5012.8 MB\n",
      "Epoch 00006 | Step 00040 | Loss 0.4275 | Train Acc 0.8916 | Speed (samples/sec) 353.7486 | GPU 5012.8 MB\n",
      "Epoch 00006 | Step 00060 | Loss 0.3128 | Train Acc 0.9160 | Speed (samples/sec) 353.9997 | GPU 5012.8 MB\n",
      "Epoch 00006 | Step 00080 | Loss 0.3609 | Train Acc 0.9033 | Speed (samples/sec) 354.1554 | GPU 5012.8 MB\n",
      "Epoch 00006 | Step 00100 | Loss 0.3012 | Train Acc 0.9189 | Speed (samples/sec) 354.3260 | GPU 5012.8 MB\n",
      "Epoch 00006 | Step 00120 | Loss 0.2863 | Train Acc 0.9248 | Speed (samples/sec) 354.5918 | GPU 5012.8 MB\n",
      "Epoch 00006 | Step 00140 | Loss 0.3633 | Train Acc 0.8994 | Speed (samples/sec) 354.7854 | GPU 5012.8 MB\n",
      "Epoch 00006 | Step 00160 | Loss 0.3735 | Train Acc 0.9111 | Speed (samples/sec) 354.8214 | GPU 5012.8 MB\n",
      "Epoch 00006 | Step 00180 | Loss 0.3958 | Train Acc 0.8926 | Speed (samples/sec) 354.8915 | GPU 5012.8 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 536.5960\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:35<00:00,  6.93it/s]\n",
      "100%|██████████| 245/245 [01:17<00:00,  3.18it/s]\n",
      "100%|██████████| 245/245 [01:17<00:00,  3.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9074, Test Acc: 0.6956\n",
      "Best val acc 0.9074, best test acc: 0.6956\n",
      "Epoch 00007 | Step 00000 | Loss 0.3200 | Train Acc 0.9209 | Speed (samples/sec) 354.8741 | GPU 5012.8 MB\n",
      "Epoch 00007 | Step 00020 | Loss 0.3712 | Train Acc 0.9209 | Speed (samples/sec) 355.0621 | GPU 5012.8 MB\n",
      "Epoch 00007 | Step 00040 | Loss 0.2806 | Train Acc 0.9307 | Speed (samples/sec) 355.2979 | GPU 5012.8 MB\n",
      "Epoch 00007 | Step 00060 | Loss 0.3278 | Train Acc 0.9199 | Speed (samples/sec) 355.3401 | GPU 5012.8 MB\n",
      "Epoch 00007 | Step 00080 | Loss 0.2921 | Train Acc 0.9229 | Speed (samples/sec) 355.1963 | GPU 5012.8 MB\n",
      "Epoch 00007 | Step 00100 | Loss 0.3122 | Train Acc 0.9141 | Speed (samples/sec) 355.1526 | GPU 5012.8 MB\n",
      "Epoch 00007 | Step 00120 | Loss 0.3387 | Train Acc 0.9170 | Speed (samples/sec) 355.3139 | GPU 5012.8 MB\n",
      "Epoch 00007 | Step 00140 | Loss 0.3341 | Train Acc 0.9150 | Speed (samples/sec) 355.3534 | GPU 5014.2 MB\n",
      "Epoch 00007 | Step 00160 | Loss 0.3895 | Train Acc 0.9023 | Speed (samples/sec) 355.4071 | GPU 5014.2 MB\n",
      "Epoch 00007 | Step 00180 | Loss 0.3374 | Train Acc 0.9102 | Speed (samples/sec) 355.3579 | GPU 5014.2 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 549.0165\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:34<00:00,  7.03it/s]\n",
      "100%|██████████| 245/245 [01:11<00:00,  3.43it/s]\n",
      "100%|██████████| 245/245 [01:14<00:00,  3.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9091, Test Acc: 0.6963\n",
      "Best val acc 0.9091, best test acc: 0.6963\n",
      "Epoch 00008 | Step 00000 | Loss 0.2441 | Train Acc 0.9297 | Speed (samples/sec) 355.2183 | GPU 5014.2 MB\n",
      "Epoch 00008 | Step 00020 | Loss 0.3319 | Train Acc 0.9150 | Speed (samples/sec) 355.3369 | GPU 5014.2 MB\n",
      "Epoch 00008 | Step 00040 | Loss 0.3548 | Train Acc 0.9199 | Speed (samples/sec) 355.8151 | GPU 5014.2 MB\n",
      "Epoch 00008 | Step 00060 | Loss 0.2782 | Train Acc 0.9404 | Speed (samples/sec) 356.2706 | GPU 5014.2 MB\n",
      "Epoch 00008 | Step 00080 | Loss 0.2971 | Train Acc 0.9277 | Speed (samples/sec) 356.3246 | GPU 5015.6 MB\n",
      "Epoch 00008 | Step 00100 | Loss 0.3488 | Train Acc 0.9102 | Speed (samples/sec) 356.2110 | GPU 5015.6 MB\n",
      "Epoch 00008 | Step 00120 | Loss 0.3234 | Train Acc 0.9004 | Speed (samples/sec) 356.3350 | GPU 5015.6 MB\n",
      "Epoch 00008 | Step 00140 | Loss 0.2580 | Train Acc 0.9326 | Speed (samples/sec) 356.7629 | GPU 5015.6 MB\n",
      "Epoch 00008 | Step 00160 | Loss 0.2846 | Train Acc 0.9219 | Speed (samples/sec) 356.7801 | GPU 5015.6 MB\n",
      "Epoch 00008 | Step 00180 | Loss 0.2825 | Train Acc 0.9258 | Speed (samples/sec) 356.7125 | GPU 5015.6 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 536.6726\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:37<00:00,  6.57it/s]\n",
      "100%|██████████| 245/245 [01:09<00:00,  3.51it/s]\n",
      "100%|██████████| 245/245 [01:11<00:00,  3.42it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9118, Test Acc: 0.7053\n",
      "Best val acc 0.9118, best test acc: 0.7053\n",
      "Epoch 00009 | Step 00000 | Loss 0.2738 | Train Acc 0.9258 | Speed (samples/sec) 356.4666 | GPU 5015.6 MB\n",
      "Epoch 00009 | Step 00020 | Loss 0.2792 | Train Acc 0.9268 | Speed (samples/sec) 356.4599 | GPU 5015.6 MB\n",
      "Epoch 00009 | Step 00040 | Loss 0.2674 | Train Acc 0.9346 | Speed (samples/sec) 356.6688 | GPU 5015.6 MB\n",
      "Epoch 00009 | Step 00060 | Loss 0.3187 | Train Acc 0.9170 | Speed (samples/sec) 357.1479 | GPU 5015.6 MB\n",
      "Epoch 00009 | Step 00080 | Loss 0.2620 | Train Acc 0.9268 | Speed (samples/sec) 357.3660 | GPU 5015.6 MB\n",
      "Epoch 00009 | Step 00100 | Loss 0.2503 | Train Acc 0.9365 | Speed (samples/sec) 357.3705 | GPU 5015.6 MB\n",
      "Epoch 00009 | Step 00120 | Loss 0.2039 | Train Acc 0.9492 | Speed (samples/sec) 357.2882 | GPU 5015.6 MB\n",
      "Epoch 00009 | Step 00140 | Loss 0.2511 | Train Acc 0.9365 | Speed (samples/sec) 357.4182 | GPU 5015.6 MB\n",
      "Epoch 00009 | Step 00160 | Loss 0.2521 | Train Acc 0.9336 | Speed (samples/sec) 357.6189 | GPU 5015.6 MB\n",
      "Epoch 00009 | Step 00180 | Loss 0.2637 | Train Acc 0.9355 | Speed (samples/sec) 357.8247 | GPU 5015.6 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 529.3374\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:34<00:00,  7.13it/s]\n",
      "100%|██████████| 245/245 [01:06<00:00,  3.68it/s]\n",
      "100%|██████████| 245/245 [01:06<00:00,  3.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9126, Test Acc: 0.7103\n",
      "Best val acc 0.9126, best test acc: 0.7103\n",
      "Epoch 00010 | Step 00000 | Loss 0.2376 | Train Acc 0.9365 | Speed (samples/sec) 358.0010 | GPU 5015.6 MB\n",
      "Epoch 00010 | Step 00020 | Loss 0.2630 | Train Acc 0.9316 | Speed (samples/sec) 358.3548 | GPU 5015.6 MB\n",
      "Epoch 00010 | Step 00040 | Loss 0.2728 | Train Acc 0.9219 | Speed (samples/sec) 358.7270 | GPU 5015.6 MB\n",
      "Epoch 00010 | Step 00060 | Loss 0.2259 | Train Acc 0.9365 | Speed (samples/sec) 359.2622 | GPU 5015.6 MB\n",
      "Epoch 00010 | Step 00080 | Loss 0.2916 | Train Acc 0.9199 | Speed (samples/sec) 359.7012 | GPU 5015.6 MB\n",
      "Epoch 00010 | Step 00100 | Loss 0.2891 | Train Acc 0.9297 | Speed (samples/sec) 360.0569 | GPU 5015.6 MB\n",
      "Epoch 00010 | Step 00120 | Loss 0.2247 | Train Acc 0.9385 | Speed (samples/sec) 360.4416 | GPU 5015.6 MB\n",
      "Epoch 00010 | Step 00140 | Loss 0.2800 | Train Acc 0.9326 | Speed (samples/sec) 360.8299 | GPU 5015.6 MB\n",
      "Epoch 00010 | Step 00160 | Loss 0.2850 | Train Acc 0.9248 | Speed (samples/sec) 361.2675 | GPU 5015.6 MB\n",
      "Epoch 00010 | Step 00180 | Loss 0.2599 | Train Acc 0.9355 | Speed (samples/sec) 361.7686 | GPU 5015.6 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 489.9601\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:32<00:00,  7.50it/s]\n",
      "100%|██████████| 245/245 [01:07<00:00,  3.64it/s]\n",
      "100%|██████████| 245/245 [01:05<00:00,  3.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9106, Test Acc: 0.7062\n",
      "Epoch 00011 | Step 00000 | Loss 0.2778 | Train Acc 0.9277 | Speed (samples/sec) 361.8817 | GPU 5015.6 MB\n",
      "Epoch 00011 | Step 00020 | Loss 0.2830 | Train Acc 0.9307 | Speed (samples/sec) 362.2018 | GPU 5015.6 MB\n",
      "Epoch 00011 | Step 00040 | Loss 0.2093 | Train Acc 0.9482 | Speed (samples/sec) 362.4462 | GPU 5016.6 MB\n",
      "Epoch 00011 | Step 00060 | Loss 0.2610 | Train Acc 0.9346 | Speed (samples/sec) 362.8728 | GPU 5016.6 MB\n",
      "Epoch 00011 | Step 00080 | Loss 0.2094 | Train Acc 0.9395 | Speed (samples/sec) 363.2865 | GPU 5016.6 MB\n",
      "Epoch 00011 | Step 00100 | Loss 0.2443 | Train Acc 0.9414 | Speed (samples/sec) 363.5724 | GPU 5016.6 MB\n",
      "Epoch 00011 | Step 00120 | Loss 0.2165 | Train Acc 0.9414 | Speed (samples/sec) 363.8819 | GPU 5016.6 MB\n",
      "Epoch 00011 | Step 00140 | Loss 0.2326 | Train Acc 0.9258 | Speed (samples/sec) 364.1816 | GPU 5016.6 MB\n",
      "Epoch 00011 | Step 00160 | Loss 0.2487 | Train Acc 0.9385 | Speed (samples/sec) 364.4403 | GPU 5016.6 MB\n",
      "Epoch 00011 | Step 00180 | Loss 0.2691 | Train Acc 0.9268 | Speed (samples/sec) 364.7562 | GPU 5016.6 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 493.9397\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:32<00:00,  7.57it/s]\n",
      "100%|██████████| 245/245 [01:09<00:00,  3.53it/s]\n",
      "100%|██████████| 245/245 [01:08<00:00,  3.59it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9133, Test Acc: 0.7207\n",
      "Best val acc 0.9133, best test acc: 0.7207\n",
      "Epoch 00012 | Step 00000 | Loss 0.1459 | Train Acc 0.9561 | Speed (samples/sec) 364.8368 | GPU 5016.6 MB\n",
      "Epoch 00012 | Step 00020 | Loss 0.1768 | Train Acc 0.9561 | Speed (samples/sec) 365.0863 | GPU 5016.6 MB\n",
      "Epoch 00012 | Step 00040 | Loss 0.2335 | Train Acc 0.9404 | Speed (samples/sec) 365.2412 | GPU 5016.6 MB\n",
      "Epoch 00012 | Step 00060 | Loss 0.2020 | Train Acc 0.9482 | Speed (samples/sec) 365.4863 | GPU 5016.6 MB\n",
      "Epoch 00012 | Step 00080 | Loss 0.1820 | Train Acc 0.9521 | Speed (samples/sec) 365.8783 | GPU 5016.6 MB\n",
      "Epoch 00012 | Step 00100 | Loss 0.2313 | Train Acc 0.9385 | Speed (samples/sec) 366.1461 | GPU 5016.6 MB\n",
      "Epoch 00012 | Step 00120 | Loss 0.2232 | Train Acc 0.9443 | Speed (samples/sec) 366.3460 | GPU 5016.6 MB\n",
      "Epoch 00012 | Step 00140 | Loss 0.1884 | Train Acc 0.9521 | Speed (samples/sec) 366.6051 | GPU 5016.6 MB\n",
      "Epoch 00012 | Step 00160 | Loss 0.1957 | Train Acc 0.9453 | Speed (samples/sec) 366.7815 | GPU 5016.6 MB\n",
      "Epoch 00012 | Step 00180 | Loss 0.2415 | Train Acc 0.9355 | Speed (samples/sec) 366.8755 | GPU 5016.6 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 502.0502\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:33<00:00,  7.28it/s]\n",
      "100%|██████████| 245/245 [01:11<00:00,  3.43it/s]\n",
      "100%|██████████| 245/245 [01:11<00:00,  3.42it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9118, Test Acc: 0.7188\n",
      "Epoch 00013 | Step 00000 | Loss 0.1854 | Train Acc 0.9453 | Speed (samples/sec) 366.8392 | GPU 5016.6 MB\n",
      "Epoch 00013 | Step 00020 | Loss 0.1941 | Train Acc 0.9521 | Speed (samples/sec) 367.0333 | GPU 5016.6 MB\n",
      "Epoch 00013 | Step 00040 | Loss 0.2089 | Train Acc 0.9473 | Speed (samples/sec) 367.3120 | GPU 5016.6 MB\n",
      "Epoch 00013 | Step 00060 | Loss 0.1870 | Train Acc 0.9502 | Speed (samples/sec) 367.5558 | GPU 5016.6 MB\n",
      "Epoch 00013 | Step 00080 | Loss 0.2402 | Train Acc 0.9326 | Speed (samples/sec) 367.9521 | GPU 5016.6 MB\n",
      "Epoch 00013 | Step 00100 | Loss 0.2120 | Train Acc 0.9395 | Speed (samples/sec) 368.2797 | GPU 5016.6 MB\n",
      "Epoch 00013 | Step 00120 | Loss 0.2106 | Train Acc 0.9492 | Speed (samples/sec) 368.5476 | GPU 5016.6 MB\n",
      "Epoch 00013 | Step 00140 | Loss 0.1646 | Train Acc 0.9521 | Speed (samples/sec) 368.7477 | GPU 5016.6 MB\n",
      "Epoch 00013 | Step 00160 | Loss 0.2089 | Train Acc 0.9443 | Speed (samples/sec) 368.9106 | GPU 5016.6 MB\n",
      "Epoch 00013 | Step 00180 | Loss 0.1942 | Train Acc 0.9482 | Speed (samples/sec) 369.0427 | GPU 5016.6 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 493.7867\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:32<00:00,  7.60it/s]\n",
      "100%|██████████| 245/245 [01:12<00:00,  3.40it/s]\n",
      "100%|██████████| 245/245 [01:09<00:00,  3.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9110, Test Acc: 0.7155\n",
      "Epoch 00014 | Step 00000 | Loss 0.1833 | Train Acc 0.9502 | Speed (samples/sec) 369.0267 | GPU 5016.6 MB\n",
      "Epoch 00014 | Step 00020 | Loss 0.2076 | Train Acc 0.9424 | Speed (samples/sec) 369.1402 | GPU 5016.6 MB\n",
      "Epoch 00014 | Step 00040 | Loss 0.1534 | Train Acc 0.9551 | Speed (samples/sec) 369.3375 | GPU 5016.6 MB\n",
      "Epoch 00014 | Step 00060 | Loss 0.1566 | Train Acc 0.9600 | Speed (samples/sec) 369.6053 | GPU 5016.6 MB\n",
      "Epoch 00014 | Step 00080 | Loss 0.1445 | Train Acc 0.9658 | Speed (samples/sec) 369.8819 | GPU 5016.6 MB\n",
      "Epoch 00014 | Step 00100 | Loss 0.2066 | Train Acc 0.9443 | Speed (samples/sec) 370.1291 | GPU 5021.6 MB\n",
      "Epoch 00014 | Step 00120 | Loss 0.1388 | Train Acc 0.9580 | Speed (samples/sec) 370.6003 | GPU 5021.6 MB\n",
      "Epoch 00014 | Step 00140 | Loss 0.1645 | Train Acc 0.9541 | Speed (samples/sec) 371.0235 | GPU 5021.6 MB\n",
      "Epoch 00014 | Step 00160 | Loss 0.1882 | Train Acc 0.9502 | Speed (samples/sec) 371.4363 | GPU 5021.6 MB\n",
      "Epoch 00014 | Step 00180 | Loss 0.1756 | Train Acc 0.9512 | Speed (samples/sec) 371.9363 | GPU 5021.6 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 474.0810\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:31<00:00,  7.79it/s]\n",
      "100%|██████████| 245/245 [01:01<00:00,  3.98it/s]\n",
      "100%|██████████| 245/245 [01:03<00:00,  3.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9095, Test Acc: 0.7135\n",
      "Epoch 00015 | Step 00000 | Loss 0.1657 | Train Acc 0.9512 | Speed (samples/sec) 372.1000 | GPU 5021.6 MB\n",
      "Epoch 00015 | Step 00020 | Loss 0.1683 | Train Acc 0.9541 | Speed (samples/sec) 372.4830 | GPU 5021.6 MB\n",
      "Epoch 00015 | Step 00040 | Loss 0.1472 | Train Acc 0.9551 | Speed (samples/sec) 372.8225 | GPU 5021.6 MB\n",
      "Epoch 00015 | Step 00060 | Loss 0.1881 | Train Acc 0.9463 | Speed (samples/sec) 373.2069 | GPU 5021.6 MB\n",
      "Epoch 00015 | Step 00080 | Loss 0.1624 | Train Acc 0.9570 | Speed (samples/sec) 373.5607 | GPU 5021.6 MB\n",
      "Epoch 00015 | Step 00100 | Loss 0.1507 | Train Acc 0.9561 | Speed (samples/sec) 374.0498 | GPU 5021.6 MB\n",
      "Epoch 00015 | Step 00120 | Loss 0.1786 | Train Acc 0.9521 | Speed (samples/sec) 374.4780 | GPU 5021.6 MB\n",
      "Epoch 00015 | Step 00140 | Loss 0.1665 | Train Acc 0.9492 | Speed (samples/sec) 374.8709 | GPU 5021.6 MB\n",
      "Epoch 00015 | Step 00160 | Loss 0.1651 | Train Acc 0.9541 | Speed (samples/sec) 375.1789 | GPU 5021.6 MB\n",
      "Epoch 00015 | Step 00180 | Loss 0.1349 | Train Acc 0.9678 | Speed (samples/sec) 375.4929 | GPU 5021.6 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 459.6553\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:30<00:00,  8.14it/s]\n",
      "100%|██████████| 245/245 [01:06<00:00,  3.71it/s]\n",
      "100%|██████████| 245/245 [01:03<00:00,  3.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9076, Test Acc: 0.7126\n",
      "Epoch 00016 | Step 00000 | Loss 0.1579 | Train Acc 0.9570 | Speed (samples/sec) 375.5443 | GPU 5021.6 MB\n",
      "Epoch 00016 | Step 00020 | Loss 0.1386 | Train Acc 0.9629 | Speed (samples/sec) 375.9087 | GPU 5021.6 MB\n",
      "Epoch 00016 | Step 00040 | Loss 0.1199 | Train Acc 0.9658 | Speed (samples/sec) 376.2108 | GPU 5021.6 MB\n",
      "Epoch 00016 | Step 00060 | Loss 0.1353 | Train Acc 0.9688 | Speed (samples/sec) 376.5496 | GPU 5021.6 MB\n",
      "Epoch 00016 | Step 00080 | Loss 0.1479 | Train Acc 0.9590 | Speed (samples/sec) 376.8630 | GPU 5021.6 MB\n",
      "Epoch 00016 | Step 00100 | Loss 0.1221 | Train Acc 0.9609 | Speed (samples/sec) 377.2272 | GPU 5021.6 MB\n",
      "Epoch 00016 | Step 00120 | Loss 0.1297 | Train Acc 0.9600 | Speed (samples/sec) 377.5431 | GPU 5021.6 MB\n",
      "Epoch 00016 | Step 00140 | Loss 0.1159 | Train Acc 0.9648 | Speed (samples/sec) 377.8111 | GPU 5021.6 MB\n",
      "Epoch 00016 | Step 00160 | Loss 0.1572 | Train Acc 0.9521 | Speed (samples/sec) 378.1555 | GPU 5021.6 MB\n",
      "Epoch 00016 | Step 00180 | Loss 0.1080 | Train Acc 0.9619 | Speed (samples/sec) 378.5969 | GPU 5021.6 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 456.1561\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:28<00:00,  8.52it/s]\n",
      "100%|██████████| 245/245 [01:01<00:00,  3.98it/s]\n",
      "100%|██████████| 245/245 [01:01<00:00,  3.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9071, Test Acc: 0.7143\n",
      "Epoch 00017 | Step 00000 | Loss 0.1310 | Train Acc 0.9629 | Speed (samples/sec) 378.7948 | GPU 5021.6 MB\n",
      "Epoch 00017 | Step 00020 | Loss 0.1320 | Train Acc 0.9639 | Speed (samples/sec) 379.2685 | GPU 5021.6 MB\n",
      "Epoch 00017 | Step 00040 | Loss 0.1098 | Train Acc 0.9678 | Speed (samples/sec) 379.7130 | GPU 5021.6 MB\n",
      "Epoch 00017 | Step 00060 | Loss 0.1107 | Train Acc 0.9658 | Speed (samples/sec) 380.1359 | GPU 5021.6 MB\n",
      "Epoch 00017 | Step 00080 | Loss 0.1214 | Train Acc 0.9619 | Speed (samples/sec) 380.6126 | GPU 5021.6 MB\n",
      "Epoch 00017 | Step 00100 | Loss 0.1509 | Train Acc 0.9609 | Speed (samples/sec) 381.1612 | GPU 5021.6 MB\n",
      "Epoch 00017 | Step 00120 | Loss 0.1023 | Train Acc 0.9697 | Speed (samples/sec) 381.7241 | GPU 5021.6 MB\n",
      "Epoch 00017 | Step 00140 | Loss 0.1341 | Train Acc 0.9678 | Speed (samples/sec) 382.2098 | GPU 5021.6 MB\n",
      "Epoch 00017 | Step 00160 | Loss 0.0981 | Train Acc 0.9688 | Speed (samples/sec) 382.7162 | GPU 5021.6 MB\n",
      "Epoch 00017 | Step 00180 | Loss 0.0842 | Train Acc 0.9736 | Speed (samples/sec) 383.1878 | GPU 5021.6 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 424.9588\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:28<00:00,  8.66it/s]\n",
      "100%|██████████| 245/245 [01:01<00:00,  3.98it/s]\n",
      "100%|██████████| 245/245 [00:59<00:00,  4.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9030, Test Acc: 0.7093\n",
      "Epoch 00018 | Step 00000 | Loss 0.0946 | Train Acc 0.9697 | Speed (samples/sec) 383.4320 | GPU 5021.6 MB\n",
      "Epoch 00018 | Step 00020 | Loss 0.0982 | Train Acc 0.9707 | Speed (samples/sec) 383.8819 | GPU 5021.6 MB\n",
      "Epoch 00018 | Step 00040 | Loss 0.1436 | Train Acc 0.9590 | Speed (samples/sec) 384.2776 | GPU 5021.6 MB\n",
      "Epoch 00018 | Step 00060 | Loss 0.1346 | Train Acc 0.9580 | Speed (samples/sec) 384.6698 | GPU 5021.6 MB\n",
      "Epoch 00018 | Step 00080 | Loss 0.1303 | Train Acc 0.9590 | Speed (samples/sec) 385.0360 | GPU 5021.6 MB\n",
      "Epoch 00018 | Step 00100 | Loss 0.1020 | Train Acc 0.9746 | Speed (samples/sec) 385.3939 | GPU 5021.6 MB\n",
      "Epoch 00018 | Step 00120 | Loss 0.1266 | Train Acc 0.9609 | Speed (samples/sec) 385.8274 | GPU 5021.6 MB\n",
      "Epoch 00018 | Step 00140 | Loss 0.1418 | Train Acc 0.9600 | Speed (samples/sec) 386.2216 | GPU 5021.6 MB\n",
      "Epoch 00018 | Step 00160 | Loss 0.1241 | Train Acc 0.9678 | Speed (samples/sec) 386.6785 | GPU 5021.6 MB\n",
      "Epoch 00018 | Step 00180 | Loss 0.1166 | Train Acc 0.9668 | Speed (samples/sec) 387.1397 | GPU 5021.6 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 428.7824\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:27<00:00,  8.92it/s]\n",
      "100%|██████████| 245/245 [01:01<00:00,  3.96it/s]\n",
      "100%|██████████| 245/245 [00:58<00:00,  4.21it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8973, Test Acc: 0.6994\n",
      "Epoch 00019 | Step 00000 | Loss 0.1229 | Train Acc 0.9619 | Speed (samples/sec) 387.3621 | GPU 5021.6 MB\n",
      "Epoch 00019 | Step 00020 | Loss 0.1587 | Train Acc 0.9648 | Speed (samples/sec) 387.7929 | GPU 5021.6 MB\n",
      "Epoch 00019 | Step 00040 | Loss 0.1253 | Train Acc 0.9648 | Speed (samples/sec) 388.2334 | GPU 5021.6 MB\n",
      "Epoch 00019 | Step 00060 | Loss 0.1077 | Train Acc 0.9697 | Speed (samples/sec) 388.6176 | GPU 5021.6 MB\n",
      "Epoch 00019 | Step 00080 | Loss 0.1291 | Train Acc 0.9609 | Speed (samples/sec) 389.0354 | GPU 5021.6 MB\n",
      "Epoch 00019 | Step 00100 | Loss 0.1170 | Train Acc 0.9727 | Speed (samples/sec) 389.4192 | GPU 5021.6 MB\n",
      "Epoch 00019 | Step 00120 | Loss 0.1297 | Train Acc 0.9619 | Speed (samples/sec) 389.8289 | GPU 5021.6 MB\n",
      "Epoch 00019 | Step 00140 | Loss 0.1054 | Train Acc 0.9648 | Speed (samples/sec) 390.2200 | GPU 5021.6 MB\n",
      "Epoch 00019 | Step 00160 | Loss 0.1022 | Train Acc 0.9707 | Speed (samples/sec) 390.5404 | GPU 5021.6 MB\n",
      "Epoch 00019 | Step 00180 | Loss 0.1523 | Train Acc 0.9600 | Speed (samples/sec) 390.8650 | GPU 5021.6 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 426.6923\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:28<00:00,  8.68it/s]\n",
      "100%|██████████| 245/245 [01:01<00:00,  4.00it/s]\n",
      "100%|██████████| 245/245 [00:58<00:00,  4.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9026, Test Acc: 0.7101\n",
      "Avg epoch time: 637.1552407741547\n",
      "Best val acc 0.9133, best test acc: 0.7207\n"
     ]
    }
   ],
   "source": [
    "g, num_labels = load_ogb('ogbn-products')\n",
    "g.ndata.pop('features')\n",
    "\n",
    "in_feats = 128\n",
    "nfeat = th.normal(0, 1, size=(g.number_of_nodes(), in_feats), dtype=th.float32)\n",
    "labels = g.ndata.pop('labels')\n",
    "\n",
    "hyperparams = {\n",
    "    'batch_size': 1024,\n",
    "    'num_workers': 0,\n",
    "    'num_hidden': 256,\n",
    "    'num_layers': 3,\n",
    "    'dropout': 0.5,\n",
    "    'lr': 0.003,\n",
    "    'sparse_lr': 0.003,\n",
    "    'num_epochs': 20,\n",
    "    'fanouts': [15,10,5],\n",
    "    'eval_batch_size': 10000\n",
    "}\n",
    "device = th.device('cuda:0')\n",
    "\n",
    "# Pack data\n",
    "data = g, nfeat, labels\n",
    "# Define model\n",
    "model = SAGE(in_feats, hyperparams['num_hidden'], num_labels,\n",
    "             hyperparams['num_layers'], F.relu, hyperparams['dropout'])\n",
    "train_ft_embed(model, data, hyperparams, device, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}