GraphSage-sample-eigen-tunable-products_gpu2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using backend: pytorch\n"
     ]
    }
   ],
   "source": [
    "import dgl\n",
    "import time\n",
    "import torch as th\n",
    "import numpy as np\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from load_data import load_ogb\n",
    "from model import SAGE, GAT, get_eigen\n",
    "from train_eval import train_ft_embed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook shows the performance of GraphSage on the OGBN-products graph with trainable embeddings on the nodes. We use scaled eigenvectors to initialize the node embeddings. The logic here is that if eigenvectors can serve as positional node embeddings, fine-tuning the node embeddings should give us even better performance. However, in this experiment, we tune the node embeddings with the same learning rate as the one for the GraphSage model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load ogbn-products\n",
      "finish loading ogbn-products\n",
      "finish constructing ogbn-products\n",
      "embedding is in  cuda:2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/numpy/core/fromnumeric.py:3373: RuntimeWarning: Mean of empty slice.\n",
      "  out=out, **kwargs)\n",
      "/usr/local/lib/python3.6/dist-packages/numpy/core/_methods.py:170: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00000 | Step 00000 | Loss 3.8519 | Train Acc 0.0205 | Speed (samples/sec) nan | GPU 4999.9 MB\n",
      "Epoch 00000 | Step 00020 | Loss 1.0807 | Train Acc 0.7715 | Speed (samples/sec) 442.8895 | GPU 5001.3 MB\n",
      "Epoch 00000 | Step 00040 | Loss 0.5557 | Train Acc 0.8838 | Speed (samples/sec) 444.9740 | GPU 5010.4 MB\n",
      "Epoch 00000 | Step 00060 | Loss 0.4304 | Train Acc 0.8984 | Speed (samples/sec) 438.2583 | GPU 5010.4 MB\n",
      "Epoch 00000 | Step 00080 | Loss 0.3049 | Train Acc 0.9160 | Speed (samples/sec) 437.0898 | GPU 5010.4 MB\n",
      "Epoch 00000 | Step 00100 | Loss 0.3824 | Train Acc 0.9033 | Speed (samples/sec) 440.6758 | GPU 5011.1 MB\n",
      "Epoch 00000 | Step 00120 | Loss 0.3783 | Train Acc 0.9102 | Speed (samples/sec) 438.4916 | GPU 5011.1 MB\n",
      "Epoch 00000 | Step 00140 | Loss 0.2801 | Train Acc 0.9287 | Speed (samples/sec) 434.6644 | GPU 5011.1 MB\n",
      "Epoch 00000 | Step 00160 | Loss 0.2674 | Train Acc 0.9395 | Speed (samples/sec) 435.9317 | GPU 5011.1 MB\n",
      "Epoch 00000 | Step 00180 | Loss 0.3101 | Train Acc 0.9238 | Speed (samples/sec) 436.3014 | GPU 5011.1 MB\n",
      "Epoch Time(s): 451.1155\n",
      "Epoch 00001 | Step 00000 | Loss 0.1602 | Train Acc 0.9512 | Speed (samples/sec) 438.4999 | GPU 5011.1 MB\n",
      "Epoch 00001 | Step 00020 | Loss 0.2105 | Train Acc 0.9434 | Speed (samples/sec) 447.6214 | GPU 5011.1 MB\n",
      "Epoch 00001 | Step 00040 | Loss 0.1865 | Train Acc 0.9561 | Speed (samples/sec) 450.5567 | GPU 5011.1 MB\n",
      "Epoch 00001 | Step 00060 | Loss 0.1821 | Train Acc 0.9473 | Speed (samples/sec) 452.9479 | GPU 5011.1 MB\n",
      "Epoch 00001 | Step 00080 | Loss 0.1826 | Train Acc 0.9453 | Speed (samples/sec) 455.5983 | GPU 5011.1 MB\n",
      "Epoch 00001 | Step 00100 | Loss 0.1738 | Train Acc 0.9512 | Speed (samples/sec) 459.6434 | GPU 5011.1 MB\n",
      "Epoch 00001 | Step 00120 | Loss 0.1847 | Train Acc 0.9482 | Speed (samples/sec) 462.1190 | GPU 5011.1 MB\n",
      "Epoch 00001 | Step 00140 | Loss 0.2147 | Train Acc 0.9443 | Speed (samples/sec) 462.4331 | GPU 5011.1 MB\n",
      "Epoch 00001 | Step 00160 | Loss 0.1519 | Train Acc 0.9561 | Speed (samples/sec) 466.9700 | GPU 5011.1 MB\n",
      "Epoch 00001 | Step 00180 | Loss 0.1805 | Train Acc 0.9531 | Speed (samples/sec) 467.8565 | GPU 5011.1 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 405.7057\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:32<00:00,  7.44it/s]\n",
      "100%|██████████| 245/245 [01:13<00:00,  3.35it/s]\n",
      "100%|██████████| 245/245 [01:10<00:00,  3.47it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9148, Test Acc: 0.7898\n",
      "Best val acc 0.9148, best test acc: 0.7898\n",
      "Epoch 00002 | Step 00000 | Loss 0.0669 | Train Acc 0.9814 | Speed (samples/sec) 463.7346 | GPU 5011.1 MB\n",
      "Epoch 00002 | Step 00020 | Loss 0.0458 | Train Acc 0.9834 | Speed (samples/sec) 459.4137 | GPU 5011.6 MB\n",
      "Epoch 00002 | Step 00040 | Loss 0.0400 | Train Acc 0.9912 | Speed (samples/sec) 454.7490 | GPU 5011.6 MB\n",
      "Epoch 00002 | Step 00060 | Loss 0.0502 | Train Acc 0.9834 | Speed (samples/sec) 450.0096 | GPU 5011.6 MB\n",
      "Epoch 00002 | Step 00080 | Loss 0.0455 | Train Acc 0.9873 | Speed (samples/sec) 447.1279 | GPU 5014.3 MB\n",
      "Epoch 00002 | Step 00100 | Loss 0.0671 | Train Acc 0.9824 | Speed (samples/sec) 443.9816 | GPU 5014.3 MB\n",
      "Epoch 00002 | Step 00120 | Loss 0.0381 | Train Acc 0.9883 | Speed (samples/sec) 439.9899 | GPU 5014.3 MB\n",
      "Epoch 00002 | Step 00140 | Loss 0.0311 | Train Acc 0.9902 | Speed (samples/sec) 436.3890 | GPU 5014.3 MB\n",
      "Epoch 00002 | Step 00160 | Loss 0.0663 | Train Acc 0.9785 | Speed (samples/sec) 433.7914 | GPU 5014.3 MB\n",
      "Epoch 00002 | Step 00180 | Loss 0.0783 | Train Acc 0.9736 | Speed (samples/sec) 430.9940 | GPU 5014.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 550.0823\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:47<00:00,  5.18it/s]\n",
      "100%|██████████| 245/245 [01:26<00:00,  2.83it/s]\n",
      "100%|██████████| 245/245 [01:21<00:00,  3.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9062, Test Acc: 0.7837\n",
      "Epoch 00003 | Step 00000 | Loss 0.0235 | Train Acc 0.9941 | Speed (samples/sec) 428.1449 | GPU 5014.3 MB\n",
      "Epoch 00003 | Step 00020 | Loss 0.0401 | Train Acc 0.9912 | Speed (samples/sec) 424.5116 | GPU 5014.3 MB\n",
      "Epoch 00003 | Step 00040 | Loss 0.0191 | Train Acc 0.9932 | Speed (samples/sec) 421.1046 | GPU 5014.3 MB\n",
      "Epoch 00003 | Step 00060 | Loss 0.0163 | Train Acc 0.9971 | Speed (samples/sec) 419.0233 | GPU 5014.3 MB\n",
      "Epoch 00003 | Step 00080 | Loss 0.0089 | Train Acc 0.9980 | Speed (samples/sec) 417.5153 | GPU 5014.3 MB\n",
      "Epoch 00003 | Step 00100 | Loss 0.0227 | Train Acc 0.9951 | Speed (samples/sec) 415.0815 | GPU 5014.3 MB\n",
      "Epoch 00003 | Step 00120 | Loss 0.0217 | Train Acc 0.9961 | Speed (samples/sec) 413.0964 | GPU 5014.3 MB\n",
      "Epoch 00003 | Step 00140 | Loss 0.0296 | Train Acc 0.9912 | Speed (samples/sec) 411.6513 | GPU 5014.3 MB\n",
      "Epoch 00003 | Step 00160 | Loss 0.0189 | Train Acc 0.9941 | Speed (samples/sec) 410.6922 | GPU 5014.3 MB\n",
      "Epoch 00003 | Step 00180 | Loss 0.0154 | Train Acc 0.9932 | Speed (samples/sec) 409.3346 | GPU 5014.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 562.5410\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:33<00:00,  7.37it/s]\n",
      "100%|██████████| 245/245 [01:20<00:00,  3.04it/s]\n",
      "100%|██████████| 245/245 [01:21<00:00,  3.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9045, Test Acc: 0.7601\n",
      "Epoch 00004 | Step 00000 | Loss 0.0047 | Train Acc 0.9990 | Speed (samples/sec) 408.6942 | GPU 5014.3 MB\n",
      "Epoch 00004 | Step 00020 | Loss 0.0103 | Train Acc 0.9961 | Speed (samples/sec) 407.0782 | GPU 5014.3 MB\n",
      "Epoch 00004 | Step 00040 | Loss 0.0103 | Train Acc 0.9971 | Speed (samples/sec) 405.7666 | GPU 5014.3 MB\n",
      "Epoch 00004 | Step 00060 | Loss 0.0069 | Train Acc 0.9971 | Speed (samples/sec) 405.3858 | GPU 5014.3 MB\n",
      "Epoch 00004 | Step 00080 | Loss 0.0088 | Train Acc 0.9951 | Speed (samples/sec) 404.1597 | GPU 5014.3 MB\n",
      "Epoch 00004 | Step 00100 | Loss 0.0164 | Train Acc 0.9951 | Speed (samples/sec) 402.9625 | GPU 5014.3 MB\n",
      "Epoch 00004 | Step 00120 | Loss 0.0066 | Train Acc 0.9971 | Speed (samples/sec) 402.2129 | GPU 5014.3 MB\n",
      "Epoch 00004 | Step 00140 | Loss 0.0155 | Train Acc 0.9951 | Speed (samples/sec) 401.8090 | GPU 5014.3 MB\n",
      "Epoch 00004 | Step 00160 | Loss 0.0052 | Train Acc 0.9990 | Speed (samples/sec) 401.1622 | GPU 5014.3 MB\n",
      "Epoch 00004 | Step 00180 | Loss 0.0157 | Train Acc 0.9961 | Speed (samples/sec) 400.3896 | GPU 5014.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 542.7859\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:41<00:00,  5.94it/s]\n",
      "100%|██████████| 245/245 [01:14<00:00,  3.30it/s]\n",
      "100%|██████████| 245/245 [01:16<00:00,  3.20it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9036, Test Acc: 0.7808\n",
      "Epoch 00005 | Step 00000 | Loss 0.0090 | Train Acc 0.9971 | Speed (samples/sec) 399.4178 | GPU 5014.3 MB\n",
      "Epoch 00005 | Step 00020 | Loss 0.0101 | Train Acc 0.9980 | Speed (samples/sec) 398.5339 | GPU 5014.3 MB\n",
      "Epoch 00005 | Step 00040 | Loss 0.0083 | Train Acc 0.9971 | Speed (samples/sec) 398.1210 | GPU 5014.3 MB\n",
      "Epoch 00005 | Step 00060 | Loss 0.0054 | Train Acc 0.9971 | Speed (samples/sec) 397.8049 | GPU 5014.3 MB\n",
      "Epoch 00005 | Step 00080 | Loss 0.0065 | Train Acc 0.9980 | Speed (samples/sec) 397.1209 | GPU 5015.5 MB\n",
      "Epoch 00005 | Step 00100 | Loss 0.0195 | Train Acc 0.9951 | Speed (samples/sec) 396.4958 | GPU 5015.5 MB\n",
      "Epoch 00005 | Step 00120 | Loss 0.0216 | Train Acc 0.9961 | Speed (samples/sec) 396.2260 | GPU 5015.5 MB\n",
      "Epoch 00005 | Step 00140 | Loss 0.0042 | Train Acc 1.0000 | Speed (samples/sec) 396.1108 | GPU 5015.5 MB\n",
      "Epoch 00005 | Step 00160 | Loss 0.0163 | Train Acc 0.9961 | Speed (samples/sec) 395.9334 | GPU 5015.5 MB\n",
      "Epoch 00005 | Step 00180 | Loss 0.0163 | Train Acc 0.9922 | Speed (samples/sec) 395.6962 | GPU 5015.5 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 526.1135\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:38<00:00,  6.35it/s]\n",
      "100%|██████████| 245/245 [01:11<00:00,  3.43it/s]\n",
      "100%|██████████| 245/245 [01:15<00:00,  3.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9045, Test Acc: 0.7563\n",
      "Epoch 00006 | Step 00000 | Loss 0.0027 | Train Acc 0.9990 | Speed (samples/sec) 395.0409 | GPU 5015.5 MB\n",
      "Epoch 00006 | Step 00020 | Loss 0.0077 | Train Acc 0.9980 | Speed (samples/sec) 393.9338 | GPU 5015.5 MB\n",
      "Epoch 00006 | Step 00040 | Loss 0.0099 | Train Acc 0.9980 | Speed (samples/sec) 392.9980 | GPU 5015.5 MB\n",
      "Epoch 00006 | Step 00060 | Loss 0.0022 | Train Acc 0.9990 | Speed (samples/sec) 392.3088 | GPU 5015.5 MB\n",
      "Epoch 00006 | Step 00080 | Loss 0.0053 | Train Acc 0.9990 | Speed (samples/sec) 391.6493 | GPU 5015.5 MB\n",
      "Epoch 00006 | Step 00100 | Loss 0.0095 | Train Acc 0.9961 | Speed (samples/sec) 390.9396 | GPU 5015.5 MB\n",
      "Epoch 00006 | Step 00120 | Loss 0.0124 | Train Acc 0.9961 | Speed (samples/sec) 390.2076 | GPU 5015.5 MB\n",
      "Epoch 00006 | Step 00140 | Loss 0.0071 | Train Acc 0.9961 | Speed (samples/sec) 389.5938 | GPU 5015.5 MB\n",
      "Epoch 00006 | Step 00160 | Loss 0.0241 | Train Acc 0.9941 | Speed (samples/sec) 389.3125 | GPU 5015.5 MB\n",
      "Epoch 00006 | Step 00180 | Loss 0.0098 | Train Acc 0.9971 | Speed (samples/sec) 388.7225 | GPU 5015.5 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 567.8470\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:43<00:00,  5.70it/s]\n",
      "100%|██████████| 245/245 [01:15<00:00,  3.24it/s]\n",
      "100%|██████████| 245/245 [01:14<00:00,  3.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9053, Test Acc: 0.7637\n",
      "Epoch 00007 | Step 00000 | Loss 0.0030 | Train Acc 0.9990 | Speed (samples/sec) 388.0241 | GPU 5015.5 MB\n",
      "Epoch 00007 | Step 00020 | Loss 0.0024 | Train Acc 0.9990 | Speed (samples/sec) 387.3515 | GPU 5015.5 MB\n",
      "Epoch 00007 | Step 00040 | Loss 0.0024 | Train Acc 1.0000 | Speed (samples/sec) 386.7622 | GPU 5015.5 MB\n",
      "Epoch 00007 | Step 00060 | Loss 0.0167 | Train Acc 0.9951 | Speed (samples/sec) 386.2498 | GPU 5015.5 MB\n",
      "Epoch 00007 | Step 00080 | Loss 0.0049 | Train Acc 0.9980 | Speed (samples/sec) 385.9149 | GPU 5015.5 MB\n",
      "Epoch 00007 | Step 00100 | Loss 0.0132 | Train Acc 0.9961 | Speed (samples/sec) 385.4193 | GPU 5015.5 MB\n",
      "Epoch 00007 | Step 00120 | Loss 0.0204 | Train Acc 0.9951 | Speed (samples/sec) 384.6991 | GPU 5015.5 MB\n",
      "Epoch 00007 | Step 00140 | Loss 0.0040 | Train Acc 0.9990 | Speed (samples/sec) 384.3522 | GPU 5015.5 MB\n",
      "Epoch 00007 | Step 00160 | Loss 0.0026 | Train Acc 1.0000 | Speed (samples/sec) 384.4901 | GPU 5015.5 MB\n",
      "Epoch 00007 | Step 00180 | Loss 0.0069 | Train Acc 0.9961 | Speed (samples/sec) 384.3811 | GPU 5015.5 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 552.6290\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:36<00:00,  6.68it/s]\n",
      "100%|██████████| 245/245 [01:09<00:00,  3.52it/s]\n",
      "100%|██████████| 245/245 [01:06<00:00,  3.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9063, Test Acc: 0.7738\n",
      "Epoch 00008 | Step 00000 | Loss 0.0056 | Train Acc 0.9980 | Speed (samples/sec) 384.0438 | GPU 5015.5 MB\n",
      "Epoch 00008 | Step 00020 | Loss 0.0042 | Train Acc 0.9990 | Speed (samples/sec) 384.0331 | GPU 5015.5 MB\n",
      "Epoch 00008 | Step 00040 | Loss 0.0163 | Train Acc 0.9961 | Speed (samples/sec) 383.8805 | GPU 5015.5 MB\n",
      "Epoch 00008 | Step 00060 | Loss 0.0039 | Train Acc 0.9990 | Speed (samples/sec) 383.7297 | GPU 5015.5 MB\n",
      "Epoch 00008 | Step 00080 | Loss 0.0114 | Train Acc 0.9971 | Speed (samples/sec) 383.5376 | GPU 5015.5 MB\n",
      "Epoch 00008 | Step 00100 | Loss 0.0039 | Train Acc 0.9990 | Speed (samples/sec) 383.6646 | GPU 5015.5 MB\n",
      "Epoch 00008 | Step 00120 | Loss 0.0023 | Train Acc 1.0000 | Speed (samples/sec) 383.5068 | GPU 5015.5 MB\n",
      "Epoch 00008 | Step 00140 | Loss 0.0061 | Train Acc 0.9980 | Speed (samples/sec) 383.2532 | GPU 5015.5 MB\n",
      "Epoch 00008 | Step 00160 | Loss 0.0017 | Train Acc 0.9990 | Speed (samples/sec) 382.9759 | GPU 5015.5 MB\n",
      "Epoch 00008 | Step 00180 | Loss 0.0033 | Train Acc 0.9980 | Speed (samples/sec) 382.9775 | GPU 5015.5 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 525.1499\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:34<00:00,  7.06it/s]\n",
      "100%|██████████| 245/245 [01:12<00:00,  3.37it/s]\n",
      "100%|██████████| 245/245 [01:12<00:00,  3.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9033, Test Acc: 0.7790\n",
      "Epoch 00009 | Step 00000 | Loss 0.0068 | Train Acc 0.9980 | Speed (samples/sec) 382.9382 | GPU 5015.5 MB\n",
      "Epoch 00009 | Step 00020 | Loss 0.0056 | Train Acc 0.9980 | Speed (samples/sec) 382.9585 | GPU 5015.5 MB\n",
      "Epoch 00009 | Step 00040 | Loss 0.0042 | Train Acc 0.9980 | Speed (samples/sec) 382.8385 | GPU 5015.5 MB\n",
      "Epoch 00009 | Step 00060 | Loss 0.0032 | Train Acc 0.9990 | Speed (samples/sec) 382.5593 | GPU 5015.5 MB\n",
      "Epoch 00009 | Step 00080 | Loss 0.0098 | Train Acc 0.9980 | Speed (samples/sec) 382.3707 | GPU 5015.5 MB\n",
      "Epoch 00009 | Step 00100 | Loss 0.0017 | Train Acc 0.9990 | Speed (samples/sec) 382.4267 | GPU 5015.5 MB\n",
      "Epoch 00009 | Step 00120 | Loss 0.0063 | Train Acc 0.9971 | Speed (samples/sec) 382.4746 | GPU 5015.5 MB\n",
      "Epoch 00009 | Step 00140 | Loss 0.0057 | Train Acc 0.9980 | Speed (samples/sec) 382.4061 | GPU 5015.5 MB\n",
      "Epoch 00009 | Step 00160 | Loss 0.0096 | Train Acc 0.9961 | Speed (samples/sec) 382.2498 | GPU 5020.5 MB\n",
      "Epoch 00009 | Step 00180 | Loss 0.0147 | Train Acc 0.9951 | Speed (samples/sec) 382.1279 | GPU 5020.5 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 527.4025\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:36<00:00,  6.71it/s]\n",
      "100%|██████████| 245/245 [01:12<00:00,  3.40it/s]\n",
      "100%|██████████| 245/245 [01:09<00:00,  3.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9066, Test Acc: 0.7774\n",
      "Epoch 00010 | Step 00000 | Loss 0.0013 | Train Acc 1.0000 | Speed (samples/sec) 381.8310 | GPU 5020.5 MB\n",
      "Epoch 00010 | Step 00020 | Loss 0.0040 | Train Acc 0.9980 | Speed (samples/sec) 381.5812 | GPU 5020.5 MB\n",
      "Epoch 00010 | Step 00040 | Loss 0.0101 | Train Acc 0.9980 | Speed (samples/sec) 381.3492 | GPU 5020.5 MB\n",
      "Epoch 00010 | Step 00060 | Loss 0.0058 | Train Acc 0.9980 | Speed (samples/sec) 381.1761 | GPU 5020.5 MB\n",
      "Epoch 00010 | Step 00080 | Loss 0.0091 | Train Acc 0.9980 | Speed (samples/sec) 381.1502 | GPU 5020.5 MB\n",
      "Epoch 00010 | Step 00100 | Loss 0.0038 | Train Acc 0.9980 | Speed (samples/sec) 381.0750 | GPU 5020.5 MB\n",
      "Epoch 00010 | Step 00120 | Loss 0.0139 | Train Acc 0.9971 | Speed (samples/sec) 380.9398 | GPU 5020.5 MB\n",
      "Epoch 00010 | Step 00140 | Loss 0.0065 | Train Acc 0.9980 | Speed (samples/sec) 380.7692 | GPU 5020.5 MB\n",
      "Epoch 00010 | Step 00160 | Loss 0.0194 | Train Acc 0.9961 | Speed (samples/sec) 380.8017 | GPU 5020.5 MB\n",
      "Epoch 00010 | Step 00180 | Loss 0.0066 | Train Acc 0.9980 | Speed (samples/sec) 380.8150 | GPU 5020.5 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 534.1409\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:35<00:00,  6.81it/s]\n",
      "100%|██████████| 245/245 [01:14<00:00,  3.30it/s]\n",
      "100%|██████████| 245/245 [01:10<00:00,  3.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9041, Test Acc: 0.7603\n",
      "Epoch 00011 | Step 00000 | Loss 0.0076 | Train Acc 0.9971 | Speed (samples/sec) 380.5338 | GPU 5020.5 MB\n",
      "Epoch 00011 | Step 00020 | Loss 0.0044 | Train Acc 0.9980 | Speed (samples/sec) 380.3363 | GPU 5020.5 MB\n",
      "Epoch 00011 | Step 00040 | Loss 0.0089 | Train Acc 0.9990 | Speed (samples/sec) 380.0096 | GPU 5020.5 MB\n",
      "Epoch 00011 | Step 00060 | Loss 0.0067 | Train Acc 0.9990 | Speed (samples/sec) 379.6831 | GPU 5020.5 MB\n",
      "Epoch 00011 | Step 00080 | Loss 0.0119 | Train Acc 0.9951 | Speed (samples/sec) 379.5635 | GPU 5020.5 MB\n",
      "Epoch 00011 | Step 00100 | Loss 0.0034 | Train Acc 0.9990 | Speed (samples/sec) 379.4835 | GPU 5020.5 MB\n",
      "Epoch 00011 | Step 00120 | Loss 0.0025 | Train Acc 0.9980 | Speed (samples/sec) 379.1482 | GPU 5020.5 MB\n",
      "Epoch 00011 | Step 00140 | Loss 0.0076 | Train Acc 0.9971 | Speed (samples/sec) 378.7897 | GPU 5020.5 MB\n",
      "Epoch 00011 | Step 00160 | Loss 0.0133 | Train Acc 0.9951 | Speed (samples/sec) 378.5893 | GPU 5020.5 MB\n",
      "Epoch 00011 | Step 00180 | Loss 0.0021 | Train Acc 1.0000 | Speed (samples/sec) 378.6078 | GPU 5020.5 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 554.8456\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:41<00:00,  5.83it/s]\n",
      "100%|██████████| 245/245 [01:17<00:00,  3.14it/s]\n",
      "100%|██████████| 245/245 [01:14<00:00,  3.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9039, Test Acc: 0.7499\n",
      "Epoch 00012 | Step 00000 | Loss 0.0038 | Train Acc 0.9980 | Speed (samples/sec) 378.3444 | GPU 5020.5 MB\n",
      "Epoch 00012 | Step 00020 | Loss 0.0102 | Train Acc 0.9961 | Speed (samples/sec) 378.1248 | GPU 5020.5 MB\n",
      "Epoch 00012 | Step 00040 | Loss 0.0052 | Train Acc 0.9971 | Speed (samples/sec) 377.9421 | GPU 5020.5 MB\n",
      "Epoch 00012 | Step 00060 | Loss 0.0023 | Train Acc 0.9990 | Speed (samples/sec) 377.7358 | GPU 5020.5 MB\n",
      "Epoch 00012 | Step 00080 | Loss 0.0086 | Train Acc 0.9980 | Speed (samples/sec) 377.6932 | GPU 5020.5 MB\n",
      "Epoch 00012 | Step 00100 | Loss 0.0148 | Train Acc 0.9951 | Speed (samples/sec) 377.7332 | GPU 5020.5 MB\n",
      "Epoch 00012 | Step 00120 | Loss 0.0082 | Train Acc 0.9941 | Speed (samples/sec) 377.5485 | GPU 5020.5 MB\n",
      "Epoch 00012 | Step 00140 | Loss 0.0145 | Train Acc 0.9971 | Speed (samples/sec) 377.3383 | GPU 5020.5 MB\n",
      "Epoch 00012 | Step 00160 | Loss 0.0199 | Train Acc 0.9961 | Speed (samples/sec) 377.1952 | GPU 5020.5 MB\n",
      "Epoch 00012 | Step 00180 | Loss 0.0149 | Train Acc 0.9961 | Speed (samples/sec) 377.1858 | GPU 5020.5 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 543.8358\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:37<00:00,  6.61it/s]\n",
      "100%|██████████| 245/245 [01:13<00:00,  3.32it/s]\n",
      "100%|██████████| 245/245 [01:11<00:00,  3.44it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9011, Test Acc: 0.7492\n",
      "Epoch 00013 | Step 00000 | Loss 0.0152 | Train Acc 0.9980 | Speed (samples/sec) 377.0421 | GPU 5020.5 MB\n",
      "Epoch 00013 | Step 00020 | Loss 0.0091 | Train Acc 0.9980 | Speed (samples/sec) 377.0438 | GPU 5020.5 MB\n",
      "Epoch 00013 | Step 00040 | Loss 0.0024 | Train Acc 0.9980 | Speed (samples/sec) 376.9770 | GPU 5020.5 MB\n",
      "Epoch 00013 | Step 00060 | Loss 0.0002 | Train Acc 1.0000 | Speed (samples/sec) 376.8470 | GPU 5020.5 MB\n",
      "Epoch 00013 | Step 00080 | Loss 0.0123 | Train Acc 0.9971 | Speed (samples/sec) 376.6845 | GPU 5020.5 MB\n",
      "Epoch 00013 | Step 00100 | Loss 0.0052 | Train Acc 0.9971 | Speed (samples/sec) 376.7267 | GPU 5020.5 MB\n",
      "Epoch 00013 | Step 00120 | Loss 0.0106 | Train Acc 0.9971 | Speed (samples/sec) 376.8264 | GPU 5020.5 MB\n",
      "Epoch 00013 | Step 00140 | Loss 0.0151 | Train Acc 0.9961 | Speed (samples/sec) 376.7123 | GPU 5020.5 MB\n",
      "Epoch 00013 | Step 00160 | Loss 0.0024 | Train Acc 1.0000 | Speed (samples/sec) 376.5700 | GPU 5020.5 MB\n",
      "Epoch 00013 | Step 00180 | Loss 0.0037 | Train Acc 0.9980 | Speed (samples/sec) 376.6228 | GPU 5020.5 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 530.1478\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:34<00:00,  7.07it/s]\n",
      "100%|██████████| 245/245 [01:14<00:00,  3.28it/s]\n",
      "100%|██████████| 245/245 [01:15<00:00,  3.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9034, Test Acc: 0.7770\n",
      "Epoch 00014 | Step 00000 | Loss 0.0025 | Train Acc 0.9990 | Speed (samples/sec) 376.5621 | GPU 5020.5 MB\n",
      "Epoch 00014 | Step 00020 | Loss 0.0047 | Train Acc 0.9980 | Speed (samples/sec) 376.5505 | GPU 5020.5 MB\n",
      "Epoch 00014 | Step 00040 | Loss 0.0080 | Train Acc 0.9971 | Speed (samples/sec) 376.5782 | GPU 5020.5 MB\n",
      "Epoch 00014 | Step 00060 | Loss 0.0143 | Train Acc 0.9971 | Speed (samples/sec) 376.4566 | GPU 5020.5 MB\n",
      "Epoch 00014 | Step 00080 | Loss 0.0138 | Train Acc 0.9951 | Speed (samples/sec) 376.2887 | GPU 5020.5 MB\n",
      "Epoch 00014 | Step 00100 | Loss 0.0083 | Train Acc 0.9971 | Speed (samples/sec) 376.3002 | GPU 5020.5 MB\n",
      "Epoch 00014 | Step 00120 | Loss 0.0037 | Train Acc 0.9980 | Speed (samples/sec) 376.2807 | GPU 5020.5 MB\n",
      "Epoch 00014 | Step 00140 | Loss 0.0208 | Train Acc 0.9941 | Speed (samples/sec) 376.2369 | GPU 5020.5 MB\n",
      "Epoch 00014 | Step 00160 | Loss 0.0094 | Train Acc 0.9961 | Speed (samples/sec) 376.0974 | GPU 5020.5 MB\n",
      "Epoch 00014 | Step 00180 | Loss 0.0182 | Train Acc 0.9951 | Speed (samples/sec) 375.9709 | GPU 5020.5 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 537.1304\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:35<00:00,  6.97it/s]\n",
      "100%|██████████| 245/245 [01:12<00:00,  3.40it/s]\n",
      "100%|██████████| 245/245 [01:13<00:00,  3.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9021, Test Acc: 0.7497\n",
      "Epoch 00015 | Step 00000 | Loss 0.0120 | Train Acc 0.9971 | Speed (samples/sec) 375.8478 | GPU 5020.5 MB\n",
      "Epoch 00015 | Step 00020 | Loss 0.0201 | Train Acc 0.9971 | Speed (samples/sec) 375.7294 | GPU 5020.5 MB\n",
      "Epoch 00015 | Step 00040 | Loss 0.0144 | Train Acc 0.9971 | Speed (samples/sec) 375.7735 | GPU 5020.5 MB\n",
      "Epoch 00015 | Step 00060 | Loss 0.0049 | Train Acc 0.9971 | Speed (samples/sec) 375.7457 | GPU 5020.5 MB\n",
      "Epoch 00015 | Step 00080 | Loss 0.0071 | Train Acc 0.9971 | Speed (samples/sec) 375.8472 | GPU 5020.5 MB\n",
      "Epoch 00015 | Step 00100 | Loss 0.0017 | Train Acc 1.0000 | Speed (samples/sec) 376.1227 | GPU 5020.5 MB\n",
      "Epoch 00015 | Step 00120 | Loss 0.0088 | Train Acc 0.9961 | Speed (samples/sec) 376.2570 | GPU 5020.5 MB\n",
      "Epoch 00015 | Step 00140 | Loss 0.0086 | Train Acc 0.9980 | Speed (samples/sec) 376.3761 | GPU 5020.5 MB\n",
      "Epoch 00015 | Step 00160 | Loss 0.0032 | Train Acc 0.9990 | Speed (samples/sec) 376.5314 | GPU 5020.5 MB\n",
      "Epoch 00015 | Step 00180 | Loss 0.0014 | Train Acc 1.0000 | Speed (samples/sec) 376.6604 | GPU 5020.5 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 506.9466\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:31<00:00,  7.87it/s]\n",
      "100%|██████████| 245/245 [01:08<00:00,  3.55it/s]\n",
      "100%|██████████| 245/245 [01:09<00:00,  3.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9036, Test Acc: 0.7786\n",
      "Epoch 00016 | Step 00000 | Loss 0.0069 | Train Acc 0.9980 | Speed (samples/sec) 376.6211 | GPU 5020.5 MB\n",
      "Epoch 00016 | Step 00020 | Loss 0.0093 | Train Acc 0.9971 | Speed (samples/sec) 376.7125 | GPU 5020.5 MB\n",
      "Epoch 00016 | Step 00040 | Loss 0.0032 | Train Acc 0.9980 | Speed (samples/sec) 376.8600 | GPU 5020.5 MB\n",
      "Epoch 00016 | Step 00060 | Loss 0.0249 | Train Acc 0.9941 | Speed (samples/sec) 376.9745 | GPU 5020.5 MB\n",
      "Epoch 00016 | Step 00080 | Loss 0.0134 | Train Acc 0.9980 | Speed (samples/sec) 377.2135 | GPU 5020.5 MB\n",
      "Epoch 00016 | Step 00100 | Loss 0.0104 | Train Acc 0.9971 | Speed (samples/sec) 377.4002 | GPU 5020.5 MB\n",
      "Epoch 00016 | Step 00120 | Loss 0.0080 | Train Acc 0.9971 | Speed (samples/sec) 377.6488 | GPU 5020.5 MB\n",
      "Epoch 00016 | Step 00140 | Loss 0.0068 | Train Acc 0.9971 | Speed (samples/sec) 377.7733 | GPU 5020.5 MB\n",
      "Epoch 00016 | Step 00160 | Loss 0.0122 | Train Acc 0.9971 | Speed (samples/sec) 377.9212 | GPU 5020.5 MB\n",
      "Epoch 00016 | Step 00180 | Loss 0.0232 | Train Acc 0.9932 | Speed (samples/sec) 378.0543 | GPU 5020.5 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 490.6963\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:32<00:00,  7.65it/s]\n",
      "100%|██████████| 245/245 [01:08<00:00,  3.59it/s]\n",
      "100%|██████████| 245/245 [01:09<00:00,  3.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9036, Test Acc: 0.7788\n",
      "Epoch 00017 | Step 00000 | Loss 0.0066 | Train Acc 0.9971 | Speed (samples/sec) 377.9873 | GPU 5020.5 MB\n",
      "Epoch 00017 | Step 00020 | Loss 0.0246 | Train Acc 0.9971 | Speed (samples/sec) 378.0818 | GPU 5020.5 MB\n",
      "Epoch 00017 | Step 00040 | Loss 0.0020 | Train Acc 1.0000 | Speed (samples/sec) 378.1776 | GPU 5020.5 MB\n",
      "Epoch 00017 | Step 00060 | Loss 0.0032 | Train Acc 0.9990 | Speed (samples/sec) 378.2941 | GPU 5020.5 MB\n",
      "Epoch 00017 | Step 00080 | Loss 0.0118 | Train Acc 0.9951 | Speed (samples/sec) 378.4173 | GPU 5020.5 MB\n",
      "Epoch 00017 | Step 00100 | Loss 0.0064 | Train Acc 0.9980 | Speed (samples/sec) 378.5905 | GPU 5020.5 MB\n",
      "Epoch 00017 | Step 00120 | Loss 0.0094 | Train Acc 0.9971 | Speed (samples/sec) 378.6997 | GPU 5020.5 MB\n",
      "Epoch 00017 | Step 00140 | Loss 0.0051 | Train Acc 0.9980 | Speed (samples/sec) 378.8872 | GPU 5020.5 MB\n",
      "Epoch 00017 | Step 00160 | Loss 0.0072 | Train Acc 0.9980 | Speed (samples/sec) 379.0796 | GPU 5020.5 MB\n",
      "Epoch 00017 | Step 00180 | Loss 0.0063 | Train Acc 0.9980 | Speed (samples/sec) 379.1760 | GPU 5020.5 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 493.2593\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:31<00:00,  7.67it/s]\n",
      "100%|██████████| 245/245 [01:05<00:00,  3.71it/s]\n",
      "100%|██████████| 245/245 [01:05<00:00,  3.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9038, Test Acc: 0.7824\n",
      "Epoch 00018 | Step 00000 | Loss 0.0092 | Train Acc 0.9961 | Speed (samples/sec) 379.0836 | GPU 5020.5 MB\n",
      "Epoch 00018 | Step 00020 | Loss 0.0031 | Train Acc 0.9990 | Speed (samples/sec) 379.2423 | GPU 5020.5 MB\n",
      "Epoch 00018 | Step 00040 | Loss 0.0061 | Train Acc 0.9980 | Speed (samples/sec) 379.3589 | GPU 5020.5 MB\n",
      "Epoch 00018 | Step 00060 | Loss 0.0122 | Train Acc 0.9971 | Speed (samples/sec) 379.5107 | GPU 5020.5 MB\n",
      "Epoch 00018 | Step 00080 | Loss 0.0049 | Train Acc 0.9980 | Speed (samples/sec) 379.5984 | GPU 5020.5 MB\n",
      "Epoch 00018 | Step 00100 | Loss 0.0085 | Train Acc 0.9971 | Speed (samples/sec) 379.7692 | GPU 5020.5 MB\n",
      "Epoch 00018 | Step 00120 | Loss 0.0028 | Train Acc 1.0000 | Speed (samples/sec) 379.9080 | GPU 5020.5 MB\n",
      "Epoch 00018 | Step 00140 | Loss 0.0079 | Train Acc 0.9971 | Speed (samples/sec) 380.0619 | GPU 5020.5 MB\n",
      "Epoch 00018 | Step 00160 | Loss 0.0126 | Train Acc 0.9951 | Speed (samples/sec) 380.1544 | GPU 5020.5 MB\n",
      "Epoch 00018 | Step 00180 | Loss 0.0107 | Train Acc 0.9971 | Speed (samples/sec) 380.2880 | GPU 5020.5 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 489.1308\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:31<00:00,  7.73it/s]\n",
      "100%|██████████| 245/245 [01:08<00:00,  3.57it/s]\n",
      "100%|██████████| 245/245 [01:06<00:00,  3.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9044, Test Acc: 0.7779\n",
      "Epoch 00019 | Step 00000 | Loss 0.0056 | Train Acc 0.9980 | Speed (samples/sec) 380.2605 | GPU 5020.5 MB\n",
      "Epoch 00019 | Step 00020 | Loss 0.0268 | Train Acc 0.9971 | Speed (samples/sec) 380.3086 | GPU 5020.5 MB\n",
      "Epoch 00019 | Step 00040 | Loss 0.0195 | Train Acc 0.9941 | Speed (samples/sec) 380.3611 | GPU 5020.5 MB\n",
      "Epoch 00019 | Step 00060 | Loss 0.0251 | Train Acc 0.9941 | Speed (samples/sec) 380.4189 | GPU 5020.5 MB\n",
      "Epoch 00019 | Step 00080 | Loss 0.0031 | Train Acc 0.9990 | Speed (samples/sec) 380.4356 | GPU 5020.5 MB\n",
      "Epoch 00019 | Step 00100 | Loss 0.0305 | Train Acc 0.9951 | Speed (samples/sec) 380.5671 | GPU 5020.5 MB\n",
      "Epoch 00019 | Step 00120 | Loss 0.0307 | Train Acc 0.9941 | Speed (samples/sec) 380.6558 | GPU 5020.5 MB\n",
      "Epoch 00019 | Step 00140 | Loss 0.0012 | Train Acc 1.0000 | Speed (samples/sec) 380.7576 | GPU 5020.5 MB\n",
      "Epoch 00019 | Step 00160 | Loss 0.0211 | Train Acc 0.9971 | Speed (samples/sec) 380.8744 | GPU 5020.5 MB\n",
      "Epoch 00019 | Step 00180 | Loss 0.0103 | Train Acc 0.9951 | Speed (samples/sec) 380.9223 | GPU 5020.5 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 500.6559\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:30<00:00,  7.95it/s]\n",
      "100%|██████████| 245/245 [01:05<00:00,  3.76it/s]\n",
      "100%|██████████| 245/245 [01:05<00:00,  3.76it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9035, Test Acc: 0.7779\n",
      "Avg epoch time: 662.7364188671112\n",
      "Best val acc 0.9148, best test acc: 0.7898\n"
     ]
    }
   ],
   "source": [
    "g, num_labels = load_ogb('ogbn-products')\n",
    "g.ndata.pop('features')\n",
    "\n",
    "in_feats = 128\n",
    "eigen_vals, eigen_vecs = get_eigen(g, in_feats, 'ogbn-products')\n",
    "nfeat = th.tensor(eigen_vecs * np.sqrt(eigen_vals).reshape((1, len(eigen_vals))), dtype=th.float32)\n",
    "labels = g.ndata.pop('labels')\n",
    "\n",
    "hyperparams = {\n",
    "    'batch_size': 1024,\n",
    "    'num_workers': 0,\n",
    "    'num_hidden': 256,\n",
    "    'num_layers': 3,\n",
    "    'dropout': 0.5,\n",
    "    'lr': 0.003,\n",
    "    'sparse_lr': 0.003,\n",
    "    'num_epochs': 20,\n",
    "    'fanouts': [15,10,5],\n",
    "    'eval_batch_size': 10000\n",
    "}\n",
    "device = th.device('cuda:2')\n",
    "\n",
    "# Pack data\n",
    "data = g, nfeat, labels\n",
    "# Define model\n",
    "model = SAGE(in_feats, hyperparams['num_hidden'], num_labels,\n",
    "             hyperparams['num_layers'], F.relu, hyperparams['dropout'])\n",
    "train_ft_embed(model, data, hyperparams, device, 1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}