Peephole_LSTM_from_scratch.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Peephole LSTM from scratch.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/tumble-weed/LSTM-from-scratch/blob/master/Peephole_LSTM_from_scratch.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G2fLbjWENgtH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "import numpy as np"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sJUYEbNfNQbj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "n_time_steps,batch_size,embedding_size = 7,32,10\n",
        "x = torch.zeros(n_time_steps,batch_size,embedding_size)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wTtjMPBSNir5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Peephole_LSTM(torch.nn.Module):\n",
        "    def __init__(self,input_size,state_size,squash_fn = torch.nn.Sigmoid):\n",
        "        Peephole_LSTM.__init__(self)\n",
        "        self.squash_fn = squash_fn\n",
        "        self.layers = {}\n",
        "\n",
        "        \n",
        "        self.layers['i from x'] = torch.nn.Linear(input_size,state_size) \n",
        "        self.layers['i from h'] = torch.nn.Linear(state_size,state_size,bias=False)\n",
        "        self.layers['i from c'] = torch.nn.Linear(state_size,state_size,bias=False)\n",
        "\n",
        "        self.layers['o from x'] = torch.nn.Linear(input_size,state_size) \n",
        "        self.layers['o from h'] = torch.nn.Linear(state_size,state_size,bias=False)\n",
        "        self.layers['o from c'] = torch.nn.Linear(state_size,state_size,bias=False)\n",
        "        \n",
        "        self.layers['r from x'] = torch.nn.Linear(input_size,state_size) \n",
        "        self.layers['r from h'] = torch.nn.Linear(state_size,state_size,bias=False)\n",
        "        self.layers['r from c'] = torch.nn.Linear(state_size,state_size,bias=False)\n",
        "        \n",
        "        self.layers['c from x'] = torch.nn.Linear(input_size,state_size) \n",
        "        \n",
        "        self.layers['c from h'] = torch.nn.Linear(state_size,state_size,bias=False)\n",
        "\n",
        "        pass\n",
        "    def forward(self,x,(h,c)):\n",
        "        outs = []\n",
        "        for x_t in x:\n",
        "            \n",
        "            i = self.layers['i from x'](x_t) + self.layers['i from h'](h) + self.layers['i from c'](c)\n",
        "            i = torch.nn.sigmoid(i)\n",
        "            \n",
        "            r = self.layers['r from x'](x_t) + self.layers['r from h'](h) + self.layers['r from c'](c)\n",
        "            r = torch.nn.sigmoid(r)\n",
        "            \n",
        "            delta_c = self.layers['c from x'](x_t) + self.layers['c from h'](h)\n",
        "            delta_c = self.squash_fn(delta_fn)\n",
        "\n",
        "            c = delta_c * i + c *r\n",
        "            \n",
        "            \n",
        "            o = self.layers['o from x'](x_t) + self.layers['o from h'](h) + self.layers['o from c'](c)\n",
        "            o = torch.nn.sigmoid(o)\n",
        "            \n",
        "            h = o* self.squash_fn(c)\n",
        "            \n",
        "            outs.append(h)\n",
        "        out = torch.stack(outs,0)\n",
        "        return = out,(h,c)\n",
        "            \n",
        "            "
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}