{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 401 CNN\n",
    "\n",
    "View more, visit my tutorial page: https://morvanzhou.github.io/tutorials/\n",
    "My Youtube Channel: https://www.youtube.com/user/MorvanZhou\n",
    "\n",
    "Dependencies:\n",
    "* torch: 0.1.11\n",
    "* torchvision\n",
    "* matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "import torch.utils.data as Data\n",
    "import torchvision\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7f33c006fe50>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.manual_seed(1)    # reproducible"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyper Parameters\n",
    "EPOCH = 1               # train the training data n times, to save time, we just train 1 epoch\n",
    "BATCH_SIZE = 50\n",
    "LR = 0.001              # learning rate\n",
    "DOWNLOAD_MNIST = True   # set to False if you have downloaded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\n",
      "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\n",
      "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz\n",
      "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz\n",
      "Processing...\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "# Mnist digits dataset\n",
    "train_data = torchvision.datasets.MNIST(\n",
    "    root='./mnist/',\n",
    "    train=True,                                     # this is training data\n",
    "    transform=torchvision.transforms.ToTensor(),    # Converts a PIL.Image or numpy.ndarray to\n",
    "                                                    # torch.FloatTensor of shape (C x H x W) and normalize in the range [0.0, 1.0]\n",
    "    download=DOWNLOAD_MNIST,                        # download it if you don't have it\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([60000, 28, 28])\n",
      "torch.Size([60000])\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 5252 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f33a14e6518>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot one example\n",
    "print(train_data.train_data.size())                 # (60000, 28, 28)\n",
    "print(train_data.train_labels.size())               # (60000)\n",
    "plt.imshow(train_data.train_data[0].numpy(), cmap='gray')\n",
    "plt.title('%i' % train_data.train_labels[0])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data Loader for easy mini-batch return in training, the image batch shape will be (50, 1, 28, 28)\n",
    "train_loader = Data.DataLoader(dataset=train_data, batch_size=BATCH_SIZE, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert test data into Variable, pick 2000 samples to speed up testing\n",
    "test_data = torchvision.datasets.MNIST(root='./mnist/', train=False)\n",
    "test_x = Variable(torch.unsqueeze(test_data.test_data, dim=1)).type(torch.FloatTensor)[:2000]/255.   # shape from (2000, 28, 28) to (2000, 1, 28, 28), value in range(0,1)\n",
    "test_y = test_data.test_labels[:2000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CNN(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(CNN, self).__init__()\n",
    "        self.conv1 = nn.Sequential(         # input shape (1, 28, 28)\n",
    "            nn.Conv2d(\n",
    "                in_channels=1,              # input height\n",
    "                out_channels=16,            # n_filters\n",
    "                kernel_size=5,              # filter size\n",
    "                stride=1,                   # filter movement/step\n",
    "                padding=2,                  # if want same width and length of this image after con2d, padding=(kernel_size-1)/2 if stride=1\n",
    "            ),                              # output shape (16, 28, 28)\n",
    "            nn.ReLU(),                      # activation\n",
    "            nn.MaxPool2d(kernel_size=2),    # choose max value in 2x2 area, output shape (16, 14, 14)\n",
    "        )\n",
    "        self.conv2 = nn.Sequential(         # input shape (1, 28, 28)\n",
    "            nn.Conv2d(16, 32, 5, 1, 2),     # output shape (32, 14, 14)\n",
    "            nn.ReLU(),                      # activation\n",
    "            nn.MaxPool2d(2),                # output shape (32, 7, 7)\n",
    "        )\n",
    "        self.out = nn.Linear(32 * 7 * 7, 10)   # fully connected layer, output 10 classes\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.conv1(x)\n",
    "        x = self.conv2(x)\n",
    "        x = x.view(x.size(0), -1)           # flatten the output of conv2 to (batch_size, 32 * 7 * 7)\n",
    "        output = self.out(x)\n",
    "        return output, x    # return x for visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CNN(\n",
      "  (conv1): Sequential(\n",
      "    (0): Conv2d(1, 16, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))\n",
      "    (1): ReLU()\n",
      "    (2): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "  )\n",
      "  (conv2): Sequential(\n",
      "    (0): Conv2d(16, 32, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))\n",
      "    (1): ReLU()\n",
      "    (2): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "  )\n",
      "  (out): Linear(in_features=1568, out_features=10, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "cnn = CNN()\n",
    "print(cnn)  # net architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = torch.optim.Adam(cnn.parameters(), lr=LR)   # optimize all cnn parameters\n",
    "loss_func = nn.CrossEntropyLoss()                       # the target label is not one-hotted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yoshitaka-i/.conda/envs/tensor/lib/python3.5/site-packages/ipykernel_launcher.py:29: UserWarning: invalid index of a 0-dim tensor. This will be an error in PyTorch 0.5. Use tensor.item() to convert a 0-dim tensor to a Python number\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  0 | train loss: 2.3101 | test accuracy: 0.20\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 24552 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f3344d2df60>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  0 | train loss: 0.6699 | test accuracy: 0.88\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 22468 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f3344d2f1d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  0 | train loss: 0.0701 | test accuracy: 0.94\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 22956 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f3340189c18>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  0 | train loss: 0.1329 | test accuracy: 0.94\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 21816 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f33401d2e10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  0 | train loss: 0.0395 | test accuracy: 0.96\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 23148 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f33504f0160>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  0 | train loss: 0.0199 | test accuracy: 0.96\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 22108 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f3350563320>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  0 | train loss: 0.1077 | test accuracy: 0.97\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 22220 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f3340435668>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  0 | train loss: 0.0391 | test accuracy: 0.97\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 22484 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f33503aa240>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  0 | train loss: 0.0789 | test accuracy: 0.97\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 21984 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f33504d67b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  0 | train loss: 0.1067 | test accuracy: 0.97\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 21524 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f33402252e8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  0 | train loss: 0.0277 | test accuracy: 0.97\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 21668 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f334ff549b0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  0 | train loss: 0.0251 | test accuracy: 0.98\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 21340 characters]\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f33504c8470>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# following function (plot_with_labels) is for visualization, can be ignored if not interested\n",
    "from matplotlib import cm\n",
    "try: from sklearn.manifold import TSNE; HAS_SK = True\n",
    "except: HAS_SK = False; print('Please install sklearn for layer visualization')\n",
    "def plot_with_labels(lowDWeights, labels):\n",
    "    plt.cla()\n",
    "    X, Y = lowDWeights[:, 0], lowDWeights[:, 1]\n",
    "    for x, y, s in zip(X, Y, labels):\n",
    "        c = cm.rainbow(int(255 * s / 9)); plt.text(x, y, s, backgroundcolor=c, fontsize=9)\n",
    "    plt.xlim(X.min(), X.max()); plt.ylim(Y.min(), Y.max()); plt.title('Visualize last layer'); plt.show(); plt.pause(0.01)\n",
    "\n",
    "plt.ion()\n",
    "# training and testing\n",
    "for epoch in range(EPOCH):\n",
    "    for step, (x, y) in enumerate(train_loader):   # gives batch data, normalize x when iterate train_loader\n",
    "        b_x = Variable(x)   # batch x\n",
    "        b_y = Variable(y)   # batch y\n",
    "\n",
    "        output = cnn(b_x)[0]               # cnn output\n",
    "        loss = loss_func(output, b_y)   # cross entropy loss\n",
    "        optimizer.zero_grad()           # clear gradients for this training step\n",
    "        loss.backward()                 # backpropagation, compute gradients\n",
    "        optimizer.step()                # apply gradients\n",
    "\n",
    "        if step % 100 == 0:\n",
    "            test_output, last_layer = cnn(test_x)\n",
    "            pred_y = torch.max(test_output, 1)[1].data.squeeze()\n",
    "            accuracy = (pred_y == test_y).sum().item() / float(test_y.size(0))\n",
    "            print('Epoch: ', epoch, '| train loss: %.4f' % loss.data[0], '| test accuracy: %.2f' % accuracy)\n",
    "            if HAS_SK:\n",
    "                # Visualization of trained flatten layer (T-SNE)\n",
    "                tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000)\n",
    "                plot_only = 500\n",
    "                low_dim_embs = tsne.fit_transform(last_layer.data.numpy()[:plot_only, :])\n",
    "                labels = test_y.numpy()[:plot_only]\n",
    "                plot_with_labels(low_dim_embs, labels)\n",
    "plt.ioff()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[7 2 1 0 4 1 4 9 5 9] prediction number\n",
      "[7 2 1 0 4 1 4 9 5 9] real number\n"
     ]
    }
   ],
   "source": [
    "# print 10 predictions from test data\n",
    "test_output, _ = cnn(test_x[:10])\n",
    "pred_y = torch.max(test_output, 1)[1].data.numpy().squeeze()\n",
    "print(pred_y, 'prediction number')\n",
    "print(test_y[:10].numpy(), 'real number')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tensor",
   "language": "python",
   "name": "tensor"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
