202003_models/MobileNetV1_CIFAR10.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": " MobileNetV1_CIFAR10.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyOhXxfxWotTtA6g0wijo51O",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "979a2646dce940cea3c9b5a2fa9bde14": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_05e46add937e44638c69f762615c18a1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1c99991e23e640bd947150c0ebd510a7",
              "IPY_MODEL_1b68b1a9a4ff4b959e6a0c465df9e9e9"
            ]
          }
        },
        "05e46add937e44638c69f762615c18a1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1c99991e23e640bd947150c0ebd510a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "IntProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_880d2159e2a94a2c9577258e124abca3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "IntProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6cf6ced087f14fc08ed34d57ddf73772"
          }
        },
        "1b68b1a9a4ff4b959e6a0c465df9e9e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fc02f6a5547a4d18a40f715c12f23e04",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "170500096it [00:30, 14348784.96it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4e8985dda761456ca90424d22fd39c07"
          }
        },
        "880d2159e2a94a2c9577258e124abca3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6cf6ced087f14fc08ed34d57ddf73772": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fc02f6a5547a4d18a40f715c12f23e04": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4e8985dda761456ca90424d22fd39c07": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/OUCTheoryGroup/colab_demo/blob/master/MobileNetV1_CIFAR10.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Vyx9j3WEsCVq",
        "colab_type": "text"
      },
      "source": [
        "## MobileNet V1\n",
        "\n",
        "MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications, 2017\n",
        "\n",
        "VGG，GoogleNet，ResNet进一步提高CNN的性能。但是到ResNet，网络已经达到152层，模型大小动辄几百300MB+。这种巨大的存储和计算开销，严重限制了CNN在某些低功耗领域的应用。在实际应用中受限于硬件运算能力与存储（比如几乎不可能在手机芯片上跑ResNet-152），**所以必须有一种能在算法层面有效的压缩存储和计算量的方法。而MobileNet/ShuffleNet正为我们打开这扇窗。**\n",
        "\n",
        "Mobilenet v1是Google于2017年发布的网络架构，旨在充分利用移动设备和嵌入式应用的有限的资源，有效地最大化模型的准确性，以满足有限资源下的各种应用案例。Mobilenet v1核心是把卷积拆分为Depthwise+Pointwise两部分。\n",
        "\n",
        "Depthwise 处理一个三通道的图像，使用3×3的卷积核，完全在二维平面上进行，卷积核的数量与输入的通道数相同，所以经过运算会生成3个feature map。卷积的参数为： 3 × 3 × 3 = 27，如下所示：\n",
        "\n",
        "![Depthwise卷积](https://gaopursuit.oss-cn-beijing.aliyuncs.com/202003/20200308135119.jpg)\n",
        "\n",
        "Pointwise 不同之处在于卷积核的尺寸为1×1×k，k为输入通道的数量。所以，这里的卷积运算会将上一层的feature map加权融合，有几个filter就有几个feature map，参数数量为：1 × 1 × 3 × 4 = 12，如下图所示：\n",
        "\n",
        "![替代文字](https://gaopursuit.oss-cn-beijing.aliyuncs.com/202003/20200308135426.jpg)\n",
        "\n",
        "因此，可以看出，同样得到4个feature map，使用Depthwise+Pointwise处理，参数数量可以大大降低。\n",
        "\n",
        "下面是 MobileNet 可分离卷积部分的代码：\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dhpe8ufGr-Jx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import torch.optim as optim\n",
        "\n",
        "class Block(nn.Module):\n",
        "    '''Depthwise conv + Pointwise conv'''\n",
        "    def __init__(self, in_planes, out_planes, stride=1):\n",
        "        super(Block, self).__init__()\n",
        "        # Depthwise 卷积，3*3 的卷积核，分为 in_planes，即各层单独进行卷积\n",
        "        self.conv1 = nn.Conv2d(in_planes, in_planes, kernel_size=3, stride=stride, padding=1, groups=in_planes, bias=False)\n",
        "        self.bn1 = nn.BatchNorm2d(in_planes)\n",
        "        # Pointwise 卷积，1*1 的卷积核\n",
        "        self.conv2 = nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1, padding=0, bias=False)\n",
        "        self.bn2 = nn.BatchNorm2d(out_planes)\n",
        "\n",
        "    def forward(self, x):\n",
        "        out = F.relu(self.bn1(self.conv1(x)))\n",
        "        out = F.relu(self.bn2(self.conv2(out)))\n",
        "        return out"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FBLkv5BBuuaE",
        "colab_type": "text"
      },
      "source": [
        "## 创建 DataLoader"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xmJhJ7C2uyvC",
        "colab_type": "code",
        "outputId": "6795f13b-65b0-4a1c-e5ae-42867bb735ef",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 100,
          "referenced_widgets": [
            "979a2646dce940cea3c9b5a2fa9bde14",
            "05e46add937e44638c69f762615c18a1",
            "1c99991e23e640bd947150c0ebd510a7",
            "1b68b1a9a4ff4b959e6a0c465df9e9e9",
            "880d2159e2a94a2c9577258e124abca3",
            "6cf6ced087f14fc08ed34d57ddf73772",
            "fc02f6a5547a4d18a40f715c12f23e04",
            "4e8985dda761456ca90424d22fd39c07"
          ]
        }
      },
      "source": [
        "# 使用GPU训练，可以在菜单 \"代码执行工具\" -> \"更改运行时类型\" 里进行设置\n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "transform_train = transforms.Compose([\n",
        "    transforms.RandomCrop(32, padding=4),\n",
        "    transforms.RandomHorizontalFlip(),\n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))])\n",
        "\n",
        "transform_test = transforms.Compose([\n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))])\n",
        "\n",
        "trainset = torchvision.datasets.CIFAR10(root='./data', train=True,  download=True, transform=transform_train)\n",
        "testset  = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test)\n",
        "\n",
        "trainloader = torch.utils.data.DataLoader(trainset, batch_size=128, shuffle=True, num_workers=2)\n",
        "testloader = torch.utils.data.DataLoader(testset, batch_size=128, shuffle=False, num_workers=2)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./data/cifar-10-python.tar.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "979a2646dce940cea3c9b5a2fa9bde14",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting ./data/cifar-10-python.tar.gz to ./data\n",
            "Files already downloaded and verified\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Bb_-Z17Eu-GY",
        "colab_type": "text"
      },
      "source": [
        "## 创建 MobileNetV1 网络\n",
        "\n",
        "32×32×3 ==> \n",
        "\n",
        "32×32×32 ==> 32×32×64 ==> 16×16×128 ==> 16×16×128 ==> \n",
        "\n",
        "8×8×256 ==> 8×8×256 ==> 4×4×512 ==> 4×4×512 ==>\n",
        "\n",
        "2×2×1024 ==> 2×2×1024\n",
        "\n",
        "接下来为均值 pooling ==> 1×1×1024\n",
        "\n",
        "最后全连接到 10个输出节点"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9s_SAyhnvEXO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class MobileNetV1(nn.Module):\n",
        "    # (128,2) means conv planes=128, stride=2\n",
        "    cfg = [(64,1), (128,2), (128,1), (256,2), (256,1), (512,2), (512,1), \n",
        "           (1024,2), (1024,1)]\n",
        "\n",
        "    def __init__(self, num_classes=10):\n",
        "        super(MobileNetV1, self).__init__()\n",
        "        self.conv1 = nn.Conv2d(3, 32, kernel_size=3, stride=1, padding=1, bias=False)\n",
        "        self.bn1 = nn.BatchNorm2d(32)\n",
        "        self.layers = self._make_layers(in_planes=32)\n",
        "        self.linear = nn.Linear(1024, num_classes)\n",
        "\n",
        "    def _make_layers(self, in_planes):\n",
        "        layers = []\n",
        "        for x in self.cfg:\n",
        "            out_planes = x[0]\n",
        "            stride = x[1]\n",
        "            layers.append(Block(in_planes, out_planes, stride))\n",
        "            in_planes = out_planes\n",
        "        return nn.Sequential(*layers)\n",
        "\n",
        "    def forward(self, x):\n",
        "        out = F.relu(self.bn1(self.conv1(x)))\n",
        "        out = self.layers(out)\n",
        "        out = F.avg_pool2d(out, 2)\n",
        "        out = out.view(out.size(0), -1)\n",
        "        out = self.linear(out)\n",
        "        return out"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KV1gTYo6YYl9",
        "colab_type": "text"
      },
      "source": [
        "实例化网络"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tr03keecYaJF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 网络放到GPU上\n",
        "net = MobileNetV1().to(device)\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "optimizer = optim.Adam(net.parameters(), lr=0.001)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "On2sLtOCYRMP",
        "colab_type": "text"
      },
      "source": [
        "## 模型训练"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yJB0a7J-YSv8",
        "colab_type": "code",
        "outputId": "5c25b46c-9865-47a4-aa0b-a46ffb2b176e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 714
        }
      },
      "source": [
        "for epoch in range(10):  # 重复多轮训练\n",
        "    for i, (inputs, labels) in enumerate(trainloader):\n",
        "        inputs = inputs.to(device)\n",
        "        labels = labels.to(device)\n",
        "        # 优化器梯度归零\n",
        "        optimizer.zero_grad()\n",
        "        # 正向传播 +　反向传播 + 优化 \n",
        "        outputs = net(inputs)\n",
        "        loss = criterion(outputs, labels)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        # 输出统计信息\n",
        "        if i % 100 == 0:   \n",
        "            print('Epoch: %d Minibatch: %5d loss: %.3f' %(epoch + 1, i + 1, loss.item()))\n",
        "\n",
        "print('Finished Training')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 1 Minibatch:     1 loss: 2.287\n",
            "Epoch: 1 Minibatch:   101 loss: 1.708\n",
            "Epoch: 1 Minibatch:   201 loss: 1.609\n",
            "Epoch: 1 Minibatch:   301 loss: 1.589\n",
            "Epoch: 2 Minibatch:     1 loss: 1.358\n",
            "Epoch: 2 Minibatch:   101 loss: 1.453\n",
            "Epoch: 2 Minibatch:   201 loss: 1.202\n",
            "Epoch: 2 Minibatch:   301 loss: 1.334\n",
            "Epoch: 3 Minibatch:     1 loss: 1.081\n",
            "Epoch: 3 Minibatch:   101 loss: 1.158\n",
            "Epoch: 3 Minibatch:   201 loss: 1.089\n",
            "Epoch: 3 Minibatch:   301 loss: 1.050\n",
            "Epoch: 4 Minibatch:     1 loss: 1.072\n",
            "Epoch: 4 Minibatch:   101 loss: 0.967\n",
            "Epoch: 4 Minibatch:   201 loss: 1.073\n",
            "Epoch: 4 Minibatch:   301 loss: 0.984\n",
            "Epoch: 5 Minibatch:     1 loss: 0.772\n",
            "Epoch: 5 Minibatch:   101 loss: 0.765\n",
            "Epoch: 5 Minibatch:   201 loss: 0.950\n",
            "Epoch: 5 Minibatch:   301 loss: 0.931\n",
            "Epoch: 6 Minibatch:     1 loss: 0.807\n",
            "Epoch: 6 Minibatch:   101 loss: 0.659\n",
            "Epoch: 6 Minibatch:   201 loss: 0.802\n",
            "Epoch: 6 Minibatch:   301 loss: 0.876\n",
            "Epoch: 7 Minibatch:     1 loss: 0.678\n",
            "Epoch: 7 Minibatch:   101 loss: 0.787\n",
            "Epoch: 7 Minibatch:   201 loss: 0.727\n",
            "Epoch: 7 Minibatch:   301 loss: 0.580\n",
            "Epoch: 8 Minibatch:     1 loss: 0.660\n",
            "Epoch: 8 Minibatch:   101 loss: 0.587\n",
            "Epoch: 8 Minibatch:   201 loss: 0.529\n",
            "Epoch: 8 Minibatch:   301 loss: 0.659\n",
            "Epoch: 9 Minibatch:     1 loss: 0.560\n",
            "Epoch: 9 Minibatch:   101 loss: 0.476\n",
            "Epoch: 9 Minibatch:   201 loss: 0.675\n",
            "Epoch: 9 Minibatch:   301 loss: 0.562\n",
            "Epoch: 10 Minibatch:     1 loss: 0.546\n",
            "Epoch: 10 Minibatch:   101 loss: 0.486\n",
            "Epoch: 10 Minibatch:   201 loss: 0.661\n",
            "Epoch: 10 Minibatch:   301 loss: 0.537\n",
            "Finished Training\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iIDFPkozZeBS",
        "colab_type": "text"
      },
      "source": [
        "## 模型测试"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kv1u01EGZgMp",
        "colab_type": "code",
        "outputId": "63523281-0f9e-40df-8399-7e098390139d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "correct = 0\n",
        "total = 0\n",
        "\n",
        "for data in testloader:\n",
        "    images, labels = data\n",
        "    images, labels = images.to(device), labels.to(device)\n",
        "    outputs = net(images)\n",
        "    _, predicted = torch.max(outputs.data, 1)\n",
        "    total += labels.size(0)\n",
        "    correct += (predicted == labels).sum().item()\n",
        "\n",
        "print('Accuracy of the network on the 10000 test images: %.2f %%' % (\n",
        "    100 * correct / total))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy of the network on the 10000 test images: 77.85 %\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}