supervised_learning_4_localization_v2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Copy of Image_Generator.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python [conda env:tensorflow113]",
      "language": "python",
      "name": "conda-env-tensorflow113-py"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.2"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/LEChaney/ML-RayCast-Experiments/blob/master/supervised_learning_4_localization_v2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "jw1arH2vxVXK",
        "outputId": "c37400f2-fd95-44ef-9b2d-232d9c4983f2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 181
        }
      },
      "source": [
        "!git clone https://github.com/LEChaney/ML-RayCast-Experiments\n",
        "%cd ML-RayCast-Experiments\n",
        "%pwd"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'ML-RayCast-Experiments'...\n",
            "remote: Enumerating objects: 73, done.\u001b[K\n",
            "remote: Counting objects:   1% (1/73)\u001b[K\rremote: Counting objects:   2% (2/73)\u001b[K\rremote: Counting objects:   4% (3/73)\u001b[K\rremote: Counting objects:   5% (4/73)\u001b[K\rremote: Counting objects:   6% (5/73)\u001b[K\rremote: Counting objects:   8% (6/73)\u001b[K\rremote: Counting objects:   9% (7/73)\u001b[K\rremote: Counting objects:  10% (8/73)\u001b[K\rremote: Counting objects:  12% (9/73)\u001b[K\rremote: Counting objects:  13% (10/73)\u001b[K\rremote: Counting objects:  15% (11/73)\u001b[K\rremote: Counting objects:  16% (12/73)\u001b[K\rremote: Counting objects:  17% (13/73)\u001b[K\rremote: Counting objects:  19% (14/73)\u001b[K\rremote: Counting objects:  20% (15/73)\u001b[K\rremote: Counting objects:  21% (16/73)\u001b[K\rremote: Counting objects:  23% (17/73)\u001b[K\rremote: Counting objects:  24% (18/73)\u001b[K\rremote: Counting objects:  26% (19/73)\u001b[K\rremote: Counting objects:  27% (20/73)\u001b[K\rremote: Counting objects:  28% (21/73)\u001b[K\rremote: Counting objects:  30% (22/73)\u001b[K\rremote: Counting objects:  31% (23/73)\u001b[K\rremote: Counting objects:  32% (24/73)\u001b[K\rremote: Counting objects:  34% (25/73)\u001b[K\rremote: Counting objects:  35% (26/73)\u001b[K\rremote: Counting objects:  36% (27/73)\u001b[K\rremote: Counting objects:  38% (28/73)\u001b[K\rremote: Counting objects:  39% (29/73)\u001b[K\rremote: Counting objects:  41% (30/73)\u001b[K\rremote: Counting objects:  42% (31/73)\u001b[K\rremote: Counting objects:  43% (32/73)\u001b[K\rremote: Counting objects:  45% (33/73)\u001b[K\rremote: Counting objects:  46% (34/73)\u001b[K\rremote: Counting objects:  47% (35/73)\u001b[K\rremote: Counting objects:  49% (36/73)\u001b[K\rremote: Counting objects:  50% (37/73)\u001b[K\rremote: Counting objects:  52% (38/73)\u001b[K\rremote: Counting objects:  53% (39/73)\u001b[K\rremote: Counting objects:  54% (40/73)\u001b[K\rremote: Counting objects:  56% (41/73)\u001b[K\rremote: Counting objects:  57% (42/73)\u001b[K\rremote: Counting objects:  58% (43/73)\u001b[K\rremote: Counting objects:  60% (44/73)\u001b[K\rremote: Counting objects:  61% (45/73)\u001b[K\rremote: Counting objects:  63% (46/73)\u001b[K\rremote: Counting objects:  64% (47/73)\u001b[K\rremote: Counting objects:  65% (48/73)\u001b[K\rremote: Counting objects:  67% (49/73)\u001b[K\rremote: Counting objects:  68% (50/73)\u001b[K\rremote: Counting objects:  69% (51/73)\u001b[K\rremote: Counting objects:  71% (52/73)\u001b[K\rremote: Counting objects:  72% (53/73)\u001b[K\rremote: Counting objects:  73% (54/73)\u001b[K\rremote: Counting objects:  75% (55/73)\u001b[K\rremote: Counting objects:  76% (56/73)\u001b[K\rremote: Counting objects:  78% (57/73)\u001b[K\rremote: Counting objects:  79% (58/73)\u001b[K\rremote: Counting objects:  80% (59/73)\u001b[K\rremote: Counting objects:  82% (60/73)\u001b[K\rremote: Counting objects:  83% (61/73)\u001b[K\rremote: Counting objects:  84% (62/73)\u001b[K\rremote: Counting objects:  86% (63/73)\u001b[K\rremote: Counting objects:  87% (64/73)\u001b[K\rremote: Counting objects:  89% (65/73)\u001b[K\rremote: Counting objects:  90% (66/73)\u001b[K\rremote: Counting objects:  91% (67/73)\u001b[K\rremote: Counting objects:  93% (68/73)\u001b[K\rremote: Counting objects:  94% (69/73)\u001b[K\rremote: Counting objects:  95% (70/73)\u001b[K\rremote: Counting objects:  97% (71/73)\u001b[K\rremote: Counting objects:  98% (72/73)\u001b[K\rremote: Counting objects: 100% (73/73)\u001b[K\rremote: Counting objects: 100% (73/73), done.\u001b[K\n",
            "remote: Compressing objects: 100% (69/69), done.\u001b[K\n",
            "remote: Total 279 (delta 29), reused 2 (delta 0), pack-reused 206\u001b[K\n",
            "Receiving objects: 100% (279/279), 44.85 MiB | 13.22 MiB/s, done.\n",
            "Resolving deltas: 100% (137/137), done.\n",
            "/content/ML-RayCast-Experiments\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'/content/ML-RayCast-Experiments'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 1
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "nR2rN0ieyYQ5",
        "colab": {}
      },
      "source": [
        "from PIL import Image\n",
        "def rotate(image_path):\n",
        "    \"\"\"\n",
        "    Rotate the given photo the amount of given degreesk, show it and save it\n",
        "    @param image_path: The path to the image to edit\n",
        "    @param degrees_to_rotate: The number of degrees to rotate the image\n",
        "    @param saved_location: Path to save the cropped image\n",
        "    \"\"\"\n",
        "    degrees_to_rotate = 180\n",
        "    image_obj = Image.open(image_path)\n",
        "    rotated_image = image_obj.rotate(degrees_to_rotate)\n",
        "    rotated_image.save(\"000.png\")\n",
        "    im = cv2.imread(\"000.png\")\n",
        "    #plt.imshow(im)\n",
        "    #plt.show()\n",
        "    return im\n",
        "    \n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "BqsexcF4qXfE",
        "outputId": "c680cc02-6d56-45b6-a996-4c42d694a164",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "import os\n",
        "os.environ['SDL_VIDEODRIVER']='dummy'\n",
        "file_directory = os.getcwd()\n",
        "os.chdir(\"/content/ML-RayCast-Experiments/Keras-FlappyBird-master/Keras-FlappyBird-master\")\n",
        "print(os.getcwd())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "error",
          "ename": "FileNotFoundError",
          "evalue": "[WinError 3] The system cannot find the path specified: '/content/ML-RayCast-Experiments/Keras-FlappyBird-master/Keras-FlappyBird-master'",
          "traceback": [
            "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
            "\u001b[1;32m<ipython-input-60-96bcbabebfe6>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0menviron\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'SDL_VIDEODRIVER'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'dummy'\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0mfile_directory\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgetcwd\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mchdir\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"/content/ML-RayCast-Experiments/Keras-FlappyBird-master/Keras-FlappyBird-master\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      5\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgetcwd\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
            "\u001b[1;31mFileNotFoundError\u001b[0m: [WinError 3] The system cannot find the path specified: '/content/ML-RayCast-Experiments/Keras-FlappyBird-master/Keras-FlappyBird-master'"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "iY7nyqTkNMef",
        "outputId": "0dd1b142-ad01-4e69-8de9-a02ef498781e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 55
        }
      },
      "source": [
        "import sys\n",
        "#sys.path.pop(1)\n",
        "sys.path.insert(1,\"/content/ML-RayCast-Experiments/Keras-FlappyBird-master/Keras-FlappyBird-master/game\")\n",
        "print(sys.path)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['C:\\\\Users\\\\Lance Chaney\\\\Desktop\\\\ml-experiments', '/content/ML-RayCast-Experiments/Keras-FlappyBird-master/Keras-FlappyBird-master/game', 'Keras-FlappyBird-master/Keras-FlappyBird-master/game', './game', './game', 'C:\\\\Anaconda3\\\\envs\\\\tensorflow113\\\\python37.zip', 'C:\\\\Anaconda3\\\\envs\\\\tensorflow113\\\\DLLs', 'C:\\\\Anaconda3\\\\envs\\\\tensorflow113\\\\lib', 'C:\\\\Anaconda3\\\\envs\\\\tensorflow113', '', 'C:\\\\Anaconda3\\\\envs\\\\tensorflow113\\\\lib\\\\site-packages', 'C:\\\\Anaconda3\\\\envs\\\\tensorflow113\\\\lib\\\\site-packages\\\\mask_rcnn-2.1-py3.7.egg', 'C:\\\\Anaconda3\\\\envs\\\\tensorflow113\\\\lib\\\\site-packages\\\\IPython\\\\extensions', 'C:\\\\Users\\\\Lance Chaney\\\\.ipython']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "F-j-vysiwaMd",
        "outputId": "92e3abef-a8fa-4c77-9210-45e2a623f1f2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 128
        }
      },
      "source": [
        "%pip install pygame"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: pygame in c:\\anaconda3\\envs\\tensorflow113\\lib\\site-packages (1.9.6)\n",
            "Note: you may need to restart the kernel to use updated packages.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "GGMYTHT4OLy4",
        "colab": {}
      },
      "source": [
        "import wrapped_flappy_bird as game\n",
        "import numpy as np\n",
        "import flappy_bird_utils\n",
        "import cv2\n",
        "import sys\n",
        "import random\n",
        "import pygame\n",
        "import pygame.surfarray as surfarray\n",
        "from pygame.locals import *\n",
        "from itertools import cycle\n",
        "import matplotlib.pyplot as plt\n",
        "import PIL.Image\n",
        "#from cStringIO import StringIO\n",
        "import IPython.display\n",
        "\n",
        "import skimage as skimage\n",
        "from skimage import transform, color, exposure\n",
        "from skimage.transform import rotate\n",
        "from skimage.viewer import ImageViewer"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "epVQmcWzpGRU",
        "outputId": "5547abaa-ec02-48c4-c7d6-b4df14788b51",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 908
        }
      },
      "source": [
        "GAME = 'bird' # the name of the game being played for log files\n",
        "CONFIG = 'nothreshold'\n",
        "ACTIONS = 2 # number of valid actions\n",
        "GAMMA = 0.99 # decay rate of past observations\n",
        "OBSERVATION = 3200. # timesteps to observe before training\n",
        "EXPLORE = 3000000. # frames over which to anneal epsilon\n",
        "FINAL_EPSILON = 0.0001 # final value of epsilon\n",
        "INITIAL_EPSILON = 0.1 # starting value of epsilon\n",
        "REPLAY_MEMORY = 50000 # number of previous transitions to remember\n",
        "BATCH = 32 # size of minibatch\n",
        "FRAME_PER_ACTION = 1\n",
        "LEARNING_RATE = 1e-4\n",
        "\n",
        "FPS = 30\n",
        "SCREENWIDTH  = 288\n",
        "SCREENHEIGHT = 512\n",
        "\n",
        "pygame.init()\n",
        "pygame.display.init()\n",
        "pygame.display.set_mode((1, 1))\n",
        "FPSCLOCK = pygame.time.Clock()\n",
        "SCREEN = pygame.Surface((SCREENWIDTH, SCREENHEIGHT)).convert_alpha()\n",
        "pygame.display.set_caption('Flappy Bird')\n",
        "\n",
        "IMAGES, SOUNDS, HITMASKS = flappy_bird_utils.load()\n",
        "PIPEGAPSIZE = 100 # gap between upper and lower part of pipe\n",
        "BASEY = SCREENHEIGHT * 0.79\n",
        "\n",
        "PLAYER_WIDTH = IMAGES['player'][0].get_width()\n",
        "PLAYER_HEIGHT = IMAGES['player'][0].get_height()\n",
        "PIPE_WIDTH = IMAGES['pipe'][0].get_width()\n",
        "PIPE_HEIGHT = IMAGES['pipe'][0].get_height()\n",
        "BACKGROUND_WIDTH = IMAGES['background'].get_width()\n",
        "\n",
        "PLAYER_INDEX_GEN = cycle([0, 1, 2, 1])\n",
        "\n",
        "img_rows , img_cols = 80, 80\n",
        "#Convert image into Black and white\n",
        "img_channels = 4 #We stack 4 frames\n",
        "\n",
        "game_state = game.GameState()\n",
        "do_nothing = np.zeros(ACTIONS)\n",
        "do_nothing[0] = 1\n",
        "game_state.frame_step(do_nothing)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([[[0, 0, 0]]], dtype=uint8), 0.1, False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 64
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GLkjBOh4Y4FG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def get_image_from_coords(data_list):\n",
        "  \n",
        "    CaseNo = int(float(data_list[0]))\n",
        "    game_state.playerx = int(float(data_list[1]))# + SCREENWIDTH /2)\n",
        "    game_state.playery = int(float(data_list[2]))# + SCREENWIDTH /2)\n",
        "    game_state.playerIndex = int(data_list[3])        \n",
        "\n",
        "    game_state.upperPipes = []\n",
        "    game_state.lowerPipes = []\n",
        "\n",
        "    for i in range(4,len(data_list),4):\n",
        "\n",
        "        newPipe = game.getRandomPipe()\n",
        "        game_state.upperPipes.append(newPipe[0])\n",
        "        game_state.upperPipes[int((i-4)/4)]['x'] = int(float(data_list[i]))# + SCREENWIDTH /2)\n",
        "        game_state.upperPipes[int((i-4)/4)]['y'] = int(float(data_list[i+1]))# + SCREENWIDTH /2)\n",
        "\n",
        "        game_state.lowerPipes.append(newPipe[1])\n",
        "        game_state.lowerPipes[int((i-4)/4)]['x'] = int(float(data_list[i+2]))# + SCREENWIDTH /2)\n",
        "        game_state.lowerPipes[int((i-4)/4)]['y'] = int(float(data_list[i+3]))# + SCREENWIDTH /2)\n",
        "\n",
        "    # draw sprites\n",
        "    SCREEN.blit(IMAGES['background'], (0,0))\n",
        "\n",
        "    for uPipe, lPipe in zip(game_state.upperPipes, game_state.lowerPipes):\n",
        "        SCREEN.blit(IMAGES['pipe'][0], (uPipe['x'], uPipe['y']))\n",
        "        SCREEN.blit(IMAGES['pipe'][1], (lPipe['x'], lPipe['y']))\n",
        "\n",
        "    SCREEN.blit(IMAGES['base'], (game_state.basex, BASEY))\n",
        "    # print score so player overlaps the score\n",
        "    # showScore(self.score)\n",
        "    SCREEN.blit(IMAGES['player'][game_state.playerIndex],\n",
        "                (game_state.playerx, game_state.playery))\n",
        "\n",
        "    image_data = pygame.surfarray.array3d(SCREEN)\n",
        "    #IPython.display.display(PIL.Image.fromarray(image_data))\n",
        "\n",
        "    image_data = skimage.color.rgb2gray(image_data)\n",
        "    image_data = skimage.transform.resize(image_data,(80,80))\n",
        "    image_data = skimage.exposure.rescale_intensity(image_data, out_range=(0,255))\n",
        "\n",
        "    image_data = image_data / 255.0\n",
        "\n",
        "    image_data = image_data.reshape(1, image_data.shape[0], image_data.shape[1], 1) #1x80x80x1\n",
        "\n",
        "    label = [game_state.playerx, game_state.playery]\n",
        "    \n",
        "    return image_data, label\n",
        "\n",
        "def gen_batch(file, count, lineNum, D):\n",
        "    \n",
        "    images=[]\n",
        "    labels=[]\n",
        "\n",
        "    count--\n",
        "\n",
        "    if count > 1:\n",
        "        continue\n",
        "    if count == 0:\n",
        "        break;\n",
        "        \n",
        "    state = []\n",
        "    state_labels = []\n",
        "\n",
        "    current = file.readline()\n",
        "\n",
        "    lineNum++\n",
        "\n",
        "    data_list = current.split(\" \")\n",
        "    data_list.pop()\n",
        "\n",
        "    image_data, state_label = get_image_from_coords(data_list)\n",
        "    state = np.stack((image_data, image_data, image_data, image_data), axis=2)\n",
        "    state = state.reshape(1, state.shape[0], state.shape[1], state.shape[2])  #1*80*80*4\n",
        "    state_label = np.array(state_label)\n",
        "    state_labels = np.stack((label, label, label, label), axis=2).flatten()\n",
        "\n",
        "    file.seek(0)\n",
        " \n",
        "    #state = state.reshape(state.shape[0],)\n",
        "            # store the transition in D\n",
        "    D.append((state, state_labels))\n",
        "\n",
        "    while len(images) < bs:  \n",
        "        \n",
        "#         num_images_in_seq = 0\n",
        "        \n",
        "#         while num_images_in_seq < 4:\n",
        "\n",
        "        current = file.readline()\n",
        "            \n",
        "#             lineNum++\n",
        "            \n",
        "            if current == \"\":\n",
        "                return\n",
        "            \n",
        "            if current[0] == \"-\":\n",
        "                continue\n",
        "\n",
        "#             if num_images_in_seq == 1:\n",
        "#                 first_line_of_next_seq = lineNum\n",
        "                \n",
        "        data_list = current.split(\" \")\n",
        "        data_list.pop()\n",
        "\n",
        "        image_data, state_label = get_image_from_coords(data_list)\n",
        "            \n",
        "#             state.append(image_data)\n",
        "#             state_labels.append(label)\n",
        "\n",
        "#             num_images_in_seq++\n",
        "            \n",
        "#             if num_images_in_seq == 4:\n",
        "#                 file.seek(first_line_of_next_seq)\n",
        "        \n",
        "        state = np.append(image_data, state[:, :, :, :3], axis=3)\n",
        "        state_labels = np.append(state_label, state_labels[:6]).flatten()\n",
        "        #state = np.array(state)\n",
        "        #state = state.reshape(state.shape[0],)\n",
        "                # store the transition in D\n",
        "        D.append((state, state_labels))\n",
        "        if len(D) > REPLAY_MEMORY:\n",
        "            D.popleft()\n",
        "\n",
        "        gen_state, gen_label = random.sample(D,1)\n",
        "        images.append(gen_state)\n",
        "        labels.append(gen_label)\n",
        "            \n",
        "    # if the data augmentation object is not None, apply it\n",
        "    if aug is not None:\n",
        "      (images, labels) = next(aug.flow(np.array(images),\n",
        "        labels, batch_size=bs))\n",
        "    \n",
        "        #running = False\n",
        "    #print(type(image_data))\n",
        "    print(\"image_data\" ,image_data)\n",
        "    print(\"image_data.shape: \", image_data.shape)\n",
        "    IPython.display.display(PIL.Image.fromarray(image_data))\n",
        "    #plt.show()\n",
        "    \n",
        "    return images, labels"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "oFd4h0qxcNXw",
        "colab": {}
      },
      "source": [
        "\n",
        "'''import numpy as np\n",
        "import sys\n",
        "import random\n",
        "import pygame\n",
        "import flappy_bird_utils\n",
        "import pygame.surfarray as surfarray\n",
        "from pygame.locals import *\n",
        "from itertools import cycle'''\n",
        "\n",
        "def csv_image_generator(filepath, bs, mode=\"train\", aug=None):\n",
        "\n",
        "    file = open(filepath,\"r\")\n",
        "    \n",
        "    count = 100\n",
        "    lineNum = 0\n",
        "    \n",
        "    # store the previous observations in replay memory\n",
        "    D = deque()\n",
        "    \n",
        "    while True:\n",
        "        \n",
        "        images, labels = get_batch(file, count, lineNum, D)\n",
        "        # yield the batch to the calling function\n",
        "        yield (np.array(images), labels)\n",
        "          \n",
        "    #print(\"bird_down: \", bird_down)      \n",
        "    file.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "RoY9XL6ERYaG",
        "outputId": "d9d2f07e-16a1-423f-9cd9-27bc80f18c47",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "file= \"data.txt\" #file path or a file in the current folder\n",
        "csv_image_generator(file, 8, mode=\"train\", aug=None)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "image_data [[[  0   0   0]\n",
            "  [  0   0   0]\n",
            "  [  0   0   0]\n",
            "  ...\n",
            "  [222 216 149]\n",
            "  [222 216 149]\n",
            "  [222 216 149]]\n",
            "\n",
            " [[  0   0   0]\n",
            "  [  0   0   0]\n",
            "  [  0   0   0]\n",
            "  ...\n",
            "  [222 216 149]\n",
            "  [222 216 149]\n",
            "  [222 216 149]]\n",
            "\n",
            " [[  0   0   0]\n",
            "  [  0   0   0]\n",
            "  [  0   0   0]\n",
            "  ...\n",
            "  [222 216 149]\n",
            "  [222 216 149]\n",
            "  [222 216 149]]\n",
            "\n",
            " ...\n",
            "\n",
            " [[  0   0   0]\n",
            "  [  0   0   0]\n",
            "  [  0   0   0]\n",
            "  ...\n",
            "  [222 216 149]\n",
            "  [222 216 149]\n",
            "  [222 216 149]]\n",
            "\n",
            " [[  0   0   0]\n",
            "  [  0   0   0]\n",
            "  [  0   0   0]\n",
            "  ...\n",
            "  [222 216 149]\n",
            "  [222 216 149]\n",
            "  [222 216 149]]\n",
            "\n",
            " [[  0   0   0]\n",
            "  [  0   0   0]\n",
            "  [  0   0   0]\n",
            "  ...\n",
            "  [222 216 149]\n",
            "  [222 216 149]\n",
            "  [222 216 149]]]\n",
            "image_data.shape:  (288, 512, 3)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAEgCAIAAABNXlwGAAAe3UlEQVR4nO3dzYskyXnH8Se1WdUv87Ij2f/AIhbba7B884DBJ7Mrg0/yghD+F/QXGINlyfgoGXTzVVqMYO3THnZ1G4Fg5yRjMMIIX+ZssfOy3dPV3SJ9iMzK54knIiuqqmd6KuP7OWh78iUysqZa8/wiIzMbAYAD9OHDD0TkX375lyLyo1/8JLnNH/3BH0dLPvvXxyLy/T98T0SO/vSvXm0X33hfue0OAABuR3vbHQCAm+drf3gkAACoFAkAwKxQ+5cjAQBApUgAAGaC2n9bJAAAqBQJAMDBK6n9f/0///0aenJYSAAAUCkSAICZo/bPIQEAQKVIAABmi9p/GgkAACpFAgAwQ9T+JUgAAFApEgCAWaH2L0cCAIBKkQAAzAS1/7ZIAABQKRIAgINXUvuH5wU9efT41XfnYJAAAKBSJAAAM8d7AnJIAABQKRIAgNmi9p9GAgCASpEAAMwQtX8JEgAAVIoEAGBWqP3LkQAAoFIkAAAzQe2/LRIAAFSKBADg4JXU/jwr1CMBAEClSAAAZo7aP4cEAACVIgHIhw8/uJF2Pv78sxtppwZ85ng9qP2nkQAAoFLtTdVih6755m922GvRtiJy+ck7N92dKnz3o/dE5GsP7opI1y9rRKTpOhHpmuHPIl0nItI0jYj84Mc/FZHu03fl5pIE5ofavwQJAAAq1e5W+c5JqCV3s1y2InJ5c52pyunxUkSWR0uRIQKEqr+Lt1QrjfDt/cZ7X1/vZXNDJ0NuME12ahvpRKQxbYf8MS5phgCy3rcz3VX90R1V/TF9DvkmZJ2xSdMr3UL4uVMnMHRtPIA5OxWaGnUW9hPo1NHjLja6Q2ptp3sl6jMRvzy0M34+2/a/KTj3X/3wLUmh9i9HAgCASjELaC/Hy4WIfHnb3ThQx0dLETlatCK23NQVX1hgCuPYV+/fFRkvFIyNdaqWN1Vt8lDDct1C2CbUy31P1FWKfhtd57q625evbkkiN+j+6Fpbp4ouPpfwX7/vkGZcJDF5S1130VW5yyLm4HpJJvLs3//8vi+jY1H7b4sEAACVIgHs5fj46La7cMCOjpYyzKTax4OQAAI1g8iO9YtINGatq8t4RLsxdbr6j8sZoq4i6OQhmRzQJwyVKnxqGWrheC9TBqta2J6vSiT6Coe6ejE0Zgr4cHi1ZXQSiapf7zw0EPd8t/4XnbtKACW1f3he0JNHjzduWQ8SAABUigRgLBZhXoEeNQ5ctSiNiJwckQB2t1y0InJ1dS0yFqXr/7EjzpnpQSIi8va9OxL/5Yyt6Skn/Y9h5D3UrXbIOdo+MbvG0bOPzNUL9SXy29hcIuNqPYdHrdXtaMNOqhJXjQ7XAPRJxDW76YLum5+/pLdxa/UVkT5n6Jk8W/a/5NzL8Z6AHBIAAFSqDTMxEtf3t5SbU+FndifqDvWz3yYxM0QPxuaOW9DDi1U8g3+5WMS7+XpHVXOnJySA3S0WrYicnV+IpEaizSTweH6I9uD+nfXPiYpbNxm2ySQAU9Rm5uT4lODnqtu5Mern3Hc4N/9H17/itk+034x9Sxwyc0SdS3T0VSuG3maWqCsHwxyqMcHs2v9u474lqP2nkQAAoFLtndNjsSVXkKjKzT/HIvbqfHb7sNwNqg97xmO17iCb+ANHR7A/66U+ARwvlxub75pxfHNIANclPUXk6vpaRFarS5H1bH0996P/r4hM/DWLyL07p+uf7Z294Zs57uVTZrb1ya+gPooex/dJt9Hla6L/eixe9TNzdH8VwZ5LpsA22+R6GJ/Xhg++i//Q39s8vVN5/4vPPYfavwQJAAAq1fbVkx9n11v5sch45XrcUGRinC73r3pox43bmp5kaqvEnIF+ydgVs00/L6IRkd9+8Tw61tHxct0T263wP+PYcWiZBLCP1dW1DDlM19GDzEC1E1JsYi/HL52uJf0Vhez2UxHFrizYMtfD7Bi676H/Fb1R08Ege22vuP8l++ZQ+5cjAQBApdowh1r6cW19Tb8RSVXobpzOVFy+fldzG0ydLhKtGCqXcZEZo9R0veB/djHEzCKQcXvvNNzZmyhj0uHlzkmoPc/SzWHS5epSRM4v1JUYXb7mvlJOP5MN1aP23xYJAAAq1X717fA+pjEBDLXuWAY3jbrKr6vs9Raynlsd/qBTQjwTw85QjkdsG72Xe/Ki32a9SGR44oqbYm1a8PsqJ/1dEfE4ZG72yOmJHn3Gdo6WSxE5v1iJiP+07d/Q1MD5MJtr89yx1NNs3DEyh0oMqieuDejZR3Ez+4xrT185yPVw2xF5vVHBAbfp5k79L9+3pPbnWaEej4KoyN88fD9a8u+f//xWegLgTdDq56gEQ6UfanCRYV6wTgl6Lo1+bol+brtPA0OFpap73Re9pbth0c6giGt8s1KNIPv+dJNFl51Povu2bsYgAewj3AFw/vJiz3bCvcSp+jd8c9w3JBMVNhWa29XE5VtP1+mJ1vRVOndHwoZ9JT5Yvp8q97u99H76vl8Tr9QnfGP9z00WnETtn0MCqEKo/f/tF5+sl/zv//2XiMi3RMgBQK36awC9/h9XXV+L2FHR7NNLdFZIPMtwTBWaechj5nmQ5m5hP8CbCAOqDyZPjM3n6ocwr3/YMNQ+6bokVC6nvA9gDxc6Abin5NvPf8rzs/Oww9CMKVJ1NtWXn7LfDZOG08v9b0ruaVFmFpz/rcktd8k7N+NuaH/8rGy9rM93/Fl/KP0Vvs63mTtriTT60+i3VL99u/W/YN8S1P7TSACV+vrv/4mI/NN//LNInwMC0gBQj/Z+uA8g9+9zpgbRlYVaYCuO0JrLEIF+bnujdzBro8XZ5xF1qgobrjT4c3EtOHfcmH6uNgkNnR4zA3134Q6As5cXYkfqB3oMevz8vefPzySqDRPfjXGFT7HmqOa6kR8mH//2cy8p0E8bNQ1kqtpstjaXBdK/Kf4e6c6tNSnKJCT32ZrfOPcL6S8FmCAzbmh7uE//J/edRO1fggQwc37mT873Xv5cROTh+0IOAOrQ3u+fBaSvy4tsuhY/WUZn9yoztUd2nUsew/ZhTHk8L7V5wrazergHdR/hDoDzl6t4xZZfmi9enIlE485jQ/a9VE3ccOLeEVEt6FH+xOWmsbN+blvu7cEmm6raNnxLG/2NVdvrVKGaH4yl+5BIVEaR8dztWau1+nqATl3mdFWCcTmgUZ+wv4K4bf/1M3c37Ruj9i9HApitUPv/7M8fi8i3f/lnt90dAG+cNjHzvbj+mn6/T2qJGocNfzbV2ShxcDUqmksnidkaCZmhXxEZE0B8Hnafce0RCWAPYf5PSAC5b45Zkyn8nj57IWKq+OH6k6vZfS0f6BlBqiedGX2Ot4kuOOhOy7pidX0wicG0FRLAuG9nLwKoXrlcqz4knUX6lvsyfeyDnttmPmd9VWA4WHyO7oi+h/YCwU79d2MNfl+P2n9bJIAq/Ex+LiLf/ou/luFugHAfwN9/6+9kPfovIlwJAGrSMoqtpeb1++ka47rVZfxOMZTTCSD3vNWSi05PX5yJJGavmxY7VXWasWZ1cNe+f5OEGdPXdataYsbIM3HVZMphGD7qYb9Sh+VuvE6Q+HxMa26BO9NEnkqHGdtkbiKduaKgluzWfynYVz3LsqT2D88LevLo8cYt60ECmCE9+i+/fLZernOAXvLr19w/AG+GVr8XNzemr2UH2F1lUbJlduXkiL/fvqiHk6P/wUk/r9+XhWovVcEtyU97CHcAvFytREQ/NbYxRfL495l7AMyzPgHE49r6fbx9y/HUldR75WTdTOrbpQtQtcRulO6nv7slV2WboXW3ZaLG17+zLmGYKys+r+gve2bLVJtxWLCfuWttp/5v2vdefJgM3hOQQwKoTqj6AaA1T1LM1AKpKRjjXItUwaNnTcRbmGkF/Y9TV/YzJXi+xjG76qpw8igiInLSXwOIj2CnfjQy1JtHC/4F3V0Y/b+4WMlYh6p6X0Q25b/gxZfnEn9N/LcjU5QWHCDflpoz47fPVbjmsOF3JC6wp2fW6crd7Jb4xcisTZzZ1FJ/RpMToBLtbNv/sn03JwBq/2n8/9fMfVviO4F9Avjeyfti5wIBqEH7IjxJUUTsNQBTTLgZBToTZCZVpyomXxP5OQmZY+XGQ/X9h/ppo4m3GRdcUVguF2obXYyl5y0sWv4F3V2fAC6vRBLzcKaf2qSFFCtq86nrPKktE6+J8+PUamUuWeSSbu642/LPOt1Hdpy9YK3fsugz36P/2+5L7V+C//+arXD3r5713y//lsh6RpC8LyIf/cNvReRv/5EcANSlffb8LFqUmCcQfs48bNP8+6+uIvhZBJLZJiGXOfzaTAKwszjGAm/ylcBytGzX3Rqei6KO6OLP1dV15gSwWUgAl5fhM/RxUoc4Gbdx9Ew2gNq/HAlgtvT7v8LT/4PwDoCQA4ba//fE5gDuAQZq0D7tn6QoImYg1j4XRc2Hcc87HEZR3V5ucq/eVz9TULeWe22Sf99Q7s3A5mkwwwmok5saVN52TF+PPmNb4Q6Aq2tS1OuzzxWINx+1/7ZIANXRaQBAzdov1JMU7a2vobof6+XEtN3chOdMDjDvZupLfFWtq/n1JlWYTm2es6wnEk8/pd27uv7dur0mDiGJyckrRp/3EO4AwOtUc+3Ps0I9EsAMhRH876hnf35HPf9HXxsAULP22YsvRWQY9x9X+Hn6dkaQWtzPBRprbc0/UCfxzk99j67OH+aZ5vHMkMTbvty0EZ05zPSlzIWAy8ur6KQ193h25p/sJdwBgDfBvK8NUPvnkABmy+cAANDaX/3wrdvuwy16N/rzKiQAfceAiIy5YayTwk/nFySA3Q13AOyu+zT+G8RuqP3rRAKYOZ0DAEBrP/78s9vuwy378OEHMtSS3/+UtwW9PpefvLNnC3x7axZ+c3Oo/UuQAKrg7+wNbw3Twv3A3AMM1IN/AKgibwGfOV4dav9y/ANQqb7S58k/QMX4BwDATFD7b4t/AKpG7Q/UjH8AABy8kto/PC/oySNm+o2+ctsdAADcDhIAgJnjPQE5JAAAqBQJAMBsUftPIwEAQKVIAABmiNq/BAkAACpFAgAwK9T+5UgAAFApEgCAmaD23xYJAAAqRQIAcPBKan+eFeqRAACgUiQAADNH7Z9DAgCASpEAAMwWtf80EgAAVIoEAGCGqP1LkAAAoFIkAACzQu1fjgQAAJUiAQCYCWr/bZEAAKBSJAAAB6+k9g/PC3ry6PGr787BIAEAQKVIAABmjvcE5JAAAKBSJAAAs0XtP40EAACVIgEAmCFq/xIkAACoFAkAwKxQ+5cjAQBApUgAAGaC2n9bJAAAqBQJAMDBK6n9eVaoRwIAgEqRAADMHLV/DgkAACpFAgAwW9T+00gAAFApEgCAGaL2L0ECAIBKkQAAzAq1fzkSAABUigQAYCao/bdFAgCASpEAABy8kto/PC/oyaPHr747B4MEAACVIgEAmDneE5BDAgCASpEAAMwWtf80EgAAVIoEAGCGqP1LkAAAoFIkAACzQu1fjgQAAJUiAQCYCWr/bZEAAKBSJAAAB6+k9udZoR4JAAAqRQIAMHPU/jkkAACoFAkAwGxR+08jAQBApUgAAGaI2r8ECQAAKkUCADAr1P7lSAAAUCkSAICZoPbfFgkAACpFAgBw8Epq//C8oCePHr/67hwMEgAAVIoEAGDmeE9ADgkAACpFAgAwW9T+00gAAFApEgCAGaL2L0ECAIBKkQAAzAq1fzkSAABUigQAYCao/bdFAgCASpEAABy8ktqfZ4V6JAAAqBQJAMDMUfvnkAAAoFIkAACzRe0/jQQAAJUiAQCYIWr/EiQAAKgUCQDArFD7lyMBAEClSAB7+fDhBzfSzseff3Yj7dSAzxw51P7bIgEAQKXam6qnatZ88zc77LVoWxG5/OSdm+5OFb770Xsi8rUHd0Wk65c1ItJ0nYh0zfBnka4TEWmaRkR+8OOfikj36btyc0kCb4KS2j88L+jJo8evvjsHgwQAAJVqd6teEYRacjfLZSsilzfXmaqcHi9FZHm0FBkiQKj6u3hLtdII3/xvvPf19V42N3Qy5AbTZKe2kU5EGtN2yB/jkmYIIOt9O9Nd1R/dUdUf0+eQb0LWGZs0vdIthJ87dQJD18YDmLNToalRZ2E/gU4dPe5iozuk1na6V6I+E/HLQzvj55Pr/69++JaU4T0BOSQAAKgUs4BuzfFyISJf3nY3DtTx0VJEjhatiC03dbUYFpjCOPbV+3dFxgsFY2OdquVNVZs81LBctxC2CfVy3xN1laLfRte5ru72pa9bksgNuj+61tapoovPJfzX7zukGRdJTN5S1110qnBZxBxcL8lEnk39fymbUPtPIwEAQKX6BJCraPDqHB8f3XYXDtjR0VKGmVT7eBASQKBmENmxfhGJxqx1ZRqPaDemTlf/cTlD1FUEnTwkkwP6hKFShU8tQy0f72VKaFVH2/NViURf4VBXL4bGTAEfDq+2jE4iUfXrnYcG4p5v6P9kAqD2L0ECAIBK9QUUtf/+FoswJ0GPGgeuWpRGRE6OSAC7Wy5aEbm6uhYZi9L1/9gR58z0IBERefveHYn/csbW9JSZ/scw8h7qVjtkHm2fmF3j6NlH5uqF+hL5bWwukXG1nsOj1up2tGEnVYmrRodrAPok4prddEH3zc9f0tu4tfqKSJ8z9CymTP9zqP3LkQAAoFJtmE2xF399f5cGUinElW6JukP97LdJzAzRg7G54xb08GIVz+BfLhbxbr7eUdXc6QkJYHeLRSsiZ+cXIqmRaDOB3M12Vx7cv7P+OVFx6ybDNpkEYIrazJwcnxL8PHc7N0b9nPsO5+b/6NpZ3PaJ9puxb4lDZo6oc4mOvmrF0NvMEnXlYJhDNSaY6f571P7bIgEAQKXaO6fH459MOZ0plW2RILbkSjQjYzNi6gU3B8BvH5a7QfVhz3is1h1kE3/g6Aj2Z73UJ4DjZZylEufVjJ/YkACuS3qKyNX1tYisVpci69n6et5L/18RmfhrFpF7d07XP9s7e8M3c9zLp8xs65NfQX0UPY7vk+7wK5Lrvx6LV/3MHN1fRbDnkgkIZptcD+Pz2vDBuytk/b3N0ztl+h+U1P48K9QjAQBApdr7d08ldZ09UePo4iA3GK8W6D/kanOzZHqML1fRqJ7n5h4kahkXb2zgGbtitunnRTQi8tsvnkfHOjpeiv0M7czocew4tEwC2Mfq6lqGHKbr6EFmoNoxCXiydPdLJ4ejE1cUsttPRRS7smDLXA+z1wB8D/2v6I2aDgbZa3sF1wByqP1zSAAAUKk2zIPWpZIqWwfTpXs/rq13btatiajFbrqAadjX76onpk4XiVYMlcu4yKcWUXu5zqZ6qE9U9zZTg5yGO3sTZUw6vNw5CbXnWbo5TLpcXYrI+YW6EqPL19xXyrmBWXB4g1H7TyMBAECl2jAPulND4P38BFO/j2Pfvu4e7txTsxrURk2jrvLrKltUC804B8A/ZcU8czwxL0LP/VB7uScv+m3Wi0SGJ664KdamBb+vchJqSTMJQ51XtFTk9ESPPmM7R8uliJxfrETEf9r2b2hq4HyYzbV57ljqaTbuGJlDJQbVE9cG9OyjuJm9xsQnrxzkerj1iLzaqOCA23Rzp+ao/UuQAACgUu3b9+6KRKPeqgZRY6mJd/pItOu60u9EzJNGdErQc2n0c0v0c9t9Ghj6o6p7fR56S3fDop1BEdf4ZqUaQfb96SaLLjufRPdt3YxBAthHuAPg/OXFnu2Ee4lT9e+YhqVfIiL5WfZq35Ttitjyrafr9ERr+iqdOseifSU+WL6fKve7vfR++r7faGBhY/9zqP3LkQAAoFLtg7fvSmr+b+duT7STWeK1Q9Ws62u1iS+73WSNvhJJPMtwTBWaechj5nmQ5m5hP8CbCAOqDyZPjM3nao8wr3/YMNQ+6QopVD2nvA9gDxc6Abin5NvPf8rzs/Oww9CMKVJ1NtWXn7LfDZOG08v9b0ruaVFmFpz/rcktd8k7N+NuaH/8rGytrc93/Fl/KP0Vvs63mTtriTT60+i3VL99k/33qP23RQIAgEqZ+wCmnzLoa6vpZ5TnahBdWagFtuIIrbkMEejntjd6B7M2Wpx9HlGnqjB95cOei2vBuePG9BO1lfqIT4+Zgb67cAfA2csLsSP1Az0GPX7+3vPnZxLVlYnvxrjC/3aYo5rrRn6YfPzbz72kQD9t1DSQqYiz2dpcFkj/pvh7pDu31qQok5DcZ2t+49wvpL8UYILMuKHt4WT/lZLaPzwv6Mmjxxu3rAcJAAAq1T8LKHvNXdLL7ZP/xpKjbN/0Nl75liV7ZNe55DFsP56jK1YStp3Vwz2o+wh3AJy/XMUrtvzSfPHiTCQadx4bsu+lauKGE/eOiGpBj/InLjeNnfVz23JvDzbZVNXF4Vva6G+s2l6nCtX8YCzdh0SiMoqM527PWq3V1wN06jKnqxKMywGN+oT9FcRN/d+M9wTkkAAAoFKtHrnO1ekl9W+iiekN42IosatdosZhw59NdTbZQzUqmksnidkaCZmhXxEZE0B8Hnafce0RCWAPYf5PSAC5b45Zkykanz57IWKq+OH6k6vZfS0f6BlBqiedGbmOt4kuOOhOy7radX0wicG0FRLAuG9nLwKoXrlcqz4kc89/f6ywydgHPbfNfM76qsBwsPgc3RF9D+0Fgqn+l6D2n0YCAIBK3cQ7gSEi6Xn9frrGuG51Gb9TDOV0Asg9b7XkotPTF2ciidnrpkX3jCz/HFnfvn+ThBnT13W3WmKSdyaumkw5DMNHPexX6rDcjdcJEp+Pac0tcGeayFPpMGObzE2kM1cU1JLp/k/Wr9T+JUgAAFCp1r/btuf+EXf/GNtR18xAn5Ydt3OVRcmW2ZWTI/5++6IeTo7+Byf9vH5fFqq9VAW3JHvtIdwB8HK1EhH91NjGFMlqvk3mr/ZZnwDicW39Pt6+5XjqSuq9crJuJvXt0sWrWmI3SvfT392Sq7LN0LrbMlHj699ZlzDM77jPK/rLntky1WYcFuxn7lrL9v9evKmIUPtvgwQAAJVqw9MQg0R57Yp8Pxtaj0VOP29H79Ho1RKvTb2PVySaVtD/ODkrIF2C52scs6uuCjfPPTjprwHER7BTP8aZIUeLdqI1TAuj/xcXKxnrUFXvi8im/Be8+PJc4q+J/3ZkitKCA+TbUnNm/Pa5Ct0cNvyOxAX29Mw6Xbmb3RK/GJm1iTObWurPaHICVKKdfP/jBEDtvy0SAABUqn1xdi7RCL6rWO043Viz5K4BmGLCzSjQmSAzqTpVMfmayM9JyBwrNx6q7z/UTxtNvM244IrCcrlQ27gP1M27WLQkgN31CeDySiQxD2f6qU2aTsCbr/Oktky8Js6Ps6uVuWSRS7q5427LP+t0H7ncXLLWb1n0mU/2v6T251mhHgkAACrVhlkQeqQ70M8K70ftVVGtnzSiJeYJ9Dvpwn5cYf79V1cR/CwCyWyTkMscfm0mAdhZHGOBN/lKYDlatutu6Zxk49LYxNXVdeYEsFlIAJeX4TP0cVKHOBm3cbKz4DAj1P45JAAAqFT79PmXMs63GQtvM++lG5fYae6urtczE0S1GWY7uOcdDs24vdzkZL2vfqagbi332iT/vqHcm4HN02CGE4jPN2PbMX09+oxthTsArq5JUa/PPlcgbgu1/zQSAABUqv3i2dn4J1XFN2rWv6yXioibuTzMbVcb5fZ1k2JSE4DSOcC8m0klkr6ZRvd53Nd2avOcZVEZaPop7d7V9e/W7TVxCElMbF4x+ryHcAcAXidq//khAQBApdqnz9Xz0EUknt2fvl1vqOj1uL9a62/jMzOC1OL+ssJYa2v+gTr2naXjfxrXqH2meTwzJPG2LzdtRGcOM30pcyHg8vIqOmnNPZ6d+Sd7CXcA4E3wpl0boPYvRwIAgEq1//mjt267Dwfq3ejPq5AA9B0DIjLmBnVHhYiInF+QAHY33AGwu+7T+G8Qu6H2P1wkAACoVPvx55/ddh8O2IcPP5Chlvz+p49vuzsVufzknT1b4Jt/6MJvX1BS+4fnBT15xO/piAQAAJXigZR7oYp8/fjMsS3eE5BDAgCASpEAAMwWtf80EgAAVIoEAGCGqP1LkAAAoFIkAACzQu1fjgQAAJUiAQCYCWr/bZEAAKBSJAAAB6+k9udZoR4JAAAqRQIAMHPU/jkkAACoFAkAwGxR+08jAQBApUgAAGaI2r8ECQAAKkUCADAr1P7lSAAAUCkSAICZoPbfFgkAACpFAgBw8Epq//C8oCePHr/67hwMEgAAVIoEAGDmeE9ADgkAACpFAgAwW9T+00gAAFApEgCAGaL2L0ECAIBKkQAAzAq1fzkSAABUigQAYCao/bdFAgCASpEAABy8ktqfZ4V6JAAAqBQJAMDMUfvnkAAAoFIkAACzRe0/jQQAAJUiAQCYIWr/EiQAAKgUCQDArFD7lyMBAEClSAAAZoLaf1skAACoFAkAwMErqf3D84KePHr86rtzMEgAAFApEgCAmeM9ATkkAACo1P8DhjjzSgjqHVgAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<PIL.Image.Image image mode=RGB size=512x288 at 0x24F6CE472B0>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "bd04oRFiRKJc",
        "colab": {}
      },
      "source": [
        "# USAGE\n",
        "# python train.py\n",
        "\n",
        "# set the matplotlib backend so figures can be saved in the background\n",
        "import matplotlib\n",
        "matplotlib.use(\"Agg\")\n",
        "\n",
        "# import the necessary packages\n",
        "import sys\n",
        "from __future__ import print_function\n",
        "\n",
        "from keras.preprocessing.image import ImageDataGenerator\n",
        "from keras.optimizers import SGD, adam\n",
        "from keras.initializers import normal, identity\n",
        "from keras.models import model_from_json\n",
        "from keras.models import Sequential\n",
        "from keras.layers.core import Dense, Dropout, Activation, Flatten\n",
        "from keras.layers.convolutional import Convolution2D, MaxPooling2D\n",
        "from tensorflow.keras import Model\n",
        "from tensorflow.keras.applications.mobilenet_v2 import MobileNetV2, preprocess_input\n",
        "from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau, Callback\n",
        "from tensorflow.keras.layers import Conv2D, Reshape\n",
        "from tensorflow.keras.utils import Sequence\n",
        "from tensorflow.keras.backend import epsilon\n",
        "\n",
        "import tensorflow as tf\n",
        "\n",
        "from sklearn.preprocessing import LabelBinarizer\n",
        "from sklearn.metrics import classification_report\n",
        "\n",
        "from pyimagesearch.minivggnet import MiniVGGNet\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import random\n",
        "import numpy as np\n",
        "\n",
        "from coord import CoordinateChannel2D\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "img_rows , img_cols = 80, 80\n",
        "#Convert image into Black and white\n",
        "img_channels = 4 #We stack 4 frames\n",
        "\n",
        "usingSimpleCNN = 1\n",
        "usingSimpleCoordConv = 0\n",
        "usingMobileNetV2Localizer = 0\n",
        "\n",
        "def buildSimpleCNN():\n",
        "    print(\"Now we build the model\")\n",
        "    model = Sequential()\n",
        "    model.add(Convolution2D(32, 8, 8, subsample=(4,4),init=lambda shape, name: normal(shape, scale=0.01, name=name), border_mode='same',input_shape=(img_channels,img_rows,img_cols)))\n",
        "    model.add(Activation('relu'))\n",
        "    model.add(Convolution2D(64, 4, 4, subsample=(2,2),init=lambda shape, name: normal(shape, scale=0.01, name=name), border_mode='same'))\n",
        "    model.add(Activation('relu'))\n",
        "    model.add(Convolution2D(64, 3, 3, subsample=(1,1),init=lambda shape, name: normal(shape, scale=0.01, name=name), border_mode='same'))\n",
        "    model.add(Activation('relu'))\n",
        "    model.add(Flatten())\n",
        "    model.add(Dense(512, init=lambda shape, name: normal(shape, scale=0.01, name=name)))\n",
        "    model.add(Activation('relu'))\n",
        "    model.add(Dense(8,init=lambda shape, name: normal(shape, scale=0.01, name=name)))\n",
        "   \n",
        "    adam = Adam(lr=1e-6)\n",
        "    model.compile(loss='mse',optimizer=adam)\n",
        "    print(\"We finish building the model\")\n",
        "    return model\n",
        "\n",
        "def create_COORDCONV_Simple_CNN():\n",
        "    print(\"Now we build the model\")\n",
        "    inputs = (img_channels,img_rows,img_cols)\n",
        "    x = CoordinateChannel2D()(ip)\n",
        "    x = Convolution2D(32, 8, 8, subsample=(4,4),init=lambda shape, name: normal(shape, scale=0.01, name=name), border_mode='same')\n",
        "    x = (Activation('relu')\n",
        "    x = CoordinateChannel2D()(x)\n",
        "    x = Convolution2D(64, 4, 4, subsample=(2,2),init=lambda shape, name: normal(shape, scale=0.01, name=name), border_mode='same')\n",
        "    x = Activation('relu')\n",
        "    x = CoordinateChannel2D()(x)\n",
        "    x = Convolution2D(64, 3, 3, subsample=(1,1),init=lambda shape, name: normal(shape, scale=0.01, name=name), border_mode='same')\n",
        "    x = Activation('relu')\n",
        "    x = Flatten()\n",
        "    x = Dense(512, init=lambda shape, name: normal(shape, scale=0.01, name=name))\n",
        "    x = Activation('relu')\n",
        "    x = Dense(8,init=lambda shape, name: normal(shape, scale=0.01, name=name))\n",
        "    model = keras.Model(inputs=inputs, outputs=x)\n",
        "    adam = Adam(lr=1e-6)\n",
        "    model.compile(loss='mse',optimizer=adam)\n",
        "    return model\n",
        "\n",
        "\"\"\"\n",
        "# prior to first conv\n",
        "ip = Input(shape=(64, 64, 2))\n",
        "x = CoordinateChannel2D()(ip)\n",
        "x = Conv2D(...)(x)  # This defines the `CoordConv` from the paper.\n",
        "...\n",
        "x = CoordinateChannel2D(use_radius=True)(x)\n",
        "x = Conv2D(...)(x)  # This adds the 3rd channel for the radius.\n",
        "\"\"\"\n",
        "def create_MobileNetV2_model(trainable=False):\n",
        "    model = MobileNetV2(input_shape=(img_channels,img_rows,img_cols), include_top=False)\n",
        "\n",
        "    # to freeze layers\n",
        "    for layer in model.layers:\n",
        "        layer.trainable = trainable\n",
        "\n",
        "    x = model.layers[-1].output\n",
        "    x = Conv2D(4, kernel_size=3, name=\"coords\")(x)\n",
        "    x = Reshape((8,))(x)\n",
        "\n",
        "    model = Model(inputs=model.input, outputs=x)\n",
        "    adam = Adam(lr=1e-6)\n",
        "    model.compile(loss='mse', optimizer=adam)\n",
        "\n",
        "    return model\n",
        "\n",
        "def get_train_test_split():\n",
        "    data = np.genfromtxt('data.txt',delimiter=' ')\n",
        "#    data = np.random.shuffle(data)\n",
        "    traintestsplit = np.split(data, int(data.shape[0]/5))\n",
        "    train = traintestsplit[0]\n",
        "    test = traintestsplit[1]\n",
        "    return train, test\n",
        "\n",
        "# initialize the paths to our training and testing CSV files\n",
        "#TRAIN_CSV = \"flowers17_training.csv\"\n",
        "#TEST_CSV = \"flowers17_testing.csv\"\n",
        "\n",
        "DATA_CSV = \"data.txt\"\n",
        "\n",
        "TRAIN, TEST = get_train_test_split()\n",
        "\n",
        "# initialize the number of epochs to train for and batch size\n",
        "NUM_EPOCHS = 75\n",
        "BS = 32\n",
        "\n",
        "# initialize the total number of training and testing image\n",
        "NUM_TRAIN_IMAGES = 0\n",
        "NUM_TEST_IMAGES = 0\n",
        "\n",
        "# open the training CSV file, then initialize the unique set of class\n",
        "# labels in the dataset along with the testing labels\n",
        "#f = open(TRAIN_CSV, \"r\")\n",
        "labels = set()\n",
        "testLabels = []\n",
        "\n",
        "'''\n",
        "# loop over all rows of the CSV file\n",
        "for row in TRAIN:\n",
        "\t# extract the class label, update the labels list, and increment\n",
        "\t# the total number of training images\n",
        "\tlabel = line.strip().split(\",\")[0]\n",
        "\tlabels.add(label)\n",
        "\tNUM_TRAIN_IMAGES += 1\n",
        "\n",
        "# close the training CSV file and open the testing CSV file\n",
        "f.close()\n",
        "f = open(TEST_CSV, \"r\")\n",
        "\n",
        "# loop over the lines in the testing file\n",
        "for row in TEST:\n",
        "\t# extract the class label, update the test labels list, and\n",
        "\t# increment the total number of testing images\n",
        "\tlabel = line.strip().split(\",\")[0]\n",
        "\ttestLabels.append(label)\n",
        "\tNUM_TEST_IMAGES += 1\n",
        "\n",
        "# close the testing CSV file\n",
        "f.close()\n",
        "'''\n",
        "\n",
        "trainLabels = TRAIN[:,1:3]\n",
        "trainData = TRAIN[:,:]\n",
        "NUM_TRAIN_IMAGES = trainData.shape[0]\n",
        "\n",
        "testLabels = TRAIN[:,1:3]\n",
        "testData = TRAIN[:,:]\n",
        "NUM_TEST_IMAGES = testData.shape[0]\n",
        "\n",
        "# create the label binarizer for one-hot encoding labels, then encode\n",
        "# the testing labels\n",
        "# lb = LabelBinarizer()\n",
        "# lb.fit(list(trainLabels))\n",
        "# testLabels = lb.transform(testLabels)\n",
        "\n",
        "# construct the training image generator for data augmentation\n",
        "aug = ImageDataGenerator(width_shift_range=0.2, height_shift_range=0.2, fill_mode=\"nearest\")\n",
        "\n",
        "# initialize both the training and testing image generators\n",
        "trainGen = csv_image_generator(trainData, BS, #lb,\n",
        "\tmode=\"train\", aug=aug)\n",
        "testGen = csv_image_generator(testData, BS, #lb,\n",
        "\tmode=\"train\", aug=None)\n",
        "\n",
        "# initialize our Keras model and compile it\n",
        "if usingSimpleCNN:\n",
        "    model = buildSimpleCNN()\n",
        "elif usingSimpleCoordConv:\n",
        "    model = create_COORDCONV_Simple_CNN()    \n",
        "elif usingMobileNetV2Localizer:\n",
        "    model = create_MobileNetV2_model()\n",
        "\n",
        "#    model = MiniVGGNet.build(64, 64, 3, len(lb.classes_))\n",
        "#   opt = SGD(lr=1e-2, momentum=0.9, decay=1e-2 / NUM_EPOCHS)\n",
        "#   model.compile(loss=\"categorical_crossentropy\", optimizer=opt, metrics=[\"accuracy\"])\n",
        "\n",
        "# train the network\n",
        "print(\"[INFO] training w/ generator...\")\n",
        "H = model.fit_generator(\n",
        "\ttrainGen,\n",
        "\tsteps_per_epoch=NUM_TRAIN_IMAGES // BS,\n",
        "\tvalidation_data=testGen,\n",
        "\tvalidation_steps=NUM_TEST_IMAGES // BS,\n",
        "\tepochs=NUM_EPOCHS)\n",
        "\n",
        "# re-initialize our testing data generator, this time for evaluating\n",
        "testGen = csv_image_generator(TEST_CSV, BS, #lb,\n",
        "\tmode=\"eval\", aug=None)\n",
        "\n",
        "# make predictions on the testing images, finding the index of the\n",
        "# label with the corresponding largest predicted probability\n",
        "predIdxs = model.predict_generator(testGen,\n",
        "\tsteps=(NUM_TEST_IMAGES // BS) + 1)\n",
        "predIdxs = np.argmax(predIdxs, axis=1)\n",
        "\n",
        "# show a nicely formatted classification report\n",
        "print(\"[INFO] evaluating network...\")\n",
        "print(classification_report(testLabels.argmax(axis=1), predIdxs,\n",
        "\ttarget_names=lb.classes_))\n",
        "\n",
        "# plot the training loss and accuracy\n",
        "N = NUM_EPOCHS\n",
        "plt.style.use(\"ggplot\")\n",
        "plt.figure()\n",
        "plt.plot(np.arange(0, N), H.history[\"loss\"], label=\"train_loss\")\n",
        "plt.plot(np.arange(0, N), H.history[\"val_loss\"], label=\"val_loss\")\n",
        "plt.plot(np.arange(0, N), H.history[\"acc\"], label=\"train_acc\")\n",
        "plt.plot(np.arange(0, N), H.history[\"val_acc\"], label=\"val_acc\")\n",
        "plt.title(\"Training Loss and Accuracy on Dataset\")\n",
        "plt.xlabel(\"Epoch #\")\n",
        "plt.ylabel(\"Loss/Accuracy\")\n",
        "plt.legend(loc=\"lower left\")\n",
        "plt.savefig(\"plot.png\")"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}