NN_C1_CNN.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "NN_C1_CNN.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dbSxElVSxrjq",
        "outputId": "ab8a72ad-e0c9-4976-9bf4-8ceed3443879"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 562
        },
        "id": "USEuKDY5xII9",
        "outputId": "c2560e1e-38c7-4b84-ff35-f901ed0abf61"
      },
      "source": [
        "# load dataset\n",
        "from numpy import genfromtxt\n",
        "import os\n",
        "from pandas import DataFrame\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import keras\n",
        "from keras.preprocessing import sequence\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense\n",
        "from keras.layers import LSTM\n",
        "from keras.optimizers import Adam\n",
        "from keras.models import load_model\n",
        "from keras.callbacks import ModelCheckpoint\n",
        "\n",
        "# load a list of files\n",
        "def load_group(filenames, prefix=''):\n",
        "    loaded = list()\n",
        "    for name in filenames:\n",
        "        data = genfromtxt(prefix + name, delimiter=',')\n",
        "        if 'x' in name:\n",
        "            loaded.append(data)\n",
        "        if 'y' in name:\n",
        "            xData = genfromtxt(prefix + name[:-5] + 'x.csv', delimiter=',')\n",
        "            interpolated_data = nn_interpolate(data, (xData.shape[0], xData.shape[1]))\n",
        "            loaded.append(interpolated_data)\n",
        "    return loaded\n",
        "\n",
        "\n",
        "def load_dataset(group, prefix=''):\n",
        "    filenamesX = list()\n",
        "    filenamesY = list()\n",
        "    # body acceleration\n",
        "    for file in os.listdir(prefix):\n",
        "        if file.startswith(group) and file.endswith('__x.csv'):\n",
        "            filenamesX += [file]\n",
        "    # load input data\n",
        "    X = load_group(filenamesX, prefix)\n",
        "    # load class output\n",
        "    for file in os.listdir(prefix):\n",
        "        if file.startswith(group) and file.endswith('__y.csv'):\n",
        "            filenamesY += [file]\n",
        "\n",
        "    y = load_group(filenamesY, prefix)\n",
        "    return X, y\n",
        "\n",
        "\n",
        "def nn_interpolate(A, new_size):\n",
        "    \"\"\"Vectorized Nearest Neighbor Interpolation\"\"\"\n",
        "\n",
        "    old_size = A.shape\n",
        "    row_ratio, col_ratio = np.array(new_size) / np.array(old_size)\n",
        "\n",
        "    # row wise interpolation\n",
        "    row_idx = (np.ceil(range(1, 1 + int(old_size[0] * row_ratio)) / row_ratio) - 1).astype(int)\n",
        "\n",
        "    final_matrix = A[row_idx]\n",
        "\n",
        "    return final_matrix\n",
        "\n",
        "\n",
        "# # summarize the balance of classes in an output variable column\n",
        "# def class_breakdown(data):\n",
        "#     # convert the numpy array into a dataframe\n",
        "#     df = DataFrame(data)\n",
        "#     # group data by the class value and calculate the number of rows\n",
        "#     counts = df.groupby(0).size()\n",
        "#     # retrieve raw rows\n",
        "#     counts = counts.values\n",
        "#     # summarize\n",
        "#     for i in range(len(counts)):\n",
        "#         percent = counts[i] / len(df) * 100\n",
        "#         print('Class=%d, total=%d, percentage=%.3f' % (i + 1, counts[i], percent))\n",
        "\n",
        "def pre_processing(X):\n",
        "    len_sequences = []\n",
        "    for one_file in X:\n",
        "        for one_seq in one_file:\n",
        "            len_sequences.append(len(one_seq))\n",
        "\n",
        "    to_pad = 70172\n",
        "    new_seq = []\n",
        "    for one_file in X:\n",
        "        for one_seq in one_file:\n",
        "            len_one_seq = len(one_seq)\n",
        "            last_val = one_seq[-1]\n",
        "            n = to_pad - len_one_seq\n",
        "\n",
        "            to_concat = np.repeat(one_seq[-1], n).reshape(6, n).transpose()\n",
        "            new_one_seq = np.concatenate([one_seq, to_concat])\n",
        "            new_seq.append(new_one_seq)\n",
        "\n",
        "        final_seq = np.stack(new_seq)\n",
        "    seq_len = 60000\n",
        "    final_seq = sequence.pad_sequences(final_seq, maxlen=seq_len, padding='post', dtype='float',\n",
        "                                       truncating='post')\n",
        "\n",
        "def data_finalize(trainX, trainy, valX, valY):\n",
        "    # train, train_target, validation, val_target = [], [], [], []\n",
        "    trainSubject, trainTarget, validationSubject, validationTarget = [], [], [], []\n",
        "    for i in range(0, len(trainX)):\n",
        "        trainSubject.append(np.vstack(trainX[i]))\n",
        "\n",
        "    train = np.vstack(trainSubject)\n",
        "\n",
        "    for i in range(0, len(trainy)):\n",
        "        trainTarget.append(np.concatenate(trainy[i]))\n",
        "\n",
        "    train_target = np.concatenate(trainTarget)\n",
        "\n",
        "    for i in range(0, len(valX)):\n",
        "        validationSubject.append(np.vstack(valX[i]))\n",
        "\n",
        "    validation = np.vstack(validationSubject)\n",
        "\n",
        "    for i in range(0, len(valY)):\n",
        "        validationTarget.append(np.concatenate(valY[i]))\n",
        "\n",
        "    val_target = np.concatenate(validationTarget)\n",
        "\n",
        "    return train, train_target, validation, val_target\n",
        "\n",
        "def make_model(input_shape, val):\n",
        "    num_classes = len(np.unique(val))\n",
        "    input_layer = keras.layers.Input(input_shape)\n",
        "\n",
        "    conv1 = keras.layers.Conv1D(filters=64, kernel_size=3, padding=\"same\")(input_layer)\n",
        "    conv1 = keras.layers.BatchNormalization()(conv1)\n",
        "    conv1 = keras.layers.ReLU()(conv1)\n",
        "\n",
        "    conv2 = keras.layers.Conv1D(filters=64, kernel_size=3, padding=\"same\")(conv1)\n",
        "    conv2 = keras.layers.BatchNormalization()(conv2)\n",
        "    conv2 = keras.layers.ReLU()(conv2)\n",
        "\n",
        "    conv3 = keras.layers.Conv1D(filters=64, kernel_size=3, padding=\"same\")(conv2)\n",
        "    conv3 = keras.layers.BatchNormalization()(conv3)\n",
        "    conv3 = keras.layers.ReLU()(conv3)\n",
        "\n",
        "    gap = keras.layers.GlobalAveragePooling1D()(conv3)\n",
        "\n",
        "    output_layer = keras.layers.Dense(num_classes, activation=\"softmax\")(gap)\n",
        "\n",
        "    return keras.models.Model(inputs=input_layer, outputs=output_layer)\n",
        "\n",
        "# load all train\n",
        "trainX = list()\n",
        "trainy = list()\n",
        "validationX = list()\n",
        "validationY = list()\n",
        "for i in range(1, 6):\n",
        "    trX, trY = load_dataset('subject_00' + str(i), '/content/gdrive/MyDrive/data/')\n",
        "    trainX.append(trX)\n",
        "    trainy.append(trY)\n",
        "for i in range(6, 9):\n",
        "    valX, valY = load_dataset('subject_00' + str(i), '/content/gdrive/MyDrive/data/')\n",
        "    validationX.append(valX)\n",
        "    validationY.append(valY)\n",
        "train, train_target, val, val_target = data_finalize(trainX, trainy, validationX, validationY)\n",
        "train = train[:-2, :]\n",
        "train = np.expand_dims(train, 2)\n",
        "print(train.shape)\n",
        "print(train_target.shape)\n",
        "print(val.shape)\n",
        "print(val_target.shape)\n",
        "\n",
        "model = make_model(input_shape=train.shape[1:], val=val)\n",
        "keras.utils.plot_model(model, show_shapes=True)\n",
        "\n",
        "epochs = 500\n",
        "batch_size = 256\n",
        "\n",
        "callbacks = [\n",
        "    keras.callbacks.ModelCheckpoint(\n",
        "        \"best_model.h5\", save_best_only=True, monitor=\"val_loss\"\n",
        "    ),\n",
        "    keras.callbacks.ReduceLROnPlateau(\n",
        "        monitor=\"val_loss\", factor=0.5, patience=20, min_lr=0.0001\n",
        "    ),\n",
        "    keras.callbacks.EarlyStopping(monitor=\"val_loss\", patience=50, verbose=1),\n",
        "]\n",
        "model.compile(\n",
        "    optimizer=\"adam\",\n",
        "    loss=\"sparse_categorical_crossentropy\",\n",
        "    metrics=[\"sparse_categorical_accuracy\"],\n",
        ")\n",
        "history = model.fit(\n",
        "    train,\n",
        "    train_target,\n",
        "    validation_data=(val, val_target),\n",
        "    batch_size=batch_size,\n",
        "    epochs=epochs,\n",
        "    callbacks=callbacks,\n",
        "    shuffle=True,\n",
        "    verbose=1\n",
        ")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(963420, 6, 1)\n",
            "(963420,)\n",
            "(378224, 6)\n",
            "(378224,)\n",
            "Epoch 1/500\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "ResourceExhaustedError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mResourceExhaustedError\u001b[0m                    Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-3-234738d7595b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    192\u001b[0m     \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    193\u001b[0m     \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 194\u001b[0;31m     \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    195\u001b[0m )\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1098\u001b[0m                 _r=1):\n\u001b[1;32m   1099\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1100\u001b[0;31m               \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1101\u001b[0m               \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1102\u001b[0m                 \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    826\u001b[0m     \u001b[0mtracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    827\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mtrace\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTrace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_name\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mtm\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 828\u001b[0;31m       \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    829\u001b[0m       \u001b[0mcompiler\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"xla\"\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_experimental_compile\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m\"nonXla\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    830\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    886\u001b[0m         \u001b[0;31m# Lifting succeeded, so variables are initialized and we can run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    887\u001b[0m         \u001b[0;31m# stateless function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 888\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    889\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    890\u001b[0m       \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfiltered_flat_args\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2941\u001b[0m        filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[1;32m   2942\u001b[0m     return graph_function._call_flat(\n\u001b[0;32m-> 2943\u001b[0;31m         filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access\n\u001b[0m\u001b[1;32m   2944\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2945\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1917\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1918\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1919\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1920\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1921\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    558\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    559\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 560\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    561\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    562\u001b[0m           outputs = execute.execute_with_cancellation(\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 60\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     61\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mResourceExhaustedError\u001b[0m:  OOM when allocating tensor with shape[256,2145105] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n\t [[node model/dense/Softmax (defined at <ipython-input-3-234738d7595b>:194) ]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n [Op:__inference_train_function_1655]\n\nFunction call stack:\ntrain_function\n"
          ]
        }
      ]
    }
  ]
}