Basic_image_FER_evaluate_on_JAFFE.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.utils import to_categorical\n",
    "from keras.preprocessing.image import ImageDataGenerator, load_img\n",
    "from keras.models import Sequential, load_model\n",
    "from keras.layers import Conv2D, MaxPooling2D, BatchNormalization, AveragePooling2D\n",
    "from keras.layers import Dense, Dropout, Activation, Flatten\n",
    "from keras.applications import VGG16\n",
    "from keras.applications.vgg16 import preprocess_input\n",
    "from keras.utils import plot_model\n",
    "from keras.optimizers import Adam\n",
    "from keras.regularizers import l2\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n",
    "from keras_vggface.vggface import VGGFace\n",
    "from sklearn.utils import shuffle\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from seaborn import heatmap\n",
    "###            ###\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random\n",
    "import os\n",
    "import cv2\n",
    "\n",
    "## PARAMETERS ##\n",
    "print_tensorflow_GPU_info = False\n",
    "tensorflow_verbosity = \"INFO\"  # DEBUG(10): All | INFO(20): Info&Warning | WARN(30)[Default]: Warning | ERROR(40): Error | FATAL(50): None\n",
    "random_state = 22\n",
    "    # General\n",
    "dataset_name = \"JAFFE\"\n",
    "multiclass = True\n",
    "num_features = 64\n",
    "num_labels = 7\n",
    "width, height = 197, 197\n",
    "channels = 3\n",
    "load_npy = True\n",
    "data_generator = False\n",
    "    # Convolution\n",
    "kernel_size = (3, 3)\n",
    "pool_size = (2, 2)\n",
    "strides = (2, 2)\n",
    "dropout = 0.3\n",
    "    # Training\n",
    "batch_size = 64\n",
    "epochs = 10\n",
    "loss = \"categorical_crossentropy\"\n",
    "optimizer = \"adam\"\n",
    "eval_metrics = \"accuracy\"\n",
    "##             ##\n",
    "\n",
    "## Reproducibility ## \n",
    "random.seed(random_state)  # Python's seed\n",
    "np.random.seed(random_state)  # Numpy's seed\n",
    "tf.set_random_seed(random_state)  # Tensorflow's seed\n",
    "##                 ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\n--CHANGED TENSORFLOW VERBOSITY FROM 2 (default) TO 2\n"
     ]
    }
   ],
   "source": [
    "## GPU usage settings for Tensorflow backend ##\n",
    "# RTX GPU Memory BUG Fix & Must also be placed at the top of the code else it doesn't work\n",
    "from keras.backend import tensorflow_backend as K\n",
    "tf_config = tf.compat.v1.ConfigProto()\n",
    "tf_config.gpu_options.allow_growth = True                     # dynamically grow the memory used on the GPU\n",
    "#tf_config.gpu_options.per_process_gpu_memory_fraction = 0.9  # fraction of the GPU to be used\n",
    "#tf_config.log_device_placement = True                        # to log device placement (on which device the operation ran)\n",
    "sess = tf.compat.v1.Session(config=tf_config)\n",
    "K.set_session(sess)                     # set this TensorFlow session as the default session for Keras\n",
    "##                                                                                          ##     \n",
    "\n",
    "## Tensorflow Verbosity Module ##\n",
    "default_verbosity = tf.compat.v1.logging.get_verbosity()\n",
    "tf.compat.v1.logging.set_verbosity(tensorflow_verbosity)\n",
    "print(f\"\\n--CHANGED TENSORFLOW VERBOSITY FROM {default_verbosity/10:.0f} (default) TO {tf.compat.v1.logging.get_verbosity()/10:.0f}\")\n",
    "##                             ##\n",
    "\n",
    "## Tensorflow GPU Information Module ##\n",
    "if print_tensorflow_GPU_info == True:\n",
    "    print(f\"\\n--AVAILABLE GPUS:\")\n",
    "    K._get_available_gpus()\n",
    "    print(f\"\\n--NUM OF GPUs AVAILABLE: {len(tf.config.experimental.list_physical_devices('GPU'))}\")\n",
    "    print(f\"\\n--IS TF BUILT WITH CUDA: {tf.test.is_built_with_cuda()}\")\n",
    "    print(f\"\\n--IS GPU AVAILABLE: {tf.test.is_gpu_available()}\")\n",
    "##                                   ##  "
   ]
  },
  {
   "source": [
    "# Load JAFFE"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Better than default preprocess_input because it applies the same way to all 3 colors (since we have grayscale images)\n",
    "def preprocess_input(x, mode='caffe'):\n",
    "    x -= 128.8006 # np.mean(train_dataset)\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Loaded the images of dataset-ANGRY with length 30\n",
      "Loaded the images of dataset-DISGUST with length 29\n",
      "Loaded the images of dataset-FEAR with length 32\n",
      "Loaded the images of dataset-HAPPY with length 31\n",
      "Loaded the images of dataset-NEUTRAL with length 30\n",
      "Loaded the images of dataset-SAD with length 31\n",
      "Loaded the images of dataset-SURPRISE with length 30\n",
      "[[103. 103. 103.]\n",
      " [107. 107. 107.]\n",
      " [114. 114. 114.]\n",
      " [113. 113. 113.]\n",
      " [115. 115. 115.]\n",
      " [117. 117. 117.]\n",
      " [115. 115. 115.]\n",
      " [124. 124. 124.]\n",
      " [124. 124. 124.]\n",
      " [119. 119. 119.]\n",
      " [120. 120. 120.]\n",
      " [119. 119. 119.]\n",
      " [121. 121. 121.]\n",
      " [125. 125. 125.]\n",
      " [121. 121. 121.]\n",
      " [124. 124. 124.]\n",
      " [124. 124. 124.]\n",
      " [127. 127. 127.]\n",
      " [125. 125. 125.]\n",
      " [126. 126. 126.]\n",
      " [120. 120. 120.]\n",
      " [122. 122. 122.]\n",
      " [124. 124. 124.]\n",
      " [124. 124. 124.]\n",
      " [122. 122. 122.]\n",
      " [133. 133. 133.]\n",
      " [128. 128. 128.]\n",
      " [124. 124. 124.]\n",
      " [130. 130. 130.]\n",
      " [132. 132. 132.]\n",
      " [143. 143. 143.]\n",
      " [132. 132. 132.]\n",
      " [137. 137. 137.]\n",
      " [139. 139. 139.]\n",
      " [137. 137. 137.]\n",
      " [143. 143. 143.]\n",
      " [135. 135. 135.]\n",
      " [145. 145. 145.]\n",
      " [140. 140. 140.]\n",
      " [147. 147. 147.]\n",
      " [140. 140. 140.]\n",
      " [146. 146. 146.]\n",
      " [141. 141. 141.]\n",
      " [140. 140. 140.]\n",
      " [137. 137. 137.]\n",
      " [133. 133. 133.]\n",
      " [133. 133. 133.]\n",
      " [135. 135. 135.]\n",
      " [133. 133. 133.]\n",
      " [139. 139. 139.]\n",
      " [149. 149. 149.]\n",
      " [143. 143. 143.]\n",
      " [148. 148. 148.]\n",
      " [147. 147. 147.]\n",
      " [151. 151. 151.]\n",
      " [154. 154. 154.]\n",
      " [149. 149. 149.]\n",
      " [157. 157. 157.]\n",
      " [154. 154. 154.]\n",
      " [153. 153. 153.]\n",
      " [155. 155. 155.]\n",
      " [153. 153. 153.]\n",
      " [152. 152. 152.]\n",
      " [160. 160. 160.]\n",
      " [157. 157. 157.]\n",
      " [153. 153. 153.]\n",
      " [159. 159. 159.]\n",
      " [158. 158. 158.]\n",
      " [154. 154. 154.]\n",
      " [163. 163. 163.]\n",
      " [156. 156. 156.]\n",
      " [163. 163. 163.]\n",
      " [159. 159. 159.]\n",
      " [163. 163. 163.]\n",
      " [158. 158. 158.]\n",
      " [156. 156. 156.]\n",
      " [159. 159. 159.]\n",
      " [163. 163. 163.]\n",
      " [166. 166. 166.]\n",
      " [164. 164. 164.]\n",
      " [164. 164. 164.]\n",
      " [163. 163. 163.]\n",
      " [159. 159. 159.]\n",
      " [159. 159. 159.]\n",
      " [168. 168. 168.]\n",
      " [160. 160. 160.]\n",
      " [157. 157. 157.]\n",
      " [162. 162. 162.]\n",
      " [163. 163. 163.]\n",
      " [161. 161. 161.]\n",
      " [163. 163. 163.]\n",
      " [157. 157. 157.]\n",
      " [161. 161. 161.]\n",
      " [156. 156. 156.]\n",
      " [158. 158. 158.]\n",
      " [158. 158. 158.]\n",
      " [153. 153. 153.]\n",
      " [162. 162. 162.]\n",
      " [165. 165. 165.]\n",
      " [169. 169. 169.]\n",
      " [162. 162. 162.]\n",
      " [163. 163. 163.]\n",
      " [162. 162. 162.]\n",
      " [170. 170. 170.]\n",
      " [160. 160. 160.]\n",
      " [163. 163. 163.]\n",
      " [160. 160. 160.]\n",
      " [160. 160. 160.]\n",
      " [160. 160. 160.]\n",
      " [165. 165. 165.]\n",
      " [171. 171. 171.]\n",
      " [168. 168. 168.]\n",
      " [169. 169. 169.]\n",
      " [160. 160. 160.]\n",
      " [160. 160. 160.]\n",
      " [162. 162. 162.]\n",
      " [160. 160. 160.]\n",
      " [159. 159. 159.]\n",
      " [155. 155. 155.]\n",
      " [156. 156. 156.]\n",
      " [158. 158. 158.]\n",
      " [155. 155. 155.]\n",
      " [161. 161. 161.]\n",
      " [160. 160. 160.]\n",
      " [162. 162. 162.]\n",
      " [160. 160. 160.]\n",
      " [157. 157. 157.]\n",
      " [166. 166. 166.]\n",
      " [166. 166. 166.]\n",
      " [169. 169. 169.]\n",
      " [163. 163. 163.]\n",
      " [170. 170. 170.]\n",
      " [163. 163. 163.]\n",
      " [168. 168. 168.]\n",
      " [166. 166. 166.]\n",
      " [167. 167. 167.]\n",
      " [168. 168. 168.]\n",
      " [172. 172. 172.]\n",
      " [169. 169. 169.]\n",
      " [165. 165. 165.]\n",
      " [163. 163. 163.]\n",
      " [159. 159. 159.]\n",
      " [159. 159. 159.]\n",
      " [161. 161. 161.]\n",
      " [160. 160. 160.]\n",
      " [169. 169. 169.]\n",
      " [158. 158. 158.]\n",
      " [155. 155. 155.]\n",
      " [162. 162. 162.]\n",
      " [160. 160. 160.]\n",
      " [161. 161. 161.]\n",
      " [164. 164. 164.]\n",
      " [164. 164. 164.]\n",
      " [159. 159. 159.]\n",
      " [163. 163. 163.]\n",
      " [157. 157. 157.]\n",
      " [152. 152. 152.]\n",
      " [157. 157. 157.]\n",
      " [155. 155. 155.]\n",
      " [156. 156. 156.]\n",
      " [155. 155. 155.]\n",
      " [153. 153. 153.]\n",
      " [150. 150. 150.]\n",
      " [156. 156. 156.]\n",
      " [156. 156. 156.]\n",
      " [159. 159. 159.]\n",
      " [152. 152. 152.]\n",
      " [155. 155. 155.]\n",
      " [152. 152. 152.]\n",
      " [145. 145. 145.]\n",
      " [146. 146. 146.]\n",
      " [155. 155. 155.]\n",
      " [151. 151. 151.]\n",
      " [148. 148. 148.]\n",
      " [147. 147. 147.]\n",
      " [150. 150. 150.]\n",
      " [153. 153. 153.]\n",
      " [147. 147. 147.]\n",
      " [139. 139. 139.]\n",
      " [147. 147. 147.]\n",
      " [145. 145. 145.]\n",
      " [142. 142. 142.]\n",
      " [140. 140. 140.]\n",
      " [139. 139. 139.]\n",
      " [140. 140. 140.]\n",
      " [141. 141. 141.]\n",
      " [153. 153. 153.]\n",
      " [131. 131. 131.]\n",
      " [140. 140. 140.]\n",
      " [141. 141. 141.]\n",
      " [135. 135. 135.]\n",
      " [137. 137. 137.]\n",
      " [141. 141. 141.]\n",
      " [148. 148. 148.]\n",
      " [134. 134. 134.]\n",
      " [132. 132. 132.]\n",
      " [132. 132. 132.]]\n",
      "[[ -0.939 -13.779 -20.68 ]\n",
      " [  3.061  -9.779 -16.68 ]\n",
      " [ 10.061  -2.779  -9.68 ]\n",
      " [  9.061  -3.779 -10.68 ]\n",
      " [ 11.061  -1.779  -8.68 ]\n",
      " [ 13.061   0.221  -6.68 ]\n",
      " [ 11.061  -1.779  -8.68 ]\n",
      " [ 20.061   7.221   0.32 ]\n",
      " [ 20.061   7.221   0.32 ]\n",
      " [ 15.061   2.221  -4.68 ]\n",
      " [ 16.061   3.221  -3.68 ]\n",
      " [ 15.061   2.221  -4.68 ]\n",
      " [ 17.061   4.221  -2.68 ]\n",
      " [ 21.061   8.221   1.32 ]\n",
      " [ 17.061   4.221  -2.68 ]\n",
      " [ 20.061   7.221   0.32 ]\n",
      " [ 20.061   7.221   0.32 ]\n",
      " [ 23.061  10.221   3.32 ]\n",
      " [ 21.061   8.221   1.32 ]\n",
      " [ 22.061   9.221   2.32 ]\n",
      " [ 16.061   3.221  -3.68 ]\n",
      " [ 18.061   5.221  -1.68 ]\n",
      " [ 20.061   7.221   0.32 ]\n",
      " [ 20.061   7.221   0.32 ]\n",
      " [ 18.061   5.221  -1.68 ]\n",
      " [ 29.061  16.221   9.32 ]\n",
      " [ 24.061  11.221   4.32 ]\n",
      " [ 20.061   7.221   0.32 ]\n",
      " [ 26.061  13.221   6.32 ]\n",
      " [ 28.061  15.221   8.32 ]\n",
      " [ 39.061  26.221  19.32 ]\n",
      " [ 28.061  15.221   8.32 ]\n",
      " [ 33.061  20.221  13.32 ]\n",
      " [ 35.061  22.221  15.32 ]\n",
      " [ 33.061  20.221  13.32 ]\n",
      " [ 39.061  26.221  19.32 ]\n",
      " [ 31.061  18.221  11.32 ]\n",
      " [ 41.061  28.221  21.32 ]\n",
      " [ 36.061  23.221  16.32 ]\n",
      " [ 43.061  30.221  23.32 ]\n",
      " [ 36.061  23.221  16.32 ]\n",
      " [ 42.061  29.221  22.32 ]\n",
      " [ 37.061  24.221  17.32 ]\n",
      " [ 36.061  23.221  16.32 ]\n",
      " [ 33.061  20.221  13.32 ]\n",
      " [ 29.061  16.221   9.32 ]\n",
      " [ 29.061  16.221   9.32 ]\n",
      " [ 31.061  18.221  11.32 ]\n",
      " [ 29.061  16.221   9.32 ]\n",
      " [ 35.061  22.221  15.32 ]\n",
      " [ 45.061  32.221  25.32 ]\n",
      " [ 39.061  26.221  19.32 ]\n",
      " [ 44.061  31.221  24.32 ]\n",
      " [ 43.061  30.221  23.32 ]\n",
      " [ 47.061  34.221  27.32 ]\n",
      " [ 50.061  37.221  30.32 ]\n",
      " [ 45.061  32.221  25.32 ]\n",
      " [ 53.061  40.221  33.32 ]\n",
      " [ 50.061  37.221  30.32 ]\n",
      " [ 49.061  36.221  29.32 ]\n",
      " [ 51.061  38.221  31.32 ]\n",
      " [ 49.061  36.221  29.32 ]\n",
      " [ 48.061  35.221  28.32 ]\n",
      " [ 56.061  43.221  36.32 ]\n",
      " [ 53.061  40.221  33.32 ]\n",
      " [ 49.061  36.221  29.32 ]\n",
      " [ 55.061  42.221  35.32 ]\n",
      " [ 54.061  41.221  34.32 ]\n",
      " [ 50.061  37.221  30.32 ]\n",
      " [ 59.061  46.221  39.32 ]\n",
      " [ 52.061  39.221  32.32 ]\n",
      " [ 59.061  46.221  39.32 ]\n",
      " [ 55.061  42.221  35.32 ]\n",
      " [ 59.061  46.221  39.32 ]\n",
      " [ 54.061  41.221  34.32 ]\n",
      " [ 52.061  39.221  32.32 ]\n",
      " [ 55.061  42.221  35.32 ]\n",
      " [ 59.061  46.221  39.32 ]\n",
      " [ 62.061  49.221  42.32 ]\n",
      " [ 60.061  47.221  40.32 ]\n",
      " [ 60.061  47.221  40.32 ]\n",
      " [ 59.061  46.221  39.32 ]\n",
      " [ 55.061  42.221  35.32 ]\n",
      " [ 55.061  42.221  35.32 ]\n",
      " [ 64.061  51.221  44.32 ]\n",
      " [ 56.061  43.221  36.32 ]\n",
      " [ 53.061  40.221  33.32 ]\n",
      " [ 58.061  45.221  38.32 ]\n",
      " [ 59.061  46.221  39.32 ]\n",
      " [ 57.061  44.221  37.32 ]\n",
      " [ 59.061  46.221  39.32 ]\n",
      " [ 53.061  40.221  33.32 ]\n",
      " [ 57.061  44.221  37.32 ]\n",
      " [ 52.061  39.221  32.32 ]\n",
      " [ 54.061  41.221  34.32 ]\n",
      " [ 54.061  41.221  34.32 ]\n",
      " [ 49.061  36.221  29.32 ]\n",
      " [ 58.061  45.221  38.32 ]\n",
      " [ 61.061  48.221  41.32 ]\n",
      " [ 65.061  52.221  45.32 ]\n",
      " [ 58.061  45.221  38.32 ]\n",
      " [ 59.061  46.221  39.32 ]\n",
      " [ 58.061  45.221  38.32 ]\n",
      " [ 66.061  53.221  46.32 ]\n",
      " [ 56.061  43.221  36.32 ]\n",
      " [ 59.061  46.221  39.32 ]\n",
      " [ 56.061  43.221  36.32 ]\n",
      " [ 56.061  43.221  36.32 ]\n",
      " [ 56.061  43.221  36.32 ]\n",
      " [ 61.061  48.221  41.32 ]\n",
      " [ 67.061  54.221  47.32 ]\n",
      " [ 64.061  51.221  44.32 ]\n",
      " [ 65.061  52.221  45.32 ]\n",
      " [ 56.061  43.221  36.32 ]\n",
      " [ 56.061  43.221  36.32 ]\n",
      " [ 58.061  45.221  38.32 ]\n",
      " [ 56.061  43.221  36.32 ]\n",
      " [ 55.061  42.221  35.32 ]\n",
      " [ 51.061  38.221  31.32 ]\n",
      " [ 52.061  39.221  32.32 ]\n",
      " [ 54.061  41.221  34.32 ]\n",
      " [ 51.061  38.221  31.32 ]\n",
      " [ 57.061  44.221  37.32 ]\n",
      " [ 56.061  43.221  36.32 ]\n",
      " [ 58.061  45.221  38.32 ]\n",
      " [ 56.061  43.221  36.32 ]\n",
      " [ 53.061  40.221  33.32 ]\n",
      " [ 62.061  49.221  42.32 ]\n",
      " [ 62.061  49.221  42.32 ]\n",
      " [ 65.061  52.221  45.32 ]\n",
      " [ 59.061  46.221  39.32 ]\n",
      " [ 66.061  53.221  46.32 ]\n",
      " [ 59.061  46.221  39.32 ]\n",
      " [ 64.061  51.221  44.32 ]\n",
      " [ 62.061  49.221  42.32 ]\n",
      " [ 63.061  50.221  43.32 ]\n",
      " [ 64.061  51.221  44.32 ]\n",
      " [ 68.061  55.221  48.32 ]\n",
      " [ 65.061  52.221  45.32 ]\n",
      " [ 61.061  48.221  41.32 ]\n",
      " [ 59.061  46.221  39.32 ]\n",
      " [ 55.061  42.221  35.32 ]\n",
      " [ 55.061  42.221  35.32 ]\n",
      " [ 57.061  44.221  37.32 ]\n",
      " [ 56.061  43.221  36.32 ]\n",
      " [ 65.061  52.221  45.32 ]\n",
      " [ 54.061  41.221  34.32 ]\n",
      " [ 51.061  38.221  31.32 ]\n",
      " [ 58.061  45.221  38.32 ]\n",
      " [ 56.061  43.221  36.32 ]\n",
      " [ 57.061  44.221  37.32 ]\n",
      " [ 60.061  47.221  40.32 ]\n",
      " [ 60.061  47.221  40.32 ]\n",
      " [ 55.061  42.221  35.32 ]\n",
      " [ 59.061  46.221  39.32 ]\n",
      " [ 53.061  40.221  33.32 ]\n",
      " [ 48.061  35.221  28.32 ]\n",
      " [ 53.061  40.221  33.32 ]\n",
      " [ 51.061  38.221  31.32 ]\n",
      " [ 52.061  39.221  32.32 ]\n",
      " [ 51.061  38.221  31.32 ]\n",
      " [ 49.061  36.221  29.32 ]\n",
      " [ 46.061  33.221  26.32 ]\n",
      " [ 52.061  39.221  32.32 ]\n",
      " [ 52.061  39.221  32.32 ]\n",
      " [ 55.061  42.221  35.32 ]\n",
      " [ 48.061  35.221  28.32 ]\n",
      " [ 51.061  38.221  31.32 ]\n",
      " [ 48.061  35.221  28.32 ]\n",
      " [ 41.061  28.221  21.32 ]\n",
      " [ 42.061  29.221  22.32 ]\n",
      " [ 51.061  38.221  31.32 ]\n",
      " [ 47.061  34.221  27.32 ]\n",
      " [ 44.061  31.221  24.32 ]\n",
      " [ 43.061  30.221  23.32 ]\n",
      " [ 46.061  33.221  26.32 ]\n",
      " [ 49.061  36.221  29.32 ]\n",
      " [ 43.061  30.221  23.32 ]\n",
      " [ 35.061  22.221  15.32 ]\n",
      " [ 43.061  30.221  23.32 ]\n",
      " [ 41.061  28.221  21.32 ]\n",
      " [ 38.061  25.221  18.32 ]\n",
      " [ 36.061  23.221  16.32 ]\n",
      " [ 35.061  22.221  15.32 ]\n",
      " [ 36.061  23.221  16.32 ]\n",
      " [ 37.061  24.221  17.32 ]\n",
      " [ 49.061  36.221  29.32 ]\n",
      " [ 27.061  14.221   7.32 ]\n",
      " [ 36.061  23.221  16.32 ]\n",
      " [ 37.061  24.221  17.32 ]\n",
      " [ 31.061  18.221  11.32 ]\n",
      " [ 33.061  20.221  13.32 ]\n",
      " [ 37.061  24.221  17.32 ]\n",
      " [ 44.061  31.221  24.32 ]\n",
      " [ 30.061  17.221  10.32 ]\n",
      " [ 28.061  15.221   8.32 ]\n",
      " [ 28.061  15.221   8.32 ]]\n",
      "(213, 197, 197, 3) \n",
      "\n",
      "Loading Done\n"
     ]
    }
   ],
   "source": [
    "if dataset_name == \"JAFFE\":\n",
    "    # Load images\n",
    "    data_path = \"./Datasets/Japanese Female Facial Expression (JAFFE) Dataset/PNG Format\"    \n",
    "    data_dir_list = os.listdir(data_path)\n",
    "\n",
    "    img_data_list=[]\n",
    "\n",
    "    for dataset in data_dir_list:\n",
    "        img_list=os.listdir(data_path+'/'+ dataset)\n",
    "        print ('Loaded the images of dataset-'+'{} with length {}'.format(dataset, len(img_list)))\n",
    "        for img in img_list:\n",
    "            input_img=cv2.imread(data_path + '/'+ dataset + '/'+ img, cv2.IMREAD_COLOR)  # Dataset is actually Grayscale\n",
    "            input_img=cv2.resize(input_img, (197,197))  # RESIZE Required\n",
    "            img_data_list.append(input_img)\n",
    "            \n",
    "    img_data = np.array(img_data_list)\n",
    "    X = img_data.astype('float64')\n",
    "    #X = X/255\n",
    "    print(X[0][0])\n",
    "\n",
    "    # Make image VGG compatible (3 channels instead of 1 channel gray scale, BGR, etc.)\n",
    "    X_new = np.empty((len(X), width, height, channels))\n",
    "    for i in range(len(X)):\n",
    "        X_new[i] = preprocess_input(X[i], mode='caffe')\n",
    "    X = X_new\n",
    "    print(X_new[0][0])    \n",
    "    del X_new\n",
    "    del img_data\n",
    "\n",
    "    print(X.shape, \"\\n\")\n",
    "\n",
    "    # Define the number of classes\n",
    "    num_of_samples = X.shape[0]\n",
    "    labels = np.ones((num_of_samples,), dtype='int64')\n",
    "    labels[0:30]=0 #30\n",
    "    labels[30:59]=1 #29\n",
    "    labels[59:91]=2 #32\n",
    "    labels[91:122]=3 #31\n",
    "    labels[122:152]=4 #30\n",
    "    labels[152:183]=5 #31\n",
    "    labels[183:]=6 #30  \n",
    "\n",
    "    # Class Label Names    \n",
    "    label_names = ['ANGRY', 'DISGUST', 'FEAR', 'HAPPY', 'NEUTRAL', 'SAD', 'SURPRISE']\n",
    "\n",
    "    # Convert class labels to on-hot encoding \n",
    "    y = to_categorical(labels, num_labels)\n",
    "\n",
    "    # Shuffle the dataset\n",
    "    X, y = shuffle(X, y, random_state=random_state)\n",
    "    print(\"Loading Done\")\n",
    "\n",
    "elif dataset_name == \"SFEW\":\n",
    "    # Load images from Train Folder\n",
    "    data_path = \"./Datasets/Static Facial Expression In The Wild (SFEW) Dataset/SFEW_2/Train\"    \n",
    "    data_dir_list = os.listdir(data_path)\n",
    "\n",
    "    img_data_list=[]\n",
    "    labels=[]\n",
    "    count = 0\n",
    "\n",
    "    for dataset in data_dir_list:\n",
    "        if (dataset.endswith(\".zip\") != True):\n",
    "            img_list=os.listdir(data_path+'/'+ dataset)\n",
    "            print ('Loaded the images of dataset-'+'{} with length {}'.format(dataset, len(img_list)))\n",
    "            for img in img_list:\n",
    "                input_img=cv2.imread(data_path + '/'+ dataset + '/'+ img, cv2.IMREAD_COLOR)  # Dataset is RGB\n",
    "                input_img = input_img[:, 72:648]  # CROP Required\n",
    "                input_img=cv2.resize(input_img, (48,48))  # RESIZE Required\n",
    "                img_data_list.append(input_img)\n",
    "                labels.append(count)\n",
    "            count += 1\n",
    "\n",
    "    # Also images from Val Folder\n",
    "    data_path = \"./Datasets/Static Facial Expression In The Wild (SFEW) Dataset/SFEW_2/Val\"    \n",
    "    data_dir_list = os.listdir(data_path)\n",
    "\n",
    "    count = 0\n",
    "\n",
    "    for dataset in data_dir_list:\n",
    "        if (dataset.endswith(\".zip\") != True):\n",
    "            img_list=os.listdir(data_path+'/'+ dataset)\n",
    "            print ('Loaded the images of dataset-'+'{} with length {}'.format(dataset, len(img_list)))\n",
    "            for img in img_list:\n",
    "                input_img=cv2.imread(data_path + '/'+ dataset + '/'+ img, cv2.IMREAD_COLOR)  # Dataset is RGB\n",
    "                input_img = input_img[:, 72:648]  # CROP Required\n",
    "                input_img=cv2.resize(input_img, (48,48))  # RESIZE Required\n",
    "                img_data_list.append(input_img)\n",
    "                labels.append(count)\n",
    "            count += 1\n",
    "\n",
    "    img_data = np.array(img_data_list)\n",
    "    img_data = img_data.astype('float32')\n",
    "    X = img_data/255\n",
    "\n",
    "    # Make image VGG compatible (3 channels instead of 1 channel gray scale, BGR, etc.)\n",
    "    X_new = np.empty((len(X), width, height, channels))\n",
    "    for i in range(len(X)):\n",
    "        X_new[i] = preprocess_input(X[i], mode='caffe')\n",
    "    X = X_new\n",
    "    del X_new\n",
    "    del img_data\n",
    "\n",
    "    print(X.shape, \"\\n\")\n",
    "    labels = np.asarray(labels, dtype='int64')\n",
    "\n",
    "    # Class Label Names    \n",
    "    label_names = ['Angry', 'Disgust', 'Fear', 'Happy', 'Neutral', 'Sad', 'Surprise']\n",
    "\n",
    "    # Convert class labels to on-hot encoding \n",
    "    y = to_categorical(labels, num_labels)\n",
    "\n",
    "    # Shuffle the dataset\n",
    "    X, y = shuffle(X, y, random_state=random_state)\n",
    "    print(\"Loading Done\")\n",
    "    print(len(X))    "
   ]
  },
  {
   "source": [
    "# Load Model"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' Load Model '''\n",
    "model = load_model('Saved Models/GitHub-FER_ResNet50_VGGFace.h5')"
   ]
  },
  {
   "source": [
    "# Evaluate on JAFFE (need to also make sure that labels match the FER dataset)"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test = X\n",
    "y_test = y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "213/213 [==============================] - 1s 3ms/step\n",
      "\n",
      "Metric - loss: 3940.29164\n",
      "Metric - accuracy: 16.43192\n",
      "\n",
      "[[19  0  0  0 11  0  0]\n",
      " [15  0  0  0 14  0  0]\n",
      " [13  0  0  0 19  0  0]\n",
      " [17  0  0  0 14  0  0]\n",
      " [14  0  0  0 16  0  0]\n",
      " [19  0  0  0 12  0  0]\n",
      " [16  0  0  0 14  0  0]]\n"
     ]
    }
   ],
   "source": [
    "''' Evaluate Model '''\n",
    "results = model.evaluate(X_test, y_test)\n",
    "print()\n",
    "\n",
    "for i in range(len(model.metrics_names)):\n",
    "    print(\"Metric - {}: {:.5f}\".format(model.metrics_names[i], results[i]*100))\n",
    "\n",
    "if multiclass == False: \n",
    "    predictions = [1 * (x[0]>=0.5) for x in model.predict(X_test)]\n",
    "    cm = confusion_matrix(y_test, predictions)\n",
    "else:    \n",
    "    predictions = np.argmax(model.predict(X_test), axis=1)\n",
    "    cm = confusion_matrix(y_test.argmax(axis=1), predictions)\n",
    "print()\n",
    "print(cm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}