fashion-mnist-fairing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 200215 14:17:36 config:123] Using preprocessor: <kubeflow.fairing.preprocessors.converted_notebook.ConvertNotebookPreprocessor object at 0x7fced5104828>\n",
      "[I 200215 14:17:36 config:125] Using builder: <kubeflow.fairing.builders.append.append.AppendBuilder object at 0x7fcec7267128>\n",
      "[I 200215 14:17:36 config:127] Using deployer: <kubeflow.fairing.builders.append.append.AppendBuilder object at 0x7fcec7267128>\n",
      "[W 200215 14:17:36 append:50] Building image using Append builder...\n",
      "[I 200215 14:17:36 base:105] Creating docker context: /tmp/fairing_context_pnv40848\n",
      "[I 200215 14:17:36 converted_notebook:127] Converting fashion-mnist-fairing.ipynb to fashion-mnist-fairing.py\n",
      "[I 200215 14:17:36 docker_creds_:234] Loading Docker credentials for repository 'brightfly/kubeflow-jupyter-lab:tf2.0-gpu'\n",
      "[W 200215 14:17:38 append:54] Image successfully built in 2.131834232946858s.\n",
      "[W 200215 14:17:38 append:94] Pushing image kubeflow-registry.default.svc.cluster.local:30000/fairing-job:4D2DC88F...\n",
      "[I 200215 14:17:38 docker_creds_:234] Loading Docker credentials for repository 'kubeflow-registry.default.svc.cluster.local:30000/fairing-job:4D2DC88F'\n",
      "[W 200215 14:17:38 append:81] Uploading kubeflow-registry.default.svc.cluster.local:30000/fairing-job:4D2DC88F\n",
      "[I 200215 14:17:38 docker_session_:280] Layer sha256:6001e1789921cf851f6fb2e5fe05be70f482fe9c2286f66892fe5a3bc404569c exists, skipping\n",
      "[I 200215 14:17:38 docker_session_:280] Layer sha256:180c1b9ae1a5a2a4aba2e08646155b250793be7a97a21556f2c4383f35510f14 exists, skipping\n",
      "[I 200215 14:17:38 docker_session_:280] Layer sha256:f6490e4900cb64767e994addad4a4329cca4ccfce37a63d21249d89b63ceb478 exists, skipping\n",
      "[I 200215 14:17:38 docker_session_:280] Layer sha256:c4e2f5cde1e102db0e852d5e60ceac5bf377621b397ef3cb7bf1587052ea9520 exists, skipping\n",
      "[I 200215 14:17:38 docker_session_:280] Layer sha256:d12137d65eaa8a740f8bc9419a0fe3e1ba963478fb9179142935e67d667c6f55 exists, skipping\n",
      "[I 200215 14:17:38 docker_session_:280] Layer sha256:db108ae5c97b9d9b4a6ab30f08b9f00de1a383adb8b2fbdad530e801661c12c5 exists, skipping\n",
      "[I 200215 14:17:38 docker_session_:280] Layer sha256:7fb8ccf83688665b4137d132962349ffeae8bed7fee8f8b878fee4ecffe1f93d exists, skipping\n",
      "[I 200215 14:17:38 docker_session_:280] Layer sha256:b0a15f35c371773fb2237091edb340a15cf0e8615de8c9d8c4b8830c35a40265 exists, skipping\n",
      "[I 200215 14:17:38 docker_session_:280] Layer sha256:947e0f532378ce4f91ff44af563f21f5679d39a28efa2541594dd3f96730edb0 exists, skipping\n",
      "[I 200215 14:17:38 docker_session_:280] Layer sha256:74a3420b0759a0426bc7a24daf48c0f376a7e8d293c015e0681884071c3ea0e1 exists, skipping\n",
      "[I 200215 14:17:38 docker_session_:280] Layer sha256:034db42ae0fa9a91805c16e78f4a6a9d8f5826c95c92713f835b2866333c786c exists, skipping\n",
      "[I 200215 14:17:38 docker_session_:280] Layer sha256:071fd910512489f1cf72f43901d730bfcf302bd46b88924e81f401810bd0e4ce exists, skipping\n",
      "[I 200215 14:17:38 docker_session_:280] Layer sha256:7d830fc21c34e37fa1d2f683b3392ce76489814b1973691f0593b2ce945b2aca exists, skipping\n",
      "[I 200215 14:17:38 docker_session_:280] Layer sha256:d1555fb0eeaac2bbfe413314cbbed0dea65a0a72312f29154e21afda1972b0d1 exists, skipping\n",
      "[I 200215 14:17:38 docker_session_:280] Layer sha256:d701a76e3193731210c61c838de0c3d8fdc8048b613ca88a58e11dc3223221ec exists, skipping\n",
      "[I 200215 14:17:38 docker_session_:280] Layer sha256:c73ad65f0763fc76b21c650ff9a5d7f4714c30a3dc27f462aa478c829a8efed1 exists, skipping\n",
      "[I 200215 14:17:38 docker_session_:280] Layer sha256:65881bda3d6dd3c81420e22d3f921824b72ae8c07137aed41d02c640a2f87f04 exists, skipping\n",
      "[I 200215 14:17:38 docker_session_:280] Layer sha256:06d09f7e28e650c4614bfeaa1336db8632aa02d48b9ff082d9cdf2edf4204376 exists, skipping\n",
      "[I 200215 14:17:38 docker_session_:280] Layer sha256:c592fdfb0e51000077def684bbdd097d6d870e62ae94418ba81a3468b03ec833 exists, skipping\n",
      "[I 200215 14:17:38 docker_session_:280] Layer sha256:57418a6dc9496eeb6c9324e7bdf6fd48b9e1ac25cb473bf7b71f40d44e2f1e64 exists, skipping\n",
      "[I 200215 14:17:38 docker_session_:280] Layer sha256:8810fcda1e6e2713f22a64b835ffa1ff15f49257f43dee869abef1929416d362 exists, skipping\n",
      "[I 200215 14:17:38 docker_session_:280] Layer sha256:0586a82d62e69da8dab24b635faf9c5da803ff9d208111a25b0e2253c97ec331 exists, skipping\n",
      "[I 200215 14:17:38 docker_session_:280] Layer sha256:35c102085707f703de2d9eaad8752d6fe1b8f02b5d2149f1d8357c9cc7fb7d0a exists, skipping\n",
      "[I 200215 14:17:38 docker_session_:280] Layer sha256:f9e7e7aaea7e58fd475e393007baf0251bad43cccaaef540a365ed1036173ae7 exists, skipping\n",
      "[I 200215 14:17:38 docker_session_:284] Layer sha256:12cc1dc4b9a41e9c4f948c1bc2d5050c72b6e411f0e7494a0b6630ce8de77506 pushed.\n",
      "[I 200215 14:17:38 docker_session_:280] Layer sha256:9f0a21d58e5dce5512db6d5595c6e9c4ab014917cf0644e2d282b8f5e3f2522a exists, skipping\n",
      "[I 200215 14:17:38 docker_session_:280] Layer sha256:251f5509d51d9e4119d4ffb70d4820f8e2d7dc72ad15df3ebd7cd755539e40fd exists, skipping\n",
      "[I 200215 14:17:38 docker_session_:280] Layer sha256:308e2d038b5334fb2ec9872230cfabd52b26102f6c8e0e257f3dcd63cc3b7177 exists, skipping\n",
      "[I 200215 14:17:38 docker_session_:280] Layer sha256:8e829fe70a46e3ac4334823560e98b257234c23629f19f05460e21a453091e6d exists, skipping\n",
      "[I 200215 14:17:38 docker_session_:284] Layer sha256:d6fee525fb1daa1b6bd5f4fe94818f8bfa478a56bc8a9a6bfa06fb25e3bb1dc2 pushed.\n",
      "[I 200215 14:17:38 docker_session_:334] Finished upload of: kubeflow-registry.default.svc.cluster.local:30000/fairing-job:4D2DC88F\n",
      "[W 200215 14:17:38 append:99] Pushed image kubeflow-registry.default.svc.cluster.local:30000/fairing-job:4D2DC88F in 0.3088517979485914s.\n",
      "[W 200215 14:17:38 job:90] The job fairing-job-82gd8 launched.\n",
      "[W 200215 14:17:38 manager:227] Waiting for fairing-job-82gd8-dp5dm to start...\n",
      "[W 200215 14:17:38 manager:227] Waiting for fairing-job-82gd8-dp5dm to start...\n",
      "[W 200215 14:17:38 manager:227] Waiting for fairing-job-82gd8-dp5dm to start...\n",
      "[W 200215 14:17:39 manager:227] Waiting for fairing-job-82gd8-dp5dm to start...\n",
      "[I 200215 14:17:41 manager:233] Pod started running True\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-labels-idx1-ubyte.gz\n",
      "32768/29515 [=================================] - 0s 0us/step\n",
      "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-images-idx3-ubyte.gz\n",
      "26427392/26421880 [==============================] - 0s 0us/step\n",
      "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-labels-idx1-ubyte.gz\n",
      "8192/5148 [===============================================] - 0s 0us/step\n",
      "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-images-idx3-ubyte.gz\n",
      "4423680/4422102 [==============================] - 0s 0us/step\n",
      "2020-02-15 14:17:44.600484: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1\n",
      "2020-02-15 14:17:44.648306: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:17:44.649457: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 0 with properties:\n",
      "name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235\n",
      "pciBusID: 0000:00:04.0\n",
      "2020-02-15 14:17:44.649549: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:17:44.650682: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 1 with properties:\n",
      "name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235\n",
      "pciBusID: 0000:00:05.0\n",
      "2020-02-15 14:17:44.680975: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.0\n",
      "2020-02-15 14:17:45.026834: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0\n",
      "2020-02-15 14:17:45.223984: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10.0\n",
      "2020-02-15 14:17:45.304531: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10.0\n",
      "2020-02-15 14:17:45.860578: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10.0\n",
      "2020-02-15 14:17:46.183176: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10.0\n",
      "2020-02-15 14:17:47.389289: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7\n",
      "2020-02-15 14:17:47.389558: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:17:47.390813: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:17:47.391822: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:17:47.392917: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:17:47.393734: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Adding visible gpu devices: 0, 1\n",
      "2020-02-15 14:17:47.406705: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n",
      "2020-02-15 14:17:47.693361: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2200000000 Hz\n",
      "2020-02-15 14:17:47.695935: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5575890 executing computations on platform Host. Devices:\n",
      "2020-02-15 14:17:47.695973: I tensorflow/compiler/xla/service/service.cc:175]   StreamExecutor device (0): Host, Default Version\n",
      "2020-02-15 14:17:48.079327: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:17:48.081566: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:17:48.082793: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x55f8100 executing computations on platform CUDA. Devices:\n",
      "2020-02-15 14:17:48.082829: I tensorflow/compiler/xla/service/service.cc:175]   StreamExecutor device (0): Tesla K80, Compute Capability 3.7\n",
      "2020-02-15 14:17:48.082838: I tensorflow/compiler/xla/service/service.cc:175]   StreamExecutor device (1): Tesla K80, Compute Capability 3.7\n",
      "2020-02-15 14:17:48.083638: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:17:48.084524: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 0 with properties:\n",
      "name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235\n",
      "pciBusID: 0000:00:04.0\n",
      "2020-02-15 14:17:48.084690: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:17:48.085682: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1618] Found device 1 with properties:\n",
      "name: Tesla K80 major: 3 minor: 7 memoryClockRate(GHz): 0.8235\n",
      "pciBusID: 0000:00:05.0\n",
      "2020-02-15 14:17:48.085748: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.0\n",
      "2020-02-15 14:17:48.085769: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0\n",
      "2020-02-15 14:17:48.085781: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10.0\n",
      "2020-02-15 14:17:48.085800: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10.0\n",
      "2020-02-15 14:17:48.085826: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10.0\n",
      "2020-02-15 14:17:48.085848: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10.0\n",
      "2020-02-15 14:17:48.085873: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7\n",
      "2020-02-15 14:17:48.085968: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:17:48.086940: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:17:48.087821: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:17:48.088733: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:17:48.089680: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1746] Adding visible gpu devices: 0, 1\n",
      "2020-02-15 14:17:48.112653: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.0\n",
      "2020-02-15 14:17:48.116019: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1159] Device interconnect StreamExecutor with strength 1 edge matrix:\n",
      "2020-02-15 14:17:48.116057: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1165]      0 1\n",
      "2020-02-15 14:17:48.116068: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1178] 0:   N Y\n",
      "2020-02-15 14:17:48.116075: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1178] 1:   Y N\n",
      "2020-02-15 14:17:48.148478: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:17:48.149619: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:17:48.150669: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:17:48.151644: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1304] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 10804 MB memory) -> physical GPU (device: 0, name: Tesla K80, pci bus id: 0000:00:04.0, compute capability: 3.7)\n",
      "2020-02-15 14:17:48.152263: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:1006] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2020-02-15 14:17:48.153302: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1304] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:1 with 10804 MB memory) -> physical GPU (device: 1, name: Tesla K80, pci bus id: 0000:00:05.0, compute capability: 3.7)\n",
      "2020-02-15 14:18:10.503259: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10.0\n",
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #\n",
      "=================================================================\n",
      "flatten (Flatten)            (None, 784)               0\n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 128)               100480\n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 128)               0\n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 10)                1290\n",
      "=================================================================\n",
      "Total params: 101,770\n",
      "Trainable params: 101,770\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 60000 samples\n",
      "Epoch 1/10\n",
      " 2912/60000 [>.............................] - ETA: 1:24 - loss: 1.0297 - accuracy: 0.64538 \n",
      " 6848/60000 [==>...........................] - ETA: 39s - loss: 0.8562 - accuracy: 0.7054\n",
      "10080/60000 [====>.........................] - ETA: 24s - loss: 0.7693 - accuracy: 0.732\n",
      "13792/60000 [=====>........................] - ETA: 17s - loss: 0.7201 - accuracy: 0.747\n",
      "17408/60000 [=======>......................] - ETA: 13s - loss: 0.6804 - accuracy: 0.759\n",
      "21184/60000 [=========>....................] - ETA: 10s - loss: 0.6558 - accuracy: 0.770\n",
      "24256/60000 [===========>..................] - ETA: 8s - loss: 0.6342 - accuracy: 0.7773\n",
      "28032/60000 [=============>................] - ETA: 7s - loss: 0.6208 - accuracy: 0.781\n",
      "32000/60000 [===============>..............] - ETA: 5s - loss: 0.6041 - accuracy: 0.787\n",
      "35712/60000 [================>.............] - ETA: 4s - loss: 0.5916 - accuracy: 0.79\n",
      "39584/60000 [==================>...........] - ETA: 3s - loss: 0.5800 - accuracy: 0.79\n",
      "42784/60000 [====================>.........] - ETA: 3s - loss: 0.5673 - accuracy: 0.80\n",
      "46624/60000 [======================>.......] - ETA: 2s - loss: 0.5586 - accuracy: 0.80\n",
      "50496/60000 [========================>.....] - ETA: 1s - loss: 0.5506 - accuracy: 0.80\n",
      "54368/60000 [=========================>....] - ETA: 0s - loss: 0.5438 - accuracy: 0.80\n",
      "57600/60000 [===========================>..] - ETA: 0s - loss: 0.5371 - accuracy: 0.81\n",
      "60000/60000 [==============================] - 9s 149us/sample - loss: 0.5327 - accuracy: 0.8117\n",
      "Epoch 2/10\n",
      " 3168/60000 [>.............................] - ETA: 4s - loss: 0.4348 - accuracy: 0.84\n",
      " 7040/60000 [==>...........................] - ETA: 4s - loss: 0.4198 - accuracy: 0.84\n",
      "10976/60000 [====>.........................] - ETA: 3s - loss: 0.4177 - accuracy: 0.85\n",
      "14816/60000 [======>.......................] - ETA: 3s - loss: 0.4125 - accuracy: 0.85\n",
      "17952/60000 [=======>......................] - ETA: 3s - loss: 0.4135 - accuracy: 0.850\n",
      "21952/60000 [=========>....................] - ETA: 3s - loss: 0.4127 - accuracy: 0.850\n",
      "25600/60000 [===========>..................] - ETA: 2s - loss: 0.4102 - accuracy: 0.851\n",
      "29504/60000 [=============>................] - ETA: 2s - loss: 0.4108 - accuracy: 0.85\n",
      "33376/60000 [===============>..............] - ETA: 2s - loss: 0.4102 - accuracy: 0.85\n",
      "36608/60000 [=================>............] - ETA: 1s - loss: 0.4061 - accuracy: 0.85\n",
      "40352/60000 [===================>..........] - ETA: 1s - loss: 0.4063 - accuracy: 0.85\n",
      "44224/60000 [=====================>........] - ETA: 1s - loss: 0.4048 - accuracy: 0.85\n",
      "48096/60000 [=======================.......] - ETA: 1s - loss: 0.4038 - accuracy: 0.85\n",
      "51424/60000 [========================>.....] - ETA: 0s - loss: 0.4021 - accuracy: 0.855\n",
      "55232/60000 [==========================>...] - ETA: 0s - loss: 0.4010 - accuracy: 0.855\n",
      "59136/60000 [============================>.] - ETA: 0s - loss: 0.3992 - accuracy: 0.856\n",
      "60000/60000 [==============================] - 5s 80us/sample - loss: 0.3989 - accuracy: 0.8560\n",
      "Epoch 3/10\n",
      " 3136/60000 [>.............................] - ETA: 4s - loss: 0.3643 - accuracy: 0.86\n",
      " 6816/60000 [==>...........................] - ETA: 4s - loss: 0.3777 - accuracy: 0.85\n",
      "10432/60000 [====>.........................] - ETA: 4s - loss: 0.3818 - accuracy: 0.85\n",
      "14240/60000 [======>.......................] - ETA: 3s - loss: 0.3800 - accuracy: 0.85\n",
      "17504/60000 [=======>......................] - ETA: 3s - loss: 0.3737 - accuracy: 0.861\n",
      "21504/60000 [=========>....................] - ETA: 3s - loss: 0.3707 - accuracy: 0.863\n",
      "25280/60000 [===========>..................] - ETA: 2s - loss: 0.3693 - accuracy: 0.865\n",
      "29056/60000 [=============>................] - ETA: 2s - loss: 0.3681 - accuracy: 0.86\n",
      "33024/60000 [===============>..............] - ETA: 2s - loss: 0.3677 - accuracy: 0.86\n",
      "36224/60000 [=================>............] - ETA: 1s - loss: 0.3693 - accuracy: 0.86\n",
      "39904/60000 [==================>...........] - ETA: 1s - loss: 0.3704 - accuracy: 0.86\n",
      "43840/60000 [====================>.........] - ETA: 1s - loss: 0.3700 - accuracy: 0.86\n",
      "47808/60000 [======================>.......] - ETA: 1s - loss: 0.3694 - accuracy: 0.86\n",
      "50976/60000 [========================>.....] - ETA: 0s - loss: 0.3682 - accuracy: 0.865\n",
      "54848/60000 [==========================>...] - ETA: 0s - loss: 0.3664 - accuracy: 0.865\n",
      "58912/60000 [============================>.] - ETA: 0s - loss: 0.3652 - accuracy: 0.865\n",
      "60000/60000 [==============================] - 5s 80us/sample - loss: 0.3655 - accuracy: 0.8660\n",
      "Epoch 4/10\n",
      " 3104/60000 [>.............................] - ETA: 4s - loss: 0.3650 - accuracy: 0.86\n",
      " 6784/60000 [==>...........................] - ETA: 4s - loss: 0.3367 - accuracy: 0.87\n",
      "10752/60000 [====>.........................] - ETA: 4s - loss: 0.3453 - accuracy: 0.87\n",
      "14528/60000 [======>.......................] - ETA: 3s - loss: 0.3428 - accuracy: 0.87\n",
      "17664/60000 [=======>......................] - ETA: 3s - loss: 0.3454 - accuracy: 0.870\n",
      "21664/60000 [=========>....................] - ETA: 3s - loss: 0.3456 - accuracy: 0.870\n",
      "25504/60000 [===========>..................] - ETA: 2s - loss: 0.3449 - accuracy: 0.871\n",
      "29248/60000 [=============>................] - ETA: 2s - loss: 0.3428 - accuracy: 0.87\n",
      "33216/60000 [===============>..............] - ETA: 2s - loss: 0.3424 - accuracy: 0.87\n",
      "36576/60000 [=================>............] - ETA: 1s - loss: 0.3420 - accuracy: 0.87\n",
      "40512/60000 [===================>..........] - ETA: 1s - loss: 0.3413 - accuracy: 0.87\n",
      "44256/60000 [=====================>........] - ETA: 1s - loss: 0.3435 - accuracy: 0.87\n",
      "48192/60000 [=======================.......] - ETA: 0s - loss: 0.3443 - accuracy: 0.87\n",
      "51488/60000 [========================>.....] - ETA: 0s - loss: 0.3456 - accuracy: 0.872\n",
      "55296/60000 [==========================>...] - ETA: 0s - loss: 0.3443 - accuracy: 0.873\n",
      "59072/60000 [============================>.] - ETA: 0s - loss: 0.3437 - accuracy: 0.874\n",
      "60000/60000 [==============================] - 5s 80us/sample - loss: 0.3431 - accuracy: 0.8744\n",
      "Epoch 5/10\n",
      " 3328/60000 [>.............................] - ETA: 4s - loss: 0.3206 - accuracy: 0.88\n",
      " 7104/60000 [==>...........................] - ETA: 4s - loss: 0.3257 - accuracy: 0.88\n",
      "11040/60000 [====>.........................] - ETA: 3s - loss: 0.3249 - accuracy: 0.88\n",
      "14976/60000 [======>.......................] - ETA: 3s - loss: 0.3241 - accuracy: 0.88\n",
      "18208/60000 [========>.....................] - ETA: 3s - loss: 0.3240 - accuracy: 0.880\n",
      "22080/60000 [==========>...................] - ETA: 3s - loss: 0.3263 - accuracy: 0.879\n",
      "26080/60000 [============>.................] - ETA: 2s - loss: 0.3244 - accuracy: 0.880\n",
      "30048/60000 [==============>...............] - ETA: 2s - loss: 0.3247 - accuracy: 0.87\n",
      "33792/60000 [===============>..............] - ETA: 2s - loss: 0.3262 - accuracy: 0.87\n",
      "37120/60000 [=================>............] - ETA: 1s - loss: 0.3278 - accuracy: 0.87\n",
      "41152/60000 [===================>..........] - ETA: 1s - loss: 0.3274 - accuracy: 0.87\n",
      "44832/60000 [=====================>........] - ETA: 1s - loss: 0.3276 - accuracy: 0.87\n",
      "48608/60000 [=======================.......] - ETA: 0s - loss: 0.3283 - accuracy: 0.87\n",
      "51904/60000 [========================>.....] - ETA: 0s - loss: 0.3271 - accuracy: 0.879\n",
      "55904/60000 [==========================>...] - ETA: 0s - loss: 0.3284 - accuracy: 0.878\n",
      "59648/60000 [============================>.] - ETA: 0s - loss: 0.3290 - accuracy: 0.878\n",
      "60000/60000 [==============================] - 5s 79us/sample - loss: 0.3288 - accuracy: 0.8787\n",
      "Epoch 6/10\n",
      " 3264/60000 [>.............................] - ETA: 4s - loss: 0.3064 - accuracy: 0.88\n",
      " 7200/60000 [==>...........................] - ETA: 4s - loss: 0.3197 - accuracy: 0.88\n",
      "10720/60000 [====>.........................] - ETA: 4s - loss: 0.3206 - accuracy: 0.88\n",
      "14624/60000 [======>.......................] - ETA: 3s - loss: 0.3177 - accuracy: 0.88\n",
      "17792/60000 [=======>......................] - ETA: 3s - loss: 0.3194 - accuracy: 0.882\n",
      "21664/60000 [=========>....................] - ETA: 3s - loss: 0.3173 - accuracy: 0.882\n",
      "25568/60000 [===========>..................] - ETA: 2s - loss: 0.3174 - accuracy: 0.882\n",
      "29408/60000 [=============>................] - ETA: 2s - loss: 0.3165 - accuracy: 0.88\n",
      "33152/60000 [===============>..............] - ETA: 2s - loss: 0.3139 - accuracy: 0.88\n",
      "36448/60000 [=================>............] - ETA: 1s - loss: 0.3148 - accuracy: 0.88\n",
      "40352/60000 [===================>..........] - ETA: 1s - loss: 0.3133 - accuracy: 0.88\n",
      "44192/60000 [=====================>........] - ETA: 1s - loss: 0.3121 - accuracy: 0.88\n",
      "47872/60000 [======================>.......] - ETA: 1s - loss: 0.3128 - accuracy: 0.88\n",
      "50976/60000 [========================>.....] - ETA: 0s - loss: 0.3138 - accuracy: 0.883\n",
      "54848/60000 [==========================>...] - ETA: 0s - loss: 0.3142 - accuracy: 0.883\n",
      "58784/60000 [============================>.] - ETA: 0s - loss: 0.3148 - accuracy: 0.883\n",
      "60000/60000 [==============================] - 5s 80us/sample - loss: 0.3148 - accuracy: 0.8831\n",
      "Epoch 7/10\n",
      " 3168/60000 [>.............................] - ETA: 4s - loss: 0.2915 - accuracy: 0.89\n",
      " 7072/60000 [==>...........................] - ETA: 4s - loss: 0.3009 - accuracy: 0.89\n",
      "10976/60000 [====>.........................] - ETA: 3s - loss: 0.2915 - accuracy: 0.89\n",
      "14688/60000 [======>.......................] - ETA: 3s - loss: 0.2968 - accuracy: 0.89\n",
      "17888/60000 [=======>......................] - ETA: 3s - loss: 0.2989 - accuracy: 0.889\n",
      "21760/60000 [=========>....................] - ETA: 3s - loss: 0.2966 - accuracy: 0.891\n",
      "25632/60000 [===========>..................] - ETA: 2s - loss: 0.2999 - accuracy: 0.889\n",
      "29504/60000 [=============>................] - ETA: 2s - loss: 0.3009 - accuracy: 0.88\n",
      "33344/60000 [===============>..............] - ETA: 2s - loss: 0.3007 - accuracy: 0.88\n",
      "36512/60000 [=================>............] - ETA: 1s - loss: 0.3036 - accuracy: 0.88\n",
      "40320/60000 [===================>..........] - ETA: 1s - loss: 0.3049 - accuracy: 0.88\n",
      "44128/60000 [=====================>........] - ETA: 1s - loss: 0.3057 - accuracy: 0.88\n",
      "47872/60000 [======================>.......] - ETA: 1s - loss: 0.3050 - accuracy: 0.88\n",
      "51072/60000 [========================>.....] - ETA: 0s - loss: 0.3062 - accuracy: 0.887\n",
      "55040/60000 [==========================>...] - ETA: 0s - loss: 0.3052 - accuracy: 0.887\n",
      "58944/60000 [============================>.] - ETA: 0s - loss: 0.3059 - accuracy: 0.887\n",
      "60000/60000 [==============================] - 5s 80us/sample - loss: 0.3057 - accuracy: 0.8874\n",
      "Epoch 8/10\n",
      " 3136/60000 [>.............................] - ETA: 4s - loss: 0.3118 - accuracy: 0.88\n",
      " 7040/60000 [==>...........................] - ETA: 4s - loss: 0.3067 - accuracy: 0.88\n",
      "10976/60000 [====>.........................] - ETA: 3s - loss: 0.2990 - accuracy: 0.88\n",
      "14528/60000 [======>.......................] - ETA: 3s - loss: 0.3024 - accuracy: 0.88\n",
      "17760/60000 [=======>......................] - ETA: 3s - loss: 0.2979 - accuracy: 0.889\n",
      "21696/60000 [=========>....................] - ETA: 3s - loss: 0.3003 - accuracy: 0.888\n",
      "25568/60000 [===========>..................] - ETA: 2s - loss: 0.2970 - accuracy: 0.890\n",
      "29344/60000 [=============>................] - ETA: 2s - loss: 0.2978 - accuracy: 0.88\n",
      "33216/60000 [===============>..............] - ETA: 2s - loss: 0.2959 - accuracy: 0.89\n",
      "36480/60000 [=================>............] - ETA: 1s - loss: 0.2946 - accuracy: 0.89\n",
      "40192/60000 [===================>..........] - ETA: 1s - loss: 0.2957 - accuracy: 0.89\n",
      "44224/60000 [=====================>........] - ETA: 1s - loss: 0.2952 - accuracy: 0.89\n",
      "48160/60000 [=======================.......] - ETA: 1s - loss: 0.2966 - accuracy: 0.89\n",
      "51328/60000 [========================>.....] - ETA: 0s - loss: 0.2956 - accuracy: 0.890\n",
      "55104/60000 [==========================>...] - ETA: 0s - loss: 0.2950 - accuracy: 0.890\n",
      "59008/60000 [============================>.] - ETA: 0s - loss: 0.2964 - accuracy: 0.890\n",
      "60000/60000 [==============================] - 5s 80us/sample - loss: 0.2960 - accuracy: 0.8901\n",
      "Epoch 9/10\n",
      " 3232/60000 [>.............................] - ETA: 4s - loss: 0.2730 - accuracy: 0.89\n",
      " 7072/60000 [==>...........................] - ETA: 4s - loss: 0.2834 - accuracy: 0.89\n",
      "10688/60000 [====>.........................] - ETA: 4s - loss: 0.2849 - accuracy: 0.89\n",
      "14336/60000 [======>.......................] - ETA: 3s - loss: 0.2843 - accuracy: 0.89\n",
      "17216/60000 [=======>......................] - ETA: 3s - loss: 0.2876 - accuracy: 0.890\n",
      "21088/60000 [=========>....................] - ETA: 3s - loss: 0.2891 - accuracy: 0.891\n",
      "24736/60000 [===========>..................] - ETA: 2s - loss: 0.2888 - accuracy: 0.892\n",
      "28544/60000 [=============>................] - ETA: 2s - loss: 0.2925 - accuracy: 0.89\n",
      "32352/60000 [==============>...............] - ETA: 2s - loss: 0.2941 - accuracy: 0.89\n",
      "35648/60000 [================>.............] - ETA: 2s - loss: 0.2918 - accuracy: 0.89\n",
      "39520/60000 [==================>...........] - ETA: 1s - loss: 0.2912 - accuracy: 0.89\n",
      "43296/60000 [====================>.........] - ETA: 1s - loss: 0.2917 - accuracy: 0.89\n",
      "47136/60000 [======================>.......] - ETA: 1s - loss: 0.2912 - accuracy: 0.89\n",
      "50368/60000 [========================>.....] - ETA: 0s - loss: 0.2899 - accuracy: 0.892\n",
      "54112/60000 [==========================>...] - ETA: 0s - loss: 0.2900 - accuracy: 0.892\n",
      "58016/60000 [============================>.] - ETA: 0s - loss: 0.2898 - accuracy: 0.892\n",
      "60000/60000 [==============================] - 5s 81us/sample - loss: 0.2893 - accuracy: 0.8921\n",
      "Epoch 10/10\n",
      " 3328/60000 [>.............................] - ETA: 4s - loss: 0.2611 - accuracy: 0.89\n",
      " 7104/60000 [==>...........................] - ETA: 4s - loss: 0.2742 - accuracy: 0.89\n",
      "11072/60000 [====>.........................] - ETA: 3s - loss: 0.2842 - accuracy: 0.88\n",
      "14976/60000 [======>.......................] - ETA: 3s - loss: 0.2841 - accuracy: 0.89\n",
      "17984/60000 [=======>......................] - ETA: 3s - loss: 0.2807 - accuracy: 0.893\n",
      "21792/60000 [=========>....................] - ETA: 3s - loss: 0.2805 - accuracy: 0.894\n",
      "25728/60000 [===========>..................] - ETA: 2s - loss: 0.2816 - accuracy: 0.894\n",
      "29504/60000 [=============>................] - ETA: 2s - loss: 0.2840 - accuracy: 0.89\n",
      "33312/60000 [===============>..............] - ETA: 2s - loss: 0.2838 - accuracy: 0.89\n",
      "36512/60000 [=================>............] - ETA: 1s - loss: 0.2834 - accuracy: 0.89\n",
      "40352/60000 [===================>..........] - ETA: 1s - loss: 0.2823 - accuracy: 0.89\n",
      "44288/60000 [=====================>........] - ETA: 1s - loss: 0.2821 - accuracy: 0.89\n",
      "48256/60000 [=======================.......] - ETA: 0s - loss: 0.2821 - accuracy: 0.89\n",
      "51456/60000 [========================>.....] - ETA: 0s - loss: 0.2819 - accuracy: 0.893\n",
      "55104/60000 [==========================>...] - ETA: 0s - loss: 0.2814 - accuracy: 0.894\n",
      "58976/60000 [============================>.] - ETA: 0s - loss: 0.2808 - accuracy: 0.894\n",
      "60000/60000 [==============================] - 5s 80us/sample - loss: 0.2818 - accuracy: 0.8942\n",
      "10000/1 - 1s - loss: 0.2460 - accuracy: 0.8822\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[W 200215 14:19:03 job:162] Cleaning up job fairing-job-82gd8...\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import os\n",
    "\n",
    "class MyFashionMnist(object):\n",
    "  def train(self):\n",
    "\n",
    "    (x_train, y_train), (x_test, y_test) = tf.keras.datasets.fashion_mnist.load_data()\n",
    "    x_train, x_test = x_train / 255.0, x_test / 255.0\n",
    "\n",
    "    model = tf.keras.models.Sequential([\n",
    "      tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
    "      tf.keras.layers.Dense(128, activation='relu'),\n",
    "      tf.keras.layers.Dropout(0.2),\n",
    "      tf.keras.layers.Dense(10, activation='softmax')\n",
    "    ])\n",
    "    model.summary()\n",
    "    model.compile(optimizer='adam',\n",
    "                  loss='sparse_categorical_crossentropy',\n",
    "                  metrics=['accuracy'])\n",
    "\n",
    "    model.fit(x_train, y_train, epochs=10)\n",
    "\n",
    "    model.evaluate(x_test,  y_test, verbose=2)\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    if os.getenv('FAIRING_RUNTIME', None) is None:\n",
    "        from kubeflow import fairing\n",
    "        from kubeflow.fairing.kubernetes import utils as k8s_utils\n",
    "\n",
    "        DOCKER_REGISTRY = 'kubeflow-registry.default.svc.cluster.local:30000'\n",
    "        fairing.config.set_builder(\n",
    "            'append',\n",
    "            image_name='fairing-job',\n",
    "            base_image='brightfly/kubeflow-jupyter-lab:tf2.0-gpu',\n",
    "            registry=DOCKER_REGISTRY, \n",
    "            push=True)\n",
    "        # cpu 2, memory 5GiB\n",
    "        fairing.config.set_deployer('job',\n",
    "                                    namespace='dudaji',\n",
    "                                    pod_spec_mutators=[\n",
    "                                        k8s_utils.get_resource_mutator(cpu=2,\n",
    "                                                                       memory=5)]\n",
    "         \n",
    "                                   )\n",
    "        fairing.config.run()\n",
    "    else:\n",
    "        remote_train = MyFashionMnist()\n",
    "        remote_train.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}