interactive_user_study.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Interactive Interface for User Study\n",
    "\n",
    "None: make sure to update the [config file](src/config.py)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for CPU only\n",
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '-1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-11-02 18:06:46.443155: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcudart.so.10.1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading BERT model... "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertModel: ['vocab_layer_norm.weight', 'vocab_projector.bias', 'vocab_transform.weight', 'vocab_projector.weight', 'vocab_transform.bias', 'vocab_layer_norm.bias']\n",
      "- This IS expected if you are initializing DistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing DistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n",
      "loading CLIP model... done\n",
      "loading precomputed CLIP text embbedings... done\n",
      "loading precomputed CLIP img embbedings... done\n",
      "DEVICE:  cpu\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import numpy as np\n",
    "from src.motion_refiner_4D import Motion_refiner, MAX_NUM_OBJS\n",
    "from src.config import *\n",
    "from src.userstudy_interface import *\n",
    "\n",
    "traj_n = 40\n",
    "mr = Motion_refiner(load_models=True ,traj_n = traj_n, locality_factor=True, clip_only=False,load_precomp_emb=True)\n",
    "feature_indices, obj_sim_indices, obj_poses_indices, traj_indices = mr.get_indices()\n",
    "embedding_indices = np.concatenate([feature_indices,obj_sim_indices, obj_poses_indices])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading dataset:  latte_100k_lf ...done\n",
      "raw X: (100000, 953) \tY: (100000, 160)\n",
      "filtered X: (96718, 953) \tY: (96718, 160)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-11-02 18:07:04.742803: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set\n",
      "2023-11-02 18:07:04.742889: I tensorflow/stream_executor/platform/default/dso_loader.cc:49] Successfully opened dynamic library libcuda.so.1\n",
      "2023-11-02 18:07:04.745428: E tensorflow/stream_executor/cuda/cuda_driver.cc:328] failed call to cuInit: CUDA_ERROR_NO_DEVICE: no CUDA-capable device is detected\n",
      "2023-11-02 18:07:04.745446: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:169] retrieving CUDA diagnostic information for host: arthur-XPS-15-9530\n",
      "2023-11-02 18:07:04.745450: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:176] hostname: arthur-XPS-15-9530\n",
      "2023-11-02 18:07:04.745493: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:200] libcuda reported version is: 535.129.3\n",
      "2023-11-02 18:07:04.745506: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:204] kernel reported version is: 535.129.3\n",
      "2023-11-02 18:07:04.745509: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:310] kernel version seems to match DSO: 535.129.3\n",
      "2023-11-02 18:07:04.745681: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  SSE4.1 SSE4.2 AVX AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-11-02 18:07:04.747479: I tensorflow/compiler/jit/xla_gpu_device.cc:99] Not creating XLA devices, tf_xla_enable_xla_devices not set\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train X: (67702, 953) \tY: (67702, 160)\n",
      "Test  X: (19344, 953) \tY: (19344, 160)\n",
      "Val   X: (9672, 953) \tY: (9672, 160)\n"
     ]
    }
   ],
   "source": [
    "dataset_name = \"latte_100k_lf\"\n",
    "\n",
    "X,Y, data = mr.load_dataset(dataset_name, filter_data = True, base_path=base_folder+\"data/\")\n",
    "X_train, X_test, X_valid, y_train, y_test, y_valid, indices_train, indices_test, indices_val = mr.split_dataset(X, Y, test_size=0.2, val_size=0.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load model for real time interactions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'num_layers_enc': 1, 'num_layers_dec': 5, 'd_model': 400, 'dff': 512, 'num_heads': 8, 'dropout_rate': 0.1, 'wp_d': 4, 'num_emb_vec': 4, 'bs': 16, 'dense_n': 512, 'num_dense': 3, 'concat_emb': True, 'features_n': 793, 'optimizer': 'adam', 'norm_layer': True, 'activation': 'tanh'}\n",
      "loading weights:  /home/arthur/Desktop/projects/LaTTe-Language-Trajectory-TransformEr/models/TF-num_layers_enc:1-num_layers_dec:5-d_model:400-dff:512-num_heads:8-dropout_rate:0.1-wp_d:4-num_emb_vec:4-bs:16-dense_n:512-num_dense:3-concat_emb:True-features_n:793-optimizer:adam-norm_layer:True-activation:tanh.h5\n",
      "DONE\n"
     ]
    }
   ],
   "source": [
    "from src.TF4D_mult_features import *\n",
    "\n",
    "model_name = \"TF-num_layers_enc:1-num_layers_dec:5-d_model:400-dff:512-num_heads:8-dropout_rate:0.1-wp_d:4-num_emb_vec:4-bs:16-dense_n:512-num_dense:3-concat_emb:True-features_n:793-optimizer:adam-norm_layer:True-activation:tanh.h5\"\n",
    "model_file = models_folder+model_name\n",
    "\n",
    "model = load_model(model_file, delimiter=\"-\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DONE - computing textual embeddings (1, 768)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1it [00:00, 2734.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DONE - computing similarity vectors \n",
      "DONE - concatenating \n",
      "X shape (1, 953)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 39/39 [00:01<00:00, 34.31it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(40, 4)\n",
      "DONE - computing textual embeddings (1, 768)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1it [00:00, 1955.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DONE - computing similarity vectors \n",
      "DONE - concatenating \n",
      "X shape (1, 953)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 39/39 [00:01<00:00, 34.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(40, 4)\n"
     ]
    }
   ],
   "source": [
    "%matplotlib qt\n",
    "user_study = User_study_interface([data], dis_names=[\"data_pred\"],samples_per_data=0, interaction_samples=3, model=model, mr=mr)\n",
    "# user_study.replay_from_file(\"user study json file\")\n",
    "# plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.13 ('py38_cu11_2')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "5de91200c9f9e1f8a0c28ceba668014be0fd55838e84400e0a7ad1d269192773"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}