Going back to RL working and understanding whats the issue

behavior_studio/brains/f1rl/train.py

@@ -31,11 +31,14 @@ def save_model(qlearn):
     date = datetime.now()
     format = date.strftime("%Y%m%d_%H%M%S")
     file_name = "_qlearn_model_e_{}_a_{}_g_{}".format(qlearn.epsilon, qlearn.alpha, qlearn.gamma)
+    if os.path.isdir('brains/f1rl/logs') is False:
+        os.mkdir('brains/f1rl/logs')
+    if os.path.isdir('brains/f1rl/logs/qlearn_models/') is False:
+        os.mkdir('brains/f1rl/logs/qlearn_models/')
     file = open("brains/f1rl/logs/qlearn_models/" + format + file_name + '.pkl', 'wb')
     pickle.dump(qlearn.q, file)
 
 # if __name__ == '__main__':
-
 print(settings.title)
 print(settings.description)
 
@@ -51,6 +54,7 @@ def save_model(qlearn):
 outdir = './logs/f1_qlearn_gym_experiments/'
 stats = {}  # epoch: steps
 
+
 env = gym.wrappers.Monitor(env, outdir, force=True)
 plotter = liveplot.LivePlot(outdir)
 last_time_steps = np.ndarray(0)

behavior_studio/brains/f1rl/utils/settings.py

@@ -3,7 +3,7 @@
 import numpy as np
 from utils.configuration import Config
 
-app_configuration = Config('default.yml')
+app_configuration = Config('default-rl.yml')
 
 debug_level = 0
 telemetry = False

behavior_studio/default-rl.yml

@@ -0,0 +1,55 @@
+Behaviors:
+    Robot:
+        Sensors:
+            Cameras:
+                Camera_0:
+                    Name: 'camera_0'
+                    Topic: '/F1ROS/cameraL/image_raw'
+            Pose3D:
+                Pose3D_0:
+                    Name: 'pose3d_0'
+                    Topic: '/F1ROS/odom'
+        Actuators:
+            Motors:
+                Motors_0:
+                    Name: 'motors_0'
+                    Topic: '/F1ROS/cmd_vel'
+                    MaxV: 3
+                    MaxW: 0.3
+                    RL: True
+        BrainPath: 'brains/f1rl/train.py'
+        Type: 'f1rl'
+        Parameters:
+            action_set: 'hard'
+            gazebo_positions_set: 'pista_simple'
+            alpha: 0.2 
+            gamma: 0.9
+            epsilon: 0.99
+            total_episodes: 20000
+            epsilon_discount: 0.9986 
+            env: 'camera'
+    Simulation:
+        World: /opt/jderobot/share/jderobot/gazebo/launch/simple_circuit.launch
+    Dataset:
+        In: '/tmp/my_bag.bag'
+        Out: ''
+    Stats:
+        Out: './'
+        PerfectLap: 'lap-simple-circuit.bag'
+    Layout:
+        Frame_0:
+            Name: frame_0
+            Geometry: [1, 1, 2, 2]
+            Data: rgbimage
+        Frame_1:
+            Name: frame_1
+            Geometry: [0, 1, 1, 1]
+            Data: rgbimage
+        Frame_2:
+            Name: frame_2
+            Geometry: [0, 2, 1, 1]
+            Data: rgbimage
+        Frame_3:
+            Name: frame_3
+            Geometry: [0, 3, 3, 1]
+            Data: rgbimage

behavior_studio/default.yml

@@ -16,8 +16,8 @@ Behaviors:
                     Topic: '/F1ROS/cmd_vel'
                     MaxV: 3
                     MaxW: 0.3
-        BrainPath: 'brains/f1/brain_f1_keras.py'
-        Model: 'model_deepest_lstm_cropped_250_norm_max_pooling.h5'
+                    
+        BrainPath: 'brains/f1/brain_f1_explicit.py'
         Type: 'f1'
     Simulation:
         World: /opt/jderobot/share/jderobot/gazebo/launch/simple_circuit.launch

behavior_studio/profiles/default.yml

@@ -19,7 +19,7 @@ Behaviors:
         BrainPath: 'brains/f1/brain_f1_opencv.py'
         Type: 'f1'
     Simulation:
-        World: /opt/jderobot/share/jderobot/gazebo/launch/f1_1_simplecircuit.launch
+        World: /opt/jderobot/share/jderobot/gazebo/launch/simple_circuit.launch
     Dataset:
         In: '/tmp/my_bag.bag'
         Out: ''

behavior_studio/robot/actuators.py

@@ -49,8 +49,14 @@ def __create_actuator(self, actuator_config, actuator_type):
             topic = actuator_config[elem]['Topic']
             vmax = actuator_config[elem]['MaxV']
             wmax = actuator_config[elem]['MaxW']
-            if actuator_type == 'motor':
-                actuator_dict[name] = PublisherMotors(topic, vmax, wmax, 0, 0)
+
+            if 'RL' in actuator_config[elem]:
+                if actuator_config[elem]['RL'] == False:
+                    if actuator_type == 'motor':
+                        actuator_dict[name] = PublisherMotors(topic, vmax, wmax, 0, 0)
+            else:
+                if actuator_type == 'motor':
+                        actuator_dict[name] = PublisherMotors(topic, vmax, wmax, 0, 0)
         return actuator_dict
 
     def __get_actuator(self, actuator_name, actuator_type):

gym-gazebo/gym_gazebo/envs/assets/worlds/simple_circuit.world

@@ -0,0 +1,28 @@
+<?xml version="1.0" ?>
+<sdf version="1.5">
+<world name="default">
+        <!-- A global light source -->
+        <include>
+                <uri>model://sun</uri>
+        </include>
+        <include>
+                <uri>model://simple_circuit</uri>
+                <pose>0 0 0 0 0 0</pose>
+        </include>
+        <include>
+                <uri>model://f1_renault</uri>
+                <pose>53.462 -10.734 0.004 0 0 -1.57</pose>
+        </include>
+
+        <scene>
+                <sky>
+                        <clouds>
+                                <speed>12</speed>
+                        </clouds>
+                </sky>
+                <ambient>1.0 1.0 1.0 1.0</ambient>
+                <shadows>false</shadows>
+                <grid>false</grid>
+        </scene>
+</world>
+</sdf>

gym-gazebo/gym_gazebo/envs/assets/worlds/simple_circuit_laser.world

@@ -0,0 +1,28 @@
+<?xml version="1.0" ?>
+<sdf version="1.5">
+<world name="default">
+        <!-- A global light source -->
+        <include>
+                <uri>model://sun</uri>
+        </include>
+        <include>
+                <uri>model://simple_circuit</uri>
+                <pose>0 0 0 0 0 0</pose>
+        </include>
+        <include>
+                <uri>model://f1_renault</uri>
+                <pose>53.462 -10.734 0.004 0 0 -1.57</pose>
+        </include>
+
+        <scene>
+                <sky>
+                        <clouds>
+                                <speed>12</speed>
+                        </clouds>
+                </sky>
+                <ambient>1.0 1.0 1.0 1.0</ambient>
+                <shadows>false</shadows>
+                <grid>false</grid>
+        </scene>
+</world>
+</sdf>

gym-gazebo/gym_gazebo/envs/installation/formula1_laser_setup.bash

@@ -6,8 +6,8 @@ else
   bash -c 'sed "s,GAZEBO_MODEL_PATH=[^;]*,'GAZEBO_MODEL_PATH=`pwd`/../assets/models'," -i ~/.bashrc'
 fi
 
-bash -c 'echo "export GYM_GAZEBO_WORLD_CIRCUIT_F1="`pwd`/../assets/worlds/f1_1_simplecircuit.world >> ~/.bashrc'
-bash -c 'echo "export GYM_GAZEBO_WORLD_CIRCUIT_F1_LASER="`pwd`/../assets/worlds/f1_1_simplecircuit_laser.world >> ~/.bashrc'
+bash -c 'echo "export GYM_GAZEBO_WORLD_CIRCUIT_F1="`pwd`/../assets/worlds/simple_circuit.world >> ~/.bashrc'
+bash -c 'echo "export GYM_GAZEBO_WORLD_CIRCUIT_F1_LASER="`pwd`/../assets/worlds/simple_circuit_laser.world >> ~/.bashrc'
 #export GYM_GAZEBO_WORLD_CIRCUIT_F1=`pwd`/../assets/worlds/f1_1_simplecircuit.world
 echo 'Formula 1 env variables loaded succesfully'
 

gym-gazebo/gym_gazebo/envs/installation/formula1_setup.bash

@@ -17,7 +17,7 @@ fi
 #else
 #  bash -c 'sed "s,GYM_GAZEBO_WORLD_CIRCUIT=[^;]*,'GYM_GAZEBO_WORLD_CIRCUIT=`pwd`/../assets/worlds/f1_1_simplecircuit.world'," -i ~/.bashrc'
 #fi
-bash -c 'echo "export GYM_GAZEBO_WORLD_CIRCUIT_F1="`pwd`/../assets/worlds/f1_1_simplecircuit.world >> ~/.bashrc'
+bash -c 'echo "export GYM_GAZEBO_WORLD_CIRCUIT_F1="`pwd`/../assets/worlds/simple_circuit.world >> ~/.bashrc'
 #export GYM_GAZEBO_WORLD_CIRCUIT_F1=`pwd`/../assets/worlds/f1_1_simplecircuit.world
 echo 'Formula 1 env variables loaded succesfully'