DQN_Pretrained_Model_for_Pong.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "DQN Pretrained Model for Pong",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "metadata": {
        "id": "NHQ8dw3q91LN",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Download Pretrained Model\n",
        "This model is only trained to play Pong"
      ]
    },
    {
      "metadata": {
        "id": "9KSRJSWT96VY",
        "colab_type": "code",
        "outputId": "c01f5f62-b051-4716-8eef-385c7fe6556a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        }
      },
      "cell_type": "code",
      "source": [
        "! git clone https://github.com/LiamHz/PretrainedModels"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'PretrainedModels'...\n",
            "remote: Enumerating objects: 7, done.\u001b[K\n",
            "remote: Counting objects: 100% (7/7), done.\u001b[K\n",
            "remote: Compressing objects: 100% (6/6), done.\u001b[K\n",
            "remote: Total 7 (delta 1), reused 7 (delta 1), pack-reused 0\u001b[K\n",
            "Unpacking objects: 100% (7/7), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "6a0xunL8GJ_X",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Colab Installs + Regular Imports\n",
        "If you're running this script outside of a notebook set Colab to 'False'\n",
        "gsync allows this notebook to save pretrained models directly to your Google Drive account"
      ]
    },
    {
      "metadata": {
        "id": "_Amw1q8s_2vt",
        "colab_type": "code",
        "outputId": "39c0695a-0c1d-4829-b0c7-bbe26d4240f4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 459
        }
      },
      "cell_type": "code",
      "source": [
        "# Google Colab PyTorch utilities\n",
        "!wget https://raw.githubusercontent.com/StefOe/colab-pytorch-utils/master/utils.py\n",
        "import utils\n",
        "gsync = utils.GDriveSync()\n",
        "\n",
        "! pip install torch\n",
        "! pip install gym\n",
        "! pip install opencv-python\n",
        "\n",
        "import time\n",
        "import numpy as np\n",
        "import collections\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "\n",
        "import cv2\n",
        "import gym\n",
        "import gym.spaces\n",
        "\n",
        "COLAB = True\n",
        "CUDA = True\n",
        "\n",
        "if not COLAB:\n",
        "    from lib import wrappers\n",
        "    from lib import dqn_model\n",
        "\n",
        "    import argparse\n",
        "    from tensorboardX import SummaryWriter"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2019-02-17 17:00:41--  https://raw.githubusercontent.com/StefOe/colab-pytorch-utils/master/utils.py\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 4519 (4.4K) [text/plain]\n",
            "Saving to: ‘utils.py’\n",
            "\n",
            "\rutils.py              0%[                    ]       0  --.-KB/s               \rutils.py            100%[===================>]   4.41K  --.-KB/s    in 0s      \n",
            "\n",
            "2019-02-17 17:00:41 (87.4 MB/s) - ‘utils.py’ saved [4519/4519]\n",
            "\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (4.28.1)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.6/dist-packages (1.0.1.post2)\n",
            "Requirement already satisfied: gym in /usr/local/lib/python3.6/dist-packages (0.10.11)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from gym) (1.1.0)\n",
            "Requirement already satisfied: numpy>=1.10.4 in /usr/local/lib/python3.6/dist-packages (from gym) (1.14.6)\n",
            "Requirement already satisfied: requests>=2.0 in /usr/local/lib/python3.6/dist-packages (from gym) (2.18.4)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from gym) (1.11.0)\n",
            "Requirement already satisfied: pyglet>=1.2.0 in /usr/local/lib/python3.6/dist-packages (from gym) (1.3.2)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0->gym) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0->gym) (2018.11.29)\n",
            "Requirement already satisfied: urllib3<1.23,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0->gym) (1.22)\n",
            "Requirement already satisfied: idna<2.7,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0->gym) (2.6)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from pyglet>=1.2.0->gym) (0.16.0)\n",
            "Requirement already satisfied: opencv-python in /usr/local/lib/python3.6/dist-packages (3.4.5.20)\n",
            "Requirement already satisfied: numpy>=1.11.3 in /usr/local/lib/python3.6/dist-packages (from opencv-python) (1.14.6)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "Fpp316gJGXZp",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# OpenAI Gym Wrappers\n",
        "These wrappers make it easier to interact with OpenAI Gym\n",
        "\n",
        "Wrappers include:\n",
        "\n",
        "\n",
        "*   Frame skipping\n",
        "*   Frame processing (downsampling and greyscaling)\n",
        "* Image normalization and converting to PyTorch\n",
        "\n"
      ]
    },
    {
      "metadata": {
        "id": "zTZyTsre_3lG",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Taken from OpenAI baseline wrappers\n",
        "# https://github.com/openai/baselines/blob/master/baselines/common/atari_wrappers.py\n",
        "\n",
        "class FireResetEnv(gym.Wrapper):\n",
        "    def __init__(self, env=None):\n",
        "        \"\"\"Take action on reset for environments that are fixed until firing.\"\"\"\n",
        "        super(FireResetEnv, self).__init__(env)\n",
        "        print(env.unwrapped.get_action_meanings())\n",
        "        assert env.unwrapped.get_action_meanings()[1] == 'FIRE'\n",
        "        assert len(env.unwrapped.get_action_meanings()) >= 3\n",
        "\n",
        "    def step(self, action):\n",
        "        return self.env.step(action)\n",
        "\n",
        "    def reset(self):\n",
        "        self.env.reset()\n",
        "        obs, _, done, _ = self.env.step(1)\n",
        "        if done:\n",
        "            self.env.reset()\n",
        "        obs, _, done, _ = self.env.step(2)\n",
        "        if done:\n",
        "            self.env.reset()\n",
        "        return obs\n",
        "\n",
        "\n",
        "class MaxAndSkipEnv(gym.Wrapper):\n",
        "    def __init__(self, env=None, skip=4):\n",
        "        \"\"\"Return only every `skip`-th frame\"\"\"\n",
        "        super(MaxAndSkipEnv, self).__init__(env)\n",
        "        # most recent raw observations (for max pooling across time steps)\n",
        "        self._obs_buffer = collections.deque(maxlen=2)\n",
        "        self._skip = skip\n",
        "\n",
        "    def step(self, action):\n",
        "        total_reward = 0.0\n",
        "        done = None\n",
        "        for _ in range(self._skip):\n",
        "            obs, reward, done, info = self.env.step(action)\n",
        "            self._obs_buffer.append(obs)\n",
        "            total_reward += reward\n",
        "            if done:\n",
        "                break\n",
        "        max_frame = np.max(np.stack(self._obs_buffer), axis=0)\n",
        "        return max_frame, total_reward, done, info\n",
        "\n",
        "    def reset(self):\n",
        "        \"\"\"Clear past frame buffer and init to first obs\"\"\"\n",
        "        self._obs_buffer.clear()\n",
        "        obs = self.env.reset()\n",
        "        self._obs_buffer.append(obs)\n",
        "        return obs\n",
        "\n",
        "\n",
        "class ProcessFrame84(gym.ObservationWrapper):\n",
        "    \"\"\"\n",
        "    Downsamples image to 84x84\n",
        "    Greyscales image\n",
        "\n",
        "    Returns numpy array\n",
        "    \"\"\"\n",
        "    def __init__(self, env=None):\n",
        "        super(ProcessFrame84, self).__init__(env)\n",
        "        self.observation_space = gym.spaces.Box(low=0, high=255, shape=(84, 84, 1), dtype=np.uint8)\n",
        "\n",
        "    def observation(self, obs):\n",
        "        return ProcessFrame84.process(obs)\n",
        "\n",
        "    @staticmethod\n",
        "    def process(frame):\n",
        "        if frame.size == 210 * 160 * 3:\n",
        "            img = np.reshape(frame, [210, 160, 3]).astype(np.float32)\n",
        "        elif frame.size == 250 * 160 * 3:\n",
        "            img = np.reshape(frame, [250, 160, 3]).astype(np.float32)\n",
        "        else:\n",
        "            assert False, \"Unknown resolution.\"\n",
        "        img = img[:, :, 0] * 0.299 + img[:, :, 1] * 0.587 + img[:, :, 2] * 0.114\n",
        "        resized_screen = cv2.resize(img, (84, 110), interpolation=cv2.INTER_AREA)\n",
        "        x_t = resized_screen[18:102, :]\n",
        "        x_t = np.reshape(x_t, [84, 84, 1])\n",
        "        return x_t.astype(np.uint8)\n",
        "\n",
        "\n",
        "class ImageToPyTorch(gym.ObservationWrapper):\n",
        "    def __init__(self, env):\n",
        "        super(ImageToPyTorch, self).__init__(env)\n",
        "        old_shape = self.observation_space.shape\n",
        "        self.observation_space = gym.spaces.Box(low=0.0, high=1.0, shape=(old_shape[-1], old_shape[0], old_shape[1]),\n",
        "                                                dtype=np.float32)\n",
        "\n",
        "    def observation(self, observation):\n",
        "        return np.moveaxis(observation, 2, 0)\n",
        "\n",
        "\n",
        "class ScaledFloatFrame(gym.ObservationWrapper):\n",
        "    \"\"\"Normalize pixel values in frame --> 0 to 1\"\"\"\n",
        "    def observation(self, obs):\n",
        "        return np.array(obs).astype(np.float32) / 255.0\n",
        "\n",
        "\n",
        "class BufferWrapper(gym.ObservationWrapper):\n",
        "    def __init__(self, env, n_steps, dtype=np.float32):\n",
        "        super(BufferWrapper, self).__init__(env)\n",
        "        self.dtype = dtype\n",
        "        old_space = env.observation_space\n",
        "        self.observation_space = gym.spaces.Box(old_space.low.repeat(n_steps, axis=0),\n",
        "                                                old_space.high.repeat(n_steps, axis=0), dtype=dtype)\n",
        "\n",
        "    def reset(self):\n",
        "        self.buffer = np.zeros_like(self.observation_space.low, dtype=self.dtype)\n",
        "        return self.observation(self.env.reset())\n",
        "\n",
        "    def observation(self, observation):\n",
        "        self.buffer[:-1] = self.buffer[1:]\n",
        "        self.buffer[-1] = observation\n",
        "        return self.buffer\n",
        "\n",
        "\n",
        "def make_env(env_name):\n",
        "    env = gym.make(env_name)\n",
        "    env = MaxAndSkipEnv(env)\n",
        "    env = FireResetEnv(env)\n",
        "    env = ProcessFrame84(env)\n",
        "    env = ImageToPyTorch(env)\n",
        "    env = BufferWrapper(env, 4)\n",
        "    return ScaledFloatFrame(env)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "MyeLIpznGhBu",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# DQN Architecture\n",
        "Deep-Q-Networks (DQNs) are composed of: \n",
        "* 3 convolution layers\n",
        "* 2 fully-connected linear layers"
      ]
    },
    {
      "metadata": {
        "id": "uyvSSjc1GjIn",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "class DQN(nn.Module):\n",
        "    def __init__(self, input_shape, n_actions):\n",
        "        super(DQN, self).__init__()\n",
        "\n",
        "        self.conv = nn.Sequential(\n",
        "            nn.Conv2d(input_shape[0], 32, kernel_size=8, stride=4),\n",
        "            nn.ReLU(),\n",
        "            nn.Conv2d(32, 64, kernel_size=4, stride=2),\n",
        "            nn.ReLU(),\n",
        "            nn.Conv2d(64, 64, kernel_size=3, stride=1),\n",
        "            nn.ReLU()\n",
        "        )\n",
        "\n",
        "        conv_out_size = self._get_conv_out(input_shape)\n",
        "        self.fc = nn.Sequential(\n",
        "            nn.Linear(conv_out_size, 512),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(512, n_actions)\n",
        "        )\n",
        "\n",
        "    def _get_conv_out(self, shape):\n",
        "        o = self.conv(torch.zeros(1, *shape))\n",
        "        return int(np.prod(o.size()))\n",
        "\n",
        "    def forward(self, x):\n",
        "        conv_out = self.conv(x).view(x.size()[0], -1)\n",
        "        return self.fc(conv_out)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "FvdTslVUGmdH",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Training Parameters\n",
        "Parameters\n",
        "* REPLAY_SIZE: Maximum number of experiences stored in replay memory\n",
        "* TARGET_UPDATE_FREQ: How many frames in between syncing target DQN with behaviour DQN\n",
        "* LEARNING_STARTS: Number of experiences to add to replay memory before training network"
      ]
    },
    {
      "metadata": {
        "id": "Tcm5N8KP0sF-",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "ENV_NAME = \"PongNoFrameskip-v4\"\n",
        "MEAN_REWARD_BOUND = 19.5\n",
        "\n",
        "GAMMA = 0.99\n",
        "BATCH_SIZE = 32\n",
        "REPLAY_SIZE = 10 ** 4 * 4\n",
        "LEARNING_RATE = 1e-4\n",
        "TARGET_UPDATE_FREQ = 1000\n",
        "LEARNING_STARTS = 50000\n",
        "\n",
        "EPSILON_DECAY = 10**5\n",
        "EPSILON_START = 1.0\n",
        "EPSILON_FINAL = 0.02\n",
        "\n",
        "MODEL = \"PretrainedModels/PongNoFrameskip-v4-407.dat\"\n",
        "LOAD_MODEL = True"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "I9fVaD2M0wLc",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Experience Replay"
      ]
    },
    {
      "metadata": {
        "id": "9KVzom9K0zIf",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "Experience = collections.namedtuple('Experience', field_names=['state', 'action', 'reward', 'done', 'new_state'])\n",
        "\n",
        "\n",
        "class ExperienceReplay:\n",
        "    def __init__(self, capacity):\n",
        "        self.buffer = collections.deque(maxlen=capacity)\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.buffer)\n",
        "\n",
        "    def append(self, experience):\n",
        "        self.buffer.append(experience)\n",
        "\n",
        "    def sample(self, batch_size):\n",
        "        indices = np.random.choice(len(self.buffer), batch_size, replace=False)\n",
        "        states, actions, rewards, dones, next_states = zip(*[self.buffer[idx] for idx in indices])\n",
        "        return np.array(states), np.array(actions), np.array(rewards, dtype=np.float32), \\\n",
        "               np.array(dones, dtype=np.uint8), np.array(next_states)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Oyc4Y9bB01t0",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Agent"
      ]
    },
    {
      "metadata": {
        "id": "iCPK9nrO05Mk",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "class Agent:\n",
        "    def __init__(self, env, replay_memory):\n",
        "        self.env = env\n",
        "        self.replay_memory = replay_memory\n",
        "        self._reset()\n",
        "        self.last_action = 0\n",
        "\n",
        "    def _reset(self):\n",
        "        self.state = env.reset()\n",
        "        self.total_reward = 0.0\n",
        "\n",
        "    def play_step(self, net, epsilon=0.0, device=\"cpu\"):\n",
        "        \"\"\"\n",
        "        Select action\n",
        "        Execute action and step environment\n",
        "        Add state/action/reward to experience replay\n",
        "        \"\"\"\n",
        "        done_reward = None\n",
        "        if np.random.random() < epsilon:\n",
        "            action = env.action_space.sample()\n",
        "        else:\n",
        "            state_a = np.array([self.state], copy=False)\n",
        "            state_v = torch.tensor(state_a).to(device)\n",
        "            q_vals_v = net(state_v)\n",
        "            _, act_v = torch.max(q_vals_v, dim=1)\n",
        "            action = int(act_v.item())\n",
        "\n",
        "        # do step in the environment\n",
        "        new_state, reward, is_done, _ = self.env.step(action)\n",
        "        self.total_reward += reward\n",
        "        new_state = new_state\n",
        "\n",
        "        exp = Experience(self.state, action, reward, is_done, new_state)\n",
        "        self.replay_memory.append(exp)\n",
        "        self.state = new_state\n",
        "        if is_done:\n",
        "            done_reward = self.total_reward\n",
        "            self._reset()\n",
        "        return done_reward"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "8dKt-mco0910",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Loss function "
      ]
    },
    {
      "metadata": {
        "id": "7ARCnK-B1C3W",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "def calculate_loss(batch, net, target_net, device=\"cpu\"):\n",
        "    \"\"\"\n",
        "    Calculate MSE between actual state action values,\n",
        "    and expected state action values from DQN\n",
        "    \"\"\"\n",
        "    states, actions, rewards, dones, next_states = batch\n",
        "\n",
        "    states_v = torch.tensor(states).to(device)\n",
        "    next_states_v = torch.tensor(next_states).to(device)\n",
        "    actions_v = torch.tensor(actions).to(device)\n",
        "    rewards_v = torch.tensor(rewards).to(device)\n",
        "    done = torch.ByteTensor(dones).to(device)\n",
        "\n",
        "    state_action_values = net(states_v).gather(1, actions_v.long().unsqueeze(-1)).squeeze(-1)\n",
        "    next_state_values = target_net(next_states_v).max(1)[0]\n",
        "    next_state_values[done] = 0.0\n",
        "    next_state_values = next_state_values.detach()\n",
        "\n",
        "    expected_state_action_values = next_state_values * GAMMA + rewards_v\n",
        "    return nn.MSELoss()(state_action_values, expected_state_action_values)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "zwYEFJAn1FOK",
        "colab_type": "text"
      },
      "cell_type": "markdown",
      "source": [
        "# Training Loop"
      ]
    },
    {
      "metadata": {
        "id": "GDZ6Kub4Gopl",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 30773
        },
        "outputId": "1caa6525-3ff6-4c2b-9433-eb3d6a8332c2"
      },
      "cell_type": "code",
      "source": [
        "print(\"ReplayMemory will require {}gb of GPU RAM\".format(round(REPLAY_SIZE * 32 * 84 * 84 / 1e+9, 2)))\n",
        "!nvcc --version\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    if COLAB:\n",
        "        \"\"\"Default argparse does not work on colab\"\"\"\n",
        "        class ColabArgParse():\n",
        "            def __init__(self, cuda, env, reward, model):\n",
        "                self.cuda = cuda\n",
        "                self.env = env\n",
        "                self.reward = reward\n",
        "                self.model = model\n",
        "\n",
        "        args = ColabArgParse(CUDA, ENV_NAME, MEAN_REWARD_BOUND, MODEL)\n",
        "    else:\n",
        "        parser = argparse.ArgumentParser()\n",
        "        parser.add_argument(\"--cuda\", default=True, action=\"store_true\", help=\"Enable cuda\")\n",
        "        parser.add_argument(\"--env\", default=ENV_NAME,\n",
        "                            help=\"Name of the environment, default=\" + ENV_NAME)\n",
        "        parser.add_argument(\"--reward\", type=float, default=MEAN_REWARD_BOUND,\n",
        "                            help=\"Mean reward to stop training, default={}\".format(round(MEAN_REWARD_BOUND, 2)))\n",
        "        parser.add_argument(\"-m\", \"--model\", help=\"Model file to load\")\n",
        "        args = parser.parse_args()\n",
        "\n",
        "    device = torch.device(\"cuda\" if args.cuda else \"cpu\")\n",
        "\n",
        "    # Make Gym environement and DQNs\n",
        "    if COLAB:\n",
        "        env = make_env(args.env)\n",
        "        net = DQN(env.observation_space.shape, env.action_space.n).to(device)\n",
        "        target_net = DQN(env.observation_space.shape, env.action_space.n).to(device)\n",
        "    else:\n",
        "        env = wrappers.make_env(args.env)\n",
        "        net = dqn_model.DQN(env.observation_space.shape, env.action_space.n).to(device)\n",
        "        target_net = dqn_model.DQN(env.observation_space.shape, env.action_space.n).to(device)\n",
        "        writer = SummaryWriter(comment=\"-\" + args.env)\n",
        "\n",
        "    print(net)\n",
        "\n",
        "    replay_memory = ExperienceReplay(REPLAY_SIZE)\n",
        "    agent = Agent(env, replay_memory)\n",
        "    epsilon = EPSILON_START\n",
        "\n",
        "    if LOAD_MODEL:\n",
        "        net.load_state_dict(torch.load(args.model, map_location=lambda storage, loc: storage))\n",
        "        target_net.load_state_dict(net.state_dict())\n",
        "        print(\"Models loaded from disk!\")\n",
        "        # Lower exploration rate IMPORTANT\n",
        "        EPSILON_START = EPSILON_FINAL\n",
        "\n",
        "    optimizer = optim.Adam(net.parameters(), lr=LEARNING_RATE)\n",
        "    total_rewards = []\n",
        "    best_mean_reward = None\n",
        "    frame_idx = 0\n",
        "    timestep_frame = 0\n",
        "    timestep = time.time()\n",
        "\n",
        "    while True:\n",
        "        frame_idx += 1\n",
        "        epsilon = max(EPSILON_FINAL, EPSILON_START - frame_idx / EPSILON_DECAY)\n",
        "\n",
        "        reward = agent.play_step(net, epsilon, device=device)\n",
        "        if reward is not None:\n",
        "            total_rewards.append(reward)\n",
        "            speed = (frame_idx - timestep_frame) / (time.time() - timestep)\n",
        "            timestep_frame = frame_idx\n",
        "            timestep = time.time()\n",
        "            mean_reward = np.mean(total_rewards[-100:])\n",
        "            print(\"{} frames: done {} games, mean reward {}, eps {}, speed {} f/s\".format(\n",
        "                frame_idx, len(total_rewards), round(mean_reward, 3), round(epsilon,2), round(speed, 2)))\n",
        "            if not COLAB:\n",
        "                writer.add_scalar(\"epsilon\", epsilon, frame_idx)\n",
        "                writer.add_scalar(\"speed\", speed, frame_idx)\n",
        "                writer.add_scalar(\"reward_100\", mean_reward, frame_idx)\n",
        "                writer.add_scalar(\"reward\", reward, frame_idx)\n",
        "            if best_mean_reward is None or best_mean_reward < mean_reward or len(total_rewards) % 25 == 0:\n",
        "                torch.save(net.state_dict(), args.env + \"-\" + str(len(total_rewards)) + \".dat\")\n",
        "                if COLAB:\n",
        "                    gsync.update_file_to_folder(args.env + \"-\" + str(len(total_rewards)) + \".dat\")\n",
        "                if best_mean_reward is not None:\n",
        "                    print(\"New best mean reward {} -> {}, model saved\".format(round(best_mean_reward, 3), round(mean_reward, 3)))\n",
        "                best_mean_reward = mean_reward\n",
        "            if mean_reward > args.reward and len(total_rewards) > 10:\n",
        "                print(\"Game solved in {} frames! Average score of {}\".format(frame_idx, mean_reward))\n",
        "                break\n",
        "\n",
        "        if len(replay_memory) < LEARNING_STARTS:\n",
        "            continue\n",
        "\n",
        "        if frame_idx % TARGET_UPDATE_FREQ == 0:\n",
        "            target_net.load_state_dict(net.state_dict())\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "        batch = replay_memory.sample(BATCH_SIZE)\n",
        "        loss_t = calculate_loss(batch, net, target_net, device=device)\n",
        "        loss_t.backward()\n",
        "        optimizer.step()\n",
        "    env.close()\n",
        "    if not COLAB:\n",
        "        writer.close()\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ReplayMemory will require 9.03gb of GPU RAM\n",
            "nvcc: NVIDIA (R) Cuda compiler driver\n",
            "Copyright (c) 2005-2018 NVIDIA Corporation\n",
            "Built on Sat_Aug_25_21:08:01_CDT_2018\n",
            "Cuda compilation tools, release 10.0, V10.0.130\n",
            "['NOOP', 'FIRE', 'RIGHT', 'LEFT', 'RIGHTFIRE', 'LEFTFIRE']\n",
            "DQN(\n",
            "  (conv): Sequential(\n",
            "    (0): Conv2d(4, 32, kernel_size=(8, 8), stride=(4, 4))\n",
            "    (1): ReLU()\n",
            "    (2): Conv2d(32, 64, kernel_size=(4, 4), stride=(2, 2))\n",
            "    (3): ReLU()\n",
            "    (4): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1))\n",
            "    (5): ReLU()\n",
            "  )\n",
            "  (fc): Sequential(\n",
            "    (0): Linear(in_features=3136, out_features=512, bias=True)\n",
            "    (1): ReLU()\n",
            "    (2): Linear(in_features=512, out_features=6, bias=True)\n",
            "  )\n",
            ")\n",
            "Models loaded from disk!\n",
            "1666 frames: done 1 games, mean reward 20.0, eps 0.02, speed 350.13 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1.dat: 100%|██████████| 100/100 [00:01<00:00, 66.97it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "3494 frames: done 2 games, mean reward 18.5, eps 0.02, speed 259.46 f/s\n",
            "5160 frames: done 3 games, mean reward 19.0, eps 0.02, speed 349.36 f/s\n",
            "6826 frames: done 4 games, mean reward 19.25, eps 0.02, speed 351.18 f/s\n",
            "8492 frames: done 5 games, mean reward 19.4, eps 0.02, speed 347.03 f/s\n",
            "10382 frames: done 6 games, mean reward 19.167, eps 0.02, speed 350.13 f/s\n",
            "12274 frames: done 7 games, mean reward 19.0, eps 0.02, speed 356.87 f/s\n",
            "14049 frames: done 8 games, mean reward 19.125, eps 0.02, speed 352.84 f/s\n",
            "15866 frames: done 9 games, mean reward 19.111, eps 0.02, speed 345.64 f/s\n",
            "17591 frames: done 10 games, mean reward 19.1, eps 0.02, speed 347.11 f/s\n",
            "19389 frames: done 11 games, mean reward 19.091, eps 0.02, speed 347.64 f/s\n",
            "21042 frames: done 12 games, mean reward 19.25, eps 0.02, speed 347.19 f/s\n",
            "23079 frames: done 13 games, mean reward 19.154, eps 0.02, speed 348.72 f/s\n",
            "24796 frames: done 14 games, mean reward 19.143, eps 0.02, speed 345.79 f/s\n",
            "26530 frames: done 15 games, mean reward 19.133, eps 0.02, speed 346.35 f/s\n",
            "28276 frames: done 16 games, mean reward 19.188, eps 0.02, speed 350.39 f/s\n",
            "30078 frames: done 17 games, mean reward 19.235, eps 0.02, speed 353.36 f/s\n",
            "31906 frames: done 18 games, mean reward 19.167, eps 0.02, speed 353.24 f/s\n",
            "33776 frames: done 19 games, mean reward 19.158, eps 0.02, speed 351.5 f/s\n",
            "35573 frames: done 20 games, mean reward 19.15, eps 0.02, speed 351.57 f/s\n",
            "37290 frames: done 21 games, mean reward 19.143, eps 0.02, speed 348.22 f/s\n",
            "39007 frames: done 22 games, mean reward 19.136, eps 0.02, speed 351.23 f/s\n",
            "40724 frames: done 23 games, mean reward 19.13, eps 0.02, speed 351.02 f/s\n",
            "42425 frames: done 24 games, mean reward 19.125, eps 0.02, speed 350.05 f/s\n",
            "44375 frames: done 25 games, mean reward 18.96, eps 0.02, speed 351.52 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-25.dat: 100%|██████████| 100/100 [00:01<00:00, 79.91it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 20.0 -> 18.96, model saved\n",
            "46129 frames: done 26 games, mean reward 18.962, eps 0.02, speed 268.8 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-26.dat: 100%|██████████| 100/100 [00:01<00:00, 94.45it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.96 -> 18.962, model saved\n",
            "47842 frames: done 27 games, mean reward 19.0, eps 0.02, speed 273.98 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-27.dat: 100%|██████████| 100/100 [00:00<00:00, 100.77it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.962 -> 19.0, model saved\n",
            "49662 frames: done 28 games, mean reward 19.0, eps 0.02, speed 283.78 f/s\n",
            "51328 frames: done 29 games, mean reward 19.034, eps 0.02, speed 356.96 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-29.dat: 100%|██████████| 100/100 [00:01<00:00, 99.04it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.0 -> 19.034, model saved\n",
            "53157 frames: done 30 games, mean reward 19.0, eps 0.02, speed 286.26 f/s\n",
            "54823 frames: done 31 games, mean reward 19.032, eps 0.02, speed 355.89 f/s\n",
            "56662 frames: done 32 games, mean reward 19.031, eps 0.02, speed 355.62 f/s\n",
            "58487 frames: done 33 games, mean reward 19.0, eps 0.02, speed 351.75 f/s\n",
            "60235 frames: done 34 games, mean reward 19.0, eps 0.02, speed 349.17 f/s\n",
            "61901 frames: done 35 games, mean reward 19.029, eps 0.02, speed 354.25 f/s\n",
            "63782 frames: done 36 games, mean reward 18.944, eps 0.02, speed 353.2 f/s\n",
            "65538 frames: done 37 games, mean reward 18.946, eps 0.02, speed 351.2 f/s\n",
            "67221 frames: done 38 games, mean reward 18.974, eps 0.02, speed 348.15 f/s\n",
            "68903 frames: done 39 games, mean reward 19.0, eps 0.02, speed 354.35 f/s\n",
            "70586 frames: done 40 games, mean reward 19.05, eps 0.02, speed 355.64 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-40.dat: 100%|██████████| 100/100 [00:01<00:00, 93.61it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.034 -> 19.05, model saved\n",
            "72332 frames: done 41 games, mean reward 19.073, eps 0.02, speed 275.22 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-41.dat: 100%|██████████| 100/100 [00:01<00:00, 94.66it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.05 -> 19.073, model saved\n",
            "74234 frames: done 42 games, mean reward 19.024, eps 0.02, speed 285.79 f/s\n",
            "76155 frames: done 43 games, mean reward 19.0, eps 0.02, speed 351.17 f/s\n",
            "77821 frames: done 44 games, mean reward 19.023, eps 0.02, speed 353.77 f/s\n",
            "79694 frames: done 45 games, mean reward 19.044, eps 0.02, speed 352.95 f/s\n",
            "81411 frames: done 46 games, mean reward 19.043, eps 0.02, speed 349.56 f/s\n",
            "83400 frames: done 47 games, mean reward 18.979, eps 0.02, speed 354.36 f/s\n",
            "85066 frames: done 48 games, mean reward 19.0, eps 0.02, speed 350.04 f/s\n",
            "86921 frames: done 49 games, mean reward 19.02, eps 0.02, speed 348.85 f/s\n",
            "88650 frames: done 50 games, mean reward 19.02, eps 0.02, speed 351.3 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-50.dat: 100%|██████████| 100/100 [00:01<00:00, 98.06it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.073 -> 19.02, model saved\n",
            "90655 frames: done 51 games, mean reward 18.98, eps 0.02, speed 282.62 f/s\n",
            "92419 frames: done 52 games, mean reward 18.962, eps 0.02, speed 351.73 f/s\n",
            "94555 frames: done 53 games, mean reward 18.925, eps 0.02, speed 359.57 f/s\n",
            "96412 frames: done 54 games, mean reward 18.926, eps 0.02, speed 357.41 f/s\n",
            "98129 frames: done 55 games, mean reward 18.927, eps 0.02, speed 355.75 f/s\n",
            "99795 frames: done 56 games, mean reward 18.946, eps 0.02, speed 351.27 f/s\n",
            "101586 frames: done 57 games, mean reward 18.947, eps 0.02, speed 354.65 f/s\n",
            "103496 frames: done 58 games, mean reward 18.914, eps 0.02, speed 351.03 f/s\n",
            "105307 frames: done 59 games, mean reward 18.881, eps 0.02, speed 356.66 f/s\n",
            "107070 frames: done 60 games, mean reward 18.9, eps 0.02, speed 353.58 f/s\n",
            "108735 frames: done 61 games, mean reward 18.918, eps 0.02, speed 350.89 f/s\n",
            "110580 frames: done 62 games, mean reward 18.887, eps 0.02, speed 348.7 f/s\n",
            "112309 frames: done 63 games, mean reward 18.889, eps 0.02, speed 354.13 f/s\n",
            "113975 frames: done 64 games, mean reward 18.906, eps 0.02, speed 355.9 f/s\n",
            "115694 frames: done 65 games, mean reward 18.908, eps 0.02, speed 354.68 f/s\n",
            "117744 frames: done 66 games, mean reward 18.818, eps 0.02, speed 356.32 f/s\n",
            "119410 frames: done 67 games, mean reward 18.836, eps 0.02, speed 350.86 f/s\n",
            "121093 frames: done 68 games, mean reward 18.853, eps 0.02, speed 352.46 f/s\n",
            "122759 frames: done 69 games, mean reward 18.87, eps 0.02, speed 353.09 f/s\n",
            "124477 frames: done 70 games, mean reward 18.871, eps 0.02, speed 353.68 f/s\n",
            "126297 frames: done 71 games, mean reward 18.873, eps 0.02, speed 352.63 f/s\n",
            "128027 frames: done 72 games, mean reward 18.875, eps 0.02, speed 353.09 f/s\n",
            "129693 frames: done 73 games, mean reward 18.89, eps 0.02, speed 350.71 f/s\n",
            "131546 frames: done 74 games, mean reward 18.892, eps 0.02, speed 352.54 f/s\n",
            "133310 frames: done 75 games, mean reward 18.88, eps 0.02, speed 351.16 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-75.dat: 100%|██████████| 100/100 [00:01<00:00, 98.97it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.02 -> 18.88, model saved\n",
            "135606 frames: done 76 games, mean reward 18.816, eps 0.02, speed 296.61 f/s\n",
            "137259 frames: done 77 games, mean reward 18.844, eps 0.02, speed 356.05 f/s\n",
            "138925 frames: done 78 games, mean reward 18.859, eps 0.02, speed 351.86 f/s\n",
            "140607 frames: done 79 games, mean reward 18.873, eps 0.02, speed 348.91 f/s\n",
            "142392 frames: done 80 games, mean reward 18.888, eps 0.02, speed 353.29 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-80.dat: 100%|██████████| 100/100 [00:00<00:00, 109.32it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.88 -> 18.888, model saved\n",
            "144153 frames: done 81 games, mean reward 18.889, eps 0.02, speed 285.16 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-81.dat: 100%|██████████| 100/100 [00:00<00:00, 109.87it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.888 -> 18.889, model saved\n",
            "145838 frames: done 82 games, mean reward 18.902, eps 0.02, speed 279.43 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-82.dat: 100%|██████████| 100/100 [00:00<00:00, 106.75it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.889 -> 18.902, model saved\n",
            "147504 frames: done 83 games, mean reward 18.916, eps 0.02, speed 281.8 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-83.dat: 100%|██████████| 100/100 [00:00<00:00, 104.25it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.902 -> 18.916, model saved\n",
            "149170 frames: done 84 games, mean reward 18.929, eps 0.02, speed 278.05 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-84.dat: 100%|██████████| 100/100 [00:00<00:00, 115.24it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.916 -> 18.929, model saved\n",
            "151048 frames: done 85 games, mean reward 18.906, eps 0.02, speed 289.75 f/s\n",
            "152743 frames: done 86 games, mean reward 18.919, eps 0.02, speed 356.8 f/s\n",
            "154427 frames: done 87 games, mean reward 18.931, eps 0.02, speed 356.08 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-87.dat: 100%|██████████| 100/100 [00:00<00:00, 105.78it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.929 -> 18.931, model saved\n",
            "156111 frames: done 88 games, mean reward 18.943, eps 0.02, speed 278.84 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-88.dat: 100%|██████████| 100/100 [00:00<00:00, 119.50it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.931 -> 18.943, model saved\n",
            "157796 frames: done 89 games, mean reward 18.955, eps 0.02, speed 287.97 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-89.dat: 100%|██████████| 100/100 [00:00<00:00, 105.95it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.943 -> 18.955, model saved\n",
            "159628 frames: done 90 games, mean reward 18.933, eps 0.02, speed 282.56 f/s\n",
            "161695 frames: done 91 games, mean reward 18.89, eps 0.02, speed 353.46 f/s\n",
            "163391 frames: done 92 games, mean reward 18.902, eps 0.02, speed 349.91 f/s\n",
            "165057 frames: done 93 games, mean reward 18.914, eps 0.02, speed 353.97 f/s\n",
            "166885 frames: done 94 games, mean reward 18.915, eps 0.02, speed 350.38 f/s\n",
            "168629 frames: done 95 games, mean reward 18.916, eps 0.02, speed 351.43 f/s\n",
            "170558 frames: done 96 games, mean reward 18.917, eps 0.02, speed 352.8 f/s\n",
            "172492 frames: done 97 games, mean reward 18.887, eps 0.02, speed 354.03 f/s\n",
            "174174 frames: done 98 games, mean reward 18.898, eps 0.02, speed 353.83 f/s\n",
            "175840 frames: done 99 games, mean reward 18.909, eps 0.02, speed 356.08 f/s\n",
            "177506 frames: done 100 games, mean reward 18.92, eps 0.02, speed 355.83 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-100.dat: 100%|██████████| 100/100 [00:01<00:00, 99.19it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.955 -> 18.92, model saved\n",
            "179202 frames: done 101 games, mean reward 18.92, eps 0.02, speed 279.12 f/s\n",
            "180868 frames: done 102 games, mean reward 18.95, eps 0.02, speed 349.29 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-102.dat: 100%|██████████| 100/100 [00:01<00:00, 91.39it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.92 -> 18.95, model saved\n",
            "182614 frames: done 103 games, mean reward 18.95, eps 0.02, speed 274.1 f/s\n",
            "184358 frames: done 104 games, mean reward 18.95, eps 0.02, speed 348.37 f/s\n",
            "186442 frames: done 105 games, mean reward 18.93, eps 0.02, speed 350.67 f/s\n",
            "188108 frames: done 106 games, mean reward 18.95, eps 0.02, speed 353.4 f/s\n",
            "189833 frames: done 107 games, mean reward 18.96, eps 0.02, speed 353.02 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-107.dat: 100%|██████████| 100/100 [00:00<00:00, 111.48it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.95 -> 18.96, model saved\n",
            "191499 frames: done 108 games, mean reward 18.96, eps 0.02, speed 281.2 f/s\n",
            "193487 frames: done 109 games, mean reward 18.94, eps 0.02, speed 350.82 f/s\n",
            "195394 frames: done 110 games, mean reward 18.92, eps 0.02, speed 351.32 f/s\n",
            "197060 frames: done 111 games, mean reward 18.93, eps 0.02, speed 353.62 f/s\n",
            "199057 frames: done 112 games, mean reward 18.89, eps 0.02, speed 350.88 f/s\n",
            "200753 frames: done 113 games, mean reward 18.91, eps 0.02, speed 353.14 f/s\n",
            "202419 frames: done 114 games, mean reward 18.92, eps 0.02, speed 349.43 f/s\n",
            "204370 frames: done 115 games, mean reward 18.9, eps 0.02, speed 353.83 f/s\n",
            "206036 frames: done 116 games, mean reward 18.9, eps 0.02, speed 351.95 f/s\n",
            "207702 frames: done 117 games, mean reward 18.9, eps 0.02, speed 353.5 f/s\n",
            "209448 frames: done 118 games, mean reward 18.92, eps 0.02, speed 347.67 f/s\n",
            "211143 frames: done 119 games, mean reward 18.93, eps 0.02, speed 352.17 f/s\n",
            "212839 frames: done 120 games, mean reward 18.94, eps 0.02, speed 351.99 f/s\n",
            "214701 frames: done 121 games, mean reward 18.91, eps 0.02, speed 353.97 f/s\n",
            "216573 frames: done 122 games, mean reward 18.9, eps 0.02, speed 357.81 f/s\n",
            "218258 frames: done 123 games, mean reward 18.91, eps 0.02, speed 349.73 f/s\n",
            "220303 frames: done 124 games, mean reward 18.9, eps 0.02, speed 348.65 f/s\n",
            "221999 frames: done 125 games, mean reward 18.95, eps 0.02, speed 348.19 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-125.dat: 100%|██████████| 100/100 [00:01<00:00, 95.23it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.96 -> 18.95, model saved\n",
            "223665 frames: done 126 games, mean reward 18.96, eps 0.02, speed 272.19 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-126.dat: 100%|██████████| 100/100 [00:00<00:00, 106.35it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.95 -> 18.96, model saved\n",
            "225331 frames: done 127 games, mean reward 18.96, eps 0.02, speed 281.16 f/s\n",
            "227263 frames: done 128 games, mean reward 18.94, eps 0.02, speed 351.97 f/s\n",
            "229133 frames: done 129 games, mean reward 18.93, eps 0.02, speed 354.38 f/s\n",
            "230996 frames: done 130 games, mean reward 18.94, eps 0.02, speed 348.72 f/s\n",
            "232662 frames: done 131 games, mean reward 18.94, eps 0.02, speed 353.71 f/s\n",
            "234328 frames: done 132 games, mean reward 18.95, eps 0.02, speed 350.56 f/s\n",
            "236553 frames: done 133 games, mean reward 18.9, eps 0.02, speed 352.08 f/s\n",
            "238421 frames: done 134 games, mean reward 18.9, eps 0.02, speed 354.68 f/s\n",
            "240087 frames: done 135 games, mean reward 18.9, eps 0.02, speed 349.74 f/s\n",
            "241925 frames: done 136 games, mean reward 18.93, eps 0.02, speed 349.08 f/s\n",
            "243610 frames: done 137 games, mean reward 18.94, eps 0.02, speed 349.17 f/s\n",
            "245555 frames: done 138 games, mean reward 18.91, eps 0.02, speed 353.8 f/s\n",
            "247272 frames: done 139 games, mean reward 18.9, eps 0.02, speed 351.37 f/s\n",
            "248956 frames: done 140 games, mean reward 18.89, eps 0.02, speed 353.29 f/s\n",
            "250705 frames: done 141 games, mean reward 18.88, eps 0.02, speed 354.38 f/s\n",
            "252371 frames: done 142 games, mean reward 18.91, eps 0.02, speed 353.42 f/s\n",
            "254135 frames: done 143 games, mean reward 18.93, eps 0.02, speed 354.93 f/s\n",
            "255854 frames: done 144 games, mean reward 18.92, eps 0.02, speed 355.21 f/s\n",
            "257630 frames: done 145 games, mean reward 18.92, eps 0.02, speed 357.09 f/s\n",
            "259432 frames: done 146 games, mean reward 18.93, eps 0.02, speed 355.19 f/s\n",
            "261115 frames: done 147 games, mean reward 18.97, eps 0.02, speed 352.75 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-147.dat: 100%|██████████| 100/100 [00:00<00:00, 107.14it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.96 -> 18.97, model saved\n",
            "262869 frames: done 148 games, mean reward 18.95, eps 0.02, speed 280.85 f/s\n",
            "264853 frames: done 149 games, mean reward 18.93, eps 0.02, speed 351.76 f/s\n",
            "267106 frames: done 150 games, mean reward 18.87, eps 0.02, speed 352.28 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-150.dat: 100%|██████████| 100/100 [00:00<00:00, 104.28it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.97 -> 18.87, model saved\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\rUploading file PongNoFrameskip-v4-151.dat:   0%|          | 0/100 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "268772 frames: done 151 games, mean reward 18.9, eps 0.02, speed 280.46 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\rUploading file PongNoFrameskip-v4-151.dat: 100%|██████████| 100/100 [00:00<00:00, 111.54it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.87 -> 18.9, model saved\n",
            "270879 frames: done 152 games, mean reward 18.88, eps 0.02, speed 299.15 f/s\n",
            "272545 frames: done 153 games, mean reward 18.91, eps 0.02, speed 352.59 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-153.dat: 100%|██████████| 100/100 [00:00<00:00, 112.25it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.9 -> 18.91, model saved\n",
            "274660 frames: done 154 games, mean reward 18.88, eps 0.02, speed 298.56 f/s\n",
            "276377 frames: done 155 games, mean reward 18.88, eps 0.02, speed 355.04 f/s\n",
            "278127 frames: done 156 games, mean reward 18.86, eps 0.02, speed 351.74 f/s\n",
            "279952 frames: done 157 games, mean reward 18.85, eps 0.02, speed 350.98 f/s\n",
            "281636 frames: done 158 games, mean reward 18.88, eps 0.02, speed 353.97 f/s\n",
            "283541 frames: done 159 games, mean reward 18.88, eps 0.02, speed 357.35 f/s\n",
            "285278 frames: done 160 games, mean reward 18.87, eps 0.02, speed 354.51 f/s\n",
            "287070 frames: done 161 games, mean reward 18.87, eps 0.02, speed 353.59 f/s\n",
            "288901 frames: done 162 games, mean reward 18.89, eps 0.02, speed 351.64 f/s\n",
            "290597 frames: done 163 games, mean reward 18.9, eps 0.02, speed 351.7 f/s\n",
            "292490 frames: done 164 games, mean reward 18.88, eps 0.02, speed 350.73 f/s\n",
            "294302 frames: done 165 games, mean reward 18.87, eps 0.02, speed 352.8 f/s\n",
            "296039 frames: done 166 games, mean reward 18.93, eps 0.02, speed 350.71 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-166.dat: 100%|██████████| 100/100 [00:00<00:00, 108.43it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.91 -> 18.93, model saved\n",
            "297887 frames: done 167 games, mean reward 18.92, eps 0.02, speed 288.78 f/s\n",
            "300024 frames: done 168 games, mean reward 18.87, eps 0.02, speed 355.54 f/s\n",
            "301740 frames: done 169 games, mean reward 18.86, eps 0.02, speed 350.86 f/s\n",
            "303476 frames: done 170 games, mean reward 18.86, eps 0.02, speed 349.7 f/s\n",
            "305515 frames: done 171 games, mean reward 18.8, eps 0.02, speed 348.66 f/s\n",
            "307229 frames: done 172 games, mean reward 18.81, eps 0.02, speed 350.84 f/s\n",
            "308895 frames: done 173 games, mean reward 18.81, eps 0.02, speed 350.89 f/s\n",
            "310560 frames: done 174 games, mean reward 18.82, eps 0.02, speed 356.49 f/s\n",
            "312226 frames: done 175 games, mean reward 18.84, eps 0.02, speed 352.79 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-175.dat: 100%|██████████| 100/100 [00:00<00:00, 112.09it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.93 -> 18.84, model saved\n",
            "314028 frames: done 176 games, mean reward 18.88, eps 0.02, speed 284.24 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-176.dat: 100%|██████████| 100/100 [00:00<00:00, 101.64it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.84 -> 18.88, model saved\n",
            "315694 frames: done 177 games, mean reward 18.87, eps 0.02, speed 275.75 f/s\n",
            "317444 frames: done 178 games, mean reward 18.85, eps 0.02, speed 354.05 f/s\n",
            "319191 frames: done 179 games, mean reward 18.84, eps 0.02, speed 352.81 f/s\n",
            "320920 frames: done 180 games, mean reward 18.83, eps 0.02, speed 352.29 f/s\n",
            "322987 frames: done 181 games, mean reward 18.8, eps 0.02, speed 352.44 f/s\n",
            "324653 frames: done 182 games, mean reward 18.8, eps 0.02, speed 353.14 f/s\n",
            "326493 frames: done 183 games, mean reward 18.78, eps 0.02, speed 347.94 f/s\n",
            "328299 frames: done 184 games, mean reward 18.76, eps 0.02, speed 352.09 f/s\n",
            "329994 frames: done 185 games, mean reward 18.79, eps 0.02, speed 351.39 f/s\n",
            "331660 frames: done 186 games, mean reward 18.79, eps 0.02, speed 350.59 f/s\n",
            "333445 frames: done 187 games, mean reward 18.79, eps 0.02, speed 354.79 f/s\n",
            "335190 frames: done 188 games, mean reward 18.79, eps 0.02, speed 355.71 f/s\n",
            "337254 frames: done 189 games, mean reward 18.76, eps 0.02, speed 353.07 f/s\n",
            "339035 frames: done 190 games, mean reward 18.79, eps 0.02, speed 357.75 f/s\n",
            "340845 frames: done 191 games, mean reward 18.83, eps 0.02, speed 353.54 f/s\n",
            "342761 frames: done 192 games, mean reward 18.81, eps 0.02, speed 351.58 f/s\n",
            "344507 frames: done 193 games, mean reward 18.81, eps 0.02, speed 353.03 f/s\n",
            "346253 frames: done 194 games, mean reward 18.82, eps 0.02, speed 349.88 f/s\n",
            "348038 frames: done 195 games, mean reward 18.83, eps 0.02, speed 350.54 f/s\n",
            "349943 frames: done 196 games, mean reward 18.8, eps 0.02, speed 351.22 f/s\n",
            "351841 frames: done 197 games, mean reward 18.82, eps 0.02, speed 353.0 f/s\n",
            "353507 frames: done 198 games, mean reward 18.82, eps 0.02, speed 353.14 f/s\n",
            "355415 frames: done 199 games, mean reward 18.82, eps 0.02, speed 351.49 f/s\n",
            "357118 frames: done 200 games, mean reward 18.82, eps 0.02, speed 352.94 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-200.dat: 100%|██████████| 100/100 [00:00<00:00, 109.83it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.88 -> 18.82, model saved\n",
            "358784 frames: done 201 games, mean reward 18.82, eps 0.02, speed 277.09 f/s\n",
            "360501 frames: done 202 games, mean reward 18.81, eps 0.02, speed 353.47 f/s\n",
            "362290 frames: done 203 games, mean reward 18.8, eps 0.02, speed 353.27 f/s\n",
            "363985 frames: done 204 games, mean reward 18.81, eps 0.02, speed 356.99 f/s\n",
            "365888 frames: done 205 games, mean reward 18.81, eps 0.02, speed 356.57 f/s\n",
            "367554 frames: done 206 games, mean reward 18.81, eps 0.02, speed 356.93 f/s\n",
            "369220 frames: done 207 games, mean reward 18.82, eps 0.02, speed 350.94 f/s\n",
            "371254 frames: done 208 games, mean reward 18.79, eps 0.02, speed 354.45 f/s\n",
            "372936 frames: done 209 games, mean reward 18.82, eps 0.02, speed 353.76 f/s\n",
            "374761 frames: done 210 games, mean reward 18.83, eps 0.02, speed 353.73 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-210.dat: 100%|██████████| 100/100 [00:00<00:00, 100.33it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.82 -> 18.83, model saved\n",
            "376807 frames: done 211 games, mean reward 18.78, eps 0.02, speed 292.38 f/s\n",
            "378674 frames: done 212 games, mean reward 18.8, eps 0.02, speed 356.03 f/s\n",
            "380340 frames: done 213 games, mean reward 18.8, eps 0.02, speed 358.2 f/s\n",
            "382110 frames: done 214 games, mean reward 18.78, eps 0.02, speed 360.59 f/s\n",
            "383776 frames: done 215 games, mean reward 18.81, eps 0.02, speed 362.12 f/s\n",
            "385472 frames: done 216 games, mean reward 18.81, eps 0.02, speed 360.3 f/s\n",
            "387256 frames: done 217 games, mean reward 18.79, eps 0.02, speed 360.93 f/s\n",
            "388987 frames: done 218 games, mean reward 18.78, eps 0.02, speed 361.04 f/s\n",
            "390671 frames: done 219 games, mean reward 18.78, eps 0.02, speed 360.67 f/s\n",
            "392352 frames: done 220 games, mean reward 18.78, eps 0.02, speed 361.29 f/s\n",
            "394205 frames: done 221 games, mean reward 18.82, eps 0.02, speed 360.95 f/s\n",
            "395922 frames: done 222 games, mean reward 18.83, eps 0.02, speed 360.95 f/s\n",
            "397706 frames: done 223 games, mean reward 18.82, eps 0.02, speed 360.15 f/s\n",
            "399443 frames: done 224 games, mean reward 18.83, eps 0.02, speed 356.92 f/s\n",
            "401128 frames: done 225 games, mean reward 18.83, eps 0.02, speed 359.67 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-225.dat: 100%|██████████| 100/100 [00:00<00:00, 103.97it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.83 -> 18.83, model saved\n",
            "402781 frames: done 226 games, mean reward 18.84, eps 0.02, speed 283.94 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-226.dat: 100%|██████████| 100/100 [00:00<00:00, 100.96it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.83 -> 18.84, model saved\n",
            "404760 frames: done 227 games, mean reward 18.8, eps 0.02, speed 295.5 f/s\n",
            "406691 frames: done 228 games, mean reward 18.81, eps 0.02, speed 357.58 f/s\n",
            "408536 frames: done 229 games, mean reward 18.81, eps 0.02, speed 352.46 f/s\n",
            "410635 frames: done 230 games, mean reward 18.77, eps 0.02, speed 351.6 f/s\n",
            "412488 frames: done 231 games, mean reward 18.76, eps 0.02, speed 350.83 f/s\n",
            "414154 frames: done 232 games, mean reward 18.76, eps 0.02, speed 353.9 f/s\n",
            "415918 frames: done 233 games, mean reward 18.81, eps 0.02, speed 355.47 f/s\n",
            "417665 frames: done 234 games, mean reward 18.81, eps 0.02, speed 349.56 f/s\n",
            "419658 frames: done 235 games, mean reward 18.8, eps 0.02, speed 350.2 f/s\n",
            "421454 frames: done 236 games, mean reward 18.8, eps 0.02, speed 355.23 f/s\n",
            "423120 frames: done 237 games, mean reward 18.8, eps 0.02, speed 357.6 f/s\n",
            "424917 frames: done 238 games, mean reward 18.82, eps 0.02, speed 353.82 f/s\n",
            "426812 frames: done 239 games, mean reward 18.81, eps 0.02, speed 349.17 f/s\n",
            "428465 frames: done 240 games, mean reward 18.82, eps 0.02, speed 351.29 f/s\n",
            "430195 frames: done 241 games, mean reward 18.82, eps 0.02, speed 351.85 f/s\n",
            "431912 frames: done 242 games, mean reward 18.81, eps 0.02, speed 348.37 f/s\n",
            "433578 frames: done 243 games, mean reward 18.81, eps 0.02, speed 350.95 f/s\n",
            "435313 frames: done 244 games, mean reward 18.81, eps 0.02, speed 355.31 f/s\n",
            "436979 frames: done 245 games, mean reward 18.81, eps 0.02, speed 352.52 f/s\n",
            "438645 frames: done 246 games, mean reward 18.81, eps 0.02, speed 355.92 f/s\n",
            "440392 frames: done 247 games, mean reward 18.8, eps 0.02, speed 354.93 f/s\n",
            "442088 frames: done 248 games, mean reward 18.82, eps 0.02, speed 354.31 f/s\n",
            "443831 frames: done 249 games, mean reward 18.83, eps 0.02, speed 351.95 f/s\n",
            "445734 frames: done 250 games, mean reward 18.87, eps 0.02, speed 352.02 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-250.dat: 100%|██████████| 100/100 [00:01<00:00, 93.91it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.84 -> 18.87, model saved\n",
            "447479 frames: done 251 games, mean reward 18.87, eps 0.02, speed 275.52 f/s\n",
            "449145 frames: done 252 games, mean reward 18.91, eps 0.02, speed 352.61 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-252.dat: 100%|██████████| 100/100 [00:00<00:00, 118.48it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.87 -> 18.91, model saved\n",
            "450864 frames: done 253 games, mean reward 18.9, eps 0.02, speed 287.28 f/s\n",
            "452560 frames: done 254 games, mean reward 18.94, eps 0.02, speed 349.17 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-254.dat: 100%|██████████| 100/100 [00:01<00:00, 97.30it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.91 -> 18.94, model saved\n",
            "454383 frames: done 255 games, mean reward 18.93, eps 0.02, speed 281.66 f/s\n",
            "456130 frames: done 256 games, mean reward 18.94, eps 0.02, speed 352.77 f/s\n",
            "458086 frames: done 257 games, mean reward 18.92, eps 0.02, speed 354.62 f/s\n",
            "459877 frames: done 258 games, mean reward 18.91, eps 0.02, speed 352.21 f/s\n",
            "461674 frames: done 259 games, mean reward 18.93, eps 0.02, speed 350.01 f/s\n",
            "463493 frames: done 260 games, mean reward 18.93, eps 0.02, speed 355.89 f/s\n",
            "465290 frames: done 261 games, mean reward 18.92, eps 0.02, speed 351.58 f/s\n",
            "467007 frames: done 262 games, mean reward 18.92, eps 0.02, speed 350.07 f/s\n",
            "468753 frames: done 263 games, mean reward 18.92, eps 0.02, speed 351.9 f/s\n",
            "470757 frames: done 264 games, mean reward 18.92, eps 0.02, speed 353.96 f/s\n",
            "472502 frames: done 265 games, mean reward 18.94, eps 0.02, speed 355.49 f/s\n",
            "474290 frames: done 266 games, mean reward 18.94, eps 0.02, speed 354.65 f/s\n",
            "475956 frames: done 267 games, mean reward 18.95, eps 0.02, speed 352.45 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-267.dat: 100%|██████████| 100/100 [00:00<00:00, 102.31it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.94 -> 18.95, model saved\n",
            "477785 frames: done 268 games, mean reward 18.98, eps 0.02, speed 281.53 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-268.dat: 100%|██████████| 100/100 [00:00<00:00, 105.31it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.95 -> 18.98, model saved\n",
            "479554 frames: done 269 games, mean reward 18.97, eps 0.02, speed 281.99 f/s\n",
            "481284 frames: done 270 games, mean reward 18.97, eps 0.02, speed 356.62 f/s\n",
            "483001 frames: done 271 games, mean reward 19.03, eps 0.02, speed 353.01 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-271.dat: 100%|██████████| 100/100 [00:00<00:00, 113.92it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.98 -> 19.03, model saved\n",
            "484667 frames: done 272 games, mean reward 19.03, eps 0.02, speed 284.31 f/s\n",
            "486392 frames: done 273 games, mean reward 19.02, eps 0.02, speed 351.89 f/s\n",
            "488227 frames: done 274 games, mean reward 19.01, eps 0.02, speed 351.43 f/s\n",
            "490019 frames: done 275 games, mean reward 19.0, eps 0.02, speed 349.9 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-275.dat: 100%|██████████| 100/100 [00:00<00:00, 102.29it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.03 -> 19.0, model saved\n",
            "491765 frames: done 276 games, mean reward 19.02, eps 0.02, speed 282.01 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-276.dat: 100%|██████████| 100/100 [00:01<00:00, 92.03it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.0 -> 19.02, model saved\n",
            "493563 frames: done 277 games, mean reward 19.0, eps 0.02, speed 278.85 f/s\n",
            "495669 frames: done 278 games, mean reward 18.97, eps 0.02, speed 350.26 f/s\n",
            "497399 frames: done 279 games, mean reward 18.97, eps 0.02, speed 349.56 f/s\n",
            "499116 frames: done 280 games, mean reward 18.97, eps 0.02, speed 354.79 f/s\n",
            "500833 frames: done 281 games, mean reward 19.0, eps 0.02, speed 354.64 f/s\n",
            "502571 frames: done 282 games, mean reward 18.99, eps 0.02, speed 351.0 f/s\n",
            "504335 frames: done 283 games, mean reward 18.99, eps 0.02, speed 356.77 f/s\n",
            "506099 frames: done 284 games, mean reward 18.99, eps 0.02, speed 354.75 f/s\n",
            "507765 frames: done 285 games, mean reward 18.99, eps 0.02, speed 354.3 f/s\n",
            "509598 frames: done 286 games, mean reward 18.96, eps 0.02, speed 355.4 f/s\n",
            "511263 frames: done 287 games, mean reward 18.96, eps 0.02, speed 353.12 f/s\n",
            "513066 frames: done 288 games, mean reward 18.97, eps 0.02, speed 352.97 f/s\n",
            "514893 frames: done 289 games, mean reward 18.98, eps 0.02, speed 358.28 f/s\n",
            "516606 frames: done 290 games, mean reward 18.98, eps 0.02, speed 359.75 f/s\n",
            "518394 frames: done 291 games, mean reward 18.97, eps 0.02, speed 359.19 f/s\n",
            "520478 frames: done 292 games, mean reward 18.96, eps 0.02, speed 361.1 f/s\n",
            "522383 frames: done 293 games, mean reward 18.95, eps 0.02, speed 359.89 f/s\n",
            "524460 frames: done 294 games, mean reward 18.91, eps 0.02, speed 360.14 f/s\n",
            "526194 frames: done 295 games, mean reward 18.9, eps 0.02, speed 356.59 f/s\n",
            "527860 frames: done 296 games, mean reward 18.94, eps 0.02, speed 361.97 f/s\n",
            "529649 frames: done 297 games, mean reward 18.95, eps 0.02, speed 360.42 f/s\n",
            "531431 frames: done 298 games, mean reward 18.93, eps 0.02, speed 361.5 f/s\n",
            "533150 frames: done 299 games, mean reward 18.92, eps 0.02, speed 360.33 f/s\n",
            "534816 frames: done 300 games, mean reward 18.92, eps 0.02, speed 360.28 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-300.dat: 100%|██████████| 100/100 [00:00<00:00, 102.95it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.02 -> 18.92, model saved\n",
            "536645 frames: done 301 games, mean reward 18.9, eps 0.02, speed 288.09 f/s\n",
            "538363 frames: done 302 games, mean reward 18.91, eps 0.02, speed 355.75 f/s\n",
            "540166 frames: done 303 games, mean reward 18.92, eps 0.02, speed 351.64 f/s\n",
            "541835 frames: done 304 games, mean reward 18.92, eps 0.02, speed 353.04 f/s\n",
            "543501 frames: done 305 games, mean reward 18.94, eps 0.02, speed 352.2 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-305.dat: 100%|██████████| 100/100 [00:00<00:00, 108.35it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.92 -> 18.94, model saved\n",
            "545197 frames: done 306 games, mean reward 18.94, eps 0.02, speed 285.44 f/s\n",
            "547207 frames: done 307 games, mean reward 18.9, eps 0.02, speed 355.22 f/s\n",
            "548873 frames: done 308 games, mean reward 18.93, eps 0.02, speed 348.58 f/s\n",
            "550767 frames: done 309 games, mean reward 18.9, eps 0.02, speed 350.13 f/s\n",
            "552657 frames: done 310 games, mean reward 18.91, eps 0.02, speed 350.24 f/s\n",
            "554323 frames: done 311 games, mean reward 18.96, eps 0.02, speed 346.87 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-311.dat: 100%|██████████| 100/100 [00:00<00:00, 110.15it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.94 -> 18.96, model saved\n",
            "556041 frames: done 312 games, mean reward 18.96, eps 0.02, speed 280.28 f/s\n",
            "558125 frames: done 313 games, mean reward 18.92, eps 0.02, speed 352.19 f/s\n",
            "560035 frames: done 314 games, mean reward 18.91, eps 0.02, speed 350.84 f/s\n",
            "561790 frames: done 315 games, mean reward 18.9, eps 0.02, speed 356.49 f/s\n",
            "563636 frames: done 316 games, mean reward 18.89, eps 0.02, speed 355.46 f/s\n",
            "565395 frames: done 317 games, mean reward 18.9, eps 0.02, speed 350.9 f/s\n",
            "567061 frames: done 318 games, mean reward 18.91, eps 0.02, speed 348.37 f/s\n",
            "568881 frames: done 319 games, mean reward 18.9, eps 0.02, speed 352.07 f/s\n",
            "570628 frames: done 320 games, mean reward 18.89, eps 0.02, speed 353.05 f/s\n",
            "572575 frames: done 321 games, mean reward 18.84, eps 0.02, speed 352.93 f/s\n",
            "574409 frames: done 322 games, mean reward 18.85, eps 0.02, speed 349.46 f/s\n",
            "576074 frames: done 323 games, mean reward 18.86, eps 0.02, speed 348.75 f/s\n",
            "577739 frames: done 324 games, mean reward 18.87, eps 0.02, speed 351.95 f/s\n",
            "579607 frames: done 325 games, mean reward 18.86, eps 0.02, speed 355.39 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-325.dat: 100%|██████████| 100/100 [00:01<00:00, 99.32it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.96 -> 18.86, model saved\n",
            "581307 frames: done 326 games, mean reward 18.86, eps 0.02, speed 276.72 f/s\n",
            "582992 frames: done 327 games, mean reward 18.9, eps 0.02, speed 353.28 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-327.dat: 100%|██████████| 100/100 [00:01<00:00, 92.82it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.86 -> 18.9, model saved\n",
            "585044 frames: done 328 games, mean reward 18.89, eps 0.02, speed 284.55 f/s\n",
            "586710 frames: done 329 games, mean reward 18.9, eps 0.02, speed 352.74 f/s\n",
            "588488 frames: done 330 games, mean reward 18.94, eps 0.02, speed 356.64 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-330.dat: 100%|██████████| 100/100 [00:01<00:00, 96.61it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.9 -> 18.94, model saved\n",
            "590242 frames: done 331 games, mean reward 18.93, eps 0.02, speed 277.95 f/s\n",
            "592019 frames: done 332 games, mean reward 18.93, eps 0.02, speed 348.69 f/s\n",
            "593868 frames: done 333 games, mean reward 18.92, eps 0.02, speed 351.98 f/s\n",
            "595534 frames: done 334 games, mean reward 18.93, eps 0.02, speed 354.21 f/s\n",
            "597278 frames: done 335 games, mean reward 18.94, eps 0.02, speed 352.07 f/s\n",
            "599146 frames: done 336 games, mean reward 18.94, eps 0.02, speed 350.08 f/s\n",
            "600875 frames: done 337 games, mean reward 18.93, eps 0.02, speed 350.53 f/s\n",
            "602747 frames: done 338 games, mean reward 18.92, eps 0.02, speed 353.64 f/s\n",
            "604464 frames: done 339 games, mean reward 18.93, eps 0.02, speed 351.9 f/s\n",
            "606211 frames: done 340 games, mean reward 18.91, eps 0.02, speed 351.05 f/s\n",
            "608067 frames: done 341 games, mean reward 18.9, eps 0.02, speed 353.4 f/s\n",
            "609733 frames: done 342 games, mean reward 18.91, eps 0.02, speed 353.73 f/s\n",
            "611399 frames: done 343 games, mean reward 18.91, eps 0.02, speed 350.12 f/s\n",
            "613189 frames: done 344 games, mean reward 18.91, eps 0.02, speed 353.07 f/s\n",
            "614914 frames: done 345 games, mean reward 18.92, eps 0.02, speed 351.18 f/s\n",
            "616763 frames: done 346 games, mean reward 18.92, eps 0.02, speed 350.44 f/s\n",
            "618637 frames: done 347 games, mean reward 18.91, eps 0.02, speed 354.05 f/s\n",
            "620499 frames: done 348 games, mean reward 18.88, eps 0.02, speed 354.94 f/s\n",
            "622246 frames: done 349 games, mean reward 18.88, eps 0.02, speed 351.79 f/s\n",
            "623942 frames: done 350 games, mean reward 18.91, eps 0.02, speed 357.24 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-350.dat: 100%|██████████| 100/100 [00:01<00:00, 95.30it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.94 -> 18.91, model saved\n",
            "625608 frames: done 351 games, mean reward 18.91, eps 0.02, speed 278.64 f/s\n",
            "627718 frames: done 352 games, mean reward 18.87, eps 0.02, speed 354.57 f/s\n",
            "629485 frames: done 353 games, mean reward 18.88, eps 0.02, speed 349.61 f/s\n",
            "631328 frames: done 354 games, mean reward 18.86, eps 0.02, speed 353.24 f/s\n",
            "633247 frames: done 355 games, mean reward 18.86, eps 0.02, speed 352.81 f/s\n",
            "634975 frames: done 356 games, mean reward 18.87, eps 0.02, speed 351.25 f/s\n",
            "636729 frames: done 357 games, mean reward 18.89, eps 0.02, speed 352.39 f/s\n",
            "638474 frames: done 358 games, mean reward 18.9, eps 0.02, speed 351.61 f/s\n",
            "640331 frames: done 359 games, mean reward 18.9, eps 0.02, speed 354.57 f/s\n",
            "641997 frames: done 360 games, mean reward 18.91, eps 0.02, speed 355.53 f/s\n",
            "643880 frames: done 361 games, mean reward 18.9, eps 0.02, speed 352.51 f/s\n",
            "645546 frames: done 362 games, mean reward 18.91, eps 0.02, speed 354.73 f/s\n",
            "647460 frames: done 363 games, mean reward 18.9, eps 0.02, speed 349.53 f/s\n",
            "649249 frames: done 364 games, mean reward 18.91, eps 0.02, speed 352.88 f/s\n",
            "650915 frames: done 365 games, mean reward 18.91, eps 0.02, speed 354.39 f/s\n",
            "652581 frames: done 366 games, mean reward 18.92, eps 0.02, speed 350.79 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-366.dat: 100%|██████████| 100/100 [00:00<00:00, 101.11it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.91 -> 18.92, model saved\n",
            "654452 frames: done 367 games, mean reward 18.9, eps 0.02, speed 284.47 f/s\n",
            "656449 frames: done 368 games, mean reward 18.89, eps 0.02, speed 349.78 f/s\n",
            "658329 frames: done 369 games, mean reward 18.9, eps 0.02, speed 347.73 f/s\n",
            "660067 frames: done 370 games, mean reward 18.9, eps 0.02, speed 349.85 f/s\n",
            "661733 frames: done 371 games, mean reward 18.91, eps 0.02, speed 350.6 f/s\n",
            "663467 frames: done 372 games, mean reward 18.9, eps 0.02, speed 352.95 f/s\n",
            "665321 frames: done 373 games, mean reward 18.89, eps 0.02, speed 353.89 f/s\n",
            "667110 frames: done 374 games, mean reward 18.88, eps 0.02, speed 354.45 f/s\n",
            "668792 frames: done 375 games, mean reward 18.89, eps 0.02, speed 354.05 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-375.dat: 100%|██████████| 100/100 [00:00<00:00, 111.09it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.92 -> 18.89, model saved\n",
            "670536 frames: done 376 games, mean reward 18.89, eps 0.02, speed 285.94 f/s\n",
            "672701 frames: done 377 games, mean reward 18.85, eps 0.02, speed 357.61 f/s\n",
            "674455 frames: done 378 games, mean reward 18.89, eps 0.02, speed 348.89 f/s\n",
            "676265 frames: done 379 games, mean reward 18.88, eps 0.02, speed 348.69 f/s\n",
            "677949 frames: done 380 games, mean reward 18.89, eps 0.02, speed 350.66 f/s\n",
            "679615 frames: done 381 games, mean reward 18.9, eps 0.02, speed 350.17 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-381.dat: 100%|██████████| 100/100 [00:00<00:00, 103.72it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.89 -> 18.9, model saved\n",
            "681490 frames: done 382 games, mean reward 18.89, eps 0.02, speed 283.95 f/s\n",
            "683156 frames: done 383 games, mean reward 18.91, eps 0.02, speed 348.74 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-383.dat: 100%|██████████| 100/100 [00:00<00:00, 117.14it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.9 -> 18.91, model saved\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\rUploading file PongNoFrameskip-v4-384.dat:   0%|          | 0/100 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "684932 frames: done 384 games, mean reward 18.93, eps 0.02, speed 291.4 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\rUploading file PongNoFrameskip-v4-384.dat: 100%|██████████| 100/100 [00:00<00:00, 104.48it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.91 -> 18.93, model saved\n",
            "686747 frames: done 385 games, mean reward 18.9, eps 0.02, speed 286.55 f/s\n",
            "688671 frames: done 386 games, mean reward 18.91, eps 0.02, speed 354.45 f/s\n",
            "690337 frames: done 387 games, mean reward 18.91, eps 0.02, speed 352.41 f/s\n",
            "692022 frames: done 388 games, mean reward 18.9, eps 0.02, speed 351.8 f/s\n",
            "693900 frames: done 389 games, mean reward 18.92, eps 0.02, speed 349.94 f/s\n",
            "695667 frames: done 390 games, mean reward 18.9, eps 0.02, speed 352.75 f/s\n",
            "697657 frames: done 391 games, mean reward 18.89, eps 0.02, speed 352.76 f/s\n",
            "699323 frames: done 392 games, mean reward 18.92, eps 0.02, speed 355.29 f/s\n",
            "701006 frames: done 393 games, mean reward 18.93, eps 0.02, speed 351.11 f/s\n",
            "702816 frames: done 394 games, mean reward 18.96, eps 0.02, speed 351.96 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-394.dat: 100%|██████████| 100/100 [00:00<00:00, 109.70it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.93 -> 18.96, model saved\n",
            "704499 frames: done 395 games, mean reward 18.97, eps 0.02, speed 282.06 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-395.dat: 100%|██████████| 100/100 [00:00<00:00, 114.90it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.96 -> 18.97, model saved\n",
            "706337 frames: done 396 games, mean reward 18.95, eps 0.02, speed 294.66 f/s\n",
            "708162 frames: done 397 games, mean reward 18.94, eps 0.02, speed 358.14 f/s\n",
            "709845 frames: done 398 games, mean reward 18.96, eps 0.02, speed 357.37 f/s\n",
            "711623 frames: done 399 games, mean reward 18.96, eps 0.02, speed 352.99 f/s\n",
            "713289 frames: done 400 games, mean reward 18.96, eps 0.02, speed 353.77 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-400.dat: 100%|██████████| 100/100 [00:00<00:00, 101.90it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.97 -> 18.96, model saved\n",
            "715312 frames: done 401 games, mean reward 18.95, eps 0.02, speed 254.25 f/s\n",
            "717320 frames: done 402 games, mean reward 18.93, eps 0.02, speed 352.29 f/s\n",
            "719423 frames: done 403 games, mean reward 18.91, eps 0.02, speed 354.19 f/s\n",
            "721322 frames: done 404 games, mean reward 18.87, eps 0.02, speed 351.41 f/s\n",
            "723246 frames: done 405 games, mean reward 18.85, eps 0.02, speed 352.93 f/s\n",
            "724993 frames: done 406 games, mean reward 18.84, eps 0.02, speed 352.85 f/s\n",
            "726659 frames: done 407 games, mean reward 18.88, eps 0.02, speed 357.3 f/s\n",
            "728731 frames: done 408 games, mean reward 18.84, eps 0.02, speed 356.24 f/s\n",
            "730413 frames: done 409 games, mean reward 18.87, eps 0.02, speed 357.38 f/s\n",
            "732155 frames: done 410 games, mean reward 18.87, eps 0.02, speed 352.11 f/s\n",
            "733821 frames: done 411 games, mean reward 18.87, eps 0.02, speed 353.63 f/s\n",
            "735572 frames: done 412 games, mean reward 18.88, eps 0.02, speed 353.46 f/s\n",
            "737238 frames: done 413 games, mean reward 18.92, eps 0.02, speed 354.1 f/s\n",
            "738963 frames: done 414 games, mean reward 18.95, eps 0.02, speed 354.72 f/s\n",
            "740723 frames: done 415 games, mean reward 18.95, eps 0.02, speed 349.75 f/s\n",
            "742489 frames: done 416 games, mean reward 18.95, eps 0.02, speed 351.7 f/s\n",
            "744276 frames: done 417 games, mean reward 18.94, eps 0.02, speed 349.81 f/s\n",
            "745993 frames: done 418 games, mean reward 18.93, eps 0.02, speed 349.11 f/s\n",
            "747659 frames: done 419 games, mean reward 18.94, eps 0.02, speed 351.22 f/s\n",
            "749325 frames: done 420 games, mean reward 18.95, eps 0.02, speed 354.42 f/s\n",
            "750991 frames: done 421 games, mean reward 19.0, eps 0.02, speed 356.39 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-421.dat: 100%|██████████| 100/100 [00:00<00:00, 104.32it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.96 -> 19.0, model saved\n",
            "752726 frames: done 422 games, mean reward 18.99, eps 0.02, speed 279.27 f/s\n",
            "754445 frames: done 423 games, mean reward 18.98, eps 0.02, speed 358.58 f/s\n",
            "756327 frames: done 424 games, mean reward 18.95, eps 0.02, speed 350.36 f/s\n",
            "757993 frames: done 425 games, mean reward 18.96, eps 0.02, speed 349.28 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-425.dat: 100%|██████████| 100/100 [00:00<00:00, 110.58it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.0 -> 18.96, model saved\n",
            "759718 frames: done 426 games, mean reward 18.96, eps 0.02, speed 278.29 f/s\n",
            "761467 frames: done 427 games, mean reward 18.95, eps 0.02, speed 350.17 f/s\n",
            "763325 frames: done 428 games, mean reward 18.97, eps 0.02, speed 348.63 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-428.dat: 100%|██████████| 100/100 [00:01<00:00, 99.74it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.96 -> 18.97, model saved\n",
            "765149 frames: done 429 games, mean reward 18.97, eps 0.02, speed 266.49 f/s\n",
            "766928 frames: done 430 games, mean reward 18.97, eps 0.02, speed 354.06 f/s\n",
            "768885 frames: done 431 games, mean reward 18.94, eps 0.02, speed 356.16 f/s\n",
            "770551 frames: done 432 games, mean reward 18.94, eps 0.02, speed 354.07 f/s\n",
            "772374 frames: done 433 games, mean reward 18.96, eps 0.02, speed 350.94 f/s\n",
            "774040 frames: done 434 games, mean reward 18.96, eps 0.02, speed 357.91 f/s\n",
            "775757 frames: done 435 games, mean reward 18.95, eps 0.02, speed 356.15 f/s\n",
            "777423 frames: done 436 games, mean reward 18.96, eps 0.02, speed 353.29 f/s\n",
            "779089 frames: done 437 games, mean reward 18.97, eps 0.02, speed 352.25 f/s\n",
            "780935 frames: done 438 games, mean reward 18.98, eps 0.02, speed 349.07 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-438.dat: 100%|██████████| 100/100 [00:00<00:00, 110.87it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.97 -> 18.98, model saved\n",
            "782836 frames: done 439 games, mean reward 18.95, eps 0.02, speed 284.67 f/s\n",
            "784553 frames: done 440 games, mean reward 18.95, eps 0.02, speed 350.96 f/s\n",
            "786331 frames: done 441 games, mean reward 18.96, eps 0.02, speed 349.7 f/s\n",
            "788172 frames: done 442 games, mean reward 18.95, eps 0.02, speed 348.3 f/s\n",
            "789838 frames: done 443 games, mean reward 18.95, eps 0.02, speed 355.23 f/s\n",
            "791689 frames: done 444 games, mean reward 18.94, eps 0.02, speed 353.91 f/s\n",
            "793605 frames: done 445 games, mean reward 18.9, eps 0.02, speed 353.67 f/s\n",
            "795405 frames: done 446 games, mean reward 18.88, eps 0.02, speed 356.37 f/s\n",
            "797273 frames: done 447 games, mean reward 18.89, eps 0.02, speed 349.56 f/s\n",
            "799136 frames: done 448 games, mean reward 18.88, eps 0.02, speed 350.07 f/s\n",
            "801063 frames: done 449 games, mean reward 18.88, eps 0.02, speed 350.47 f/s\n",
            "802980 frames: done 450 games, mean reward 18.85, eps 0.02, speed 353.76 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-450.dat: 100%|██████████| 100/100 [00:00<00:00, 110.84it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.98 -> 18.85, model saved\n",
            "804646 frames: done 451 games, mean reward 18.85, eps 0.02, speed 281.38 f/s\n",
            "806327 frames: done 452 games, mean reward 18.89, eps 0.02, speed 350.94 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-452.dat: 100%|██████████| 100/100 [00:00<00:00, 102.49it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.85 -> 18.89, model saved\n",
            "808322 frames: done 453 games, mean reward 18.85, eps 0.02, speed 292.33 f/s\n",
            "810196 frames: done 454 games, mean reward 18.86, eps 0.02, speed 352.48 f/s\n",
            "811891 frames: done 455 games, mean reward 18.89, eps 0.02, speed 350.36 f/s\n",
            "813750 frames: done 456 games, mean reward 18.87, eps 0.02, speed 351.25 f/s\n",
            "815416 frames: done 457 games, mean reward 18.89, eps 0.02, speed 352.45 f/s\n",
            "817160 frames: done 458 games, mean reward 18.89, eps 0.02, speed 353.89 f/s\n",
            "819004 frames: done 459 games, mean reward 18.88, eps 0.02, speed 351.79 f/s\n",
            "820741 frames: done 460 games, mean reward 18.87, eps 0.02, speed 349.1 f/s\n",
            "822476 frames: done 461 games, mean reward 18.88, eps 0.02, speed 351.29 f/s\n",
            "824250 frames: done 462 games, mean reward 18.87, eps 0.02, speed 349.65 f/s\n",
            "826073 frames: done 463 games, mean reward 18.87, eps 0.02, speed 351.88 f/s\n",
            "827756 frames: done 464 games, mean reward 18.88, eps 0.02, speed 352.42 f/s\n",
            "829452 frames: done 465 games, mean reward 18.88, eps 0.02, speed 355.05 f/s\n",
            "831262 frames: done 466 games, mean reward 18.86, eps 0.02, speed 354.85 f/s\n",
            "832928 frames: done 467 games, mean reward 18.88, eps 0.02, speed 356.9 f/s\n",
            "834736 frames: done 468 games, mean reward 18.9, eps 0.02, speed 353.15 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-468.dat: 100%|██████████| 100/100 [00:01<00:00, 94.80it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.89 -> 18.9, model saved\n",
            "836461 frames: done 469 games, mean reward 18.9, eps 0.02, speed 276.04 f/s\n",
            "838127 frames: done 470 games, mean reward 18.91, eps 0.02, speed 351.92 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-470.dat: 100%|██████████| 100/100 [00:00<00:00, 102.22it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.9 -> 18.91, model saved\n",
            "840000 frames: done 471 games, mean reward 18.88, eps 0.02, speed 283.72 f/s\n",
            "841802 frames: done 472 games, mean reward 18.88, eps 0.02, speed 349.88 f/s\n",
            "843468 frames: done 473 games, mean reward 18.9, eps 0.02, speed 345.54 f/s\n",
            "845321 frames: done 474 games, mean reward 18.91, eps 0.02, speed 351.12 f/s\n",
            "847335 frames: done 475 games, mean reward 18.89, eps 0.02, speed 352.17 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-475.dat: 100%|██████████| 100/100 [00:00<00:00, 112.92it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.91 -> 18.89, model saved\n",
            "849276 frames: done 476 games, mean reward 18.88, eps 0.02, speed 291.27 f/s\n",
            "851380 frames: done 477 games, mean reward 18.91, eps 0.02, speed 349.87 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-477.dat: 100%|██████████| 100/100 [00:00<00:00, 111.35it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.89 -> 18.91, model saved\n",
            "853579 frames: done 478 games, mean reward 18.87, eps 0.02, speed 301.75 f/s\n",
            "855314 frames: done 479 games, mean reward 18.88, eps 0.02, speed 355.7 f/s\n",
            "857088 frames: done 480 games, mean reward 18.87, eps 0.02, speed 351.29 f/s\n",
            "858926 frames: done 481 games, mean reward 18.85, eps 0.02, speed 351.7 f/s\n",
            "860729 frames: done 482 games, mean reward 18.87, eps 0.02, speed 349.0 f/s\n",
            "862473 frames: done 483 games, mean reward 18.87, eps 0.02, speed 348.17 f/s\n",
            "864139 frames: done 484 games, mean reward 18.87, eps 0.02, speed 350.28 f/s\n",
            "866063 frames: done 485 games, mean reward 18.88, eps 0.02, speed 348.37 f/s\n",
            "867746 frames: done 486 games, mean reward 18.9, eps 0.02, speed 353.41 f/s\n",
            "869543 frames: done 487 games, mean reward 18.89, eps 0.02, speed 358.33 f/s\n",
            "871209 frames: done 488 games, mean reward 18.89, eps 0.02, speed 354.59 f/s\n",
            "872969 frames: done 489 games, mean reward 18.88, eps 0.02, speed 355.71 f/s\n",
            "874944 frames: done 490 games, mean reward 18.85, eps 0.02, speed 350.16 f/s\n",
            "876626 frames: done 491 games, mean reward 18.88, eps 0.02, speed 355.0 f/s\n",
            "878551 frames: done 492 games, mean reward 18.85, eps 0.02, speed 351.16 f/s\n",
            "880217 frames: done 493 games, mean reward 18.85, eps 0.02, speed 350.02 f/s\n",
            "882077 frames: done 494 games, mean reward 18.83, eps 0.02, speed 351.4 f/s\n",
            "883742 frames: done 495 games, mean reward 18.83, eps 0.02, speed 353.84 f/s\n",
            "885408 frames: done 496 games, mean reward 18.85, eps 0.02, speed 350.44 f/s\n",
            "887146 frames: done 497 games, mean reward 18.86, eps 0.02, speed 349.51 f/s\n",
            "888812 frames: done 498 games, mean reward 18.86, eps 0.02, speed 354.48 f/s\n",
            "890686 frames: done 499 games, mean reward 18.86, eps 0.02, speed 354.72 f/s\n",
            "892352 frames: done 500 games, mean reward 18.86, eps 0.02, speed 357.14 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-500.dat: 100%|██████████| 100/100 [00:01<00:00, 93.89it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.91 -> 18.86, model saved\n",
            "894108 frames: done 501 games, mean reward 18.88, eps 0.02, speed 277.7 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-501.dat: 100%|██████████| 100/100 [00:00<00:00, 116.60it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.86 -> 18.88, model saved\n",
            "895774 frames: done 502 games, mean reward 18.9, eps 0.02, speed 286.92 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-502.dat: 100%|██████████| 100/100 [00:01<00:00, 55.41it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.88 -> 18.9, model saved\n",
            "897440 frames: done 503 games, mean reward 18.92, eps 0.02, speed 243.56 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-503.dat: 100%|██████████| 100/100 [00:00<00:00, 102.40it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.9 -> 18.92, model saved\n",
            "899204 frames: done 504 games, mean reward 18.93, eps 0.02, speed 281.22 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-504.dat: 100%|██████████| 100/100 [00:00<00:00, 105.59it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.92 -> 18.93, model saved\n",
            "901033 frames: done 505 games, mean reward 18.93, eps 0.02, speed 287.24 f/s\n",
            "902867 frames: done 506 games, mean reward 18.93, eps 0.02, speed 354.33 f/s\n",
            "904674 frames: done 507 games, mean reward 18.92, eps 0.02, speed 352.89 f/s\n",
            "906514 frames: done 508 games, mean reward 18.94, eps 0.02, speed 352.68 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-508.dat: 100%|██████████| 100/100 [00:01<00:00, 87.03it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.93 -> 18.94, model saved\n",
            "908390 frames: done 509 games, mean reward 18.92, eps 0.02, speed 279.79 f/s\n",
            "910055 frames: done 510 games, mean reward 18.93, eps 0.02, speed 354.67 f/s\n",
            "911721 frames: done 511 games, mean reward 18.93, eps 0.02, speed 353.92 f/s\n",
            "913387 frames: done 512 games, mean reward 18.93, eps 0.02, speed 354.89 f/s\n",
            "915053 frames: done 513 games, mean reward 18.93, eps 0.02, speed 350.0 f/s\n",
            "916719 frames: done 514 games, mean reward 18.93, eps 0.02, speed 347.59 f/s\n",
            "918385 frames: done 515 games, mean reward 18.94, eps 0.02, speed 349.98 f/s\n",
            "920163 frames: done 516 games, mean reward 18.95, eps 0.02, speed 346.31 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-516.dat: 100%|██████████| 100/100 [00:00<00:00, 119.44it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.94 -> 18.95, model saved\n",
            "922001 frames: done 517 games, mean reward 18.96, eps 0.02, speed 291.12 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-517.dat: 100%|██████████| 100/100 [00:00<00:00, 109.43it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.95 -> 18.96, model saved\n",
            "923667 frames: done 518 games, mean reward 18.97, eps 0.02, speed 282.46 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-518.dat: 100%|██████████| 100/100 [00:00<00:00, 118.09it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.96 -> 18.97, model saved\n",
            "925440 frames: done 519 games, mean reward 18.96, eps 0.02, speed 288.21 f/s\n",
            "927106 frames: done 520 games, mean reward 18.96, eps 0.02, speed 352.71 f/s\n",
            "929113 frames: done 521 games, mean reward 18.93, eps 0.02, speed 354.73 f/s\n",
            "930779 frames: done 522 games, mean reward 18.94, eps 0.02, speed 355.0 f/s\n",
            "932542 frames: done 523 games, mean reward 18.93, eps 0.02, speed 354.66 f/s\n",
            "934260 frames: done 524 games, mean reward 18.95, eps 0.02, speed 354.64 f/s\n",
            "936073 frames: done 525 games, mean reward 18.93, eps 0.02, speed 355.34 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-525.dat: 100%|██████████| 100/100 [00:00<00:00, 111.45it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.97 -> 18.93, model saved\n",
            "937878 frames: done 526 games, mean reward 18.91, eps 0.02, speed 289.41 f/s\n",
            "939544 frames: done 527 games, mean reward 18.92, eps 0.02, speed 355.49 f/s\n",
            "941226 frames: done 528 games, mean reward 18.93, eps 0.02, speed 352.74 f/s\n",
            "943080 frames: done 529 games, mean reward 18.91, eps 0.02, speed 354.05 f/s\n",
            "944798 frames: done 530 games, mean reward 18.91, eps 0.02, speed 351.96 f/s\n",
            "946481 frames: done 531 games, mean reward 18.96, eps 0.02, speed 351.86 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-531.dat: 100%|██████████| 100/100 [00:00<00:00, 108.35it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.93 -> 18.96, model saved\n",
            "948147 frames: done 532 games, mean reward 18.96, eps 0.02, speed 255.93 f/s\n",
            "949979 frames: done 533 games, mean reward 18.95, eps 0.02, speed 355.67 f/s\n",
            "951753 frames: done 534 games, mean reward 18.95, eps 0.02, speed 352.47 f/s\n",
            "953436 frames: done 535 games, mean reward 18.96, eps 0.02, speed 357.72 f/s\n",
            "955120 frames: done 536 games, mean reward 18.96, eps 0.02, speed 355.76 f/s\n",
            "956786 frames: done 537 games, mean reward 18.96, eps 0.02, speed 355.26 f/s\n",
            "958520 frames: done 538 games, mean reward 18.96, eps 0.02, speed 354.94 f/s\n",
            "960389 frames: done 539 games, mean reward 18.98, eps 0.02, speed 355.44 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-539.dat: 100%|██████████| 100/100 [00:00<00:00, 104.83it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.96 -> 18.98, model saved\n",
            "962055 frames: done 540 games, mean reward 18.99, eps 0.02, speed 281.11 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-540.dat: 100%|██████████| 100/100 [00:00<00:00, 109.94it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.98 -> 18.99, model saved\n",
            "963844 frames: done 541 games, mean reward 18.99, eps 0.02, speed 288.21 f/s\n",
            "965838 frames: done 542 games, mean reward 18.97, eps 0.02, speed 350.09 f/s\n",
            "967504 frames: done 543 games, mean reward 18.97, eps 0.02, speed 348.87 f/s\n",
            "969503 frames: done 544 games, mean reward 18.97, eps 0.02, speed 355.07 f/s\n",
            "971559 frames: done 545 games, mean reward 18.96, eps 0.02, speed 356.44 f/s\n",
            "973387 frames: done 546 games, mean reward 18.96, eps 0.02, speed 353.88 f/s\n",
            "975646 frames: done 547 games, mean reward 18.93, eps 0.02, speed 350.12 f/s\n",
            "977330 frames: done 548 games, mean reward 18.97, eps 0.02, speed 349.99 f/s\n",
            "979137 frames: done 549 games, mean reward 18.97, eps 0.02, speed 348.75 f/s\n",
            "980819 frames: done 550 games, mean reward 19.0, eps 0.02, speed 351.5 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-550.dat: 100%|██████████| 100/100 [00:00<00:00, 114.63it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.99 -> 19.0, model saved\n",
            "982603 frames: done 551 games, mean reward 19.0, eps 0.02, speed 288.01 f/s\n",
            "984268 frames: done 552 games, mean reward 19.0, eps 0.02, speed 351.11 f/s\n",
            "985934 frames: done 553 games, mean reward 19.04, eps 0.02, speed 349.49 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-553.dat: 100%|██████████| 100/100 [00:00<00:00, 102.91it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.0 -> 19.04, model saved\n",
            "987599 frames: done 554 games, mean reward 19.05, eps 0.02, speed 282.7 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-554.dat: 100%|██████████| 100/100 [00:00<00:00, 117.45it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.04 -> 19.05, model saved\n",
            "989398 frames: done 555 games, mean reward 19.02, eps 0.02, speed 292.12 f/s\n",
            "991064 frames: done 556 games, mean reward 19.04, eps 0.02, speed 353.82 f/s\n",
            "992730 frames: done 557 games, mean reward 19.04, eps 0.02, speed 354.07 f/s\n",
            "994396 frames: done 558 games, mean reward 19.04, eps 0.02, speed 353.6 f/s\n",
            "996233 frames: done 559 games, mean reward 19.05, eps 0.02, speed 355.1 f/s\n",
            "998050 frames: done 560 games, mean reward 19.05, eps 0.02, speed 351.36 f/s\n",
            "1000105 frames: done 561 games, mean reward 19.03, eps 0.02, speed 353.44 f/s\n",
            "1002061 frames: done 562 games, mean reward 19.02, eps 0.02, speed 352.45 f/s\n",
            "1004375 frames: done 563 games, mean reward 18.98, eps 0.02, speed 350.34 f/s\n",
            "1006058 frames: done 564 games, mean reward 18.98, eps 0.02, speed 354.05 f/s\n",
            "1008020 frames: done 565 games, mean reward 18.93, eps 0.02, speed 356.69 f/s\n",
            "1009839 frames: done 566 games, mean reward 18.94, eps 0.02, speed 349.19 f/s\n",
            "1011652 frames: done 567 games, mean reward 18.92, eps 0.02, speed 351.08 f/s\n",
            "1013318 frames: done 568 games, mean reward 18.93, eps 0.02, speed 352.7 f/s\n",
            "1014984 frames: done 569 games, mean reward 18.94, eps 0.02, speed 351.43 f/s\n",
            "1016685 frames: done 570 games, mean reward 18.93, eps 0.02, speed 353.07 f/s\n",
            "1018460 frames: done 571 games, mean reward 18.95, eps 0.02, speed 355.26 f/s\n",
            "1020177 frames: done 572 games, mean reward 18.95, eps 0.02, speed 352.72 f/s\n",
            "1021926 frames: done 573 games, mean reward 18.94, eps 0.02, speed 349.67 f/s\n",
            "1023872 frames: done 574 games, mean reward 18.91, eps 0.02, speed 349.53 f/s\n",
            "1025568 frames: done 575 games, mean reward 18.93, eps 0.02, speed 354.2 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-575.dat: 100%|██████████| 100/100 [00:01<00:00, 83.22it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.05 -> 18.93, model saved\n",
            "1027233 frames: done 576 games, mean reward 18.94, eps 0.02, speed 260.47 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-576.dat: 100%|██████████| 100/100 [00:00<00:00, 101.92it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.93 -> 18.94, model saved\n",
            "1028950 frames: done 577 games, mean reward 18.96, eps 0.02, speed 279.89 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-577.dat: 100%|██████████| 100/100 [00:00<00:00, 105.73it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.94 -> 18.96, model saved\n",
            "1030691 frames: done 578 games, mean reward 19.0, eps 0.02, speed 283.38 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-578.dat: 100%|██████████| 100/100 [00:00<00:00, 103.55it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.96 -> 19.0, model saved\n",
            "1032689 frames: done 579 games, mean reward 18.99, eps 0.02, speed 290.12 f/s\n",
            "1034355 frames: done 580 games, mean reward 19.0, eps 0.02, speed 353.88 f/s\n",
            "1036050 frames: done 581 games, mean reward 19.02, eps 0.02, speed 352.45 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-581.dat: 100%|██████████| 100/100 [00:00<00:00, 113.53it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.0 -> 19.02, model saved\n",
            "1037716 frames: done 582 games, mean reward 19.02, eps 0.02, speed 284.22 f/s\n",
            "1039535 frames: done 583 games, mean reward 18.99, eps 0.02, speed 347.73 f/s\n",
            "1041323 frames: done 584 games, mean reward 18.99, eps 0.02, speed 349.2 f/s\n",
            "1043075 frames: done 585 games, mean reward 19.01, eps 0.02, speed 347.24 f/s\n",
            "1044975 frames: done 586 games, mean reward 18.98, eps 0.02, speed 350.71 f/s\n",
            "1046641 frames: done 587 games, mean reward 18.99, eps 0.02, speed 347.74 f/s\n",
            "1048294 frames: done 588 games, mean reward 19.0, eps 0.02, speed 349.31 f/s\n",
            "1050107 frames: done 589 games, mean reward 18.99, eps 0.02, speed 349.93 f/s\n",
            "1051854 frames: done 590 games, mean reward 19.03, eps 0.02, speed 353.84 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-590.dat: 100%|██████████| 100/100 [00:00<00:00, 111.77it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.02 -> 19.03, model saved\n",
            "1053585 frames: done 591 games, mean reward 19.02, eps 0.02, speed 277.08 f/s\n",
            "1055460 frames: done 592 games, mean reward 19.02, eps 0.02, speed 351.75 f/s\n",
            "1057235 frames: done 593 games, mean reward 19.02, eps 0.02, speed 352.69 f/s\n",
            "1059042 frames: done 594 games, mean reward 19.05, eps 0.02, speed 357.7 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-594.dat: 100%|██████████| 100/100 [00:00<00:00, 111.23it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.03 -> 19.05, model saved\n",
            "1060708 frames: done 595 games, mean reward 19.05, eps 0.02, speed 285.99 f/s\n",
            "1062425 frames: done 596 games, mean reward 19.04, eps 0.02, speed 357.03 f/s\n",
            "1064091 frames: done 597 games, mean reward 19.05, eps 0.02, speed 349.12 f/s\n",
            "1065757 frames: done 598 games, mean reward 19.05, eps 0.02, speed 352.71 f/s\n",
            "1067501 frames: done 599 games, mean reward 19.06, eps 0.02, speed 350.35 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-599.dat: 100%|██████████| 100/100 [00:01<00:00, 95.34it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.05 -> 19.06, model saved\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\rUploading file PongNoFrameskip-v4-600.dat:   0%|          | 0/100 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "1069186 frames: done 600 games, mean reward 19.06, eps 0.02, speed 251.03 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\rUploading file PongNoFrameskip-v4-600.dat: 100%|██████████| 100/100 [00:00<00:00, 109.46it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.06 -> 19.06, model saved\n",
            "1070852 frames: done 601 games, mean reward 19.07, eps 0.02, speed 283.36 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-601.dat: 100%|██████████| 100/100 [00:00<00:00, 120.67it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.06 -> 19.07, model saved\n",
            "1072712 frames: done 602 games, mean reward 19.05, eps 0.02, speed 293.48 f/s\n",
            "1074531 frames: done 603 games, mean reward 19.04, eps 0.02, speed 354.81 f/s\n",
            "1076197 frames: done 604 games, mean reward 19.06, eps 0.02, speed 352.64 f/s\n",
            "1077915 frames: done 605 games, mean reward 19.07, eps 0.02, speed 353.48 f/s\n",
            "1079758 frames: done 606 games, mean reward 19.06, eps 0.02, speed 355.48 f/s\n",
            "1081483 frames: done 607 games, mean reward 19.06, eps 0.02, speed 354.96 f/s\n",
            "1083149 frames: done 608 games, mean reward 19.08, eps 0.02, speed 351.25 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-608.dat: 100%|██████████| 100/100 [00:00<00:00, 110.37it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.07 -> 19.08, model saved\n",
            "1085033 frames: done 609 games, mean reward 19.09, eps 0.02, speed 284.25 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-609.dat: 100%|██████████| 100/100 [00:00<00:00, 102.32it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.08 -> 19.09, model saved\n",
            "1086899 frames: done 610 games, mean reward 19.07, eps 0.02, speed 284.44 f/s\n",
            "1088565 frames: done 611 games, mean reward 19.07, eps 0.02, speed 350.38 f/s\n",
            "1090350 frames: done 612 games, mean reward 19.07, eps 0.02, speed 353.52 f/s\n",
            "1092016 frames: done 613 games, mean reward 19.07, eps 0.02, speed 351.75 f/s\n",
            "1093835 frames: done 614 games, mean reward 19.06, eps 0.02, speed 354.15 f/s\n",
            "1095500 frames: done 615 games, mean reward 19.06, eps 0.02, speed 355.88 f/s\n",
            "1097246 frames: done 616 games, mean reward 19.06, eps 0.02, speed 355.65 f/s\n",
            "1098963 frames: done 617 games, mean reward 19.06, eps 0.02, speed 355.2 f/s\n",
            "1100629 frames: done 618 games, mean reward 19.06, eps 0.02, speed 353.53 f/s\n",
            "1102295 frames: done 619 games, mean reward 19.07, eps 0.02, speed 354.08 f/s\n",
            "1103961 frames: done 620 games, mean reward 19.07, eps 0.02, speed 354.25 f/s\n",
            "1105627 frames: done 621 games, mean reward 19.1, eps 0.02, speed 350.15 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-621.dat: 100%|██████████| 100/100 [00:00<00:00, 101.47it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.09 -> 19.1, model saved\n",
            "1107344 frames: done 622 games, mean reward 19.09, eps 0.02, speed 275.13 f/s\n",
            "1109334 frames: done 623 games, mean reward 19.08, eps 0.02, speed 346.03 f/s\n",
            "1111124 frames: done 624 games, mean reward 19.08, eps 0.02, speed 350.03 f/s\n",
            "1112809 frames: done 625 games, mean reward 19.1, eps 0.02, speed 350.82 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-625.dat: 100%|██████████| 100/100 [00:00<00:00, 109.67it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.1 -> 19.1, model saved\n",
            "1114475 frames: done 626 games, mean reward 19.11, eps 0.02, speed 283.4 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-626.dat: 100%|██████████| 100/100 [00:00<00:00, 124.70it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.1 -> 19.11, model saved\n",
            "1116191 frames: done 627 games, mean reward 19.1, eps 0.02, speed 289.55 f/s\n",
            "1117908 frames: done 628 games, mean reward 19.09, eps 0.02, speed 356.91 f/s\n",
            "1119939 frames: done 629 games, mean reward 19.07, eps 0.02, speed 357.13 f/s\n",
            "1121683 frames: done 630 games, mean reward 19.08, eps 0.02, speed 351.45 f/s\n",
            "1123417 frames: done 631 games, mean reward 19.07, eps 0.02, speed 347.26 f/s\n",
            "1125178 frames: done 632 games, mean reward 19.06, eps 0.02, speed 349.67 f/s\n",
            "1127014 frames: done 633 games, mean reward 19.06, eps 0.02, speed 352.72 f/s\n",
            "1128727 frames: done 634 games, mean reward 19.06, eps 0.02, speed 355.08 f/s\n",
            "1130559 frames: done 635 games, mean reward 19.04, eps 0.02, speed 354.52 f/s\n",
            "1132457 frames: done 636 games, mean reward 19.01, eps 0.02, speed 351.98 f/s\n",
            "1134123 frames: done 637 games, mean reward 19.01, eps 0.02, speed 348.76 f/s\n",
            "1135911 frames: done 638 games, mean reward 19.0, eps 0.02, speed 352.64 f/s\n",
            "1137577 frames: done 639 games, mean reward 19.02, eps 0.02, speed 351.79 f/s\n",
            "1139436 frames: done 640 games, mean reward 19.0, eps 0.02, speed 348.89 f/s\n",
            "1141406 frames: done 641 games, mean reward 18.98, eps 0.02, speed 348.1 f/s\n",
            "1143397 frames: done 642 games, mean reward 18.96, eps 0.02, speed 351.72 f/s\n",
            "1145062 frames: done 643 games, mean reward 18.96, eps 0.02, speed 348.8 f/s\n",
            "1147062 frames: done 644 games, mean reward 18.93, eps 0.02, speed 349.64 f/s\n",
            "1148786 frames: done 645 games, mean reward 18.97, eps 0.02, speed 350.85 f/s\n",
            "1150564 frames: done 646 games, mean reward 18.98, eps 0.02, speed 350.45 f/s\n",
            "1152437 frames: done 647 games, mean reward 18.98, eps 0.02, speed 346.83 f/s\n",
            "1154235 frames: done 648 games, mean reward 18.95, eps 0.02, speed 350.82 f/s\n",
            "1155982 frames: done 649 games, mean reward 18.96, eps 0.02, speed 358.29 f/s\n",
            "1157699 frames: done 650 games, mean reward 18.95, eps 0.02, speed 353.97 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-650.dat: 100%|██████████| 100/100 [00:01<00:00, 65.96it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.11 -> 18.95, model saved\n",
            "1159365 frames: done 651 games, mean reward 18.95, eps 0.02, speed 258.82 f/s\n",
            "1161031 frames: done 652 games, mean reward 18.95, eps 0.02, speed 359.35 f/s\n",
            "1162875 frames: done 653 games, mean reward 18.92, eps 0.02, speed 354.4 f/s\n",
            "1164541 frames: done 654 games, mean reward 18.92, eps 0.02, speed 351.72 f/s\n",
            "1166424 frames: done 655 games, mean reward 18.92, eps 0.02, speed 347.89 f/s\n",
            "1168090 frames: done 656 games, mean reward 18.92, eps 0.02, speed 352.37 f/s\n",
            "1169814 frames: done 657 games, mean reward 18.92, eps 0.02, speed 351.83 f/s\n",
            "1171479 frames: done 658 games, mean reward 18.92, eps 0.02, speed 349.21 f/s\n",
            "1173267 frames: done 659 games, mean reward 18.92, eps 0.02, speed 353.19 f/s\n",
            "1174933 frames: done 660 games, mean reward 18.93, eps 0.02, speed 348.79 f/s\n",
            "1176650 frames: done 661 games, mean reward 18.95, eps 0.02, speed 352.13 f/s\n",
            "1178316 frames: done 662 games, mean reward 18.97, eps 0.02, speed 350.29 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-662.dat: 100%|██████████| 100/100 [00:00<00:00, 117.01it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.95 -> 18.97, model saved\n",
            "1180112 frames: done 663 games, mean reward 19.0, eps 0.02, speed 289.11 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-663.dat: 100%|██████████| 100/100 [00:00<00:00, 117.94it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.97 -> 19.0, model saved\n",
            "1182072 frames: done 664 games, mean reward 18.96, eps 0.02, speed 294.99 f/s\n",
            "1183738 frames: done 665 games, mean reward 19.01, eps 0.02, speed 349.46 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-665.dat: 100%|██████████| 100/100 [00:00<00:00, 110.51it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.0 -> 19.01, model saved\n",
            "1185404 frames: done 666 games, mean reward 19.02, eps 0.02, speed 283.27 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-666.dat: 100%|██████████| 100/100 [00:00<00:00, 114.91it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.01 -> 19.02, model saved\n",
            "1187696 frames: done 667 games, mean reward 19.0, eps 0.02, speed 299.35 f/s\n",
            "1189362 frames: done 668 games, mean reward 19.0, eps 0.02, speed 353.18 f/s\n",
            "1191058 frames: done 669 games, mean reward 19.0, eps 0.02, speed 351.81 f/s\n",
            "1192818 frames: done 670 games, mean reward 19.0, eps 0.02, speed 349.44 f/s\n",
            "1194535 frames: done 671 games, mean reward 19.0, eps 0.02, speed 351.25 f/s\n",
            "1196593 frames: done 672 games, mean reward 18.99, eps 0.02, speed 354.86 f/s\n",
            "1198412 frames: done 673 games, mean reward 18.99, eps 0.02, speed 355.62 f/s\n",
            "1200129 frames: done 674 games, mean reward 19.02, eps 0.02, speed 353.29 f/s\n",
            "1202103 frames: done 675 games, mean reward 18.96, eps 0.02, speed 348.58 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-675.dat: 100%|██████████| 100/100 [00:01<00:00, 97.91it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.02 -> 18.96, model saved\n",
            "1204000 frames: done 676 games, mean reward 18.94, eps 0.02, speed 286.43 f/s\n",
            "1205666 frames: done 677 games, mean reward 18.95, eps 0.02, speed 348.71 f/s\n",
            "1207604 frames: done 678 games, mean reward 18.93, eps 0.02, speed 354.69 f/s\n",
            "1209392 frames: done 679 games, mean reward 18.93, eps 0.02, speed 356.61 f/s\n",
            "1211214 frames: done 680 games, mean reward 18.92, eps 0.02, speed 353.82 f/s\n",
            "1212933 frames: done 681 games, mean reward 18.91, eps 0.02, speed 348.05 f/s\n",
            "1214760 frames: done 682 games, mean reward 18.9, eps 0.02, speed 351.41 f/s\n",
            "1216610 frames: done 683 games, mean reward 18.91, eps 0.02, speed 350.46 f/s\n",
            "1218306 frames: done 684 games, mean reward 18.91, eps 0.02, speed 351.86 f/s\n",
            "1220117 frames: done 685 games, mean reward 18.9, eps 0.02, speed 354.98 f/s\n",
            "1222078 frames: done 686 games, mean reward 18.9, eps 0.02, speed 356.06 f/s\n",
            "1223795 frames: done 687 games, mean reward 18.89, eps 0.02, speed 353.93 f/s\n",
            "1225557 frames: done 688 games, mean reward 18.87, eps 0.02, speed 353.82 f/s\n",
            "1227221 frames: done 689 games, mean reward 18.89, eps 0.02, speed 349.82 f/s\n",
            "1229032 frames: done 690 games, mean reward 18.88, eps 0.02, speed 350.03 f/s\n",
            "1230931 frames: done 691 games, mean reward 18.86, eps 0.02, speed 351.59 f/s\n",
            "1232879 frames: done 692 games, mean reward 18.85, eps 0.02, speed 348.95 f/s\n",
            "1234714 frames: done 693 games, mean reward 18.85, eps 0.02, speed 351.43 f/s\n",
            "1236380 frames: done 694 games, mean reward 18.85, eps 0.02, speed 353.83 f/s\n",
            "1238046 frames: done 695 games, mean reward 18.85, eps 0.02, speed 360.03 f/s\n",
            "1239876 frames: done 696 games, mean reward 18.84, eps 0.02, speed 358.82 f/s\n",
            "1241755 frames: done 697 games, mean reward 18.82, eps 0.02, speed 354.02 f/s\n",
            "1243501 frames: done 698 games, mean reward 18.81, eps 0.02, speed 352.74 f/s\n",
            "1245427 frames: done 699 games, mean reward 18.77, eps 0.02, speed 355.01 f/s\n",
            "1247092 frames: done 700 games, mean reward 18.77, eps 0.02, speed 349.99 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-700.dat: 100%|██████████| 100/100 [00:00<00:00, 105.50it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.96 -> 18.77, model saved\n",
            "1248841 frames: done 701 games, mean reward 18.76, eps 0.02, speed 279.65 f/s\n",
            "1250587 frames: done 702 games, mean reward 18.78, eps 0.02, speed 349.83 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-702.dat: 100%|██████████| 100/100 [00:00<00:00, 114.50it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.77 -> 18.78, model saved\n",
            "1252253 frames: done 703 games, mean reward 18.79, eps 0.02, speed 282.61 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-703.dat: 100%|██████████| 100/100 [00:00<00:00, 108.13it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.78 -> 18.79, model saved\n",
            "1254161 frames: done 704 games, mean reward 18.76, eps 0.02, speed 293.03 f/s\n",
            "1255846 frames: done 705 games, mean reward 18.77, eps 0.02, speed 352.89 f/s\n",
            "1257512 frames: done 706 games, mean reward 18.79, eps 0.02, speed 356.68 f/s\n",
            "1259361 frames: done 707 games, mean reward 18.78, eps 0.02, speed 353.99 f/s\n",
            "1261163 frames: done 708 games, mean reward 18.78, eps 0.02, speed 353.51 f/s\n",
            "1262859 frames: done 709 games, mean reward 18.79, eps 0.02, speed 346.58 f/s\n",
            "1264542 frames: done 710 games, mean reward 18.81, eps 0.02, speed 347.53 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-710.dat: 100%|██████████| 100/100 [00:01<00:00, 97.30it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.79 -> 18.81, model saved\n",
            "1266272 frames: done 711 games, mean reward 18.8, eps 0.02, speed 277.84 f/s\n",
            "1268009 frames: done 712 games, mean reward 18.79, eps 0.02, speed 352.27 f/s\n",
            "1269826 frames: done 713 games, mean reward 18.78, eps 0.02, speed 352.68 f/s\n",
            "1271636 frames: done 714 games, mean reward 18.78, eps 0.02, speed 350.57 f/s\n",
            "1273385 frames: done 715 games, mean reward 18.77, eps 0.02, speed 351.07 f/s\n",
            "1275080 frames: done 716 games, mean reward 18.78, eps 0.02, speed 352.61 f/s\n",
            "1276776 frames: done 717 games, mean reward 18.79, eps 0.02, speed 352.82 f/s\n",
            "1278441 frames: done 718 games, mean reward 18.79, eps 0.02, speed 356.85 f/s\n",
            "1280170 frames: done 719 games, mean reward 18.78, eps 0.02, speed 354.15 f/s\n",
            "1281836 frames: done 720 games, mean reward 18.78, eps 0.02, speed 353.88 f/s\n",
            "1283518 frames: done 721 games, mean reward 18.78, eps 0.02, speed 355.67 f/s\n",
            "1285488 frames: done 722 games, mean reward 18.76, eps 0.02, speed 352.71 f/s\n",
            "1287154 frames: done 723 games, mean reward 18.79, eps 0.02, speed 351.33 f/s\n",
            "1288939 frames: done 724 games, mean reward 18.8, eps 0.02, speed 353.72 f/s\n",
            "1290814 frames: done 725 games, mean reward 18.77, eps 0.02, speed 350.13 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-725.dat: 100%|██████████| 100/100 [00:01<00:00, 93.98it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.81 -> 18.77, model saved\n",
            "1292497 frames: done 726 games, mean reward 18.77, eps 0.02, speed 276.32 f/s\n",
            "1294162 frames: done 727 games, mean reward 18.78, eps 0.02, speed 352.96 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-727.dat: 100%|██████████| 100/100 [00:00<00:00, 104.86it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.77 -> 18.78, model saved\n",
            "1295907 frames: done 728 games, mean reward 18.79, eps 0.02, speed 281.63 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-728.dat: 100%|██████████| 100/100 [00:00<00:00, 110.21it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.78 -> 18.79, model saved\n",
            "1297626 frames: done 729 games, mean reward 18.82, eps 0.02, speed 282.33 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-729.dat: 100%|██████████| 100/100 [00:00<00:00, 110.63it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.79 -> 18.82, model saved\n",
            "1299533 frames: done 730 games, mean reward 18.79, eps 0.02, speed 292.44 f/s\n",
            "1301199 frames: done 731 games, mean reward 18.8, eps 0.02, speed 351.13 f/s\n",
            "1302946 frames: done 732 games, mean reward 18.8, eps 0.02, speed 352.43 f/s\n",
            "1304751 frames: done 733 games, mean reward 18.79, eps 0.02, speed 350.75 f/s\n",
            "1306476 frames: done 734 games, mean reward 18.79, eps 0.02, speed 351.23 f/s\n",
            "1308142 frames: done 735 games, mean reward 18.81, eps 0.02, speed 355.44 f/s\n",
            "1309824 frames: done 736 games, mean reward 18.85, eps 0.02, speed 353.22 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-736.dat: 100%|██████████| 100/100 [00:01<00:00, 98.23it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.82 -> 18.85, model saved\n",
            "1311603 frames: done 737 games, mean reward 18.84, eps 0.02, speed 278.25 f/s\n",
            "1313269 frames: done 738 games, mean reward 18.86, eps 0.02, speed 352.36 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-738.dat: 100%|██████████| 100/100 [00:01<00:00, 78.27it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.85 -> 18.86, model saved\n",
            "1314970 frames: done 739 games, mean reward 18.85, eps 0.02, speed 268.96 f/s\n",
            "1316856 frames: done 740 games, mean reward 18.83, eps 0.02, speed 352.0 f/s\n",
            "1318520 frames: done 741 games, mean reward 18.86, eps 0.02, speed 352.8 f/s\n",
            "1320201 frames: done 742 games, mean reward 18.91, eps 0.02, speed 353.56 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-742.dat: 100%|██████████| 100/100 [00:01<00:00, 93.14it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.86 -> 18.91, model saved\n",
            "1322233 frames: done 743 games, mean reward 18.87, eps 0.02, speed 285.81 f/s\n",
            "1324255 frames: done 744 games, mean reward 18.9, eps 0.02, speed 352.15 f/s\n",
            "1326045 frames: done 745 games, mean reward 18.89, eps 0.02, speed 347.76 f/s\n",
            "1327872 frames: done 746 games, mean reward 18.89, eps 0.02, speed 349.29 f/s\n",
            "1329538 frames: done 747 games, mean reward 18.93, eps 0.02, speed 344.98 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-747.dat: 100%|██████████| 100/100 [00:00<00:00, 108.55it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.91 -> 18.93, model saved\n",
            "1331204 frames: done 748 games, mean reward 18.96, eps 0.02, speed 280.54 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-748.dat: 100%|██████████| 100/100 [00:00<00:00, 114.56it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.93 -> 18.96, model saved\n",
            "1333163 frames: done 749 games, mean reward 18.93, eps 0.02, speed 294.18 f/s\n",
            "1334895 frames: done 750 games, mean reward 18.95, eps 0.02, speed 355.91 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-750.dat: 100%|██████████| 100/100 [00:00<00:00, 110.15it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.96 -> 18.95, model saved\n",
            "1336934 frames: done 751 games, mean reward 18.91, eps 0.02, speed 294.9 f/s\n",
            "1338723 frames: done 752 games, mean reward 18.9, eps 0.02, speed 355.44 f/s\n",
            "1340389 frames: done 753 games, mean reward 18.93, eps 0.02, speed 354.12 f/s\n",
            "1342133 frames: done 754 games, mean reward 18.93, eps 0.02, speed 350.62 f/s\n",
            "1344014 frames: done 755 games, mean reward 18.94, eps 0.02, speed 354.06 f/s\n",
            "1345697 frames: done 756 games, mean reward 18.94, eps 0.02, speed 350.94 f/s\n",
            "1347363 frames: done 757 games, mean reward 18.94, eps 0.02, speed 353.46 f/s\n",
            "1349124 frames: done 758 games, mean reward 18.93, eps 0.02, speed 351.78 f/s\n",
            "1350790 frames: done 759 games, mean reward 18.94, eps 0.02, speed 348.08 f/s\n",
            "1352456 frames: done 760 games, mean reward 18.94, eps 0.02, speed 351.17 f/s\n",
            "1354173 frames: done 761 games, mean reward 18.94, eps 0.02, speed 352.54 f/s\n",
            "1355952 frames: done 762 games, mean reward 18.93, eps 0.02, speed 353.18 f/s\n",
            "1357831 frames: done 763 games, mean reward 18.94, eps 0.02, speed 354.42 f/s\n",
            "1359548 frames: done 764 games, mean reward 18.97, eps 0.02, speed 348.09 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-764.dat: 100%|██████████| 100/100 [00:00<00:00, 102.38it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.95 -> 18.97, model saved\n",
            "1361214 frames: done 765 games, mean reward 18.97, eps 0.02, speed 279.11 f/s\n",
            "1363002 frames: done 766 games, mean reward 18.96, eps 0.02, speed 355.29 f/s\n",
            "1364766 frames: done 767 games, mean reward 18.98, eps 0.02, speed 350.59 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-767.dat: 100%|██████████| 100/100 [00:00<00:00, 104.44it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.97 -> 18.98, model saved\n",
            "1366700 frames: done 768 games, mean reward 18.95, eps 0.02, speed 288.9 f/s\n",
            "1368366 frames: done 769 games, mean reward 18.95, eps 0.02, speed 351.22 f/s\n",
            "1370062 frames: done 770 games, mean reward 18.96, eps 0.02, speed 348.9 f/s\n",
            "1371728 frames: done 771 games, mean reward 18.97, eps 0.02, speed 350.46 f/s\n",
            "1373646 frames: done 772 games, mean reward 18.97, eps 0.02, speed 353.7 f/s\n",
            "1375457 frames: done 773 games, mean reward 18.96, eps 0.02, speed 357.44 f/s\n",
            "1377336 frames: done 774 games, mean reward 18.96, eps 0.02, speed 356.92 f/s\n",
            "1379083 frames: done 775 games, mean reward 19.02, eps 0.02, speed 354.65 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-775.dat: 100%|██████████| 100/100 [00:00<00:00, 109.08it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.98 -> 19.02, model saved\n",
            "1380880 frames: done 776 games, mean reward 19.02, eps 0.02, speed 286.27 f/s\n",
            "1382665 frames: done 777 games, mean reward 19.02, eps 0.02, speed 355.92 f/s\n",
            "1384574 frames: done 778 games, mean reward 19.02, eps 0.02, speed 356.15 f/s\n",
            "1386424 frames: done 779 games, mean reward 19.03, eps 0.02, speed 353.02 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-779.dat: 100%|██████████| 100/100 [00:00<00:00, 115.07it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.02 -> 19.03, model saved\n",
            "1388349 frames: done 780 games, mean reward 19.01, eps 0.02, speed 294.86 f/s\n",
            "1390162 frames: done 781 games, mean reward 19.0, eps 0.02, speed 351.5 f/s\n",
            "1391879 frames: done 782 games, mean reward 19.0, eps 0.02, speed 348.02 f/s\n",
            "1393545 frames: done 783 games, mean reward 19.02, eps 0.02, speed 349.5 f/s\n",
            "1395264 frames: done 784 games, mean reward 19.01, eps 0.02, speed 351.42 f/s\n",
            "1396930 frames: done 785 games, mean reward 19.02, eps 0.02, speed 351.65 f/s\n",
            "1398782 frames: done 786 games, mean reward 19.03, eps 0.02, speed 358.34 f/s\n",
            "1400448 frames: done 787 games, mean reward 19.04, eps 0.02, speed 350.96 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-787.dat: 100%|██████████| 100/100 [00:00<00:00, 112.78it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.03 -> 19.04, model saved\n",
            "1402114 frames: done 788 games, mean reward 19.05, eps 0.02, speed 277.08 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-788.dat: 100%|██████████| 100/100 [00:00<00:00, 110.56it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.04 -> 19.05, model saved\n",
            "1403905 frames: done 789 games, mean reward 19.04, eps 0.02, speed 288.29 f/s\n",
            "1405729 frames: done 790 games, mean reward 19.06, eps 0.02, speed 355.44 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-790.dat: 100%|██████████| 100/100 [00:00<00:00, 103.11it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.05 -> 19.06, model saved\n",
            "1407414 frames: done 791 games, mean reward 19.09, eps 0.02, speed 280.32 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-791.dat: 100%|██████████| 100/100 [00:00<00:00, 117.39it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.06 -> 19.09, model saved\n",
            "1409275 frames: done 792 games, mean reward 19.1, eps 0.02, speed 293.54 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-792.dat: 100%|██████████| 100/100 [00:01<00:00, 63.06it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.09 -> 19.1, model saved\n",
            "1410941 frames: done 793 games, mean reward 19.1, eps 0.02, speed 254.23 f/s\n",
            "1413054 frames: done 794 games, mean reward 19.06, eps 0.02, speed 352.41 f/s\n",
            "1414800 frames: done 795 games, mean reward 19.05, eps 0.02, speed 353.2 f/s\n",
            "1416543 frames: done 796 games, mean reward 19.08, eps 0.02, speed 355.46 f/s\n",
            "1418516 frames: done 797 games, mean reward 19.07, eps 0.02, speed 355.95 f/s\n",
            "1420233 frames: done 798 games, mean reward 19.07, eps 0.02, speed 354.54 f/s\n",
            "1422047 frames: done 799 games, mean reward 19.09, eps 0.02, speed 350.61 f/s\n",
            "1423731 frames: done 800 games, mean reward 19.09, eps 0.02, speed 351.07 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-800.dat: 100%|██████████| 100/100 [00:00<00:00, 108.50it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.1 -> 19.09, model saved\n",
            "1425478 frames: done 801 games, mean reward 19.09, eps 0.02, speed 285.17 f/s\n",
            "1427239 frames: done 802 games, mean reward 19.09, eps 0.02, speed 350.74 f/s\n",
            "1428905 frames: done 803 games, mean reward 19.09, eps 0.02, speed 349.33 f/s\n",
            "1430690 frames: done 804 games, mean reward 19.12, eps 0.02, speed 347.96 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-804.dat: 100%|██████████| 100/100 [00:00<00:00, 102.35it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.09 -> 19.12, model saved\n",
            "1432386 frames: done 805 games, mean reward 19.12, eps 0.02, speed 279.66 f/s\n",
            "1434070 frames: done 806 games, mean reward 19.12, eps 0.02, speed 349.84 f/s\n",
            "1435913 frames: done 807 games, mean reward 19.11, eps 0.02, speed 353.2 f/s\n",
            "1437710 frames: done 808 games, mean reward 19.11, eps 0.02, speed 355.53 f/s\n",
            "1439392 frames: done 809 games, mean reward 19.11, eps 0.02, speed 352.55 f/s\n",
            "1441348 frames: done 810 games, mean reward 19.08, eps 0.02, speed 353.3 f/s\n",
            "1443014 frames: done 811 games, mean reward 19.09, eps 0.02, speed 351.45 f/s\n",
            "1444731 frames: done 812 games, mean reward 19.09, eps 0.02, speed 350.49 f/s\n",
            "1446397 frames: done 813 games, mean reward 19.1, eps 0.02, speed 348.68 f/s\n",
            "1448080 frames: done 814 games, mean reward 19.11, eps 0.02, speed 354.3 f/s\n",
            "1449776 frames: done 815 games, mean reward 19.12, eps 0.02, speed 351.5 f/s\n",
            "1451493 frames: done 816 games, mean reward 19.1, eps 0.02, speed 353.08 f/s\n",
            "1453306 frames: done 817 games, mean reward 19.09, eps 0.02, speed 355.19 f/s\n",
            "1454972 frames: done 818 games, mean reward 19.09, eps 0.02, speed 353.42 f/s\n",
            "1456638 frames: done 819 games, mean reward 19.1, eps 0.02, speed 355.89 f/s\n",
            "1458517 frames: done 820 games, mean reward 19.08, eps 0.02, speed 353.42 f/s\n",
            "1460264 frames: done 821 games, mean reward 19.07, eps 0.02, speed 353.48 f/s\n",
            "1462087 frames: done 822 games, mean reward 19.09, eps 0.02, speed 349.17 f/s\n",
            "1463906 frames: done 823 games, mean reward 19.08, eps 0.02, speed 348.67 f/s\n",
            "1465735 frames: done 824 games, mean reward 19.06, eps 0.02, speed 350.5 f/s\n",
            "1467548 frames: done 825 games, mean reward 19.07, eps 0.02, speed 350.81 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-825.dat: 100%|██████████| 100/100 [00:00<00:00, 101.77it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.12 -> 19.07, model saved\n",
            "1469333 frames: done 826 games, mean reward 19.06, eps 0.02, speed 282.28 f/s\n",
            "1471586 frames: done 827 games, mean reward 18.99, eps 0.02, speed 350.76 f/s\n",
            "1473252 frames: done 828 games, mean reward 18.99, eps 0.02, speed 348.65 f/s\n",
            "1474969 frames: done 829 games, mean reward 18.99, eps 0.02, speed 349.94 f/s\n",
            "1476830 frames: done 830 games, mean reward 19.0, eps 0.02, speed 354.29 f/s\n",
            "1479116 frames: done 831 games, mean reward 18.93, eps 0.02, speed 355.77 f/s\n",
            "1480957 frames: done 832 games, mean reward 18.93, eps 0.02, speed 353.42 f/s\n",
            "1482658 frames: done 833 games, mean reward 18.96, eps 0.02, speed 351.71 f/s\n",
            "1484370 frames: done 834 games, mean reward 18.96, eps 0.02, speed 352.81 f/s\n",
            "1486038 frames: done 835 games, mean reward 18.97, eps 0.02, speed 353.58 f/s\n",
            "1487764 frames: done 836 games, mean reward 18.96, eps 0.02, speed 356.4 f/s\n",
            "1489430 frames: done 837 games, mean reward 18.97, eps 0.02, speed 353.09 f/s\n",
            "1491147 frames: done 838 games, mean reward 18.96, eps 0.02, speed 349.67 f/s\n",
            "1492813 frames: done 839 games, mean reward 18.97, eps 0.02, speed 348.08 f/s\n",
            "1494690 frames: done 840 games, mean reward 19.0, eps 0.02, speed 352.63 f/s\n",
            "1496542 frames: done 841 games, mean reward 18.97, eps 0.02, speed 353.67 f/s\n",
            "1498238 frames: done 842 games, mean reward 18.97, eps 0.02, speed 352.05 f/s\n",
            "1500050 frames: done 843 games, mean reward 18.99, eps 0.02, speed 353.31 f/s\n",
            "1502011 frames: done 844 games, mean reward 18.96, eps 0.02, speed 353.45 f/s\n",
            "1503830 frames: done 845 games, mean reward 18.96, eps 0.02, speed 353.63 f/s\n",
            "1505602 frames: done 846 games, mean reward 18.95, eps 0.02, speed 355.76 f/s\n",
            "1507410 frames: done 847 games, mean reward 18.94, eps 0.02, speed 350.25 f/s\n",
            "1509255 frames: done 848 games, mean reward 18.92, eps 0.02, speed 348.14 f/s\n",
            "1510921 frames: done 849 games, mean reward 18.95, eps 0.02, speed 349.07 f/s\n",
            "1512587 frames: done 850 games, mean reward 18.94, eps 0.02, speed 351.8 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-850.dat: 100%|██████████| 100/100 [00:00<00:00, 114.28it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 19.07 -> 18.94, model saved\n",
            "1514304 frames: done 851 games, mean reward 18.97, eps 0.02, speed 285.73 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-851.dat: 100%|██████████| 100/100 [00:00<00:00, 101.40it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.94 -> 18.97, model saved\n",
            "1516089 frames: done 852 games, mean reward 18.98, eps 0.02, speed 276.94 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-852.dat: 100%|██████████| 100/100 [00:00<00:00, 113.41it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.97 -> 18.98, model saved\n",
            "1518035 frames: done 853 games, mean reward 18.97, eps 0.02, speed 290.81 f/s\n",
            "1519983 frames: done 854 games, mean reward 18.96, eps 0.02, speed 354.41 f/s\n",
            "1521831 frames: done 855 games, mean reward 18.96, eps 0.02, speed 359.14 f/s\n",
            "1523577 frames: done 856 games, mean reward 18.96, eps 0.02, speed 356.34 f/s\n",
            "1525373 frames: done 857 games, mean reward 18.94, eps 0.02, speed 356.36 f/s\n",
            "1527454 frames: done 858 games, mean reward 18.89, eps 0.02, speed 355.0 f/s\n",
            "1529106 frames: done 859 games, mean reward 18.9, eps 0.02, speed 351.37 f/s\n",
            "1530997 frames: done 860 games, mean reward 18.88, eps 0.02, speed 349.85 f/s\n",
            "1532838 frames: done 861 games, mean reward 18.88, eps 0.02, speed 351.95 f/s\n",
            "1534619 frames: done 862 games, mean reward 18.87, eps 0.02, speed 349.51 f/s\n",
            "1536586 frames: done 863 games, mean reward 18.84, eps 0.02, speed 353.18 f/s\n",
            "1538252 frames: done 864 games, mean reward 18.85, eps 0.02, speed 349.81 f/s\n",
            "1540790 frames: done 865 games, mean reward 18.81, eps 0.02, speed 352.69 f/s\n",
            "1542851 frames: done 866 games, mean reward 18.79, eps 0.02, speed 350.89 f/s\n",
            "1544774 frames: done 867 games, mean reward 18.79, eps 0.02, speed 355.05 f/s\n",
            "1546534 frames: done 868 games, mean reward 18.81, eps 0.02, speed 352.1 f/s\n",
            "1548296 frames: done 869 games, mean reward 18.8, eps 0.02, speed 349.07 f/s\n",
            "1549962 frames: done 870 games, mean reward 18.8, eps 0.02, speed 349.41 f/s\n",
            "1551628 frames: done 871 games, mean reward 18.8, eps 0.02, speed 348.44 f/s\n",
            "1553376 frames: done 872 games, mean reward 18.82, eps 0.02, speed 348.25 f/s\n",
            "1555130 frames: done 873 games, mean reward 18.82, eps 0.02, speed 349.3 f/s\n",
            "1556915 frames: done 874 games, mean reward 18.83, eps 0.02, speed 350.88 f/s\n",
            "1558598 frames: done 875 games, mean reward 18.83, eps 0.02, speed 353.01 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-875.dat: 100%|██████████| 100/100 [00:00<00:00, 119.89it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.98 -> 18.83, model saved\n",
            "1560359 frames: done 876 games, mean reward 18.84, eps 0.02, speed 290.83 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-876.dat: 100%|██████████| 100/100 [00:00<00:00, 119.03it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.83 -> 18.84, model saved\n",
            "1562200 frames: done 877 games, mean reward 18.82, eps 0.02, speed 293.51 f/s\n",
            "1563866 frames: done 878 games, mean reward 18.85, eps 0.02, speed 350.62 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-878.dat: 100%|██████████| 100/100 [00:00<00:00, 110.68it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.84 -> 18.85, model saved\n",
            "1565989 frames: done 879 games, mean reward 18.83, eps 0.02, speed 293.06 f/s\n",
            "1567830 frames: done 880 games, mean reward 18.85, eps 0.02, speed 354.38 f/s\n",
            "1569682 frames: done 881 games, mean reward 18.84, eps 0.02, speed 346.62 f/s\n",
            "1571348 frames: done 882 games, mean reward 18.85, eps 0.02, speed 348.4 f/s\n",
            "1573236 frames: done 883 games, mean reward 18.83, eps 0.02, speed 351.58 f/s\n",
            "1574990 frames: done 884 games, mean reward 18.82, eps 0.02, speed 350.77 f/s\n",
            "1576957 frames: done 885 games, mean reward 18.81, eps 0.02, speed 352.12 f/s\n",
            "1578817 frames: done 886 games, mean reward 18.81, eps 0.02, speed 354.1 f/s\n",
            "1580732 frames: done 887 games, mean reward 18.79, eps 0.02, speed 354.87 f/s\n",
            "1582529 frames: done 888 games, mean reward 18.78, eps 0.02, speed 351.57 f/s\n",
            "1584277 frames: done 889 games, mean reward 18.78, eps 0.02, speed 348.39 f/s\n",
            "1586099 frames: done 890 games, mean reward 18.76, eps 0.02, speed 354.62 f/s\n",
            "1587765 frames: done 891 games, mean reward 18.76, eps 0.02, speed 356.07 f/s\n",
            "1589431 frames: done 892 games, mean reward 18.79, eps 0.02, speed 354.01 f/s\n",
            "1591274 frames: done 893 games, mean reward 18.77, eps 0.02, speed 352.29 f/s\n",
            "1592940 frames: done 894 games, mean reward 18.81, eps 0.02, speed 354.47 f/s\n",
            "1594772 frames: done 895 games, mean reward 18.8, eps 0.02, speed 353.61 f/s\n",
            "1596595 frames: done 896 games, mean reward 18.76, eps 0.02, speed 348.91 f/s\n",
            "1598261 frames: done 897 games, mean reward 18.79, eps 0.02, speed 354.05 f/s\n",
            "1599973 frames: done 898 games, mean reward 18.8, eps 0.02, speed 351.45 f/s\n",
            "1601657 frames: done 899 games, mean reward 18.82, eps 0.02, speed 350.83 f/s\n",
            "1603624 frames: done 900 games, mean reward 18.8, eps 0.02, speed 355.49 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-900.dat: 100%|██████████| 100/100 [00:00<00:00, 107.71it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.85 -> 18.8, model saved\n",
            "1605658 frames: done 901 games, mean reward 18.79, eps 0.02, speed 291.48 f/s\n",
            "1607570 frames: done 902 games, mean reward 18.77, eps 0.02, speed 353.5 f/s\n",
            "1609331 frames: done 903 games, mean reward 18.76, eps 0.02, speed 352.87 f/s\n",
            "1611404 frames: done 904 games, mean reward 18.72, eps 0.02, speed 349.32 f/s\n",
            "1613262 frames: done 905 games, mean reward 18.68, eps 0.02, speed 349.12 f/s\n",
            "1614947 frames: done 906 games, mean reward 18.68, eps 0.02, speed 346.31 f/s\n",
            "1616950 frames: done 907 games, mean reward 18.69, eps 0.02, speed 349.9 f/s\n",
            "1618694 frames: done 908 games, mean reward 18.69, eps 0.02, speed 351.21 f/s\n",
            "1620477 frames: done 909 games, mean reward 18.68, eps 0.02, speed 353.35 f/s\n",
            "1622231 frames: done 910 games, mean reward 18.69, eps 0.02, speed 348.74 f/s\n",
            "1623956 frames: done 911 games, mean reward 18.68, eps 0.02, speed 353.5 f/s\n",
            "1625622 frames: done 912 games, mean reward 18.69, eps 0.02, speed 350.53 f/s\n",
            "1627579 frames: done 913 games, mean reward 18.65, eps 0.02, speed 352.73 f/s\n",
            "1629275 frames: done 914 games, mean reward 18.65, eps 0.02, speed 346.1 f/s\n",
            "1631116 frames: done 915 games, mean reward 18.64, eps 0.02, speed 355.03 f/s\n",
            "1632930 frames: done 916 games, mean reward 18.63, eps 0.02, speed 355.38 f/s\n",
            "1634596 frames: done 917 games, mean reward 18.64, eps 0.02, speed 348.5 f/s\n",
            "1636390 frames: done 918 games, mean reward 18.63, eps 0.02, speed 348.66 f/s\n",
            "1638288 frames: done 919 games, mean reward 18.62, eps 0.02, speed 349.13 f/s\n",
            "1640131 frames: done 920 games, mean reward 18.62, eps 0.02, speed 349.33 f/s\n",
            "1641921 frames: done 921 games, mean reward 18.62, eps 0.02, speed 351.12 f/s\n",
            "1643705 frames: done 922 games, mean reward 18.63, eps 0.02, speed 351.34 f/s\n",
            "1645400 frames: done 923 games, mean reward 18.64, eps 0.02, speed 355.87 f/s\n",
            "1647133 frames: done 924 games, mean reward 18.65, eps 0.02, speed 354.75 f/s\n",
            "1648910 frames: done 925 games, mean reward 18.66, eps 0.02, speed 354.13 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-925.dat: 100%|██████████| 100/100 [00:00<00:00, 105.39it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.8 -> 18.66, model saved\n",
            "1650639 frames: done 926 games, mean reward 18.66, eps 0.02, speed 280.13 f/s\n",
            "1652352 frames: done 927 games, mean reward 18.73, eps 0.02, speed 349.07 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-927.dat: 100%|██████████| 100/100 [00:00<00:00, 114.67it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.66 -> 18.73, model saved\n",
            "1654153 frames: done 928 games, mean reward 18.73, eps 0.02, speed 290.56 f/s\n",
            "1656072 frames: done 929 games, mean reward 18.71, eps 0.02, speed 357.55 f/s\n",
            "1657789 frames: done 930 games, mean reward 18.72, eps 0.02, speed 349.94 f/s\n",
            "1659519 frames: done 931 games, mean reward 18.79, eps 0.02, speed 347.79 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-931.dat: 100%|██████████| 100/100 [00:00<00:00, 116.89it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.73 -> 18.79, model saved\n",
            "1661185 frames: done 932 games, mean reward 18.8, eps 0.02, speed 285.89 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-932.dat: 100%|██████████| 100/100 [00:00<00:00, 109.09it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.79 -> 18.8, model saved\n",
            "1663142 frames: done 933 games, mean reward 18.78, eps 0.02, speed 292.3 f/s\n",
            "1664838 frames: done 934 games, mean reward 18.78, eps 0.02, speed 353.31 f/s\n",
            "1666695 frames: done 935 games, mean reward 18.75, eps 0.02, speed 351.91 f/s\n",
            "1668426 frames: done 936 games, mean reward 18.74, eps 0.02, speed 352.46 f/s\n",
            "1670399 frames: done 937 games, mean reward 18.7, eps 0.02, speed 350.31 f/s\n",
            "1672065 frames: done 938 games, mean reward 18.71, eps 0.02, speed 350.71 f/s\n",
            "1673731 frames: done 939 games, mean reward 18.71, eps 0.02, speed 349.54 f/s\n",
            "1675397 frames: done 940 games, mean reward 18.72, eps 0.02, speed 350.8 f/s\n",
            "1677336 frames: done 941 games, mean reward 18.72, eps 0.02, speed 352.77 f/s\n",
            "1679002 frames: done 942 games, mean reward 18.72, eps 0.02, speed 349.45 f/s\n",
            "1680721 frames: done 943 games, mean reward 18.73, eps 0.02, speed 353.86 f/s\n",
            "1682620 frames: done 944 games, mean reward 18.74, eps 0.02, speed 351.86 f/s\n",
            "1684472 frames: done 945 games, mean reward 18.74, eps 0.02, speed 351.28 f/s\n",
            "1686138 frames: done 946 games, mean reward 18.76, eps 0.02, speed 354.74 f/s\n",
            "1687863 frames: done 947 games, mean reward 18.77, eps 0.02, speed 354.41 f/s\n",
            "1689721 frames: done 948 games, mean reward 18.77, eps 0.02, speed 351.25 f/s\n",
            "1691482 frames: done 949 games, mean reward 18.76, eps 0.02, speed 350.4 f/s\n",
            "1693219 frames: done 950 games, mean reward 18.75, eps 0.02, speed 353.39 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-950.dat: 100%|██████████| 100/100 [00:00<00:00, 112.26it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.8 -> 18.75, model saved\n",
            "1694963 frames: done 951 games, mean reward 18.76, eps 0.02, speed 286.38 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-951.dat: 100%|██████████| 100/100 [00:00<00:00, 120.59it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.75 -> 18.76, model saved\n",
            "1696818 frames: done 952 games, mean reward 18.73, eps 0.02, speed 282.68 f/s\n",
            "1698517 frames: done 953 games, mean reward 18.74, eps 0.02, speed 352.35 f/s\n",
            "1700510 frames: done 954 games, mean reward 18.73, eps 0.02, speed 351.32 f/s\n",
            "1702452 frames: done 955 games, mean reward 18.71, eps 0.02, speed 352.52 f/s\n",
            "1704118 frames: done 956 games, mean reward 18.71, eps 0.02, speed 348.25 f/s\n",
            "1706161 frames: done 957 games, mean reward 18.71, eps 0.02, speed 351.42 f/s\n",
            "1707827 frames: done 958 games, mean reward 18.77, eps 0.02, speed 350.71 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-958.dat: 100%|██████████| 100/100 [00:00<00:00, 106.23it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.76 -> 18.77, model saved\n",
            "1709715 frames: done 959 games, mean reward 18.74, eps 0.02, speed 278.35 f/s\n",
            "1711668 frames: done 960 games, mean reward 18.72, eps 0.02, speed 350.57 f/s\n",
            "1713334 frames: done 961 games, mean reward 18.73, eps 0.02, speed 354.72 f/s\n",
            "1715128 frames: done 962 games, mean reward 18.73, eps 0.02, speed 351.62 f/s\n",
            "1716794 frames: done 963 games, mean reward 18.77, eps 0.02, speed 351.49 f/s\n",
            "1718677 frames: done 964 games, mean reward 18.74, eps 0.02, speed 350.18 f/s\n",
            "1720691 frames: done 965 games, mean reward 18.75, eps 0.02, speed 349.5 f/s\n",
            "1722511 frames: done 966 games, mean reward 18.77, eps 0.02, speed 348.26 f/s\n",
            "1724626 frames: done 967 games, mean reward 18.74, eps 0.02, speed 354.96 f/s\n",
            "1726491 frames: done 968 games, mean reward 18.74, eps 0.02, speed 353.24 f/s\n",
            "1728835 frames: done 969 games, mean reward 18.69, eps 0.02, speed 349.64 f/s\n",
            "1730501 frames: done 970 games, mean reward 18.69, eps 0.02, speed 347.47 f/s\n",
            "1732265 frames: done 971 games, mean reward 18.67, eps 0.02, speed 350.33 f/s\n",
            "1734079 frames: done 972 games, mean reward 18.65, eps 0.02, speed 351.92 f/s\n",
            "1735745 frames: done 973 games, mean reward 18.67, eps 0.02, speed 349.5 f/s\n",
            "1737876 frames: done 974 games, mean reward 18.63, eps 0.02, speed 354.47 f/s\n",
            "1739542 frames: done 975 games, mean reward 18.63, eps 0.02, speed 352.45 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-975.dat: 100%|██████████| 100/100 [00:01<00:00, 97.22it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.77 -> 18.63, model saved\n",
            "1741225 frames: done 976 games, mean reward 18.64, eps 0.02, speed 273.17 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-976.dat: 100%|██████████| 100/100 [00:00<00:00, 101.49it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.63 -> 18.64, model saved\n",
            "1743208 frames: done 977 games, mean reward 18.63, eps 0.02, speed 289.42 f/s\n",
            "1745164 frames: done 978 games, mean reward 18.59, eps 0.02, speed 351.7 f/s\n",
            "1746881 frames: done 979 games, mean reward 18.61, eps 0.02, speed 345.62 f/s\n",
            "1748547 frames: done 980 games, mean reward 18.62, eps 0.02, speed 351.03 f/s\n",
            "1750529 frames: done 981 games, mean reward 18.6, eps 0.02, speed 353.09 f/s\n",
            "1752195 frames: done 982 games, mean reward 18.6, eps 0.02, speed 353.03 f/s\n",
            "1753861 frames: done 983 games, mean reward 18.62, eps 0.02, speed 349.17 f/s\n",
            "1755527 frames: done 984 games, mean reward 18.64, eps 0.02, speed 351.5 f/s\n",
            "1757381 frames: done 985 games, mean reward 18.63, eps 0.02, speed 351.11 f/s\n",
            "1759111 frames: done 986 games, mean reward 18.64, eps 0.02, speed 352.17 f/s\n",
            "1760794 frames: done 987 games, mean reward 18.66, eps 0.02, speed 349.27 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-987.dat: 100%|██████████| 100/100 [00:00<00:00, 112.83it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.64 -> 18.66, model saved\n",
            "1762613 frames: done 988 games, mean reward 18.66, eps 0.02, speed 289.22 f/s\n",
            "1764266 frames: done 989 games, mean reward 18.68, eps 0.02, speed 349.24 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-989.dat: 100%|██████████| 100/100 [00:00<00:00, 112.68it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.66 -> 18.68, model saved\n",
            "1766032 frames: done 990 games, mean reward 18.69, eps 0.02, speed 289.0 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-990.dat: 100%|██████████| 100/100 [00:00<00:00, 110.69it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.68 -> 18.69, model saved\n",
            "1767871 frames: done 991 games, mean reward 18.68, eps 0.02, speed 289.94 f/s\n",
            "1769766 frames: done 992 games, mean reward 18.67, eps 0.02, speed 350.0 f/s\n",
            "1771512 frames: done 993 games, mean reward 18.68, eps 0.02, speed 350.36 f/s\n",
            "1773620 frames: done 994 games, mean reward 18.66, eps 0.02, speed 351.07 f/s\n",
            "1775286 frames: done 995 games, mean reward 18.68, eps 0.02, speed 348.19 f/s\n",
            "1776981 frames: done 996 games, mean reward 18.71, eps 0.02, speed 352.14 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-996.dat: 100%|██████████| 100/100 [00:00<00:00, 117.69it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.69 -> 18.71, model saved\n",
            "1778647 frames: done 997 games, mean reward 18.71, eps 0.02, speed 285.35 f/s\n",
            "1780366 frames: done 998 games, mean reward 18.7, eps 0.02, speed 350.04 f/s\n",
            "1782032 frames: done 999 games, mean reward 18.7, eps 0.02, speed 351.72 f/s\n",
            "1783890 frames: done 1000 games, mean reward 18.7, eps 0.02, speed 356.41 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1000.dat: 100%|██████████| 100/100 [00:00<00:00, 102.26it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.71 -> 18.7, model saved\n",
            "1785878 frames: done 1001 games, mean reward 18.68, eps 0.02, speed 288.11 f/s\n",
            "1787603 frames: done 1002 games, mean reward 18.69, eps 0.02, speed 351.62 f/s\n",
            "1789409 frames: done 1003 games, mean reward 18.69, eps 0.02, speed 352.51 f/s\n",
            "1791230 frames: done 1004 games, mean reward 18.73, eps 0.02, speed 352.07 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1004.dat: 100%|██████████| 100/100 [00:00<00:00, 101.83it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.7 -> 18.73, model saved\n",
            "1792912 frames: done 1005 games, mean reward 18.77, eps 0.02, speed 271.93 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1005.dat: 100%|██████████| 100/100 [00:00<00:00, 102.59it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.73 -> 18.77, model saved\n",
            "1794629 frames: done 1006 games, mean reward 18.76, eps 0.02, speed 283.66 f/s\n",
            "1796509 frames: done 1007 games, mean reward 18.76, eps 0.02, speed 352.53 f/s\n",
            "1798464 frames: done 1008 games, mean reward 18.74, eps 0.02, speed 355.81 f/s\n",
            "1800209 frames: done 1009 games, mean reward 18.75, eps 0.02, speed 353.79 f/s\n",
            "1801875 frames: done 1010 games, mean reward 18.77, eps 0.02, speed 347.59 f/s\n",
            "1803656 frames: done 1011 games, mean reward 18.76, eps 0.02, speed 348.58 f/s\n",
            "1805596 frames: done 1012 games, mean reward 18.72, eps 0.02, speed 348.03 f/s\n",
            "1807356 frames: done 1013 games, mean reward 18.76, eps 0.02, speed 350.32 f/s\n",
            "1809357 frames: done 1014 games, mean reward 18.72, eps 0.02, speed 350.33 f/s\n",
            "1811174 frames: done 1015 games, mean reward 18.7, eps 0.02, speed 353.26 f/s\n",
            "1812869 frames: done 1016 games, mean reward 18.73, eps 0.02, speed 350.13 f/s\n",
            "1814839 frames: done 1017 games, mean reward 18.69, eps 0.02, speed 354.82 f/s\n",
            "1816505 frames: done 1018 games, mean reward 18.7, eps 0.02, speed 349.53 f/s\n",
            "1818335 frames: done 1019 games, mean reward 18.69, eps 0.02, speed 352.22 f/s\n",
            "1820173 frames: done 1020 games, mean reward 18.69, eps 0.02, speed 353.62 f/s\n",
            "1821951 frames: done 1021 games, mean reward 18.69, eps 0.02, speed 348.51 f/s\n",
            "1823753 frames: done 1022 games, mean reward 18.69, eps 0.02, speed 351.85 f/s\n",
            "1825419 frames: done 1023 games, mean reward 18.69, eps 0.02, speed 348.75 f/s\n",
            "1827655 frames: done 1024 games, mean reward 18.64, eps 0.02, speed 352.31 f/s\n",
            "1829356 frames: done 1025 games, mean reward 18.64, eps 0.02, speed 355.31 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1025.dat: 100%|██████████| 100/100 [00:00<00:00, 110.14it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.77 -> 18.64, model saved\n",
            "1831085 frames: done 1026 games, mean reward 18.64, eps 0.02, speed 287.84 f/s\n",
            "1833015 frames: done 1027 games, mean reward 18.6, eps 0.02, speed 353.58 f/s\n",
            "1835049 frames: done 1028 games, mean reward 18.56, eps 0.02, speed 352.14 f/s\n",
            "1836715 frames: done 1029 games, mean reward 18.59, eps 0.02, speed 348.78 f/s\n",
            "1838476 frames: done 1030 games, mean reward 18.59, eps 0.02, speed 350.61 f/s\n",
            "1840142 frames: done 1031 games, mean reward 18.59, eps 0.02, speed 349.47 f/s\n",
            "1841970 frames: done 1032 games, mean reward 18.59, eps 0.02, speed 354.62 f/s\n",
            "1843636 frames: done 1033 games, mean reward 18.61, eps 0.02, speed 354.71 f/s\n",
            "1846015 frames: done 1034 games, mean reward 18.55, eps 0.02, speed 352.21 f/s\n",
            "1847937 frames: done 1035 games, mean reward 18.54, eps 0.02, speed 349.01 f/s\n",
            "1849603 frames: done 1036 games, mean reward 18.55, eps 0.02, speed 352.89 f/s\n",
            "1851318 frames: done 1037 games, mean reward 18.59, eps 0.02, speed 356.78 f/s\n",
            "1853052 frames: done 1038 games, mean reward 18.58, eps 0.02, speed 354.38 f/s\n",
            "1854717 frames: done 1039 games, mean reward 18.58, eps 0.02, speed 349.17 f/s\n",
            "1856567 frames: done 1040 games, mean reward 18.57, eps 0.02, speed 351.67 f/s\n",
            "1858448 frames: done 1041 games, mean reward 18.59, eps 0.02, speed 347.58 f/s\n",
            "1860114 frames: done 1042 games, mean reward 18.59, eps 0.02, speed 350.76 f/s\n",
            "1862119 frames: done 1043 games, mean reward 18.54, eps 0.02, speed 349.81 f/s\n",
            "1863785 frames: done 1044 games, mean reward 18.58, eps 0.02, speed 346.42 f/s\n",
            "1865468 frames: done 1045 games, mean reward 18.59, eps 0.02, speed 352.15 f/s\n",
            "1867212 frames: done 1046 games, mean reward 18.59, eps 0.02, speed 355.48 f/s\n",
            "1869264 frames: done 1047 games, mean reward 18.54, eps 0.02, speed 352.66 f/s\n",
            "1871323 frames: done 1048 games, mean reward 18.53, eps 0.02, speed 349.37 f/s\n",
            "1873040 frames: done 1049 games, mean reward 18.53, eps 0.02, speed 353.3 f/s\n",
            "1874952 frames: done 1050 games, mean reward 18.51, eps 0.02, speed 355.7 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1050.dat: 100%|██████████| 100/100 [00:01<00:00, 89.78it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.64 -> 18.51, model saved\n",
            "1876820 frames: done 1051 games, mean reward 18.49, eps 0.02, speed 277.01 f/s\n",
            "1878486 frames: done 1052 games, mean reward 18.52, eps 0.02, speed 350.56 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1052.dat: 100%|██████████| 100/100 [00:00<00:00, 106.15it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.51 -> 18.52, model saved\n",
            "1880258 frames: done 1053 games, mean reward 18.5, eps 0.02, speed 284.35 f/s\n",
            "1882022 frames: done 1054 games, mean reward 18.5, eps 0.02, speed 352.43 f/s\n",
            "1883688 frames: done 1055 games, mean reward 18.53, eps 0.02, speed 353.43 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1055.dat: 100%|██████████| 100/100 [00:01<00:00, 99.69it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.52 -> 18.53, model saved\n",
            "1885452 frames: done 1056 games, mean reward 18.51, eps 0.02, speed 282.39 f/s\n",
            "1887275 frames: done 1057 games, mean reward 18.5, eps 0.02, speed 353.62 f/s\n",
            "1888941 frames: done 1058 games, mean reward 18.5, eps 0.02, speed 351.53 f/s\n",
            "1890624 frames: done 1059 games, mean reward 18.52, eps 0.02, speed 351.72 f/s\n",
            "1892468 frames: done 1060 games, mean reward 18.54, eps 0.02, speed 352.69 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1060.dat: 100%|██████████| 100/100 [00:00<00:00, 116.15it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.53 -> 18.54, model saved\n",
            "1894185 frames: done 1061 games, mean reward 18.53, eps 0.02, speed 281.67 f/s\n",
            "1895902 frames: done 1062 games, mean reward 18.54, eps 0.02, speed 353.71 f/s\n",
            "1897731 frames: done 1063 games, mean reward 18.53, eps 0.02, speed 353.5 f/s\n",
            "1899471 frames: done 1064 games, mean reward 18.55, eps 0.02, speed 355.12 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1064.dat: 100%|██████████| 100/100 [00:00<00:00, 110.68it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.54 -> 18.55, model saved\n",
            "1901205 frames: done 1065 games, mean reward 18.58, eps 0.02, speed 277.94 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1065.dat: 100%|██████████| 100/100 [00:00<00:00, 106.90it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.55 -> 18.58, model saved\n",
            "1902871 frames: done 1066 games, mean reward 18.59, eps 0.02, speed 275.7 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1066.dat: 100%|██████████| 100/100 [00:00<00:00, 109.59it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.58 -> 18.59, model saved\n",
            "1904536 frames: done 1067 games, mean reward 18.64, eps 0.02, speed 278.51 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1067.dat: 100%|██████████| 100/100 [00:01<00:00, 98.64it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.59 -> 18.64, model saved\n",
            "1906420 frames: done 1068 games, mean reward 18.63, eps 0.02, speed 280.43 f/s\n",
            "1908199 frames: done 1069 games, mean reward 18.67, eps 0.02, speed 345.83 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1069.dat: 100%|██████████| 100/100 [00:00<00:00, 109.79it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.64 -> 18.67, model saved\n",
            "1909935 frames: done 1070 games, mean reward 18.65, eps 0.02, speed 281.71 f/s\n",
            "1911601 frames: done 1071 games, mean reward 18.67, eps 0.02, speed 352.79 f/s\n",
            "1913411 frames: done 1072 games, mean reward 18.68, eps 0.02, speed 349.34 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1072.dat: 100%|██████████| 100/100 [00:00<00:00, 101.76it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.67 -> 18.68, model saved\n",
            "1915077 frames: done 1073 games, mean reward 18.68, eps 0.02, speed 271.92 f/s\n",
            "1917073 frames: done 1074 games, mean reward 18.69, eps 0.02, speed 353.02 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1074.dat: 100%|██████████| 100/100 [00:00<00:00, 110.11it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.68 -> 18.69, model saved\n",
            "1919325 frames: done 1075 games, mean reward 18.64, eps 0.02, speed 294.91 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1075.dat: 100%|██████████| 100/100 [00:00<00:00, 100.60it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.69 -> 18.64, model saved\n",
            "1921237 frames: done 1076 games, mean reward 18.62, eps 0.02, speed 285.24 f/s\n",
            "1922956 frames: done 1077 games, mean reward 18.64, eps 0.02, speed 352.31 f/s\n",
            "1924796 frames: done 1078 games, mean reward 18.66, eps 0.02, speed 348.85 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1078.dat: 100%|██████████| 100/100 [00:00<00:00, 102.93it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.64 -> 18.66, model saved\n",
            "1926693 frames: done 1079 games, mean reward 18.65, eps 0.02, speed 283.44 f/s\n",
            "1928422 frames: done 1080 games, mean reward 18.64, eps 0.02, speed 353.64 f/s\n",
            "1930088 frames: done 1081 games, mean reward 18.69, eps 0.02, speed 355.95 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1081.dat: 100%|██████████| 100/100 [00:00<00:00, 107.11it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.66 -> 18.69, model saved\n",
            "1931852 frames: done 1082 games, mean reward 18.67, eps 0.02, speed 277.63 f/s\n",
            "1933518 frames: done 1083 games, mean reward 18.67, eps 0.02, speed 354.27 f/s\n",
            "1935765 frames: done 1084 games, mean reward 18.63, eps 0.02, speed 351.49 f/s\n",
            "1937496 frames: done 1085 games, mean reward 18.64, eps 0.02, speed 350.49 f/s\n",
            "1939213 frames: done 1086 games, mean reward 18.64, eps 0.02, speed 349.72 f/s\n",
            "1940879 frames: done 1087 games, mean reward 18.64, eps 0.02, speed 350.22 f/s\n",
            "1942623 frames: done 1088 games, mean reward 18.65, eps 0.02, speed 349.79 f/s\n",
            "1944340 frames: done 1089 games, mean reward 18.64, eps 0.02, speed 351.79 f/s\n",
            "1946006 frames: done 1090 games, mean reward 18.65, eps 0.02, speed 355.12 f/s\n",
            "1947672 frames: done 1091 games, mean reward 18.66, eps 0.02, speed 352.59 f/s\n",
            "1949457 frames: done 1092 games, mean reward 18.67, eps 0.02, speed 353.3 f/s\n",
            "1951123 frames: done 1093 games, mean reward 18.68, eps 0.02, speed 353.75 f/s\n",
            "1952789 frames: done 1094 games, mean reward 18.7, eps 0.02, speed 352.15 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1094.dat: 100%|██████████| 100/100 [00:00<00:00, 112.05it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.69 -> 18.7, model saved\n",
            "1954455 frames: done 1095 games, mean reward 18.7, eps 0.02, speed 279.69 f/s\n",
            "1956583 frames: done 1096 games, mean reward 18.64, eps 0.02, speed 351.72 f/s\n",
            "1958300 frames: done 1097 games, mean reward 18.63, eps 0.02, speed 348.9 f/s\n",
            "1959966 frames: done 1098 games, mean reward 18.64, eps 0.02, speed 347.22 f/s\n",
            "1961632 frames: done 1099 games, mean reward 18.64, eps 0.02, speed 354.49 f/s\n",
            "1963456 frames: done 1100 games, mean reward 18.64, eps 0.02, speed 353.43 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1100.dat: 100%|██████████| 100/100 [00:00<00:00, 106.31it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.7 -> 18.64, model saved\n",
            "1965171 frames: done 1101 games, mean reward 18.68, eps 0.02, speed 277.81 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1101.dat: 100%|██████████| 100/100 [00:00<00:00, 109.47it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.64 -> 18.68, model saved\n",
            "1966837 frames: done 1102 games, mean reward 18.69, eps 0.02, speed 278.02 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1102.dat: 100%|██████████| 100/100 [00:00<00:00, 109.42it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.68 -> 18.69, model saved\n",
            "1968503 frames: done 1103 games, mean reward 18.7, eps 0.02, speed 279.6 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1103.dat: 100%|██████████| 100/100 [00:00<00:00, 101.21it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.69 -> 18.7, model saved\n",
            "1970220 frames: done 1104 games, mean reward 18.69, eps 0.02, speed 276.58 f/s\n",
            "1972016 frames: done 1105 games, mean reward 18.67, eps 0.02, speed 352.11 f/s\n",
            "1973883 frames: done 1106 games, mean reward 18.67, eps 0.02, speed 355.42 f/s\n",
            "1975760 frames: done 1107 games, mean reward 18.66, eps 0.02, speed 358.67 f/s\n",
            "1977548 frames: done 1108 games, mean reward 18.67, eps 0.02, speed 354.44 f/s\n",
            "1979472 frames: done 1109 games, mean reward 18.66, eps 0.02, speed 352.62 f/s\n",
            "1981191 frames: done 1110 games, mean reward 18.65, eps 0.02, speed 350.37 f/s\n",
            "1983061 frames: done 1111 games, mean reward 18.64, eps 0.02, speed 349.79 f/s\n",
            "1985112 frames: done 1112 games, mean reward 18.64, eps 0.02, speed 350.92 f/s\n",
            "1986778 frames: done 1113 games, mean reward 18.64, eps 0.02, speed 351.45 f/s\n",
            "1988444 frames: done 1114 games, mean reward 18.68, eps 0.02, speed 348.77 f/s\n",
            "1990423 frames: done 1115 games, mean reward 18.69, eps 0.02, speed 352.99 f/s\n",
            "1992238 frames: done 1116 games, mean reward 18.65, eps 0.02, speed 354.91 f/s\n",
            "1994264 frames: done 1117 games, mean reward 18.65, eps 0.02, speed 356.01 f/s\n",
            "1996106 frames: done 1118 games, mean reward 18.62, eps 0.02, speed 355.67 f/s\n",
            "1997852 frames: done 1119 games, mean reward 18.64, eps 0.02, speed 352.22 f/s\n",
            "1999569 frames: done 1120 games, mean reward 18.65, eps 0.02, speed 352.2 f/s\n",
            "2001234 frames: done 1121 games, mean reward 18.66, eps 0.02, speed 356.49 f/s\n",
            "2003028 frames: done 1122 games, mean reward 18.64, eps 0.02, speed 353.13 f/s\n",
            "2004813 frames: done 1123 games, mean reward 18.64, eps 0.02, speed 351.27 f/s\n",
            "2006479 frames: done 1124 games, mean reward 18.7, eps 0.02, speed 349.48 f/s\n",
            "2008175 frames: done 1125 games, mean reward 18.71, eps 0.02, speed 353.71 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1125.dat: 100%|██████████| 100/100 [00:00<00:00, 115.72it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.7 -> 18.71, model saved\n",
            "2009989 frames: done 1126 games, mean reward 18.7, eps 0.02, speed 289.64 f/s\n",
            "2011707 frames: done 1127 games, mean reward 18.73, eps 0.02, speed 356.46 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1127.dat: 100%|██████████| 100/100 [00:00<00:00, 106.30it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.71 -> 18.73, model saved\n",
            "2013526 frames: done 1128 games, mean reward 18.76, eps 0.02, speed 288.83 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1128.dat: 100%|██████████| 100/100 [00:00<00:00, 111.08it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.73 -> 18.76, model saved\n",
            "2015452 frames: done 1129 games, mean reward 18.75, eps 0.02, speed 289.09 f/s\n",
            "2017224 frames: done 1130 games, mean reward 18.74, eps 0.02, speed 351.31 f/s\n",
            "2019078 frames: done 1131 games, mean reward 18.73, eps 0.02, speed 351.85 f/s\n",
            "2020860 frames: done 1132 games, mean reward 18.71, eps 0.02, speed 350.79 f/s\n",
            "2022526 frames: done 1133 games, mean reward 18.71, eps 0.02, speed 349.22 f/s\n",
            "2024227 frames: done 1134 games, mean reward 18.76, eps 0.02, speed 350.24 f/s\n",
            "2026072 frames: done 1135 games, mean reward 18.76, eps 0.02, speed 349.99 f/s\n",
            "2027830 frames: done 1136 games, mean reward 18.76, eps 0.02, speed 353.68 f/s\n",
            "2029629 frames: done 1137 games, mean reward 18.75, eps 0.02, speed 350.14 f/s\n",
            "2031483 frames: done 1138 games, mean reward 18.75, eps 0.02, speed 348.27 f/s\n",
            "2033326 frames: done 1139 games, mean reward 18.73, eps 0.02, speed 355.11 f/s\n",
            "2035275 frames: done 1140 games, mean reward 18.7, eps 0.02, speed 352.82 f/s\n",
            "2036970 frames: done 1141 games, mean reward 18.72, eps 0.02, speed 353.04 f/s\n",
            "2038814 frames: done 1142 games, mean reward 18.71, eps 0.02, speed 348.67 f/s\n",
            "2040654 frames: done 1143 games, mean reward 18.75, eps 0.02, speed 352.95 f/s\n",
            "2042418 frames: done 1144 games, mean reward 18.73, eps 0.02, speed 354.68 f/s\n",
            "2044137 frames: done 1145 games, mean reward 18.72, eps 0.02, speed 355.21 f/s\n",
            "2045857 frames: done 1146 games, mean reward 18.72, eps 0.02, speed 347.95 f/s\n",
            "2047738 frames: done 1147 games, mean reward 18.73, eps 0.02, speed 349.91 f/s\n",
            "2049481 frames: done 1148 games, mean reward 18.76, eps 0.02, speed 351.28 f/s\n",
            "2051462 frames: done 1149 games, mean reward 18.73, eps 0.02, speed 354.99 f/s\n",
            "2053425 frames: done 1150 games, mean reward 18.73, eps 0.02, speed 353.67 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1150.dat: 100%|██████████| 100/100 [00:00<00:00, 105.75it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.76 -> 18.73, model saved\n",
            "2055340 frames: done 1151 games, mean reward 18.72, eps 0.02, speed 286.47 f/s\n",
            "2057006 frames: done 1152 games, mean reward 18.72, eps 0.02, speed 350.54 f/s\n",
            "2058803 frames: done 1153 games, mean reward 18.74, eps 0.02, speed 352.17 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1153.dat: 100%|██████████| 100/100 [00:00<00:00, 100.26it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.73 -> 18.74, model saved\n",
            "2060468 frames: done 1154 games, mean reward 18.76, eps 0.02, speed 280.24 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1154.dat: 100%|██████████| 100/100 [00:01<00:00, 94.15it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.74 -> 18.76, model saved\n",
            "2062322 frames: done 1155 games, mean reward 18.74, eps 0.02, speed 281.22 f/s\n",
            "2064194 frames: done 1156 games, mean reward 18.74, eps 0.02, speed 348.77 f/s\n",
            "2065860 frames: done 1157 games, mean reward 18.77, eps 0.02, speed 354.09 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1157.dat: 100%|██████████| 100/100 [00:00<00:00, 115.96it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.76 -> 18.77, model saved\n",
            "2067673 frames: done 1158 games, mean reward 18.75, eps 0.02, speed 279.78 f/s\n",
            "2069398 frames: done 1159 games, mean reward 18.75, eps 0.02, speed 351.77 f/s\n",
            "2071064 frames: done 1160 games, mean reward 18.77, eps 0.02, speed 349.61 f/s\n",
            "2072900 frames: done 1161 games, mean reward 18.77, eps 0.02, speed 349.67 f/s\n",
            "2074583 frames: done 1162 games, mean reward 18.78, eps 0.02, speed 351.61 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1162.dat: 100%|██████████| 100/100 [00:00<00:00, 116.91it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.77 -> 18.78, model saved\n",
            "2076532 frames: done 1163 games, mean reward 18.75, eps 0.02, speed 288.23 f/s\n",
            "2078628 frames: done 1164 games, mean reward 18.73, eps 0.02, speed 350.18 f/s\n",
            "2080363 frames: done 1165 games, mean reward 18.72, eps 0.02, speed 348.49 f/s\n",
            "2082256 frames: done 1166 games, mean reward 18.69, eps 0.02, speed 349.71 f/s\n",
            "2084051 frames: done 1167 games, mean reward 18.68, eps 0.02, speed 350.2 f/s\n",
            "2085931 frames: done 1168 games, mean reward 18.69, eps 0.02, speed 353.18 f/s\n",
            "2087597 frames: done 1169 games, mean reward 18.71, eps 0.02, speed 356.61 f/s\n",
            "2089550 frames: done 1170 games, mean reward 18.69, eps 0.02, speed 347.58 f/s\n",
            "2091436 frames: done 1171 games, mean reward 18.67, eps 0.02, speed 353.04 f/s\n",
            "2093337 frames: done 1172 games, mean reward 18.65, eps 0.02, speed 352.79 f/s\n",
            "2095344 frames: done 1173 games, mean reward 18.63, eps 0.02, speed 351.98 f/s\n",
            "2097010 frames: done 1174 games, mean reward 18.66, eps 0.02, speed 346.94 f/s\n",
            "2098676 frames: done 1175 games, mean reward 18.71, eps 0.02, speed 353.03 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1175.dat: 100%|██████████| 100/100 [00:00<00:00, 110.85it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.78 -> 18.71, model saved\n",
            "2100739 frames: done 1176 games, mean reward 18.7, eps 0.02, speed 294.0 f/s\n",
            "2102405 frames: done 1177 games, mean reward 18.71, eps 0.02, speed 353.2 f/s\n",
            "2104227 frames: done 1178 games, mean reward 18.72, eps 0.02, speed 349.27 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1178.dat: 100%|██████████| 100/100 [00:00<00:00, 105.03it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.71 -> 18.72, model saved\n",
            "2105893 frames: done 1179 games, mean reward 18.74, eps 0.02, speed 283.68 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1179.dat: 100%|██████████| 100/100 [00:00<00:00, 111.50it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.72 -> 18.74, model saved\n",
            "2107559 frames: done 1180 games, mean reward 18.75, eps 0.02, speed 287.47 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1180.dat: 100%|██████████| 100/100 [00:00<00:00, 118.94it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.74 -> 18.75, model saved\n",
            "2109225 frames: done 1181 games, mean reward 18.75, eps 0.02, speed 284.54 f/s\n",
            "2110909 frames: done 1182 games, mean reward 18.77, eps 0.02, speed 351.64 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1182.dat: 100%|██████████| 100/100 [00:00<00:00, 104.84it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.75 -> 18.77, model saved\n",
            "2112612 frames: done 1183 games, mean reward 18.77, eps 0.02, speed 275.51 f/s\n",
            "2114742 frames: done 1184 games, mean reward 18.79, eps 0.02, speed 351.9 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1184.dat: 100%|██████████| 100/100 [00:00<00:00, 115.94it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.77 -> 18.79, model saved\n",
            "2116408 frames: done 1185 games, mean reward 18.8, eps 0.02, speed 281.07 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1185.dat: 100%|██████████| 100/100 [00:00<00:00, 113.07it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.79 -> 18.8, model saved\n",
            "2118104 frames: done 1186 games, mean reward 18.81, eps 0.02, speed 270.93 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1186.dat: 100%|██████████| 100/100 [00:00<00:00, 110.47it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.8 -> 18.81, model saved\n",
            "2119770 frames: done 1187 games, mean reward 18.81, eps 0.02, speed 270.64 f/s\n",
            "2121453 frames: done 1188 games, mean reward 18.81, eps 0.02, speed 349.74 f/s\n",
            "2123119 frames: done 1189 games, mean reward 18.81, eps 0.02, speed 353.08 f/s\n",
            "2124848 frames: done 1190 games, mean reward 18.8, eps 0.02, speed 356.65 f/s\n",
            "2126787 frames: done 1191 games, mean reward 18.77, eps 0.02, speed 356.38 f/s\n",
            "2128453 frames: done 1192 games, mean reward 18.77, eps 0.02, speed 352.12 f/s\n",
            "2130119 frames: done 1193 games, mean reward 18.77, eps 0.02, speed 357.83 f/s\n",
            "2131803 frames: done 1194 games, mean reward 18.77, eps 0.02, speed 350.48 f/s\n",
            "2133469 frames: done 1195 games, mean reward 18.77, eps 0.02, speed 350.64 f/s\n",
            "2135292 frames: done 1196 games, mean reward 18.83, eps 0.02, speed 355.5 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1196.dat: 100%|██████████| 100/100 [00:00<00:00, 104.84it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.81 -> 18.83, model saved\n",
            "2137167 frames: done 1197 games, mean reward 18.82, eps 0.02, speed 286.41 f/s\n",
            "2138900 frames: done 1198 games, mean reward 18.81, eps 0.02, speed 348.62 f/s\n",
            "2140583 frames: done 1199 games, mean reward 18.81, eps 0.02, speed 354.65 f/s\n",
            "2142679 frames: done 1200 games, mean reward 18.81, eps 0.02, speed 354.79 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1200.dat: 100%|██████████| 100/100 [00:00<00:00, 110.51it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.83 -> 18.81, model saved\n",
            "2144345 frames: done 1201 games, mean reward 18.81, eps 0.02, speed 284.27 f/s\n",
            "2146334 frames: done 1202 games, mean reward 18.78, eps 0.02, speed 358.46 f/s\n",
            "2148415 frames: done 1203 games, mean reward 18.73, eps 0.02, speed 356.74 f/s\n",
            "2150081 frames: done 1204 games, mean reward 18.74, eps 0.02, speed 354.56 f/s\n",
            "2151800 frames: done 1205 games, mean reward 18.75, eps 0.02, speed 354.1 f/s\n",
            "2153770 frames: done 1206 games, mean reward 18.74, eps 0.02, speed 355.94 f/s\n",
            "2155565 frames: done 1207 games, mean reward 18.74, eps 0.02, speed 356.5 f/s\n",
            "2157231 frames: done 1208 games, mean reward 18.75, eps 0.02, speed 355.42 f/s\n",
            "2158897 frames: done 1209 games, mean reward 18.76, eps 0.02, speed 357.83 f/s\n",
            "2160563 frames: done 1210 games, mean reward 18.77, eps 0.02, speed 356.04 f/s\n",
            "2162229 frames: done 1211 games, mean reward 18.8, eps 0.02, speed 351.81 f/s\n",
            "2164078 frames: done 1212 games, mean reward 18.81, eps 0.02, speed 355.36 f/s\n",
            "2165744 frames: done 1213 games, mean reward 18.81, eps 0.02, speed 354.05 f/s\n",
            "2167427 frames: done 1214 games, mean reward 18.81, eps 0.02, speed 354.21 f/s\n",
            "2169093 frames: done 1215 games, mean reward 18.83, eps 0.02, speed 349.67 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1215.dat: 100%|██████████| 100/100 [00:01<00:00, 99.33it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.81 -> 18.83, model saved\n",
            "2170759 frames: done 1216 games, mean reward 18.86, eps 0.02, speed 278.67 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1216.dat: 100%|██████████| 100/100 [00:01<00:00, 99.47it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.83 -> 18.86, model saved\n",
            "2172489 frames: done 1217 games, mean reward 18.89, eps 0.02, speed 280.34 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1217.dat: 100%|██████████| 100/100 [00:00<00:00, 111.02it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.86 -> 18.89, model saved\n",
            "2174184 frames: done 1218 games, mean reward 18.92, eps 0.02, speed 281.6 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1218.dat: 100%|██████████| 100/100 [00:00<00:00, 103.31it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.89 -> 18.92, model saved\n",
            "2175902 frames: done 1219 games, mean reward 18.91, eps 0.02, speed 278.51 f/s\n",
            "2177930 frames: done 1220 games, mean reward 18.87, eps 0.02, speed 347.15 f/s\n",
            "2179689 frames: done 1221 games, mean reward 18.86, eps 0.02, speed 347.47 f/s\n",
            "2181401 frames: done 1222 games, mean reward 18.88, eps 0.02, speed 350.37 f/s\n",
            "2183071 frames: done 1223 games, mean reward 18.89, eps 0.02, speed 351.15 f/s\n",
            "2184806 frames: done 1224 games, mean reward 18.88, eps 0.02, speed 347.64 f/s\n",
            "2186618 frames: done 1225 games, mean reward 18.86, eps 0.02, speed 353.13 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1225.dat: 100%|██████████| 100/100 [00:00<00:00, 109.83it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.92 -> 18.86, model saved\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\rUploading file PongNoFrameskip-v4-1226.dat:   0%|          | 0/100 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2188335 frames: done 1226 games, mean reward 18.87, eps 0.02, speed 275.85 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\rUploading file PongNoFrameskip-v4-1226.dat: 100%|██████████| 100/100 [00:00<00:00, 120.14it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.86 -> 18.87, model saved\n",
            "2190332 frames: done 1227 games, mean reward 18.85, eps 0.02, speed 299.27 f/s\n",
            "2191997 frames: done 1228 games, mean reward 18.86, eps 0.02, speed 352.49 f/s\n",
            "2193693 frames: done 1229 games, mean reward 18.87, eps 0.02, speed 351.85 f/s\n",
            "2195595 frames: done 1230 games, mean reward 18.84, eps 0.02, speed 352.96 f/s\n",
            "2197261 frames: done 1231 games, mean reward 18.85, eps 0.02, speed 353.41 f/s\n",
            "2198994 frames: done 1232 games, mean reward 18.86, eps 0.02, speed 350.96 f/s\n",
            "2200711 frames: done 1233 games, mean reward 18.85, eps 0.02, speed 352.95 f/s\n",
            "2202739 frames: done 1234 games, mean reward 18.81, eps 0.02, speed 346.69 f/s\n",
            "2204473 frames: done 1235 games, mean reward 18.83, eps 0.02, speed 351.83 f/s\n",
            "2206139 frames: done 1236 games, mean reward 18.83, eps 0.02, speed 349.72 f/s\n",
            "2207804 frames: done 1237 games, mean reward 18.84, eps 0.02, speed 354.13 f/s\n",
            "2209521 frames: done 1238 games, mean reward 18.84, eps 0.02, speed 348.13 f/s\n",
            "2211389 frames: done 1239 games, mean reward 18.85, eps 0.02, speed 354.69 f/s\n",
            "2213055 frames: done 1240 games, mean reward 18.89, eps 0.02, speed 354.53 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1240.dat: 100%|██████████| 100/100 [00:00<00:00, 107.08it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.87 -> 18.89, model saved\n",
            "2214772 frames: done 1241 games, mean reward 18.87, eps 0.02, speed 281.5 f/s\n",
            "2216907 frames: done 1242 games, mean reward 18.84, eps 0.02, speed 348.5 f/s\n",
            "2218573 frames: done 1243 games, mean reward 18.86, eps 0.02, speed 345.51 f/s\n",
            "2220390 frames: done 1244 games, mean reward 18.85, eps 0.02, speed 350.0 f/s\n",
            "2222119 frames: done 1245 games, mean reward 18.85, eps 0.02, speed 351.53 f/s\n",
            "2223836 frames: done 1246 games, mean reward 18.84, eps 0.02, speed 349.9 f/s\n",
            "2225502 frames: done 1247 games, mean reward 18.88, eps 0.02, speed 346.16 f/s\n",
            "2227167 frames: done 1248 games, mean reward 18.88, eps 0.02, speed 350.63 f/s\n",
            "2229307 frames: done 1249 games, mean reward 18.87, eps 0.02, speed 350.35 f/s\n",
            "2231127 frames: done 1250 games, mean reward 18.89, eps 0.02, speed 347.25 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1250.dat: 100%|██████████| 100/100 [00:00<00:00, 101.16it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.89 -> 18.89, model saved\n",
            "2233295 frames: done 1251 games, mean reward 18.88, eps 0.02, speed 295.34 f/s\n",
            "2235132 frames: done 1252 games, mean reward 18.86, eps 0.02, speed 352.26 f/s\n",
            "2237049 frames: done 1253 games, mean reward 18.83, eps 0.02, speed 352.79 f/s\n",
            "2238745 frames: done 1254 games, mean reward 18.83, eps 0.02, speed 353.23 f/s\n",
            "2240398 frames: done 1255 games, mean reward 18.86, eps 0.02, speed 349.96 f/s\n",
            "2242083 frames: done 1256 games, mean reward 18.88, eps 0.02, speed 351.6 f/s\n",
            "2243828 frames: done 1257 games, mean reward 18.88, eps 0.02, speed 353.56 f/s\n",
            "2245589 frames: done 1258 games, mean reward 18.89, eps 0.02, speed 353.29 f/s\n",
            "2247255 frames: done 1259 games, mean reward 18.89, eps 0.02, speed 351.8 f/s\n",
            "2249239 frames: done 1260 games, mean reward 18.87, eps 0.02, speed 356.77 f/s\n",
            "2250905 frames: done 1261 games, mean reward 18.88, eps 0.02, speed 355.46 f/s\n",
            "2252704 frames: done 1262 games, mean reward 18.86, eps 0.02, speed 354.76 f/s\n",
            "2254496 frames: done 1263 games, mean reward 18.88, eps 0.02, speed 351.99 f/s\n",
            "2256213 frames: done 1264 games, mean reward 18.9, eps 0.02, speed 351.39 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1264.dat: 100%|██████████| 100/100 [00:00<00:00, 117.41it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.89 -> 18.9, model saved\n",
            "2257879 frames: done 1265 games, mean reward 18.91, eps 0.02, speed 285.37 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1265.dat: 100%|██████████| 100/100 [00:01<00:00, 65.43it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.9 -> 18.91, model saved\n",
            "2259861 frames: done 1266 games, mean reward 18.9, eps 0.02, speed 259.12 f/s\n",
            "2261527 frames: done 1267 games, mean reward 18.91, eps 0.02, speed 349.87 f/s\n",
            "2263246 frames: done 1268 games, mean reward 18.91, eps 0.02, speed 348.3 f/s\n",
            "2264971 frames: done 1269 games, mean reward 18.9, eps 0.02, speed 347.91 f/s\n",
            "2266718 frames: done 1270 games, mean reward 18.93, eps 0.02, speed 352.56 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1270.dat: 100%|██████████| 100/100 [00:00<00:00, 108.18it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.91 -> 18.93, model saved\n",
            "2268384 frames: done 1271 games, mean reward 18.95, eps 0.02, speed 282.95 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1271.dat: 100%|██████████| 100/100 [00:00<00:00, 109.88it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.93 -> 18.95, model saved\n",
            "2270067 frames: done 1272 games, mean reward 18.98, eps 0.02, speed 286.05 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1272.dat: 100%|██████████| 100/100 [00:00<00:00, 117.80it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.95 -> 18.98, model saved\n",
            "2271877 frames: done 1273 games, mean reward 18.99, eps 0.02, speed 288.57 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1273.dat: 100%|██████████| 100/100 [00:00<00:00, 101.18it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.98 -> 18.99, model saved\n",
            "2273902 frames: done 1274 games, mean reward 18.96, eps 0.02, speed 291.45 f/s\n",
            "2275567 frames: done 1275 games, mean reward 18.96, eps 0.02, speed 353.57 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1275.dat: 100%|██████████| 100/100 [00:00<00:00, 105.91it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.99 -> 18.96, model saved\n",
            "2277609 frames: done 1276 games, mean reward 18.94, eps 0.02, speed 292.86 f/s\n",
            "2279275 frames: done 1277 games, mean reward 18.94, eps 0.02, speed 352.44 f/s\n",
            "2280992 frames: done 1278 games, mean reward 18.94, eps 0.02, speed 352.58 f/s\n",
            "2282687 frames: done 1279 games, mean reward 18.94, eps 0.02, speed 352.56 f/s\n",
            "2284467 frames: done 1280 games, mean reward 18.92, eps 0.02, speed 348.75 f/s\n",
            "2286149 frames: done 1281 games, mean reward 18.92, eps 0.02, speed 352.58 f/s\n",
            "2287815 frames: done 1282 games, mean reward 18.92, eps 0.02, speed 356.53 f/s\n",
            "2289608 frames: done 1283 games, mean reward 18.92, eps 0.02, speed 356.72 f/s\n",
            "2291274 frames: done 1284 games, mean reward 18.94, eps 0.02, speed 352.22 f/s\n",
            "2293180 frames: done 1285 games, mean reward 18.9, eps 0.02, speed 354.87 f/s\n",
            "2295157 frames: done 1286 games, mean reward 18.87, eps 0.02, speed 355.15 f/s\n",
            "2296841 frames: done 1287 games, mean reward 18.87, eps 0.02, speed 356.87 f/s\n",
            "2298625 frames: done 1288 games, mean reward 18.85, eps 0.02, speed 351.9 f/s\n",
            "2300468 frames: done 1289 games, mean reward 18.84, eps 0.02, speed 351.23 f/s\n",
            "2302267 frames: done 1290 games, mean reward 18.82, eps 0.02, speed 354.63 f/s\n",
            "2304099 frames: done 1291 games, mean reward 18.83, eps 0.02, speed 351.22 f/s\n",
            "2306007 frames: done 1292 games, mean reward 18.81, eps 0.02, speed 351.93 f/s\n",
            "2307754 frames: done 1293 games, mean reward 18.8, eps 0.02, speed 348.55 f/s\n",
            "2309420 frames: done 1294 games, mean reward 18.8, eps 0.02, speed 354.22 f/s\n",
            "2311243 frames: done 1295 games, mean reward 18.77, eps 0.02, speed 357.33 f/s\n",
            "2313168 frames: done 1296 games, mean reward 18.76, eps 0.02, speed 356.23 f/s\n",
            "2314863 frames: done 1297 games, mean reward 18.78, eps 0.02, speed 353.68 f/s\n",
            "2316529 frames: done 1298 games, mean reward 18.79, eps 0.02, speed 353.12 f/s\n",
            "2318390 frames: done 1299 games, mean reward 18.77, eps 0.02, speed 353.4 f/s\n",
            "2320257 frames: done 1300 games, mean reward 18.78, eps 0.02, speed 352.47 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1300.dat: 100%|██████████| 100/100 [00:00<00:00, 105.92it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.96 -> 18.78, model saved\n",
            "2321974 frames: done 1301 games, mean reward 18.77, eps 0.02, speed 280.36 f/s\n",
            "2323736 frames: done 1302 games, mean reward 18.8, eps 0.02, speed 355.52 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1302.dat: 100%|██████████| 100/100 [00:00<00:00, 113.88it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.78 -> 18.8, model saved\n",
            "2325863 frames: done 1303 games, mean reward 18.81, eps 0.02, speed 298.78 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1303.dat: 100%|██████████| 100/100 [00:00<00:00, 109.72it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.8 -> 18.81, model saved\n",
            "2327580 frames: done 1304 games, mean reward 18.8, eps 0.02, speed 286.18 f/s\n",
            "2329428 frames: done 1305 games, mean reward 18.79, eps 0.02, speed 352.1 f/s\n",
            "2331123 frames: done 1306 games, mean reward 18.81, eps 0.02, speed 348.38 f/s\n",
            "2332840 frames: done 1307 games, mean reward 18.83, eps 0.02, speed 350.71 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1307.dat: 100%|██████████| 100/100 [00:00<00:00, 117.25it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.81 -> 18.83, model saved\n",
            "2334506 frames: done 1308 games, mean reward 18.83, eps 0.02, speed 280.65 f/s\n",
            "2336172 frames: done 1309 games, mean reward 18.83, eps 0.02, speed 355.05 f/s\n",
            "2337917 frames: done 1310 games, mean reward 18.83, eps 0.02, speed 352.64 f/s\n",
            "2339583 frames: done 1311 games, mean reward 18.83, eps 0.02, speed 351.93 f/s\n",
            "2341248 frames: done 1312 games, mean reward 18.86, eps 0.02, speed 354.3 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1312.dat: 100%|██████████| 100/100 [00:00<00:00, 112.35it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.83 -> 18.86, model saved\n",
            "2343027 frames: done 1313 games, mean reward 18.85, eps 0.02, speed 285.52 f/s\n",
            "2345170 frames: done 1314 games, mean reward 18.81, eps 0.02, speed 354.38 f/s\n",
            "2347077 frames: done 1315 games, mean reward 18.78, eps 0.02, speed 353.87 f/s\n",
            "2348838 frames: done 1316 games, mean reward 18.79, eps 0.02, speed 351.28 f/s\n",
            "2350504 frames: done 1317 games, mean reward 18.8, eps 0.02, speed 354.45 f/s\n",
            "2352323 frames: done 1318 games, mean reward 18.79, eps 0.02, speed 352.88 f/s\n",
            "2354294 frames: done 1319 games, mean reward 18.75, eps 0.02, speed 356.08 f/s\n",
            "2355960 frames: done 1320 games, mean reward 18.8, eps 0.02, speed 351.59 f/s\n",
            "2357922 frames: done 1321 games, mean reward 18.79, eps 0.02, speed 351.98 f/s\n",
            "2359911 frames: done 1322 games, mean reward 18.76, eps 0.02, speed 355.8 f/s\n",
            "2361806 frames: done 1323 games, mean reward 18.73, eps 0.02, speed 352.88 f/s\n",
            "2363472 frames: done 1324 games, mean reward 18.74, eps 0.02, speed 357.39 f/s\n",
            "2365138 frames: done 1325 games, mean reward 18.76, eps 0.02, speed 356.19 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1325.dat: 100%|██████████| 100/100 [00:00<00:00, 107.15it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.86 -> 18.76, model saved\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\rUploading file PongNoFrameskip-v4-1326.dat:   0%|          | 0/100 [00:00<?, ?it/s]"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "2366803 frames: done 1326 games, mean reward 18.77, eps 0.02, speed 278.5 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "\rUploading file PongNoFrameskip-v4-1326.dat: 100%|██████████| 100/100 [00:00<00:00, 116.38it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.76 -> 18.77, model saved\n",
            "2368748 frames: done 1327 games, mean reward 18.75, eps 0.02, speed 294.48 f/s\n",
            "2370674 frames: done 1328 games, mean reward 18.72, eps 0.02, speed 348.54 f/s\n",
            "2372847 frames: done 1329 games, mean reward 18.68, eps 0.02, speed 353.73 f/s\n",
            "2374611 frames: done 1330 games, mean reward 18.71, eps 0.02, speed 351.69 f/s\n",
            "2376421 frames: done 1331 games, mean reward 18.7, eps 0.02, speed 344.68 f/s\n",
            "2378087 frames: done 1332 games, mean reward 18.71, eps 0.02, speed 353.63 f/s\n",
            "2379825 frames: done 1333 games, mean reward 18.71, eps 0.02, speed 351.05 f/s\n",
            "2381605 frames: done 1334 games, mean reward 18.75, eps 0.02, speed 351.79 f/s\n",
            "2383322 frames: done 1335 games, mean reward 18.75, eps 0.02, speed 353.42 f/s\n",
            "2385309 frames: done 1336 games, mean reward 18.71, eps 0.02, speed 354.0 f/s\n",
            "2387118 frames: done 1337 games, mean reward 18.7, eps 0.02, speed 357.69 f/s\n",
            "2388771 frames: done 1338 games, mean reward 18.72, eps 0.02, speed 355.13 f/s\n",
            "2390718 frames: done 1339 games, mean reward 18.7, eps 0.02, speed 355.12 f/s\n",
            "2392435 frames: done 1340 games, mean reward 18.69, eps 0.02, speed 355.03 f/s\n",
            "2394189 frames: done 1341 games, mean reward 18.68, eps 0.02, speed 352.19 f/s\n",
            "2395974 frames: done 1342 games, mean reward 18.72, eps 0.02, speed 352.73 f/s\n",
            "2397640 frames: done 1343 games, mean reward 18.72, eps 0.02, speed 355.07 f/s\n",
            "2399584 frames: done 1344 games, mean reward 18.72, eps 0.02, speed 354.12 f/s\n",
            "2401250 frames: done 1345 games, mean reward 18.73, eps 0.02, speed 356.73 f/s\n",
            "2403061 frames: done 1346 games, mean reward 18.72, eps 0.02, speed 351.8 f/s\n",
            "2404743 frames: done 1347 games, mean reward 18.72, eps 0.02, speed 352.93 f/s\n",
            "2406487 frames: done 1348 games, mean reward 18.72, eps 0.02, speed 354.48 f/s\n",
            "2408153 frames: done 1349 games, mean reward 18.77, eps 0.02, speed 351.09 f/s\n",
            "2409819 frames: done 1350 games, mean reward 18.78, eps 0.02, speed 355.44 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1350.dat: 100%|██████████| 100/100 [00:00<00:00, 103.94it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.77 -> 18.78, model saved\n",
            "2411485 frames: done 1351 games, mean reward 18.82, eps 0.02, speed 278.45 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1351.dat: 100%|██████████| 100/100 [00:01<00:00, 97.37it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.78 -> 18.82, model saved\n",
            "2413372 frames: done 1352 games, mean reward 18.82, eps 0.02, speed 283.0 f/s\n",
            "2415161 frames: done 1353 games, mean reward 18.84, eps 0.02, speed 358.12 f/s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Uploading file PongNoFrameskip-v4-1353.dat: 100%|██████████| 100/100 [00:00<00:00, 103.17it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "New best mean reward 18.82 -> 18.84, model saved\n",
            "2416969 frames: done 1354 games, mean reward 18.83, eps 0.02, speed 284.19 f/s\n",
            "2418733 frames: done 1355 games, mean reward 18.82, eps 0.02, speed 351.41 f/s\n",
            "2420399 frames: done 1356 games, mean reward 18.82, eps 0.02, speed 352.41 f/s\n",
            "2422173 frames: done 1357 games, mean reward 18.83, eps 0.02, speed 349.32 f/s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "qGOQvzRp_DfR",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}