Fix white-space merge conflict

.landscape.yml

@@ -0,0 +1,12 @@
+doc-warnings: true
+test-warnings: false
+strictness: veryhigh
+max-line-length: 100
+pep8:
+  full: true
+ignore-paths: 
+    - ci_scripts
+ignore-patterns:
+    - (^|/)docs(/|$)
+python-targets:
+    - 3

.travis.yml

@@ -7,19 +7,26 @@ matrix:
     env: PYTHON_VERSION="3.4" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh"
   - os: linux
     env: PYTHON_VERSION="3.5" COVERAGE="true" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh"
+  - os: linux
+    env: PYTHON_VERSION="3.6" COVERAGE="true" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh"
 
+  # Disable OSX building because it takes too long and hinders progress
   # Set language to generic to not break travis-ci
   # https://github.com/travis-ci/travis-ci/issues/2312#issuecomment-195620855
   # so far, this issue is still open and there is no good solution
   # python will then be installed by anaconda
-  - os: osx
-    sudo: required
-    language: generic
-    env: PYTHON_VERSION="3.4" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh"
-  - os: osx
-    sudo: required
-    language: generic
-    env: PYTHON_VERSION="3.5" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh"
+  #- os: osx
+  #  sudo: required
+  #  language: generic
+  #  env: PYTHON_VERSION="3.4" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh"
+  #- os: osx
+  #  sudo: required
+  #  language: generic
+  #  env: PYTHON_VERSION="3.5" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh"
+  #- os: osx
+  #  sudo: required
+  #  language: generic
+  #  env: PYTHON_VERSION="3.6" MINICONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh"
 
 cache:
   directories:

changelog.md

@@ -1,3 +1,20 @@
+# 0.3
+
+* Major speed improvements when sampling new configurations:
+    * Improved conditional hyperparameter imputation (PR #176)
+    * Faster generation of the one exchange neighborhood (PR #174)
+* FIX #171 potential bug with pSMAC
+* FIX #175 backwards compability for reading runhistory files
+
+# 0.2.4
+
+* CI only check code quality for python3
+* Perform local search on configurations from previous runs as proposed in the
+  original paper from 2011 instead of random configurations as implemented
+  before
+* CI run travis-ci unit tests with python3.6
+* FIX #167, remove an endless loop which occured when using pSMAC
+
 # 0.2.3
 
 * MAINT refactor Intensifcation and adding unit tests

requirements.txt

@@ -5,7 +5,7 @@ six
 Cython
 psutil
 pynisher>=0.4.1
-ConfigSpace>=0.2.1
+ConfigSpace>=0.3.0
 pyrfr
 scikit-learn
 typing

smac/__version__.py

@@ -1,4 +1,4 @@
 """Version information."""
 
 # The following line *must* be the last in the module, exactly as formatted:
-__version__ = "0.2.3"
+__version__ = "0.3.0"

smac/configspace/__init__.py

@@ -1,3 +1,4 @@
 from ConfigSpace import ConfigurationSpace, Configuration
 from ConfigSpace.io import pcs
-from ConfigSpace.util import impute_inactive_values, get_random_neighbor, get_one_exchange_neighbourhood
+from ConfigSpace.util import get_random_neighbor, get_one_exchange_neighbourhood
+from smac.configspace.util import convert_configurations_to_array

smac/configspace/util.py

@@ -0,0 +1,37 @@
+from typing import List
+
+import numpy as np
+
+from smac.configspace import Configuration
+
+
+def convert_configurations_to_array(configs: List[Configuration]) -> np.ndarray:
+    """Impute inactive hyperparameters in configurations with their default.
+
+    Necessary to apply an EPM to the data.
+
+    Parameters
+    ----------
+    configs : List[Configuration]
+        List of configuration objects.
+
+    Returns
+    np.ndarray
+        Array with configuration hyperparameters. Inactive values are imputed
+        with their default value.
+    """
+    configs_array = np.array([config.get_array() for config in configs],
+                             dtype=np.float64)
+    configuration_space = configs[0].configuration_space
+    for hp in configuration_space.get_hyperparameters():
+        default = hp._inverse_transform(hp.default)
+        idx = configuration_space.get_idx_by_hyperparameter_name(hp.name)
+
+        # Create a mask which is True for all non-finite entries in column idx!
+        column_mask = np.zeros(configs_array.shape, dtype=np.bool)
+        column_mask[:, idx] = True
+        nonfinite_mask = ~np.isfinite(configs_array)
+        mask = column_mask & nonfinite_mask
+
+        configs_array[mask] = default
+    return configs_array

smac/epm/rf_with_instances.py

@@ -108,10 +108,8 @@ def train(self, X, y, **kwargs):
         """
 
         self.X = X
-        self.y = y
-
-        y = y.flatten()
-        data = pyrfr.regression.numpy_data_container(self.X, y, self.types)
+        self.y = y.flatten()
+        data = pyrfr.regression.numpy_data_container(self.X, self.y, self.types)
 
         self.rf.fit(data)
         return self

smac/intensification/intensification.py

@@ -413,12 +413,13 @@ def _compare_configs(self, incumbent: Configuration,
 
         inc_runs = run_history.get_runs_for_config(incumbent)
         chall_runs = run_history.get_runs_for_config(challenger)
+        to_compare_runs = set(inc_runs).intersection(chall_runs)
 
         # performance on challenger runs
         chal_perf = aggregate_func(
-            challenger, run_history, chall_runs)
+            challenger, run_history, to_compare_runs)
         inc_perf = aggregate_func(
-            incumbent, run_history, chall_runs)
+            incumbent, run_history, to_compare_runs)
 
         # Line 15
         if chal_perf > inc_perf and len(chall_runs) >= self.minR:
@@ -428,9 +429,9 @@ def _compare_configs(self, incumbent: Configuration,
             return incumbent
 
         # Line 16
-        if len(chall_runs) >= len(inc_runs):
+        if not set(inc_runs) - set(chall_runs):
             # Challenger is as good as incumbent
-            # and has the same number of runs
+            # and has at least the same runs as inc
             # -> change incumbent
 
             n_samples = len(chall_runs)

smac/runhistory/runhistory.py

@@ -99,17 +99,21 @@ def add(self, config, cost, time,
 
         k = RunKey(config_id, instance_id, seed)
         v = RunValue(cost, time, status, additional_info)
-        self.data[k] = v
 
-        if not external_data:
-            # also add to fast data structure
-            is_k = InstSeedKey(instance_id, seed)
-            self._configid_to_inst_seed[
-                config_id] = self._configid_to_inst_seed.get(config_id, [])
-            self._configid_to_inst_seed[config_id].append(is_k)
+        # Each runkey is supposed to be used only once. Repeated tries to add
+        # the same runkey will be ignored silently.
+        if self.data.get(k) is None:
+            self.data[k] = v
 
-        # assumes an average across runs as cost function
-        self.incremental_update_cost(config, cost)
+            if not external_data:
+                # also add to fast data structure
+                is_k = InstSeedKey(instance_id, seed)
+                self._configid_to_inst_seed[
+                    config_id] = self._configid_to_inst_seed.get(config_id, [])
+                self._configid_to_inst_seed[config_id].append(is_k)
+
+            # assumes an average across runs as cost function
+            self.incremental_update_cost(config, cost)
 
     def update_cost(self, config):
         '''
@@ -191,11 +195,17 @@ def get_runs_for_config(self, config):
             list: tuples of instance, seed
         """
         config_id = self.config_ids.get(config)
-        is_list = self._configid_to_inst_seed.get(config_id)
-        if is_list is None:
-            return []
-        else:
-            return is_list
+        return self._configid_to_inst_seed.get(config_id, [])
+
+    def get_all_configs(self):
+        """ Return all configurations in this RunHistory object
+
+        Returns
+        -------
+            list: parameter configurations
+
+        """
+        return list(self.config_ids.keys())
 
     def empty(self):
         """
@@ -272,7 +282,7 @@ def load_json(self, fn, cs):
             self.add(config=self.ids_config[int(k[0])],
                      cost=float(v[0]),
                      time=float(v[1]),
-                     status=v[2],
+                     status=StatusType(v[2]),
                      instance_id=k[1],
                      seed=int(k[2]),
                      additional_info=v[3])

smac/runhistory/runhistory2epm.py

@@ -8,7 +8,7 @@
 
 from smac.tae.execute_ta_run import StatusType
 from smac.runhistory.runhistory import RunHistory, RunKey, RunValue
-from smac.configspace import impute_inactive_values
+from smac.configspace import convert_configurations_to_array
 import smac.epm.base_imputor
 from smac.utils import constants
 
@@ -295,12 +295,12 @@ def _build_matrix(self, run_dict, runhistory, instances=None, par_factor=1):
         for row, (key, run) in enumerate(run_dict.items()):
             # Scaling is automatically done in configSpace
             conf = runhistory.ids_config[key.config_id]
-            conf = impute_inactive_values(conf)
+            conf_vector = convert_configurations_to_array([conf])[0]
             if self.n_feats:
                 feats = self.instance_features[key.instance_id]
-                X[row, :] = np.hstack((conf.get_array(), feats))
+                X[row, :] = np.hstack((conf_vector, feats))
             else:
-                X[row, :] = conf.get_array()
+                X[row, :] = conf_vector
             # run_array[row, -1] = instances[row]
             if self.scenario.run_obj == "runtime":
                 if run.status != StatusType.SUCCESS:

smac/smbo/local_search.py

@@ -1,9 +1,9 @@
 import logging
 import time
-import random
 import numpy as np
 
-from smac.configspace import impute_inactive_values, get_one_exchange_neighbourhood, Configuration
+from smac.configspace import get_one_exchange_neighbourhood, \
+    convert_configurations_to_array
 
 __author__ = "Aaron Klein, Marius Lindauer"
 __copyright__ = "Copyright 2015, ML4AAD"
@@ -71,10 +71,8 @@ def maximize(self, start_point, *args):
         """
         incumbent = start_point
         # Compute the acquisition value of the incumbent
-        incumbent_ = impute_inactive_values(incumbent)
-        acq_val_incumbent = self.acquisition_function(
-                                            incumbent_.get_array(),
-                                            *args)
+        incumbent_array = convert_configurations_to_array([incumbent])
+        acq_val_incumbent = self.acquisition_function(incumbent_array, *args)
 
         local_search_steps = 0
         neighbors_looked_at = 0
@@ -83,23 +81,24 @@ def maximize(self, start_point, *args):
 
             local_search_steps += 1
             if local_search_steps % 1000 == 0:
-                self.logger.warn("Local search took already %d iterations." \
-                "Is it maybe stuck in a infinite loop?", local_search_steps)
+                self.logger.warn("Local search took already %d iterations."
+                                 "Is it maybe stuck in a infinite loop?",
+                                 local_search_steps)
 
             # Get neighborhood of the current incumbent
             # by randomly drawing configurations
             changed_inc = False
 
-            all_neighbors = get_one_exchange_neighbourhood(incumbent,
-                                                    seed=self.rng.seed())
-            self.rng.shuffle(all_neighbors)
+            # Get one exchange neighborhood returns an iterator (in contrast of
+            # the previously returned list).
+            all_neighbors = get_one_exchange_neighbourhood(
+                incumbent, seed=self.rng.seed())
 
             for neighbor in all_neighbors:
                 s_time = time.time()
-                neighbor_ = impute_inactive_values(neighbor)
-                n_array = neighbor_.get_array()
+                neighbor_array_ = convert_configurations_to_array([neighbor])
 
-                acq_val = self.acquisition_function(n_array, *args)
+                acq_val = self.acquisition_function(neighbor_array_, *args)
 
                 neighbors_looked_at += 1