Merge pull request #872 from automl/fix_mo_cache

Fix Multi-Objective cache

changelog.md

@@ -1,3 +1,11 @@
+# 1.4.0
+
+## Improvements
+* Enabled caching for multi-objectives (#872). Costs are now normalized in `get_cost` 
+or optionally in `average_cost`/`sum_cost`/`min_cost` to receive a single float value. Therefore,
+the cached cost values do not need to be updated everytime a new entry to the runhistory was added.
+
+
 # 1.3.4
 * Added reference to JMLR paper.
 * Typos in documentations.

examples/python/plot_simple_multi_objective.py

@@ -27,7 +27,10 @@ def plot(all_x):
     plt.figure()
     for x in all_x:
         f1, f2 = schaffer(x)
-        plt.scatter(f1, f2, c="blue", alpha=0.1)
+        plt.scatter(f1, f2, c="blue", alpha=0.1, zorder=3000)
+
+    plt.vlines([1], 0, 4, linestyles="dashed", colors=["red"])
+    plt.hlines([1], 0, 4, linestyles="dashed", colors=["red"])
 
     plt.show()
 

smac/facade/smac_ac_facade.py

@@ -530,7 +530,7 @@ def __init__(
 
         if scenario.multi_objectives is not None and num_obj > 1:  # type: ignore[attr-defined] # noqa F821
             # define any defaults here
-            _multi_objective_kwargs = {"rng": rng, "num_obj": num_obj}
+            _multi_objective_kwargs = {"rng": rng}
 
             if multi_objective_kwargs is not None:
                 _multi_objective_kwargs.update(multi_objective_kwargs)

smac/intensification/abstract_racer.py

@@ -89,7 +89,6 @@ def __init__(
         maxR: int = 2000,
         adaptive_capping_slackfactor: float = 1.2,
         min_chall: int = 1,
-        num_obj: int = 1,
     ):
 
         self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__)
@@ -129,12 +128,6 @@ def __init__(
         # to mark the end of an iteration
         self.iteration_done = False
 
-        if num_obj > 1:
-            raise ValueError(
-                "Intensifiers only support single objective optimization. For multi-objective problems,"
-                "please refer to multi-objective intensifiers"
-            )
-
     def get_next_run(
         self,
         challengers: Optional[List[Configuration]],
@@ -303,9 +296,10 @@ def _adapt_cutoff(self, challenger: Configuration, run_history: RunHistory, inc_
         # reasons)
         chall_inst_seeds = run_history.get_runs_for_config(challenger, only_max_observed_budget=True)
         chal_sum_cost = run_history.sum_cost(
-            config=challenger,
-            instance_seed_budget_keys=chall_inst_seeds,
+            config=challenger, instance_seed_budget_keys=chall_inst_seeds, normalize=True
         )
+        assert type(chal_sum_cost) == float
+
         cutoff = min(curr_cutoff, inc_sum_cost * self.adaptive_capping_slackfactor - chal_sum_cost)
         return cutoff
 
@@ -348,8 +342,11 @@ def _compare_configs(
 
         # performance on challenger runs, the challenger only becomes incumbent
         # if it dominates the incumbent
-        chal_perf = run_history.average_cost(challenger, to_compare_runs)
-        inc_perf = run_history.average_cost(incumbent, to_compare_runs)
+        chal_perf = run_history.average_cost(challenger, to_compare_runs, normalize=True)
+        inc_perf = run_history.average_cost(incumbent, to_compare_runs, normalize=True)
+
+        assert type(chal_perf) == float
+        assert type(inc_perf) == float
 
         # Line 15
         if np.any(chal_perf > inc_perf) and len(chall_runs) >= self.minR:

smac/intensification/hyperband.py

@@ -98,7 +98,6 @@ def __init__(
         min_chall: int = 1,
         incumbent_selection: str = "highest_executed_budget",
         identifier: int = 0,
-        num_obj: int = 1,
     ) -> None:
 
         super().__init__(
@@ -119,7 +118,6 @@ def __init__(
             adaptive_capping_slackfactor=adaptive_capping_slackfactor,
             min_chall=min_chall,
             incumbent_selection=incumbent_selection,
-            num_obj=num_obj,
         )
 
         self.identifier = identifier

smac/intensification/intensification.py

@@ -144,7 +144,6 @@ def __init__(
         maxR: int = 2000,
         adaptive_capping_slackfactor: float = 1.2,
         min_chall: int = 2,
-        num_obj: int = 1,
     ):
         super().__init__(
             stats=stats,
@@ -159,7 +158,6 @@ def __init__(
             maxR=maxR,
             adaptive_capping_slackfactor=adaptive_capping_slackfactor,
             min_chall=min_chall,
-            num_obj=num_obj,
         )
 
         self.logger = logging.getLogger(self.__module__ + "." + self.__class__.__name__)
@@ -795,6 +793,7 @@ def _process_racer_results(
         """
         chal_runs = run_history.get_runs_for_config(challenger, only_max_observed_budget=True)
         chal_perf = run_history.get_cost(challenger)
+
         # if all <instance, seed> have been run, compare challenger performance
         if not self.to_run:
             new_incumbent = self._compare_configs(
@@ -896,11 +895,8 @@ def _get_instances_to_run(
         # because of efficiency computed here
         inst_seed_pairs = list(inc_inst_seeds - set(missing_runs))
         # cost used by incumbent for going over all runs in inst_seed_pairs
-        inc_sum_cost = run_history.sum_cost(
-            config=incumbent,
-            instance_seed_budget_keys=inst_seed_pairs,
-        )
-
+        inc_sum_cost = run_history.sum_cost(config=incumbent, instance_seed_budget_keys=inst_seed_pairs, normalize=True)
+        assert type(inc_sum_cost) == float
         return to_run, inc_sum_cost
 
     def get_next_challenger(

smac/intensification/parallel_scheduling.py

@@ -89,7 +89,6 @@ def __init__(
         inst_seed_pairs: Optional[List[Tuple[str, int]]] = None,
         min_chall: int = 1,
         incumbent_selection: str = "highest_executed_budget",
-        num_obj: int = 1,
     ) -> None:
 
         super().__init__(
@@ -103,7 +102,6 @@ def __init__(
             run_obj_time=run_obj_time,
             adaptive_capping_slackfactor=adaptive_capping_slackfactor,
             min_chall=min_chall,
-            num_obj=num_obj,
         )
 
         # We have a pool of instances that yield configurations ot run

smac/intensification/simple_intensifier.py

@@ -46,7 +46,6 @@ def __init__(
         cutoff: Optional[float] = None,
         deterministic: bool = False,
         run_obj_time: bool = True,
-        num_obj: int = 1,
         **kwargs: Any,
     ) -> None:
 
@@ -61,10 +60,7 @@ def __init__(
             run_obj_time=run_obj_time,
             adaptive_capping_slackfactor=1.0,
             min_chall=1,
-            num_obj=num_obj,
         )
-        # Simple intensifier does not require comparing run results, thus we could simply ignore num_obj here
-
         # We want to control the number of runs that are sent to
         # the workers. At any time, we want to make sure that if there
         # are just W workers, there should be at max W active runs

smac/intensification/successive_halving.py

@@ -137,7 +137,6 @@ def __init__(
         min_chall: int = 1,
         incumbent_selection: str = "highest_executed_budget",
         identifier: int = 0,
-        num_obj: int = 1,
     ) -> None:
         super().__init__(
             stats=stats,
@@ -150,7 +149,6 @@ def __init__(
             run_obj_time=run_obj_time,
             adaptive_capping_slackfactor=adaptive_capping_slackfactor,
             min_chall=min_chall,
-            num_obj=num_obj,
         )
 
         self.identifier = identifier
@@ -629,14 +627,16 @@ def get_next_run(
         #   - during the 1st intensify run, the incumbent shouldn't be capped after being compared against itself
         if incumbent and incumbent != challenger:
             inc_runs = run_history.get_runs_for_config(incumbent, only_max_observed_budget=True)
-            inc_sum_cost = run_history.sum_cost(config=incumbent, instance_seed_budget_keys=inc_runs)
+            inc_sum_cost = run_history.sum_cost(config=incumbent, instance_seed_budget_keys=inc_runs, normalize=True)
         else:
             inc_sum_cost = np.inf
             if self.first_run:
                 self.logger.info("First run, no incumbent provided; challenger is assumed to be the incumbent")
                 incumbent = challenger
 
-        # selecting instance-seed subset for this budget, depending on the kind of budget
+        assert type(inc_sum_cost) == float
+
+        # Selecting instance-seed subset for this budget, depending on the kind of budget
         if self.instance_as_budget:
             prev_budget = int(self.all_budgets[self.stage - 1]) if self.stage > 0 else 0
             curr_insts = self.inst_seed_pairs[int(prev_budget) : int(curr_budget)]

smac/optimizer/multi_objective/abstract_multi_objective_algorithm.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from abc import ABC
 from typing import Optional
 
@@ -10,10 +12,8 @@ class AbstractMultiObjectiveAlgorithm(ABC):
     It can be applied to rh2epm or epmchooser.
     """
 
-    def __init__(self, num_obj: int, rng: Optional[np.random.RandomState] = None):
-
+    def __init__(self, rng: Optional[np.random.RandomState] = None):
         if rng is None:
             rng = np.random.RandomState(0)
 
-        self.num_obj = num_obj
         self.rng = rng

smac/optimizer/multi_objective/aggregation_strategy.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from abc import abstractmethod
 
 import numpy as np
@@ -9,22 +11,24 @@
 
 class AggregationStrategy(AbstractMultiObjectiveAlgorithm):
     """
-    An abstract class to aggregate multi-objective losses to a single objective losses, which can then be utilized
-    by the single-objective optimizer.
+    An abstract class to aggregate multi-objective losses to a single objective loss,
+    which can then be utilized by the single-objective optimizer.
     """
 
     @abstractmethod
-    def __call__(self, values: np.ndarray) -> float:
+    def __call__(self, values: list[float]) -> float:
         """
         Transform a multi-objective loss to a single loss.
 
         Parameters
         ----------
-            values: np.ndarray[num_evaluations, num_obj].
+        values : list[float]
+            Normalized values.
 
         Returns
         -------
-            cost: float.
+        cost : float
+            Combined cost.
         """
         raise NotImplementedError
 
@@ -35,16 +39,18 @@ class MeanAggregationStrategy(AggregationStrategy):
     which can then be utilized by the single-objective optimizer.
     """
 
-    def __call__(self, values: np.ndarray) -> float:
+    def __call__(self, values: list[float]) -> float:
         """
         Transform a multi-objective loss to a single loss.
 
         Parameters
         ----------
-            values (np.ndarray): Normalized values.
+        values : list[float]
+            Normalized values.
 
         Returns
         -------
-            cost (float): Combined cost.
+        cost : float
+            Combined cost.
         """
-        return np.mean(values, axis=1)
+        return np.mean(values, axis=0)

smac/optimizer/multi_objective/parego.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from typing import Optional
 
 import numpy as np
@@ -8,32 +10,32 @@
 class ParEGO(AggregationStrategy):
     def __init__(
         self,
-        num_obj: int,
         rng: Optional[np.random.RandomState] = None,
         rho: float = 0.05,
     ):
-        super(ParEGO, self).__init__(num_obj=num_obj, rng=rng)
+        super(ParEGO, self).__init__(rng=rng)
         self.rho = rho
 
-    def __call__(self, values: np.ndarray) -> float:
+    def __call__(self, values: list[float]) -> float:
         """
         Transform a multi-objective loss to a single loss.
 
         Parameters
         ----------
-            values (np.ndarray): Normalized values.
+        values : list[float]
+            Normalized values.
 
         Returns
         -------
-            cost (float): Combined cost.
+        cost : float
+            Combined cost.
         """
         # Then we have to compute the weight
-        theta = self.rng.rand(self.num_obj)
+        theta = self.rng.rand(len(values))
 
         # Normalize st all theta values sum up to 1
         theta = theta / (np.sum(theta) + 1e-10)
 
         # Weight the values
         theta_f = theta * values
-
-        return np.max(theta_f, axis=1) + self.rho * np.sum(theta_f, axis=1)
+        return np.max(theta_f, axis=0) + self.rho * np.sum(theta_f, axis=0)