🐎 Change bootstrap sample size for better results

deepsig/__init__.py

@@ -5,5 +5,5 @@
 from deepsig.permutation import permutation_test
 from deepsig.sample_size import aso_uncertainty_reduction, bootstrap_power_analysis
 
-__version__ = "1.2.2"
+__version__ = "1.2.3"
 __author__ = "Dennis Ulmer"

deepsig/aso.py

@@ -94,6 +94,13 @@ def aso(
         num_jobs
     )
 
+    # TODO: Remove in future version
+    if num_samples != 1000:
+        warn(
+            "'num_samples' argument is being ignored in the current version and will be deprecated in version 1.3!",
+            DeprecationWarning,
+        )
+
     violation_ratio = compute_violation_ratio(scores_a, scores_b, dt)
     # Based on the actual number of samples
     quantile_func_a = get_quantile_function(scores_a)
@@ -151,8 +158,8 @@ def _bootstrap_iter(seed: Optional[int] = None):
         if seed is not None:
             np.random.seed(seed)
 
-        sampled_scores_a = quantile_func_a(np.random.uniform(0, 1, num_samples))
-        sampled_scores_b = quantile_func_b(np.random.uniform(0, 1, num_samples))
+        sampled_scores_a = quantile_func_a(np.random.uniform(0, 1, len(scores_a)))
+        sampled_scores_b = quantile_func_b(np.random.uniform(0, 1, len(scores_b)))
         sample = compute_violation_ratio(
             sampled_scores_a,
             sampled_scores_b,
@@ -229,6 +236,13 @@ def multi_aso(
     Union[np.array, pd.DataFrame]
         2D numpy array or pandas Dataframe (if scores is dictionary and return_df=True) with result of ASO.
     """
+    # TODO: Remove in future version
+    if num_samples != 1000:
+        warn(
+            "'num_samples' argument is being ignored in the current version and will be deprecated in version 1.3!",
+            DeprecationWarning,
+        )
+
     num_models = _get_num_models(scores)
     num_comparisons = num_models * (num_models - 1) / 2
     eps_min = np.eye(num_models)  # Initialize score matrix
@@ -257,7 +271,7 @@ def multi_aso(
                 scores_a,
                 scores_b,
                 confidence_level=confidence_level,
-                num_samples=num_samples,
+                num_samples=1000,  # TODO: Avoid double warning, remove in future version
                 num_bootstrap_iterations=num_bootstrap_iterations,
                 dt=dt,
                 num_jobs=num_jobs,
@@ -276,7 +290,7 @@ def multi_aso(
                     scores_b,
                     scores_a,
                     confidence_level=confidence_level,
-                    num_samples=num_samples,
+                    num_samples=1000,  # TODO: Avoid double warning, remove in future version
                     num_bootstrap_iterations=num_bootstrap_iterations,
                     dt=dt,
                     num_jobs=num_jobs,

deepsig/tests/test_aso.py

@@ -152,7 +152,7 @@ def setUp(self) -> None:
         self.aso_kwargs = {
             "num_samples": 100,
             "num_bootstrap_iterations": 100,
-            "num_jobs": 2,
+            "num_jobs": 4,
         }
         self.num_models = 3
         self.num_seeds = 100
@@ -276,7 +276,7 @@ class ASOSanityChecks(unittest.TestCase):
 
     def setUp(self) -> None:
         self.num_samples = 1000
-        self.num_bootstrap_iters = 500
+        self.num_bootstrap_iters = 1000
 
     def test_extreme_cases(self):
         """
@@ -295,7 +295,7 @@ def test_extreme_cases(self):
             samples_normal1 + 1e-8,
             num_bootstrap_iterations=self.num_bootstrap_iters,
             show_progress=False,
-            num_jobs=2,
+            num_jobs=4,
         )
         self.assertAlmostEqual(eps_min, 1, delta=0.001)
 
@@ -308,7 +308,7 @@ def test_extreme_cases(self):
             samples_normal2,
             num_bootstrap_iterations=self.num_bootstrap_iters,
             show_progress=False,
-            num_jobs=2,
+            num_jobs=4,
         )
         self.assertAlmostEqual(eps_min2, 0, delta=0.01)
 
@@ -332,7 +332,7 @@ def test_dependency_on_alpha(self):
                 confidence_level=alpha,
                 num_bootstrap_iterations=100,
                 show_progress=False,
-                num_jobs=2,
+                num_jobs=4,
                 seed=seed,
             )
             min_epsilons.append(min_eps)
@@ -359,7 +359,7 @@ def test_dependency_on_samples(self):
                 samples_normal2,
                 num_bootstrap_iterations=100,
                 show_progress=False,
-                num_jobs=2,
+                num_jobs=4,
                 seed=seed,
             )
             min_epsilons.append(min_eps)
@@ -391,14 +391,14 @@ def test_symmetry(self):
                 samples_normal1,
                 samples_normal2,
                 show_progress=True,  # Show progress so travis CI build doesn't time out
-                num_jobs=2,
+                num_jobs=4,
                 num_bootstrap_iterations=1000,
             )
             eps_min2 = aso(
                 samples_normal2,
                 samples_normal1,
                 show_progress=True,  # Show progress so travis CI build doesn't time out
-                num_jobs=2,
+                num_jobs=4,
                 num_bootstrap_iterations=1000,
             )
             self.assertAlmostEqual(eps_min1, 1 - eps_min2, delta=0.2)

setup.py

@@ -8,7 +8,7 @@
 
 setup(
     name="deepsig",
-    version="1.2.2",
+    version="1.2.3",
     author="Dennis Ulmer",
     description="Easy Significance Testing for Deep Neural Networks.",
     long_description=long_description,