Setup benchmarks w/ asv #71

bench/.gitignore

@@ -0,0 +1 @@
+yatsm

bench/README.md

@@ -0,0 +1,3 @@
+# Benchmarks
+
+Performance benchmarks and benchmark tracking using [Airpspeed Velocity](https://github.com/spacetelescope/asv).

bench/asv.conf.json

@@ -0,0 +1,92 @@
+{
+    // The version of the config file format.  Do not change, unless
+    // you know what you are doing.
+    "version": 1,
+
+    // The name of the project being benchmarked
+    "project": "yatsm",
+
+    // The project's homepage
+    "project_url": "https://github.com/ceholden/yatsm",
+
+    // The URL or local path of the source code repository for the
+    // project being benchmarked
+    "repo": "https://github.com/ceholden/yatsm.git",
+
+    // List of branches to benchmark. If not provided, defaults to "master"
+    // (for git) or "tip" (for mercurial).
+    "branches": ["master"], // for git
+
+    // The DVCS being used.  If not set, it will be automatically
+    // determined from "repo" by looking at the protocol in the URL
+    // (if remote), or by looking for special directories, such as
+    // ".git" (if local).
+    // "dvcs": "git",
+
+    // The tool to use to create environments.  May be "conda",
+    // "virtualenv" or other value depending on the plugins in use.
+    // If missing or the empty string, the tool will be automatically
+    // determined by looking for tools on the PATH environment
+    // variable.
+    "environment_type": "conda",
+
+    // the base URL to show a commit for the project.
+    "show_commit_url": "http://github.com/ceholden/yatsm/commit/",
+
+    // The Pythons you'd like to test against.  If not provided, defaults
+    // to the current version of Python used to run `asv`.
+    "pythons": ["2.7"],
+
+    // The matrix of dependencies to test.  Each key is the name of a
+    // package (in PyPI) and the values are version numbers.  An empty
+    // list indicates to just test against the default (latest)
+    // version.
+    "matrix": {
+        "numpy": ["1.8", "1.9", "1.10"],
+        "scipy": ["0.14.0"],
+        "Cython": ["0.20"],
+        "statsmodels": ["0.5.0"],
+        "scikit-learn": ["0.15"],
+        "pandas": ["0.16.0"],
+        "patsy": ["0.3.0"],
+        "gdal": ["1.11"],
+        "click": ["4.0"],
+        "pyyaml": ["3.11"],
+        "matplotlib": ["1.4.3"],
+        "numba": ["0.20"],
+        // asv needs to support adding conda channels before rpy2 can be used
+        // "rpy2": ["2.5.6"]
+    },
+    "exclude": [
+        {"numpy": "1.8", "gdal": "1.11"},
+        {"numpy": "1.10", "gdal": "1.11"}
+    ],
+
+    "regressions_first_commits": {
+        ".*": "v0.5.0",
+    },
+
+    // The directory (relative to the current directory) that benchmarks are
+    // stored in.  If not provided, defaults to "benchmarks"
+    // "benchmark_dir": "benchmarks",
+
+    // The directory (relative to the current directory) to cache the Python
+    // environments in.  If not provided, defaults to "env"
+    // "env_dir": "env",
+
+    // The directory (relative to the current directory) that raw benchmark
+    // results are stored in.  If not provided, defaults to "results".
+    // "results_dir": "results",
+
+    // The directory (relative to the current directory) that the html tree
+    // should be written to.  If not provided, defaults to "html".
+    // "html_dir": "html",
+
+    // The number of characters to retain in the commit hashes.
+    // "hash_length": 8,
+
+    // `asv` will cache wheels of the recent builds in each
+    // environment, making them faster to install next time.  This is
+    // number of builds to keep, per environment.
+    // "wheel_cache_size": 0
+}

bench/benchmarks/__init__.py

@@ -0,0 +1,6 @@
+import os
+
+# Fix OPENLBAS threads to 1
+NP_THREAD_VARS = ['OPENBLAS_NUM_THREADS', 'MKL_NUM_THREADS', 'OPM_NUM_THREADS']
+for ev in NP_THREAD_VARS:
+    os.environ[ev] = '1'

bench/benchmarks/algorithms/bench_ccdcesque.py

@@ -0,0 +1,83 @@
+import os
+
+import numpy as np
+import sklearn.linear_model
+
+from yatsm import __version__ as yatsm_version
+from yatsm.algorithms import CCDCesque
+
+n = 50
+# Hack for goof up in API previous to v0.6.0
+if yatsm_version.split('.')[1] == '5':
+    est = 'lm'
+else:
+    est = 'estimator'
+
+
+class CCDCesqueSuite(object):
+
+    def setup_cache(self):
+        example_data = os.path.join(
+            os.path.dirname(__file__),
+            '../../../tests/algorithms/data/example_timeseries_masked.npz')
+
+        dat = np.load(example_data)['arr_0'].item()
+        X = dat['X']
+        Y = dat['Y']
+        dates = dat['dates']
+
+        kwargs = {
+            'test_indices': np.array([2, 3, 4, 5]),
+            est: sklearn.linear_model.Lasso(alpha=[20]),
+            'consecutive': 5,
+            'threshold': 4,
+            'min_obs': 24,
+            'min_rmse': 100,
+            'retrain_time': 365.25,
+            'screening': 'RLM',
+            'screening_crit': 400.0,
+            'green_band': 1,
+            'swir1_band': 4,
+            'remove_noise': False,
+            'dynamic_rmse': False,
+            'slope_test': False,
+            'idx_slope': 1
+        }
+        return {'X': X, 'Y': Y, 'dates': dates, 'kwargs': kwargs}
+
+    def time_ccdcesque1(self, setup):
+        """ Bench with 'defaults' defined in setup with most tests turned off
+        """
+        for i in range(n):
+            model = CCDCesque(**setup['kwargs'])
+            model.fit(setup['X'], setup['Y'], setup['dates'])
+
+    def time_ccdcesque2(self, setup):
+        """ Bench with remove_noise turned on
+        """
+        kwargs = setup['kwargs']
+        kwargs.update({'remove_noise': True})
+        for i in range(n):
+            model = CCDCesque(**kwargs)
+            model.fit(setup['X'], setup['Y'], setup['dates'])
+
+    def time_ccdcesque3(self, setup):
+        """ Bench with remove_noise, dynamic_rmse turned on
+        """
+        kwargs = setup['kwargs']
+        kwargs.update({'remove_noise': True,
+                       'dynamic_rmse': True})
+        for i in range(n):
+            model = CCDCesque(**kwargs)
+            model.fit(setup['X'], setup['Y'], setup['dates'])
+
+    def time_ccdcesque4(self, setup):
+        """ Bench with remove_noise, dynamic_rmse, slope_test turned on
+        """
+        kwargs = setup['kwargs']
+        kwargs.update({'remove_noise': True,
+                       'dynamic_rmse': True,
+                       'slope_test': True})
+        for i in range(n):
+            model = CCDCesque(**kwargs)
+            model.fit(setup['X'], setup['Y'], setup['dates'])

requirements/dev.txt

@@ -6,3 +6,4 @@ sphinxcontrib-napoleon>=0.2.8
 sphinxcontrib-programoutput>=0.8
 sphinxcontrib-bibtex>=0.3.1
 coverage>=3.7.1
+git+git://github.com/spacetelescope/asv