diff --git a/bench/.gitignore b/bench/.gitignore new file mode 100644 index 00000000..a5e573f0 --- /dev/null +++ b/bench/.gitignore @@ -0,0 +1 @@ +yatsm diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 00000000..7fc9c074 --- /dev/null +++ b/bench/README.md @@ -0,0 +1,3 @@ +# Benchmarks + +Performance benchmarks and benchmark tracking using [Airpspeed Velocity](https://github.com/spacetelescope/asv). diff --git a/bench/asv.conf.json b/bench/asv.conf.json new file mode 100644 index 00000000..d9bf8b1d --- /dev/null +++ b/bench/asv.conf.json @@ -0,0 +1,92 @@ +{ + // The version of the config file format. Do not change, unless + // you know what you are doing. + "version": 1, + + // The name of the project being benchmarked + "project": "yatsm", + + // The project's homepage + "project_url": "https://github.com/ceholden/yatsm", + + // The URL or local path of the source code repository for the + // project being benchmarked + "repo": "https://github.com/ceholden/yatsm.git", + + // List of branches to benchmark. If not provided, defaults to "master" + // (for git) or "tip" (for mercurial). + "branches": ["master"], // for git + + // The DVCS being used. If not set, it will be automatically + // determined from "repo" by looking at the protocol in the URL + // (if remote), or by looking for special directories, such as + // ".git" (if local). + // "dvcs": "git", + + // The tool to use to create environments. May be "conda", + // "virtualenv" or other value depending on the plugins in use. + // If missing or the empty string, the tool will be automatically + // determined by looking for tools on the PATH environment + // variable. + "environment_type": "conda", + + // the base URL to show a commit for the project. + "show_commit_url": "http://github.com/ceholden/yatsm/commit/", + + // The Pythons you'd like to test against. If not provided, defaults + // to the current version of Python used to run `asv`. + "pythons": ["2.7"], + + // The matrix of dependencies to test. Each key is the name of a + // package (in PyPI) and the values are version numbers. An empty + // list indicates to just test against the default (latest) + // version. + "matrix": { + "numpy": ["1.8", "1.9", "1.10"], + "scipy": ["0.14.0"], + "Cython": ["0.20"], + "statsmodels": ["0.5.0"], + "scikit-learn": ["0.15"], + "pandas": ["0.16.0"], + "patsy": ["0.3.0"], + "gdal": ["1.11"], + "click": ["4.0"], + "pyyaml": ["3.11"], + "matplotlib": ["1.4.3"], + "numba": ["0.20"], + // asv needs to support adding conda channels before rpy2 can be used + // "rpy2": ["2.5.6"] + }, + "exclude": [ + {"numpy": "1.8", "gdal": "1.11"}, + {"numpy": "1.10", "gdal": "1.11"} + ], + + "regressions_first_commits": { + ".*": "v0.5.0", + }, + + // The directory (relative to the current directory) that benchmarks are + // stored in. If not provided, defaults to "benchmarks" + // "benchmark_dir": "benchmarks", + + // The directory (relative to the current directory) to cache the Python + // environments in. If not provided, defaults to "env" + // "env_dir": "env", + + // The directory (relative to the current directory) that raw benchmark + // results are stored in. If not provided, defaults to "results". + // "results_dir": "results", + + // The directory (relative to the current directory) that the html tree + // should be written to. If not provided, defaults to "html". + // "html_dir": "html", + + // The number of characters to retain in the commit hashes. + // "hash_length": 8, + + // `asv` will cache wheels of the recent builds in each + // environment, making them faster to install next time. This is + // number of builds to keep, per environment. + // "wheel_cache_size": 0 +} diff --git a/bench/benchmarks/__init__.py b/bench/benchmarks/__init__.py new file mode 100644 index 00000000..72e439c8 --- /dev/null +++ b/bench/benchmarks/__init__.py @@ -0,0 +1,6 @@ +import os + +# Fix OPENLBAS threads to 1 +NP_THREAD_VARS = ['OPENBLAS_NUM_THREADS', 'MKL_NUM_THREADS', 'OPM_NUM_THREADS'] +for ev in NP_THREAD_VARS: + os.environ[ev] = '1' diff --git a/bench/benchmarks/algorithms/__init__.py b/bench/benchmarks/algorithms/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/bench/benchmarks/algorithms/bench_ccdcesque.py b/bench/benchmarks/algorithms/bench_ccdcesque.py new file mode 100644 index 00000000..dc23d1ea --- /dev/null +++ b/bench/benchmarks/algorithms/bench_ccdcesque.py @@ -0,0 +1,83 @@ +import os + +import numpy as np +import sklearn.linear_model + +from yatsm import __version__ as yatsm_version +from yatsm.algorithms import CCDCesque + +n = 50 +# Hack for goof up in API previous to v0.6.0 +if yatsm_version.split('.')[1] == '5': + est = 'lm' +else: + est = 'estimator' + + +class CCDCesqueSuite(object): + + def setup_cache(self): + example_data = os.path.join( + os.path.dirname(__file__), + '../../../tests/algorithms/data/example_timeseries_masked.npz') + + dat = np.load(example_data)['arr_0'].item() + X = dat['X'] + Y = dat['Y'] + dates = dat['dates'] + + kwargs = { + 'test_indices': np.array([2, 3, 4, 5]), + est: sklearn.linear_model.Lasso(alpha=[20]), + 'consecutive': 5, + 'threshold': 4, + 'min_obs': 24, + 'min_rmse': 100, + 'retrain_time': 365.25, + 'screening': 'RLM', + 'screening_crit': 400.0, + 'green_band': 1, + 'swir1_band': 4, + 'remove_noise': False, + 'dynamic_rmse': False, + 'slope_test': False, + 'idx_slope': 1 + } + return {'X': X, 'Y': Y, 'dates': dates, 'kwargs': kwargs} + + def time_ccdcesque1(self, setup): + """ Bench with 'defaults' defined in setup with most tests turned off + """ + for i in range(n): + model = CCDCesque(**setup['kwargs']) + model.fit(setup['X'], setup['Y'], setup['dates']) + + def time_ccdcesque2(self, setup): + """ Bench with remove_noise turned on + """ + kwargs = setup['kwargs'] + kwargs.update({'remove_noise': True}) + for i in range(n): + model = CCDCesque(**kwargs) + model.fit(setup['X'], setup['Y'], setup['dates']) + + def time_ccdcesque3(self, setup): + """ Bench with remove_noise, dynamic_rmse turned on + """ + kwargs = setup['kwargs'] + kwargs.update({'remove_noise': True, + 'dynamic_rmse': True}) + for i in range(n): + model = CCDCesque(**kwargs) + model.fit(setup['X'], setup['Y'], setup['dates']) + + def time_ccdcesque4(self, setup): + """ Bench with remove_noise, dynamic_rmse, slope_test turned on + """ + kwargs = setup['kwargs'] + kwargs.update({'remove_noise': True, + 'dynamic_rmse': True, + 'slope_test': True}) + for i in range(n): + model = CCDCesque(**kwargs) + model.fit(setup['X'], setup['Y'], setup['dates']) diff --git a/requirements/dev.txt b/requirements/dev.txt index f375f680..8fda3686 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -6,3 +6,4 @@ sphinxcontrib-napoleon>=0.2.8 sphinxcontrib-programoutput>=0.8 sphinxcontrib-bibtex>=0.3.1 coverage>=3.7.1 +git+git://github.com/spacetelescope/asv