From 83fbcf1741655dde639046dbd1dbc6ce3dfff120 Mon Sep 17 00:00:00 2001 From: Stefan Appelhoff Date: Mon, 5 Aug 2019 22:19:21 +0200 Subject: [PATCH 01/11] ENH: func to extract list of instances of a certain entity from BIDS ds Co-authored-by: Mainak Jas --- mne_bids/utils.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/mne_bids/utils.py b/mne_bids/utils.py index 0d180bb26..293d33611 100644 --- a/mne_bids/utils.py +++ b/mne_bids/utils.py @@ -16,6 +16,7 @@ import re from datetime import datetime from collections import defaultdict +from pathlib import Path import numpy as np from mne import read_events, find_events, events_from_annotations @@ -28,6 +29,23 @@ from mne_bids.tsv_handler import _to_tsv, _tsv_to_str +def get_list_of_entity(bids_root, key='sub'): + """Get list of a particular entity.""" + accepted_keys = ('sub', 'ses', 'run', 'acq') + if key not in accepted_keys: + raise ValueError('Key must be one of {}. Got {}' + .format(accepted_keys, key)) + + p = re.compile(r'%s-(.*?)_' % key) + entities = list() + for filename in Path(bids_root).rglob('*{}-*_*'.format(key)): + match = p.search(filename.stem) + entity = match.group(1) + if entity not in entities: + entities.append(entity) + return entities + + def _get_ch_type_mapping(fro='mne', to='bids'): """Map between BIDS and MNE nomenclatures for channel types. From 4dafc042646e8fdc709dc3e1a0cc3b9ac3070db7 Mon Sep 17 00:00:00 2001 From: Stefan Appelhoff Date: Mon, 5 Aug 2019 22:21:48 +0200 Subject: [PATCH 02/11] add test --- mne_bids/tests/test_utils.py | 55 ++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/mne_bids/tests/test_utils.py b/mne_bids/tests/test_utils.py index b90fb433f..f4be0fc49 100644 --- a/mne_bids/tests/test_utils.py +++ b/mne_bids/tests/test_utils.py @@ -19,7 +19,7 @@ _infer_eeg_placement_scheme, _handle_kind, _find_matching_sidecar, _parse_ext, _get_ch_type_mapping, _parse_bids_filename, - _find_best_candidates) + _find_best_candidates, get_list_of_entity) base_path = op.join(op.dirname(mne.__file__), 'io') @@ -34,6 +34,40 @@ task=task) +@pytest.fixture(scope='session') +def return_bids_test_dir(tmpdir_factory): + """Return path to a written test BIDS dir.""" + output_path = str(tmpdir_factory.mktemp('mnebids_utils_test_bids_ds')) + data_path = testing.data_path() + raw_fname = op.join(data_path, 'MEG', 'sample', + 'sample_audvis_trunc_raw.fif') + + event_id = {'Auditory/Left': 1, 'Auditory/Right': 2, 'Visual/Left': 3, + 'Visual/Right': 4, 'Smiley': 5, 'Button': 32} + events_fname = op.join(data_path, 'MEG', 'sample', + 'sample_audvis_trunc_raw-eve.fif') + + raw = mne.io.read_raw_fif(raw_fname) + with pytest.warns(UserWarning, match='No line frequency'): + write_raw_bids(raw, bids_basename, output_path, + events_data=events_fname, event_id=event_id, + overwrite=False) + + return output_path + + +def test_get_list_of_entity(return_bids_test_dir): + """Test getting a list of entities.""" + bids_root = return_bids_test_dir + with pytest.raises(ValueError, match='Key must be one of'): + get_list_of_entity(bids_root, key='bogus') + + assert get_list_of_entity(bids_root, 'sub') == [subject_id] + assert get_list_of_entity(bids_root, 'ses') == [session_id] + assert get_list_of_entity(bids_root, 'run') == [run] + assert get_list_of_entity(bids_root, 'acq') == [acq] + + def test_get_ch_type_mapping(): """Test getting a correct channel mapping.""" with pytest.raises(ValueError, match='specified from "bogus" to "mne"'): @@ -239,24 +273,9 @@ def test_find_best_candidates(candidate_list, best_candidates): assert _find_best_candidates(params, candidate_list) == best_candidates -def test_find_matching_sidecar(): +def test_find_matching_sidecar(return_bids_test_dir): """Test finding a sidecar file from a BIDS dir.""" - # First write a BIDS dir - output_path = _TempDir() - data_path = testing.data_path() - raw_fname = op.join(data_path, 'MEG', 'sample', - 'sample_audvis_trunc_raw.fif') - - event_id = {'Auditory/Left': 1, 'Auditory/Right': 2, 'Visual/Left': 3, - 'Visual/Right': 4, 'Smiley': 5, 'Button': 32} - events_fname = op.join(data_path, 'MEG', 'sample', - 'sample_audvis_trunc_raw-eve.fif') - - raw = mne.io.read_raw_fif(raw_fname) - with pytest.warns(UserWarning, match='No line frequency'): - write_raw_bids(raw, bids_basename, output_path, - events_data=events_fname, event_id=event_id, - overwrite=False) + output_path = return_bids_test_dir # Now find a sidecar sidecar_fname = _find_matching_sidecar(bids_basename, output_path, From 5378ca947ebed8b0ccb284d738a1d587a99c68a8 Mon Sep 17 00:00:00 2001 From: Stefan Appelhoff Date: Mon, 5 Aug 2019 22:25:41 +0200 Subject: [PATCH 03/11] change str formatting --- mne_bids/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mne_bids/utils.py b/mne_bids/utils.py index 293d33611..b4fbd28ce 100644 --- a/mne_bids/utils.py +++ b/mne_bids/utils.py @@ -36,7 +36,7 @@ def get_list_of_entity(bids_root, key='sub'): raise ValueError('Key must be one of {}. Got {}' .format(accepted_keys, key)) - p = re.compile(r'%s-(.*?)_' % key) + p = re.compile(r'{}-(.*?)_'.format(key)) entities = list() for filename in Path(bids_root).rglob('*{}-*_*'.format(key)): match = p.search(filename.stem) From 3216ad41d19c173290ce0c0fc5426bb0cd8e628a Mon Sep 17 00:00:00 2001 From: Stefan Appelhoff Date: Mon, 5 Aug 2019 22:31:25 +0200 Subject: [PATCH 04/11] add whatsnew and api --- doc/api.rst | 1 + doc/whats_new.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/doc/api.rst b/doc/api.rst index 43ced2ed6..f17e7f676 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -37,6 +37,7 @@ Utils (:py:mod:`mne_bids.utils`) :toctree: generated/ print_dir_tree + get_list_of_entity Copyfiles (:py:mod:`mne_bids.copyfiles`) ======================================== diff --git a/doc/whats_new.rst b/doc/whats_new.rst index dd17abaeb..447cac9d7 100644 --- a/doc/whats_new.rst +++ b/doc/whats_new.rst @@ -22,6 +22,7 @@ Current Changelog ~~~~~~~~~ +- New function :func:`mne_bids.utils.get_list_of_entity` allows to get a list of instances of a certain entity in a BIDS directory, by `Mainak Jas`_ and `Stefan Appelhoff`_ (`#252 `_) - :func:`mne_bids.utils.print_dir_tree` now accepts an argument :code:`max_depth` which can limit the depth until which the directory tree is printed, by `Stefan Appelhoff`_ (`#245 `_) - New command line function exposed :code:`cp` for renaming/copying files including automatic doc generation "CLI", by `Stefan Appelhoff`_ (`#225 `_) - :func:`read_raw_bids` now also reads channels.tsv files accompanying a raw BIDS file and sets the channel types accordingly, by `Stefan Appelhoff`_ (`#219 `_) From a9e054eafbd654bf5fdd839432a15242e150ef44 Mon Sep 17 00:00:00 2001 From: Stefan Appelhoff Date: Mon, 5 Aug 2019 22:35:59 +0200 Subject: [PATCH 05/11] add docstr --- mne_bids/utils.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/mne_bids/utils.py b/mne_bids/utils.py index b4fbd28ce..7bd9e8355 100644 --- a/mne_bids/utils.py +++ b/mne_bids/utils.py @@ -30,10 +30,26 @@ def get_list_of_entity(bids_root, key='sub'): - """Get list of a particular entity.""" + """Get list of instances of a particular entity. + + Parameters + ---------- + bids_root : str + Path to the root of the BIDS directory. + key : str + The name of the entity to search for. Can be one of + ['sub', 'ses', 'run', 'acq']. Defaults to 'sub'. + + Returns + ------- + entities : list of str + List of the instances of the entity given by `key` in the BIDS + dataset pointed to by `bids_root`. + + """ accepted_keys = ('sub', 'ses', 'run', 'acq') if key not in accepted_keys: - raise ValueError('Key must be one of {}. Got {}' + raise ValueError('`key` must be one of "{}". Got "{}"' .format(accepted_keys, key)) p = re.compile(r'{}-(.*?)_'.format(key)) From 641833781035951b7c8ac1b57c4fd45f68fad9a7 Mon Sep 17 00:00:00 2001 From: Stefan Appelhoff Date: Tue, 6 Aug 2019 11:17:07 +0200 Subject: [PATCH 06/11] rename func, improve tests, return sorted vals --- doc/api.rst | 2 +- doc/whats_new.rst | 2 +- mne_bids/tests/test_utils.py | 33 +++++++++++++++++++-------------- mne_bids/utils.py | 30 ++++++++++++++++++------------ 4 files changed, 39 insertions(+), 28 deletions(-) diff --git a/doc/api.rst b/doc/api.rst index f17e7f676..5d95f9d70 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -37,7 +37,7 @@ Utils (:py:mod:`mne_bids.utils`) :toctree: generated/ print_dir_tree - get_list_of_entity + get_values_for_key Copyfiles (:py:mod:`mne_bids.copyfiles`) ======================================== diff --git a/doc/whats_new.rst b/doc/whats_new.rst index 447cac9d7..426afbd53 100644 --- a/doc/whats_new.rst +++ b/doc/whats_new.rst @@ -22,7 +22,7 @@ Current Changelog ~~~~~~~~~ -- New function :func:`mne_bids.utils.get_list_of_entity` allows to get a list of instances of a certain entity in a BIDS directory, by `Mainak Jas`_ and `Stefan Appelhoff`_ (`#252 `_) +- New function :func:`mne_bids.utils.get_values_for_key` allows to get a list of instances of a certain entity in a BIDS directory, by `Mainak Jas`_ and `Stefan Appelhoff`_ (`#252 `_) - :func:`mne_bids.utils.print_dir_tree` now accepts an argument :code:`max_depth` which can limit the depth until which the directory tree is printed, by `Stefan Appelhoff`_ (`#245 `_) - New command line function exposed :code:`cp` for renaming/copying files including automatic doc generation "CLI", by `Stefan Appelhoff`_ (`#225 `_) - :func:`read_raw_bids` now also reads channels.tsv files accompanying a raw BIDS file and sets the channel types accordingly, by `Stefan Appelhoff`_ (`#219 `_) diff --git a/mne_bids/tests/test_utils.py b/mne_bids/tests/test_utils.py index f4be0fc49..71597ad49 100644 --- a/mne_bids/tests/test_utils.py +++ b/mne_bids/tests/test_utils.py @@ -19,14 +19,14 @@ _infer_eeg_placement_scheme, _handle_kind, _find_matching_sidecar, _parse_ext, _get_ch_type_mapping, _parse_bids_filename, - _find_best_candidates, get_list_of_entity) + _find_best_candidates, get_values_for_key) base_path = op.join(op.dirname(mne.__file__), 'io') subject_id = '01' session_id = '01' run = '01' -acq = '01' +acq = None task = 'testing' bids_basename = make_bids_basename( @@ -48,24 +48,29 @@ def return_bids_test_dir(tmpdir_factory): 'sample_audvis_trunc_raw-eve.fif') raw = mne.io.read_raw_fif(raw_fname) - with pytest.warns(UserWarning, match='No line frequency'): - write_raw_bids(raw, bids_basename, output_path, - events_data=events_fname, event_id=event_id, - overwrite=False) + # Write multiple runs for test_purposes + bids_basename2 = bids_basename.replace('run-{}'.format(run), 'run-02') + for name in [bids_basename, + bids_basename2, + ]: + with pytest.warns(UserWarning, match='No line frequency'): + write_raw_bids(raw, name, output_path, + events_data=events_fname, event_id=event_id, + overwrite=True) return output_path -def test_get_list_of_entity(return_bids_test_dir): +def test_get_values_for_key(return_bids_test_dir): """Test getting a list of entities.""" bids_root = return_bids_test_dir - with pytest.raises(ValueError, match='Key must be one of'): - get_list_of_entity(bids_root, key='bogus') + with pytest.raises(ValueError, match='`key` must be one of'): + get_values_for_key(bids_root, key='bogus') - assert get_list_of_entity(bids_root, 'sub') == [subject_id] - assert get_list_of_entity(bids_root, 'ses') == [session_id] - assert get_list_of_entity(bids_root, 'run') == [run] - assert get_list_of_entity(bids_root, 'acq') == [acq] + assert get_values_for_key(bids_root, 'sub') == [subject_id] + assert get_values_for_key(bids_root, 'ses') == [session_id] + assert get_values_for_key(bids_root, 'run') == [run, '02'] + assert get_values_for_key(bids_root, 'acq') == [] def test_get_ch_type_mapping(): @@ -281,7 +286,7 @@ def test_find_matching_sidecar(return_bids_test_dir): sidecar_fname = _find_matching_sidecar(bids_basename, output_path, 'coordsystem.json') expected_file = op.join('sub-01', 'ses-01', 'meg', - 'sub-01_ses-01_acq-01_coordsystem.json') + 'sub-01_ses-01_coordsystem.json') assert sidecar_fname.endswith(expected_file) # Find multiple sidecars, tied in score, triggering an error diff --git a/mne_bids/utils.py b/mne_bids/utils.py index 7bd9e8355..c03bc8de3 100644 --- a/mne_bids/utils.py +++ b/mne_bids/utils.py @@ -29,22 +29,28 @@ from mne_bids.tsv_handler import _to_tsv, _tsv_to_str -def get_list_of_entity(bids_root, key='sub'): - """Get list of instances of a particular entity. +def get_values_for_key(bids_root, key): + """Get list of values for a key in a BIDS dataset. + + BIDS file names are organized by key-value pairs called "entities" [1]_. Parameters ---------- bids_root : str Path to the root of the BIDS directory. key : str - The name of the entity to search for. Can be one of - ['sub', 'ses', 'run', 'acq']. Defaults to 'sub'. + The name of the key to search for. Can be one of + ['sub', 'ses', 'run', 'acq']. Returns ------- - entities : list of str - List of the instances of the entity given by `key` in the BIDS - dataset pointed to by `bids_root`. + value_list : list of str + List of the values associated with a `key` in the BIDS dataset pointed + to by `bids_root`. + + References + ---------- + .. [1] https://bids-specification.rtfd.io/en/latest/02-common-principles.html#file-name-structure # noqa: E501 """ accepted_keys = ('sub', 'ses', 'run', 'acq') @@ -53,13 +59,13 @@ def get_list_of_entity(bids_root, key='sub'): .format(accepted_keys, key)) p = re.compile(r'{}-(.*?)_'.format(key)) - entities = list() + value_list = list() for filename in Path(bids_root).rglob('*{}-*_*'.format(key)): match = p.search(filename.stem) - entity = match.group(1) - if entity not in entities: - entities.append(entity) - return entities + value = match.group(1) + if value not in value_list: + value_list.append(value) + return sorted(value_list) def _get_ch_type_mapping(fro='mne', to='bids'): From 3dc4472028b1a4bcca5b21b52e1b10edecf656ab Mon Sep 17 00:00:00 2001 From: Stefan Appelhoff Date: Tue, 6 Aug 2019 11:17:38 +0200 Subject: [PATCH 07/11] bump nodejs in CIs to 10.16.1 (LTS) --- .travis.yml | 4 ++-- appveyor.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1b23a5f72..c90069dad 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ node_js: - - "10.0.0" + - "10.16.1" language: python # Specify version of BIDS-validator to be used, 'master', or 'stable' @@ -16,7 +16,7 @@ before_install: - export PATH=/home/travis/miniconda/bin:$PATH - conda update --yes --quiet conda - npm install -g npm stable - - npm install -g node@10.0.0 + - npm install -g node@10.16.1 - npm --version - node --version - yarn --version diff --git a/appveyor.yml b/appveyor.yml index d4518df84..3501556d0 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -7,7 +7,7 @@ environment: VALIDATOR_VERSION: "master" MNE_VERSION: "master" # can be "maint/0.18" ... or "master" VALIDATOR_EXECUTABLE: "C:\\projects\\mne-bids\\bids-validator\\bids-validator\\bin\\bids-validator" - NODEJS_VERSION: "10.0.0" + NODEJS_VERSION: "10.16.1" PYTHON: "C:\\conda" matrix: - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 From 57bb685f63e57772a9cfeb9dc527e90f156a5725 Mon Sep 17 00:00:00 2001 From: Stefan Appelhoff Date: Tue, 6 Aug 2019 16:34:22 +0200 Subject: [PATCH 08/11] try to update LTS nodejs on appveyor --- appveyor.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 3501556d0..fcadf6621 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -15,7 +15,8 @@ environment: PYTHON_ARCH: "64" install: - - ps: Install-Product node $env:NODEJS_VERSION + # https://www.appveyor.com/docs/lang/nodejs-iojs/ + - ps: Update-NodeJsInstallation node $env:NODEJS_VERSION - node --version - npm --version - yarn --version From 9f3e9882f3d18ab8eb2827288696897119423178 Mon Sep 17 00:00:00 2001 From: Stefan Appelhoff Date: Tue, 6 Aug 2019 16:37:49 +0200 Subject: [PATCH 09/11] fix typo --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index fcadf6621..5120756df 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -16,7 +16,7 @@ environment: install: # https://www.appveyor.com/docs/lang/nodejs-iojs/ - - ps: Update-NodeJsInstallation node $env:NODEJS_VERSION + - ps: Update-NodeJsInstallation $env:NODEJS_VERSION - node --version - npm --version - yarn --version From 227ecb43ead0df70545e4a099cfa35c265d1568e Mon Sep 17 00:00:00 2001 From: Stefan Appelhoff Date: Wed, 7 Aug 2019 14:40:43 +0200 Subject: [PATCH 10/11] try downgrading bids-validator version --- .travis.yml | 18 +++++++++--------- appveyor.yml | 21 ++++++++++++--------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index c90069dad..f5f5e8820 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,10 @@ language: python # Specify version of BIDS-validator to be used, 'master', or 'stable' env: global: - - VALIDATOR_VERSION="master" - - MNE_VERSION="master" # can be "maint/0.18" ... or "master" + # can be "stable", or anything that can be used with git checkout + - VALIDATOR_VERSION="a7e2fddeec489a8d5db81c561a5adc89363a306e" + # can be "maint/0.18" ... or "master" + - MNE_VERSION="master" before_install: @@ -24,18 +26,16 @@ before_install: install: - echo $PATH - | - if [ $VALIDATOR_VERSION == 'master' ];then + if [ $VALIDATOR_VERSION == 'stable' ];then + npm install -g bids-validator + else pushd ~ - git clone --depth 1 https://github.com/bids-standard/bids-validator + git clone https://github.com/bids-standard/bids-validator cd bids-validator + git checkout $VALIDATOR_VERSION yarn export PATH=~/bids-validator/bids-validator/bin:$PATH popd - elif [ $VALIDATOR_VERSION == 'stable' ];then - npm install -g bids-validator - else - echo "VALIDATOR_VERSION should be set to master or stable" - echo "but VALIDATOR_VERSION=$VALIDATOR_VERSION" fi - echo $PATH - bids-validator --version diff --git a/appveyor.yml b/appveyor.yml index 5120756df..d86f0c11c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,11 +1,13 @@ environment: -# Specify version of BIDS-validator to be used, "master", or "stable" +# Specify version of BIDS-validator to be used, "stable", or anything that can be used with git checkout # NOTE: For "master" you MUST adjust VALIDATOR_EXECUTABLE to the following value: # "C:\\projects\\mne-bids\\bids-validator\\bids-validator\\bin\\bids-validator" # ... whereas for "stable", VALIDATOR_EXECUTABLE MUST be set to "n/a" global: - VALIDATOR_VERSION: "master" - MNE_VERSION: "master" # can be "maint/0.18" ... or "master" + # can be + VALIDATOR_VERSION: "a7e2fddeec489a8d5db81c561a5adc89363a306e" + # can be "maint/0.18" ... or "master" + MNE_VERSION: "master" VALIDATOR_EXECUTABLE: "C:\\projects\\mne-bids\\bids-validator\\bids-validator\\bin\\bids-validator" NODEJS_VERSION: "10.16.1" PYTHON: "C:\\conda" @@ -20,17 +22,18 @@ install: - node --version - npm --version - yarn --version - - cmd: if [%VALIDATOR_VERSION%]==[master] ( - git clone --depth 1 https://github.com/bids-standard/bids-validator && - cd bids-validator && - yarn && - cd .. && - set PATH="%PATH%;C:\projects\mne-bids\bids-validator\bids-validator\bin\" ) - cmd: if [%VALIDATOR_VERSION%]==[stable] ( npm install -g bids-validator bids-validator --version which bids-validator ) + - cmd: if [%VALIDATOR_VERSION%]!=[stable] ( + git clone https://github.com/bids-standard/bids-validator && + cd bids-validator && + git checkout %VALIDATOR_VERSION% && + yarn && + cd .. && + set PATH="%PATH%;C:\projects\mne-bids\bids-validator\bids-validator\bin\" ) - "git clone --depth 1 git://github.com/astropy/ci-helpers.git" - "powershell ci-helpers/appveyor/install-miniconda.ps1" - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" From ab45051dcc9082b8e3aa43a15c5b9474972a2baf Mon Sep 17 00:00:00 2001 From: Stefan Appelhoff Date: Wed, 7 Aug 2019 14:44:18 +0200 Subject: [PATCH 11/11] use else --- appveyor.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index d86f0c11c..34e78c45e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -26,8 +26,7 @@ install: npm install -g bids-validator bids-validator --version which bids-validator - ) - - cmd: if [%VALIDATOR_VERSION%]!=[stable] ( + ) else ( git clone https://github.com/bids-standard/bids-validator && cd bids-validator && git checkout %VALIDATOR_VERSION% &&