diff --git a/.markdownlint.yaml b/.markdownlint.yaml new file mode 100644 index 0000000..7420359 --- /dev/null +++ b/.markdownlint.yaml @@ -0,0 +1,16 @@ +# Markdown Linter configuration for docs +# https://github.com/DavidAnson/markdownlint +# https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md +MD009: false # permit trailing spaces +MD007: false # List indenting - permit 4 spaces +MD013: + line_length: "88" # Line length limits + tables: false # disable for tables + headings: false # disable for headings +MD030: false # Number of spaces after a list +MD033: # HTML elements allowed + allowed_elements: + - "br" +MD034: false # Permit bare URLs +MD031: false # Spacing w/code blocks. Conflicts with `??? Note` and code tab styling +MD046: false # Spacing w/code blocks. Conflicts with `??? Note` and code tab styling diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..2c685f6 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,56 @@ +default_stages: [commit, push] +exclude: (^.github/|^docs/|^images/) + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files # prevent giant files from being committed + - id: requirements-txt-fixer + - id: mixed-line-ending + args: ["--fix=lf"] + description: Forces to replace line ending by the UNIX 'lf' character. + + # black + - repo: https://github.com/psf/black + rev: 22.12.0 + hooks: + - id: black + - id: black-jupyter + args: + - --line-length=88 + + # isort + - repo: https://github.com/pycqa/isort + rev: 5.11.2 + hooks: + - id: isort + args: ["--profile", "black"] + description: Sorts imports in an alphabetical order + + # flake8 + - repo: https://github.com/pycqa/flake8 + rev: 4.0.1 + hooks: + - id: flake8 + args: # arguments to configure flake8 + # making isort line length compatible with black + - "--max-line-length=88" + - "--max-complexity=18" + - "--select=B,C,E,F,W,T4,B9" + + # these are errors that will be ignored by flake8 + # https://www.flake8rules.com/rules/{code}.html + - "--ignore=E203,E501,W503,W605" + # E203 - Colons should not have any space before them. + # Needed for list indexing + # E501 - Line lengths are recommended to be no greater than 79 characters. + # Needed as we conform to 88 + # W503 - Line breaks should occur after the binary operator. + # Needed because not compatible with black + # W605 - a backslash-character pair that is not a valid escape sequence now + # generates a DeprecationWarning. This will eventually become a SyntaxError. + # Needed because we use \d as an escape sequence diff --git a/CHANGELOG.md b/CHANGELOG.md index 74a7bd6..867c5f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,16 @@ # Changelog -Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention. +Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and +[Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention. + +## [0.5.0] - 2023-01-09 + ++ Remove - `recursive_search` function ++ Add - pre-commit checks to the repo to observe flake8, black, isort ++ Add - `value_to_bool` and `QuietStdOut` utilities ## [0.4.2] - 2022-12-16 + + Update - PrairieView loader checks for multi-plane vs single-plane scans. ## [0.4.1] - 2022-12-15 @@ -17,10 +25,10 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and ## [0.3.0] - 2022-10-7 -+ Add - Function `prairieviewreader` to parse metadata from Bruker PrarieView acquisition system ++ Add - Function `prairieviewreader` to parse metadata from Bruker PrarieView acquisition + system + Update - Changelog with tag links - ## [0.2.1] - 2022-07-13 + Add - Adopt `black` formatting @@ -33,7 +41,8 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and + Add - Function `run_caiman` to trigger CNMF algorithm. + Add - Function `ingest_csv_to_table` to insert data from CSV files into tables. + Add - Function `recursive_search` to search through nested dictionary for a key. -+ Add - Function `upload_to_dandi` to upload Neurodata Without Borders file to the DANDI platform. ++ Add - Function `upload_to_dandi` to upload Neurodata Without Borders file to the DANDI + platform. + Update - Remove `extras_require` feature to allow this package to be published to PyPI. ## [0.1.0a1] - 2022-01-12 @@ -44,6 +53,7 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and + Add - Readers for: `ScanImage`, `Suite2p`, `CaImAn`. +[0.5.0]: https://github.com/datajoint/element-interface/releases/tag/0.5.0 [0.4.2]: https://github.com/datajoint/element-interface/releases/tag/0.4.2 [0.4.1]: https://github.com/datajoint/element-interface/releases/tag/0.4.1 [0.4.0]: https://github.com/datajoint/element-interface/releases/tag/0.4.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f07cecb..e04d170 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,3 +1,5 @@ # Contribution Guidelines -This project follows the [DataJoint Contribution Guidelines](https://docs.datajoint.io/python/community/02-Contribute.html). Please reference the link for more full details. \ No newline at end of file +This project follows the +[DataJoint Contribution Guidelines](https://datajoint.com/docs/community/contribute/). +Please reference the link for more full details. diff --git a/LICENSE b/LICENSE index b844d82..d394fe3 100644 --- a/LICENSE +++ b/LICENSE @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file +SOFTWARE. diff --git a/README.md b/README.md index 1d6c4eb..cf8342c 100644 --- a/README.md +++ b/README.md @@ -7,5 +7,5 @@ corresponding database tables that can be combined with other Elements to assemb fully functional pipeline. Element Interface is home to a number of utilities that make this possible. -Installation and usage instructions can be found at the +Installation and usage instructions can be found at the [Element documentation](https://datajoint.com/docs/elements/element-interface). diff --git a/cspell.json b/cspell.json new file mode 100644 index 0000000..d715152 --- /dev/null +++ b/cspell.json @@ -0,0 +1,25 @@ +// cSpell Settings +//https://github.com/streetsidesoftware/vscode-spell-checker +{ + "version": "0.2", // Version of the setting file. Always 0.2 + "language": "en", // language - current active spelling language + "enabledLanguageIds": [ + "markdown", + "yaml" + ], + // flagWords - list of words to be always considered incorrect + // This is useful for offensive words and common spelling errors. + // For example "hte" should be "the" + "flagWords": [], + "allowCompoundWords": true, + "ignorePaths": [ + ], + "words": [ + "isort", + "Bruker", + "Neurodata", + "Prairie", + "CNMF", + "deconvolution" + ] +} diff --git a/docs/mkdocs.yaml b/docs/mkdocs.yaml index 74a68e6..e435e30 100644 --- a/docs/mkdocs.yaml +++ b/docs/mkdocs.yaml @@ -43,7 +43,7 @@ nav: # HOST_UID=$(id -u) docker compose -f docs/docker-compose.yaml up --build # ``` # 02. Site analytics depend on a local environment variable GOOGLE_ANALYTICS_KEY -# You can find this in LastPass or declare with any string to suprress errors +# You can find this in LastPass or declare with any string to suppress errors # 03. The API section will pull docstrings. # A. Follow google styleguide e.g., # https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html diff --git a/docs/src/citation.md b/docs/src/citation.md index 7aa6762..4b1ef55 100644 --- a/docs/src/citation.md +++ b/docs/src/citation.md @@ -8,4 +8,4 @@ Resource Identifier (RRID). Neurophysiology. bioRxiv. 2021 Jan 1. doi: https://doi.org/10.1101/2021.03.30.437358 + DataJoint Elements ([RRID:SCR_021894](https://scicrunch.org/resolver/SCR_021894)) - - Element Interface (version {{ PATCH_VERSION }}) \ No newline at end of file + Element Interface (version {{ PATCH_VERSION }}) diff --git a/docs/src/concepts.md b/docs/src/concepts.md index b940c72..c7a592c 100644 --- a/docs/src/concepts.md +++ b/docs/src/concepts.md @@ -11,26 +11,30 @@ across other packages, without causing issues in the respective Element. ### General utilities -`utils.find_full_path` and `utils.find_root_directory` are used -across many Elements and Workflows to allow for the flexibility of providing +`utils.find_full_path` and `utils.find_root_directory` are used +across many Elements and Workflows to allow for the flexibility of providing one or more root directories in the user's config, and extrapolating from a relative path at runtime. -`utils.ingest_csv_to_table` is used across workflow examples to ingest from sample data from -local CSV files into sets of manual tables. While researchers may wish to manually +`utils.ingest_csv_to_table` is used across workflow examples to ingest from sample data +from local CSV files into sets of manual tables. While researchers may wish to manually insert for day-to-day operations, it helps to have a more complete dataset when learning how to use various Elements. +`utils.str_to_bool` converts a set of strings to boolean True or False. This is implemented +as the equivalent item in Python's `distutils` which will be removed in future versions. + ### Suite2p This Element provides functions to independently run Suite2p's motion correction, segmentation, and deconvolution steps. These functions currently work for single plane -tiff files. If one is running all Suite2p pre-processing steps concurrently, these functions -are not required and one can run `suite2p.run_s2p()`. The wrapper functions here were developed primarily because `run_s2p` cannot individually -run deconvolution using the `spikedetect` flag ( +tiff files. If one is running all Suite2p pre-processing steps concurrently, these +functions are not required and one can run `suite2p.run_s2p()`. The wrapper functions +here were developed primarily because `run_s2p` cannot individually run deconvolution +using the `spikedetect` flag ( [Suite2p Issue #718](https://github.com/MouseLand/suite2p/issues/718)). -**Requirements** +Requirements: - [ops dictionary](https://suite2p.readthedocs.io/en/latest/settings.html) @@ -42,13 +46,13 @@ run deconvolution using the `spikedetect` flag ( ### PrairieView Reader -This Element provides a function to read the PrairieView Scanner's metadata -file. The PrairieView software generates one `.ome.tif` imaging file per frame acquired. The -metadata for all frames is contained in one `.xml` file. This function locates the `.xml` -file and generates a dictionary necessary to populate the DataJoint ScanInfo and -Field tables. PrairieView works with resonance scanners with a single field, -does not support bidirectional x and y scanning, and the `.xml` file does not -contain ROI information. +This Element provides a function to read the PrairieView Scanner's metadata file. The +PrairieView software generates one `.ome.tif` imaging file per frame acquired. The +metadata for all frames is contained in one `.xml` file. This function locates the +`.xml` file and generates a dictionary necessary to populate the DataJoint ScanInfo and +Field tables. PrairieView works with resonance scanners with a single field, does not +support bidirectional x and y scanning, and the `.xml` file does not contain ROI +information. ## Element Architecture @@ -58,9 +62,9 @@ module. - Acquisition packages: [ScanImage](../api/element_interface/scanimage_utils) - Analysis packages: - - Suite2p [loader](../api/element_interface/suite2p_loader) and [trigger](../api/element_interface/suite2p_trigger) - - - CaImAn [loader](../api/element_interface/caiman_loader) and [trigger](../api/element_interface/run_caiman) + - Suite2p [loader](../api/element_interface/suite2p_loader) and [trigger](../api/element_interface/suite2p_trigger) + + - CaImAn [loader](../api/element_interface/caiman_loader) and [trigger](../api/element_interface/run_caiman) - Data upload: [DANDI](../api/element_interface/dandi/) @@ -68,4 +72,4 @@ module. Further development of this Element is community driven. Upon user requests and based on guidance from the Scientific Steering Group we will additional features to -this Element. \ No newline at end of file +this Element. diff --git a/element_interface/caiman_loader.py b/element_interface/caiman_loader.py index 2fcab79..3726afd 100644 --- a/element_interface/caiman_loader.py +++ b/element_interface/caiman_loader.py @@ -1,12 +1,12 @@ -import h5py -import caiman as cm -import scipy -import numpy as np -from datetime import datetime import os import pathlib -from tqdm import tqdm +from datetime import datetime +import caiman as cm +import h5py +import numpy as np +import scipy +from tqdm import tqdm _required_hdf5_fields = [ "/motion_correction/reference_image", @@ -122,7 +122,7 @@ def extract_masks(self) -> dict: mask_xpix, mask_ypix, mask_zpix, inferred_trace, dff, spikes """ if self.params.motion["is3D"]: - raise NotImplemented( + raise NotImplementedError( "CaImAn mask extraction for volumetric data not yet implemented" ) @@ -166,8 +166,8 @@ def _process_scanimage_tiff(scan_filenames, output_dir="./"): Read ScanImage TIFF - reshape into volumetric data based on scanning depths/channels Save new TIFF files for each channel - with shape (frame x height x width x depth) """ - from tifffile import imsave import scanreader + from tifffile import imsave # ------------ CaImAn multi-channel multi-plane tiff file ------------ for scan_filename in tqdm(scan_filenames): diff --git a/element_interface/dandi.py b/element_interface/dandi.py index 2c2a2f5..decaeaa 100644 --- a/element_interface/dandi.py +++ b/element_interface/dandi.py @@ -1,5 +1,6 @@ import os import subprocess + from dandi.download import download from dandi.upload import upload diff --git a/element_interface/extract_loader.py b/element_interface/extract_loader.py index ca31949..278ee5b 100644 --- a/element_interface/extract_loader.py +++ b/element_interface/extract_loader.py @@ -1,7 +1,8 @@ import os -import numpy as np -from pathlib import Path from datetime import datetime +from pathlib import Path + +import numpy as np class EXTRACT_loader: @@ -18,7 +19,7 @@ def __init__(self, extract_dir: str): try: extract_file = next(Path(extract_dir).glob("*_extract_output.mat")) - except StopInteration: + except StopInteration: # noqa F821 raise FileNotFoundError( f"EXTRACT output result file is not found at {extract_dir}." ) @@ -31,7 +32,7 @@ def __init__(self, extract_dir: str): def load_results(self): """Load the EXTRACT results - + Returns: masks (dict): Details of the masks identified with the EXTRACT segmentation package. """ diff --git a/element_interface/extract_trigger.py b/element_interface/extract_trigger.py index 7541204..103e3e0 100644 --- a/element_interface/extract_trigger.py +++ b/element_interface/extract_trigger.py @@ -1,8 +1,7 @@ import os -from typing import Union from pathlib import Path from textwrap import dedent -from datetime import datetime +from typing import Union class EXTRACT_trigger: @@ -11,11 +10,11 @@ class EXTRACT_trigger: % Load Data data = load('{scanfile}'); M = data.M; - + % Input Paramaters config = struct(); {parameters_list_string} - + % Run EXTRACT output = extractor(M, config); save('{output_fullpath}', 'output'); diff --git a/element_interface/prairieviewreader.py b/element_interface/prairieviewreader.py index ef09f18..a4c6979 100644 --- a/element_interface/prairieviewreader.py +++ b/element_interface/prairieviewreader.py @@ -1,6 +1,7 @@ import pathlib import xml.etree.ElementTree as ET from datetime import datetime + import numpy as np @@ -43,8 +44,7 @@ def get_pv_metadata(pvtiffile: str) -> dict: bidirectional_scan = False # Does not support bidirectional roi = 1 n_fields = 1 # Always contains 1 field - record_start_time = root.find( - ".//Sequence/[@cycle='1']").attrib.get("time") + record_start_time = root.find(".//Sequence/[@cycle='1']").attrib.get("time") # Get all channels and find unique values channel_list = [ @@ -54,8 +54,7 @@ def get_pv_metadata(pvtiffile: str) -> dict: n_channels = len(set(channel_list)) n_frames = len(root.findall(".//Sequence/Frame")) framerate = 1 / float( - root.findall( - './/PVStateValue/[@key="framePeriod"]')[0].attrib.get("value") + root.findall('.//PVStateValue/[@key="framePeriod"]')[0].attrib.get("value") ) # rate = 1/framePeriod usec_per_line = ( @@ -67,16 +66,14 @@ def get_pv_metadata(pvtiffile: str) -> dict: * 1e6 ) # Convert from seconds to microseconds - scan_datetime = datetime.strptime( - root.attrib.get("date"), "%m/%d/%Y %I:%M:%S %p") + scan_datetime = datetime.strptime(root.attrib.get("date"), "%m/%d/%Y %I:%M:%S %p") total_duration = float( root.findall(".//Sequence/Frame")[-1].attrib.get("relativeTime") ) px_height = int( - root.findall( - ".//PVStateValue/[@key='pixelsPerLine']")[0].attrib.get("value") + root.findall(".//PVStateValue/[@key='pixelsPerLine']")[0].attrib.get("value") ) # All PrairieView-acquired images have square dimensions (512 x 512; 1024 x 1024) px_width = px_height @@ -100,11 +97,17 @@ def get_pv_metadata(pvtiffile: str) -> dict: ".//PVStateValue/[@key='currentScanCenter']/IndexedValue/[@index='YAxis']" ).attrib.get("value") ) - if root.find(".//Sequence/[@cycle='1']/Frame/PVStateShard/PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']") is None: + if ( + root.find( + ".//Sequence/[@cycle='1']/Frame/PVStateShard/PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']" + ) + is None + ): z_fields = np.float64( root.find( - ".//PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']/SubindexedValue").attrib.get("value") + ".//PVStateValue/[@key='positionCurrent']/SubindexedValues/[@index='ZAxis']/SubindexedValue" + ).attrib.get("value") ) n_depths = 1 assert z_fields.size == n_depths @@ -112,8 +115,7 @@ def get_pv_metadata(pvtiffile: str) -> dict: else: - bidirection_z = bool( - root.find(".//Sequence").attrib.get("bidirectionalZ")) + bidirection_z = bool(root.find(".//Sequence").attrib.get("bidirectionalZ")) # One "Frame" per depth. Gets number of frames in first sequence planes = [ diff --git a/element_interface/run_caiman.py b/element_interface/run_caiman.py index 31ad6de..eb480a9 100644 --- a/element_interface/run_caiman.py +++ b/element_interface/run_caiman.py @@ -1,14 +1,15 @@ -import cv2 import pathlib +import cv2 + try: cv2.setNumThreads(0) -except: - pass +except: # noqa E722 + pass # TODO: remove bare except import caiman as cm -from caiman.source_extraction.cnmf.cnmf import * from caiman.source_extraction.cnmf import params as params +from caiman.source_extraction.cnmf.cnmf import CNMF from .caiman_loader import _save_mc diff --git a/element_interface/scanimage_utils.py b/element_interface/scanimage_utils.py index 43764c6..29eac26 100644 --- a/element_interface/scanimage_utils.py +++ b/element_interface/scanimage_utils.py @@ -22,8 +22,8 @@ def parse_scanimage_header(scan): key, value = item.split(" = ") key = re.sub("^scanimage_", "", key.replace(".", "_")) header[key] = value - except: - pass + except: # noqa E722 + pass # TODO: remove bare except return header diff --git a/element_interface/suite2p_loader.py b/element_interface/suite2p_loader.py index ff2a1b1..07dbff1 100644 --- a/element_interface/suite2p_loader.py +++ b/element_interface/suite2p_loader.py @@ -1,8 +1,8 @@ -import numpy as np import pathlib -from datetime import datetime from collections import OrderedDict +from datetime import datetime +import numpy as np _suite2p_ftypes = ( "ops", diff --git a/element_interface/suite2p_trigger.py b/element_interface/suite2p_trigger.py index 75134ba..e4e984d 100644 --- a/element_interface/suite2p_trigger.py +++ b/element_interface/suite2p_trigger.py @@ -1,8 +1,9 @@ -import suite2p -import numpy as np import os import warnings +import numpy as np +import suite2p + def motion_correction_suite2p(ops: dict, db: dict) -> tuple: """Performs motion correction (i.e. registration) using the Suite2p package. diff --git a/element_interface/utils.py b/element_interface/utils.py index 22e44ed..14d4eee 100644 --- a/element_interface/utils.py +++ b/element_interface/utils.py @@ -1,7 +1,14 @@ +import csv +import hashlib +import logging +import os import pathlib +import sys import uuid -import hashlib -import csv + +from datajoint.utils import to_camel_case + +logger = logging.getLogger("datajoint") def find_full_path(root_directories: list, relative_path: str) -> pathlib.PosixPath: @@ -140,31 +147,43 @@ def ingest_csv_to_table( ) if verbose: insert_len = len(table) - prev_len - print( + logger.info( f"\n---- Inserting {insert_len} entry(s) " - + f"into {table.table_name} ----" + + f"into {to_camel_case(table.table_name)} ----" ) -def recursive_search(key: str, dictionary: dict) -> any: - """Return value for key in a nested dictionary - - Search through a nested dictionary for a key and returns its value. If there are - more than one key with the same name at different depths, the algorithm returns the - value of the least nested key. +def value_to_bool(value) -> bool: + """Return whether the provided value represents true. Otherwise false. Args: - key (str): Key used to search through a nested dictionary - dictionary (dict): Nested dictionary + value (str, bool, int): Any input Returns: - value (any): value of the input argument `key` + bool (bool): True if value in ("y", "yes", "t", "true", "on", "1") """ - if key in dictionary: - return dictionary[key] - for value in dictionary.values(): - if isinstance(value, dict): - a = recursive_search(key, value) - if a is not None: - return a - return None + if not value: + return False + return str(value).lower() in ("y", "yes", "t", "true", "on", "1") + + +class QuietStdOut: + """Context for quieting standard output, and setting datajoint loglevel to warning + + Used in pytest functions to render clear output showing only pass/fail + + Example: + with QuietStdOut(): + table.delete(safemode=False) + """ + + def __enter__(self): + self.prev_log_level = logger.level + logger.setLevel(30) # set DataJoint logger to warning + self._original_stdout = sys.stdout + sys.stdout = open(os.devnull, "w") + + def __exit__(self, *args): + logger.setLevel(self.prev_log_level) + sys.stdout.close() + sys.stdout = self._original_stdout diff --git a/element_interface/version.py b/element_interface/version.py index 2f158f7..20b1b3e 100644 --- a/element_interface/version.py +++ b/element_interface/version.py @@ -1,3 +1,3 @@ """Package metadata""" -__version__ = "0.4.2" +__version__ = "0.5.0" diff --git a/requirements.txt b/requirements.txt index 3c0f334..75d95e8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -numpy dandi +numpy diff --git a/setup.py b/setup.py index 40658d8..9f874b0 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,8 @@ #!/usr/bin/env python -from setuptools import setup, find_packages -from os import path import urllib.request +from os import path + +from setuptools import find_packages, setup pkg_name = next(p for p in find_packages() if "." not in p) here = path.abspath(path.dirname(__file__)) @@ -17,7 +18,7 @@ setup( name=pkg_name.replace("_", "-"), - version=__version__, + version=__version__, # noqa F821 description="Loaders of neurophysiological data into the DataJoint Elements", long_description=long_description, long_description_content_type="text/markdown",