Merge branch 'development' into modern_path_handling

Makefile

@@ -0,0 +1,160 @@
+#!/usr/bin/make
+# 2024 Nicolas Gampierakis
+
+LIBNAME=cosipy
+
+ifeq (, $(shell which python ))
+  $(error "PYTHON=$(PYTHON) not found in $(PATH)")
+endif
+PYTHON=$(shell which python)
+PYTHON_VERSION=$(shell $(PYTHON) -c "import sys;\
+	version='{v[0]}.{v[1]}'.format(v=list(sys.version_info[:2]));\
+	sys.stdout.write(version)")
+PYTHON_CHECK_MAJOR=$(shell $(PYTHON) -c 'import sys;\
+  	print(int(float("%d"% sys.version_info.major) >= 3))')
+PYTHON_CHECK_MINOR=$(shell $(PYTHON) -c 'import sys;\
+  	print(int(12 >= float("%d"% sys.version_info.minor) >= 9))' )
+PYTHON_SOURCES=src/$(LIBNAME)/[a-z]*.py
+
+TEST_SOURCES=tests/[a-z]*.py
+DOCS_SOURCES=docs
+
+LOG_DATE=$(shell date +%Y%m%d_%H%M%S)
+
+.PHONY:	help
+help:	## Display this help screen
+		@grep -hE '^[A-Za-z0-9_ \-]*?:.*##.*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+.PHONY:	install
+install:	install-conda-env	## Install editable package using conda/mamba
+		@echo "\nInstalling editable..."
+		@pip install -e .
+
+install-conda-env:	--check-python --hook-manager	## Install conda/mamba dependencies
+		@echo "\nInstalling dependencies (core)..."
+		@$(pkg-manager) install --file conda_requirements.txt -c conda-forge
+
+install-pip:	## Install editable package using pip
+		@echo "\nInstalling editable..."
+		$(PYTHON) -m pip install --upgrade gdal==`gdal-config --version` pybind11
+		$(PYTHON) -m pip install -e .
+
+install-pip-tests:	install-pip	## Install editable package with tests using pip
+		@echo "\nInstalling editable with tests..."
+		$(PYTHON) -m pip install --upgrade gdal==`gdal-config --version` pybind11
+		@pip install -e .[tests]
+
+install-pip-docs:	## Install editable package with local documentation using pip
+		@echo "\nInstalling editable with documentation..."
+		$(PYTHON) -m pip install --upgrade gdal==`gdal-config --version` pybind11
+		@pip install -e .[docs]
+		@make docs
+
+install-pip-all:	## Install editable package with tests & documentation using pip
+		@echo "\nInstalling editable with tests & documentation..."
+		$(PYTHON) -m pip install --upgrade gdal==`gdal-config --version` pybind11
+		@pip install -e .[tests,docs]
+		@make docs
+
+install-pip-dev:	--check-python --hook-manager	## Install editable package in development mode using pip
+		@echo "\nInstalling editable in development mode..."
+		$(PYTHON) -m pip install --upgrade gdal==`gdal-config --version` pybind11
+		@pip install -e .[dev]
+
+.PHONY:	tests
+tests:	flake8 coverage pylint	## Run tests
+
+.PHONY:	commit
+commit:	tests	## Test, then commit
+		@echo "\nCommitting..."
+		@git commit
+
+.PHONY: docs
+docs:	## Build documentation
+		@echo "\nBuilding documentation..."
+		@cd $(DOCS_SOURCES); make clean && make html
+
+format:	isort black	## Format all python files
+
+setup-cosipy:	## Generate COSIPY configuration files
+		@$(PYTHON) -m cosipy.utilities.setup_cosipy.setup_cosipy
+
+create-static:
+		@$(PYTHON) -m cosipy.utilities.createStatic.create_static_file
+
+commands:	## Display help for COSIPY
+		@$(PYTHON) -m COSIPY.py -h
+
+.PHONY: run
+run:	commands	## Alias for `make commands`
+
+flake8:	## Lint with flake8
+		@echo "\nLinting with flake8..."
+		@flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+		@flake8 . --count --exit-zero --max-complexity=10 --max-line-length=88 --statistics
+
+.PHONY:	coverage
+coverage:	## Run pytest with coverage
+		@echo "\nRunning tests..."
+		@mkdir -p "./logs/coverage"
+		@coverage run --rcfile .coveragerc -m pytest && coverage html
+
+.PHONY: pylint
+pylint:	## Lint with Pylint
+		@echo "\nLinting with pylint..."
+		@pylint --rcfile .pylintrc **/*.py
+
+black:	## Format all python files with black
+		@echo "\nFormatting..."
+		@black $(PYTHON_SOURCES) --line-length=80
+		@black $(TEST_SOURCES) --line-length=80
+
+isort:	## Optimise python imports
+		@isort $(PYTHON_SOURCES)
+		@isort $(TEST_SOURCES)
+
+.PHONY:	pkg
+pkg:	tests docs build	## Run tests, build documentation, build package
+
+.PHONY: build
+build:	--install-build-deps	## Build COSIPY package
+		$(PYTHON) -m build
+		@twine check dist/*
+
+# .PHONY
+# bump-version:
+# 		@grep -Po '\bversion\s*=\s*"\K.*?(?=")' pyproject.toml
+
+.PHONY:
+upload-pypi:	# Private: upload COSIPY package
+		@twine check dist/*
+		@twine upload dist/*
+
+
+--install-pip-deps:	--check-python  # Private: install core dependencies with pip
+		@echo "\nInstalling dependencies with pip..."
+		$(PYTHON) -m pip install --r requirements.txt
+
+--install-build-deps:	--check-python	--hook-manager	# Private: install build dependencies
+		@echo "\nInstalling build dependencies..."
+		$(PYTHON) -m pip install --upgrade build hatchling twine
+
+--check-python:	# Private: check Python is >=3.9
+ifeq ($(PYTHON_CHECK_MAJOR),0)
+	$(error "Python version is $(PYTHON_VERSION). Requires Python >= 3.9")
+else ifeq ($(PYTHON_CHECK_MINOR),0)
+	$(error "Python version is $(PYTHON_VERSION). Requires Python >= 3.9")
+endif
+
+--hook-manager:	# Private: hook package manager
+ifneq (,$(findstring mamba, ${CONDA_EXE}))
+pkg-manager := @mamba
+else ifneq (,$(findstring miniforge, ${CONDA_EXE}))
+pkg-manager := @mamba
+else ifeq (,$(findstring conda, ${CONDA_EXE}))
+pkg-manager := @conda
+else
+	$(error "No conda/mamba installation found. Try pip install -e . instead")
+endif
+
+

README.rst

@@ -39,6 +39,7 @@ Install GDAL:
 
     sudo apt-get install gdal-bin libgdal-dev
     pip install --upgrade gdal==`gdal-config --version` pybind11  # with pip
+    conda install gdal  # with conda
 
 Install COSIPY with pip (for general use):
 
@@ -48,15 +49,31 @@ Install COSIPY with pip (for general use):
     cosipy-setup  # generate template configuration files
     cosipy-help   # view help
 
+Install COSIPY as an editable (recommended for development):
+
+.. code-block:: console
+
+    git clone https://github.com/cryotools/cosipy.git
+    cd cosipy    
+    make install          # using conda/mamba
+    make install-pip-dev  # using pip
+
+    cosipy-help           # view help
+
 Install COSIPY from source (for development):
 
 .. code-block:: console
 
     git clone https://github.com/cryotools/cosipy.git
+    cd cosipy
+    make install-conda-envs                      # install using conda/mamba
+    conda install --file conda_requirements.txt  # install with conda
+
     pip install -r requirements.txt              # install default environment
     pip install -r dev_requirements.txt          # install dev environment
-    conda install --file conda_requirements.txt  # install using conda/mamba
+
     python3 COSIPY.py -h
+    make commands
 
 Communication and Support
 -------------------------

cosipy/config.py

@@ -3,6 +3,8 @@
 """
 
 import argparse
+import os
+import pathlib
 import sys
 from importlib.metadata import entry_points
 from pathlib import Path
@@ -20,12 +22,36 @@
 default_utilities_path = cwd / "utilities_config.toml"
 
 
+def get_cosipy_path_from_env(name: str = "COSIPY_DIR") -> pathlib.Path:
+    """Get path to COSIPY directory.
+
+    When using WRFxCSPY, the coupler will default to searching for
+    config files in the current working directory, which may contain the
+    COSIPY source code. This function instead loads an environment
+    variable.
+
+    Args:
+        name: Name of environment variable pointing to the COSIPY
+            directory.
+
+    Returns:
+        Path to the COSIPY directory.
+
+    Raises:
+        NotADirectoryError: Invalid path.
+    """
+    cosipy_path = pathlib.Path(os.environ.get(name, os.getcwd()))
+    if not cosipy_path.is_dir():
+        raise NotADirectoryError(f"Invalid path at: {cosipy_path}")
+
+    return cosipy_path
+
+
 def set_parser() -> argparse.ArgumentParser:
     """Set argument parser for COSIPY."""
-    tagline = (
-        "Coupled snowpack and ice surface energy and mass balance model in Python."
-    )
+    tagline = "Coupled snowpack and ice surface energy and mass balance model in Python."
     parser = argparse.ArgumentParser(prog="COSIPY", description=tagline)
+    cosipy_path = get_cosipy_path_from_env()
 
     # Optional arguments
     parser.add_argument(
@@ -53,9 +79,9 @@ def set_parser() -> argparse.ArgumentParser:
     parser.add_argument(
         "-s",
         "--slurm",
-        default=default_slurm_path,
+        default=cosipy_path / "slurm_config.toml",
         dest="slurm_path",
-        type=Path,
+        type=pathlib.Path,
         metavar="<path>",
         required=False,
         help="relative path to Slurm configuration file",

cosipy/cpkernel/io.py

@@ -25,6 +25,10 @@ def __init__(self, data: xr.Dataset | None = None):
         """
 
         self.atm = self.get_output_variables(Config.output_atm)
+        # filter out input data, otherwise they get replaced by empty arrays
+        for input_name in ["T2", "RH2", "U2", "RRR", "N", "G", "PRES"]:
+            self.atm = [value for value in self.atm if value!=input_name]
+
         self.internal = self.get_output_variables(Config.output_internal)
         self.full = self.get_output_variables(Config.output_full)
 

cosipy/tests/conftest.py

@@ -37,6 +37,15 @@ def conftest_mock_check_file_exists():
     mock_exists.return_value = True
 
 
+@pytest.fixture(scope="function", autouse=False)
+def conftest_mock_check_directory_exists():
+    """Override checks when mocking directories."""
+
+    patcher = patch("pathlib.Path.is_dir")
+    mock_exists = patcher.start()
+    mock_exists.return_value = True
+
+
 @pytest.fixture(scope="function", autouse=False)
 def conftest_disable_jit():
     # numba.config.DISABLE_JIT = True

cosipy/tests/test_config.py

@@ -1,4 +1,9 @@
 import argparse
+import os
+import pathlib
+from unittest.mock import patch
+
+import pytest
 
 import cosipy.config
 from cosipy.config import Config
@@ -25,6 +30,56 @@ def test_set_parser(self):
         for name in ["help", "config", "constants", "slurm"]:
             assert name in actions
 
+    @pytest.mark.dependency(name="TestConfigParser::test_set_parser")
+    @pytest.mark.parametrize("arg_type", (str, pathlib.Path))
+    def test_user_arguments(self, arg_type):
+        test_parser = cosipy.config.set_parser()
+        test_path = "./some/path/config.toml"
+        assert isinstance(test_path, str)
+
+        test_args = [
+            "--config",
+            test_path,
+            "--constants",
+            test_path,
+            "--slurm",
+            test_path,
+        ]
+        arguments, unknown = test_parser.parse_known_args(test_args)
+        assert isinstance(arguments, argparse.Namespace)
+
+        for user_path in [
+            arguments.config_path,
+            arguments.constants_path,
+            arguments.slurm_path,
+        ]:
+            assert isinstance(user_path, pathlib.Path)
+            assert user_path == pathlib.Path(test_path)
+
+    @patch.dict(os.environ, {"COSIPY_DIR": "./path/to/wrong/cosipy/"})
+    def test_check_directory_exists(self):
+        """Raise error if directory not found."""
+        wrong_path = "./path/to/wrong/cosipy/"
+        error_message = f"Invalid path at: {pathlib.Path(wrong_path)}"
+        with pytest.raises(NotADirectoryError, match=error_message):
+            cosipy.config.get_cosipy_path_from_env(name="COSIPY_DIR")
+
+    @pytest.mark.parametrize(
+        "arg_env", ((True, "COSIPY_DIR"), (True, "XFAIL"), (False, ""))
+    )
+    @patch.dict(os.environ, {"COSIPY_DIR": "./path/to/cosipy/"})
+    def test_get_cosipy_path(
+        self, arg_env, conftest_mock_check_directory_exists
+    ):
+        _ = conftest_mock_check_directory_exists
+        test_name = arg_env[1]
+        compare_path = cosipy.config.get_cosipy_path_from_env(name=test_name)
+
+        if arg_env[1] == "COSIPY_DIR":
+            assert compare_path == pathlib.Path("./path/to/cosipy/")
+        else:
+            assert compare_path == pathlib.Path.cwd()
+
 
 class TestConfig:
     """Test rtoml support."""

cosipy/utilities/config_utils.py

@@ -3,6 +3,7 @@
 """
 
 import argparse
+import pathlib
 import sys
 from collections import namedtuple
 
@@ -73,9 +74,9 @@ def set_arg_parser(cls) -> argparse.ArgumentParser:
         parser.add_argument(
             "-u",
             "--utilities",
-            default="./utilities_config.toml",
+            default=pathlib.Path("./utilities_config.toml"),
             dest="utilities_path",
-            type=str,
+            type=pathlib.Path,
             metavar="<path>",
             required=False,
             help="relative path to utilities' configuration file",