diff --git a/grpc4bmi/bmi_client_docker.py b/grpc4bmi/bmi_client_docker.py index 44e6e19..cc12604 100644 --- a/grpc4bmi/bmi_client_docker.py +++ b/grpc4bmi/bmi_client_docker.py @@ -11,6 +11,7 @@ class LogsException(Exception): pass + class DeadDockerContainerException(ChildProcessError): """ Exception for when a Docker container has died. @@ -115,7 +116,7 @@ def initialize(self, filename): fn = stage_config_file(filename, self.input_dir, self.input_mount_point) super(BmiClientDocker, self).initialize(fn) - def get_value_ref(self, var_name): + def get_value_ptr(self, var_name): raise NotImplementedError("Cannot exchange memory references across process boundary") def logs(self): diff --git a/grpc4bmi/bmi_client_singularity.py b/grpc4bmi/bmi_client_singularity.py index 5a9677e..8ba6a62 100644 --- a/grpc4bmi/bmi_client_singularity.py +++ b/grpc4bmi/bmi_client_singularity.py @@ -41,12 +41,22 @@ class BmiClientSingularity(BmiClient): input_dir (str): Directory for input files of model output_dir (str): Directory for input files of model timeout (int): Seconds to wait for gRPC client to connect to server + extra_volumes (Dict[str,str]): Extra volumes to attach to Singularity container. + + The key is the hosts path and the value the mounted volume inside the container. + Contrary to Docker client, extra volumes are always read/write + + For example: + + .. code-block:: python + + {'/data/shared/forcings/': /data/forcings'} """ INPUT_MOUNT_POINT = "/data/input" OUTPUT_MOUNT_POINT = "/data/output" - def __init__(self, image, input_dir=None, output_dir=None, timeout=None): + def __init__(self, image, input_dir=None, output_dir=None, timeout=None, extra_volumes=None): check_singularity_version() host = 'localhost' port = BmiClient.get_unique_port(host) @@ -54,9 +64,12 @@ def __init__(self, image, input_dir=None, output_dir=None, timeout=None): "singularity", "run", ] + mount_points = {} if extra_volumes is None else extra_volumes if input_dir is not None: + mount_points[input_dir] = BmiClientSingularity.INPUT_MOUNT_POINT self.input_dir = abspath(input_dir) - args += ["--bind", input_dir + ':' + BmiClientSingularity.INPUT_MOUNT_POINT] + if any(mount_points): + args += ["--bind", ','.join([hp + ':' + ip for hp, ip in mount_points.items()])] if output_dir is not None: self.output_dir = abspath(output_dir) try: @@ -82,5 +95,5 @@ def initialize(self, filename): fn = stage_config_file(filename, self.input_dir, self.INPUT_MOUNT_POINT, home_mounted=True) super(BmiClientSingularity, self).initialize(fn) - def get_value_ref(self, var_name): + def get_value_ptr(self, var_name): raise NotImplementedError("Cannot exchange memory references across process boundary") diff --git a/setup.py b/setup.py index a68fa51..0d4549b 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ def read(fname): "grpcio-reflection==1.27.2", "grpcio-status==1.27.2", "googleapis-common-protos>=1.5.5", - "protobuf", + "protobuf==3.12.2", "numpy", "docker", "bmipy", diff --git a/test/test_docker.py b/test/test_docker.py index 5147463..f95dff8 100644 --- a/test/test_docker.py +++ b/test/test_docker.py @@ -52,9 +52,9 @@ def test_initialize(self, walrus_input, walrus_model): walrus_model.initialize(str(walrus_input)) assert walrus_model.get_current_time() == walrus_model.get_start_time() - def test_get_value_ref(self, walrus_model): + def test_get_value_ptr(self, walrus_model): with pytest.raises(NotImplementedError): - walrus_model.get_value_ref('Q') + walrus_model.get_value_ptr('Q') def test_extra_volume(self, walrus_model_with_extra_volume): walrus_model_with_extra_volume.initialize('/data/input/config.yml') diff --git a/test/test_singularity.py b/test/test_singularity.py index 39732e2..3706be8 100644 --- a/test/test_singularity.py +++ b/test/test_singularity.py @@ -6,7 +6,7 @@ from nbformat.v4 import new_notebook, new_code_cell from grpc4bmi.bmi_client_singularity import BmiClientSingularity -from grpc4bmi.reserve import reserve_grid_padding +from grpc4bmi.reserve import reserve_grid_padding, reserve_values IMAGE_NAME = "docker://ewatercycle/walrus-grpc4bmi:v0.3.1" @@ -18,6 +18,15 @@ def walrus_model(tmp_path, walrus_input): del model +@pytest.fixture() +def walrus_model_with_extra_volume(walrus_input_on_extra_volume): + (input_dir, docker_extra_volumes) = walrus_input_on_extra_volume + extra_volumes = {str(k): str(v['bind']) for k, v in docker_extra_volumes.items()} + model = BmiClientSingularity(image=IMAGE_NAME, input_dir=str(input_dir), extra_volumes=extra_volumes) + yield model + del model + + class TestBmiClientDocker: def test_component_name(self, walrus_model): assert walrus_model.get_component_name() == 'WALRUS' @@ -26,15 +35,23 @@ def test_initialize(self, walrus_input, walrus_model): walrus_model.initialize(str(walrus_input)) assert walrus_model.get_current_time() == walrus_model.get_start_time() - def test_get_value_ref(self, walrus_model): + def test_get_value_ptr(self, walrus_model): with pytest.raises(NotImplementedError): - walrus_model.get_value_ref('Q') + walrus_model.get_value_ptr('Q') def test_get_grid_origin(self, walrus_input, walrus_model): walrus_model.initialize(str(walrus_input)) grid_id = walrus_model.get_var_grid('Q') assert len(walrus_model.get_grid_origin(grid_id, reserve_grid_padding(walrus_model, grid_id))) == 2 + def test_extra_volumes(self, walrus_model_with_extra_volume): + walrus_model_with_extra_volume.initialize('/data/input/config.yml') + walrus_model_with_extra_volume.update() + + # After initialization and update the forcings have been read from the extra volume + result = reserve_values(walrus_model_with_extra_volume, 'Q') + assert len(walrus_model_with_extra_volume.get_value('Q', result)) == 1 + @pytest.fixture def notebook():