Added extra volumes to singularity client

grpc4bmi/bmi_client_docker.py

@@ -11,6 +11,7 @@
 class LogsException(Exception):
     pass
 
+
 class DeadDockerContainerException(ChildProcessError):
     """
     Exception for when a Docker container has died.
@@ -115,7 +116,7 @@ def initialize(self, filename):
         fn = stage_config_file(filename, self.input_dir, self.input_mount_point)
         super(BmiClientDocker, self).initialize(fn)
 
-    def get_value_ref(self, var_name):
+    def get_value_ptr(self, var_name):
         raise NotImplementedError("Cannot exchange memory references across process boundary")
 
     def logs(self):

grpc4bmi/bmi_client_singularity.py

@@ -41,22 +41,35 @@ class BmiClientSingularity(BmiClient):
         input_dir (str): Directory for input files of model
         output_dir (str): Directory for input files of model
         timeout (int): Seconds to wait for gRPC client to connect to server
+        extra_volumes (Dict[str,str]): Extra volumes to attach to Singularity container.
+
+            The key is the hosts path and the value the mounted volume inside the container.
+            Contrary to Docker client, extra volumes are always read/write
+
+            For example:
+
+            .. code-block:: python
+
+                    {'/data/shared/forcings/': /data/forcings'}
 
     """
     INPUT_MOUNT_POINT = "/data/input"
     OUTPUT_MOUNT_POINT = "/data/output"
 
-    def __init__(self, image, input_dir=None, output_dir=None, timeout=None):
+    def __init__(self, image, input_dir=None, output_dir=None, timeout=None, extra_volumes=None):
         check_singularity_version()
         host = 'localhost'
         port = BmiClient.get_unique_port(host)
         args = [
             "singularity",
             "run",
         ]
+        mount_points = {} if extra_volumes is None else extra_volumes
         if input_dir is not None:
+            mount_points[input_dir] = BmiClientSingularity.INPUT_MOUNT_POINT
             self.input_dir = abspath(input_dir)
-            args += ["--bind", input_dir + ':' + BmiClientSingularity.INPUT_MOUNT_POINT]
+        if any(mount_points):
+            args += ["--bind", ','.join([hp + ':' + ip for hp, ip in mount_points.items()])]
         if output_dir is not None:
             self.output_dir = abspath(output_dir)
             try:
@@ -82,5 +95,5 @@ def initialize(self, filename):
         fn = stage_config_file(filename, self.input_dir, self.INPUT_MOUNT_POINT, home_mounted=True)
         super(BmiClientSingularity, self).initialize(fn)
 
-    def get_value_ref(self, var_name):
+    def get_value_ptr(self, var_name):
         raise NotImplementedError("Cannot exchange memory references across process boundary")

setup.py

@@ -29,7 +29,7 @@ def read(fname):
           "grpcio-reflection==1.27.2",
           "grpcio-status==1.27.2",
           "googleapis-common-protos>=1.5.5",
-          "protobuf",
+          "protobuf==3.12.2",
           "numpy",
           "docker",
           "bmipy",

test/test_docker.py

@@ -52,9 +52,9 @@ def test_initialize(self, walrus_input, walrus_model):
         walrus_model.initialize(str(walrus_input))
         assert walrus_model.get_current_time() == walrus_model.get_start_time()
 
-    def test_get_value_ref(self, walrus_model):
+    def test_get_value_ptr(self, walrus_model):
         with pytest.raises(NotImplementedError):
-            walrus_model.get_value_ref('Q')
+            walrus_model.get_value_ptr('Q')
 
     def test_extra_volume(self, walrus_model_with_extra_volume):
         walrus_model_with_extra_volume.initialize('/data/input/config.yml')

test/test_singularity.py

@@ -6,7 +6,7 @@
 from nbformat.v4 import new_notebook, new_code_cell
 
 from grpc4bmi.bmi_client_singularity import BmiClientSingularity
-from grpc4bmi.reserve import reserve_grid_padding
+from grpc4bmi.reserve import reserve_grid_padding, reserve_values
 
 IMAGE_NAME = "docker://ewatercycle/walrus-grpc4bmi:v0.3.1"
 
@@ -18,6 +18,15 @@ def walrus_model(tmp_path, walrus_input):
     del model
 
 
+@pytest.fixture()
+def walrus_model_with_extra_volume(walrus_input_on_extra_volume):
+    (input_dir, docker_extra_volumes) = walrus_input_on_extra_volume
+    extra_volumes = {str(k): str(v['bind']) for k, v in docker_extra_volumes.items()}
+    model = BmiClientSingularity(image=IMAGE_NAME, input_dir=str(input_dir), extra_volumes=extra_volumes)
+    yield model
+    del model
+
+
 class TestBmiClientDocker:
     def test_component_name(self, walrus_model):
         assert walrus_model.get_component_name() == 'WALRUS'
@@ -26,15 +35,23 @@ def test_initialize(self, walrus_input, walrus_model):
         walrus_model.initialize(str(walrus_input))
         assert walrus_model.get_current_time() == walrus_model.get_start_time()
 
-    def test_get_value_ref(self, walrus_model):
+    def test_get_value_ptr(self, walrus_model):
         with pytest.raises(NotImplementedError):
-            walrus_model.get_value_ref('Q')
+            walrus_model.get_value_ptr('Q')
 
     def test_get_grid_origin(self, walrus_input, walrus_model):
         walrus_model.initialize(str(walrus_input))
         grid_id = walrus_model.get_var_grid('Q')
         assert len(walrus_model.get_grid_origin(grid_id, reserve_grid_padding(walrus_model, grid_id))) == 2
 
+    def test_extra_volumes(self, walrus_model_with_extra_volume):
+        walrus_model_with_extra_volume.initialize('/data/input/config.yml')
+        walrus_model_with_extra_volume.update()
+
+        # After initialization and update the forcings have been read from the extra volume
+        result = reserve_values(walrus_model_with_extra_volume, 'Q')
+        assert len(walrus_model_with_extra_volume.get_value('Q', result)) == 1
+
 
 @pytest.fixture
 def notebook():