nextcloud: fix ixvolume storage with deprecated layout (#1434)

* nextcloud: fix ixvolume storage with deprecated layout

* empty space

* bump

* reset test file

ix-dev/stable/nextcloud/app.yaml

@@ -28,8 +28,8 @@ keywords:
 - http
 - web
 - php
-lib_version: 2.1.9
-lib_version_hash: 6bd4d433db7dce2d4b8cc456a0f7874b45d52a6e2b145d5a1f48f327654a7125
+lib_version: 2.1.13
+lib_version_hash: 19a001bec1a3bede7f6f55a757b4088b81e31e92a48e8e89d23d064b059a65d0
 maintainers:
 - email: [email protected]
   name: truenas
@@ -66,4 +66,4 @@ sources:
 - https://github.com/truenas/charts/tree/master/charts/nextcloud
 title: Nextcloud
 train: stable
-version: 1.5.14
+version: 1.5.15

ix-dev/stable/nextcloud/templates/docker-compose.yaml

@@ -70,15 +70,17 @@
 ]) %}
 {% if values.storage.is_data_in_the_same_volume %}
   {% do tpl.notes.add_warning(values.consts.deprecation_warning) %}
-  {% set data_host_path = values.storage.data.get("host_path_config", {}).get("acl", {}).get("path", "") or values.storage.data.get("host_path_config", {}).get("path", "") %}
-  {% set html_host_path = values.storage.html.get("host_path_config", {}).get("acl", {}).get("path", "") or values.storage.html.get("host_path_config", {}).get("path", "") %}
-  {% set html_create_host_path = values.storage.html.get("host_path_config", {}).get("create_host_path", False) %}
+  {% set data_host_path = tpl.funcs.get_host_path(values.storage.data) %}
   {% set data_create_host_path = values.storage.data.get("host_path_config", {}).get("create_host_path", False) %}
-  {% do nc_storage.x.append((values.nextcloud.data_dir_path, dict(values.storage.data, **{"host_path_config": {"path": "%s/data"|format(data_host_path), "create_host_path": data_create_host_path}} ))) %}
-  {% do nc_storage.x.append(("/var/www/html", dict(values.storage.html, **{"host_path_config": {"path": "%s/html"|format(html_host_path), "create_host_path": html_create_host_path}} ))) %}
-  {% do nc_storage.x.append(("/var/www/html/config", dict(values.storage.html, **{"host_path_config": {"path": "%s/config"|format(html_host_path), "create_host_path": html_create_host_path}} ))) %}
-  {% do nc_storage.x.append(("/var/www/html/custom_apps", dict(values.storage.html, **{"host_path_config": {"path": "%s/custom_apps"|format(html_host_path), "create_host_path": html_create_host_path}} ))) %}
-  {% do nc_storage.x.append(("/var/www/html/themes", dict(values.storage.html, **{"host_path_config": {"path": "%s/themes"|format(html_host_path), "create_host_path": html_create_host_path}} ))) %}
+
+  {% set html_host_path = tpl.funcs.get_host_path(values.storage.html) %}
+  {% set html_create_host_path = values.storage.html.get("host_path_config", {}).get("create_host_path", False) %}
+
+  {% do nc_storage.x.append((values.nextcloud.data_dir_path, dict(values.storage.data, **{"type": "host_path", "host_path_config": {"path": "%s/data"|format(data_host_path), "create_host_path": data_create_host_path, "allow_unsafe_ix_volume": true}} ))) %}
+  {% do nc_storage.x.append(("/var/www/html", dict(values.storage.html, **{"type": "host_path", "host_path_config": {"path": "%s/html"|format(html_host_path), "create_host_path": html_create_host_path, "allow_unsafe_ix_volume": true}} ))) %}
+  {% do nc_storage.x.append(("/var/www/html/config", dict(values.storage.html, **{"type": "host_path", "host_path_config": {"path": "%s/config"|format(html_host_path), "create_host_path": html_create_host_path, "allow_unsafe_ix_volume": true}} ))) %}
+  {% do nc_storage.x.append(("/var/www/html/custom_apps", dict(values.storage.html, **{"type": "host_path", "host_path_config": {"path": "%s/custom_apps"|format(html_host_path), "create_host_path": html_create_host_path, "allow_unsafe_ix_volume": true}} ))) %}
+  {% do nc_storage.x.append(("/var/www/html/themes", dict(values.storage.html, **{"type": "host_path", "host_path_config": {"path": "%s/themes"|format(html_host_path), "create_host_path": html_create_host_path, "allow_unsafe_ix_volume": true}} ))) %}
 {% else %}
   {% do nc_storage.x.append(("/var/www/html", values.storage.html)) %}
   {% do nc_storage.x.append((values.nextcloud.data_dir_path, values.storage.data)) %}

ix-dev/stable/nextcloud/templates/library/base_v2_1_9/volume_sources.py → ix-dev/stable/nextcloud/templates/library/base_v2_1_13/volume_sources.py

@@ -32,7 +32,9 @@ def __init__(self, render_instance: "Render", config: "IxStorageHostPathConfig")
             path = valid_fs_path_or_raise(config.get("path", ""))
 
         path = path.rstrip("/")
-        self.source = allowed_fs_host_path_or_raise(path)
+        # TODO: Hack for Nextcloud deprecated config. Remove once we remove support for it
+        allow_unsafe_ix_volume = config.get("allow_unsafe_ix_volume", False)
+        self.source = allowed_fs_host_path_or_raise(path, allow_unsafe_ix_volume)
 
     def get(self):
         return self.source

ix-dev/stable/nextcloud/templates/library/base_v2_1_9/container.py → library/2.1.13/container.py

@@ -8,43 +8,49 @@
     from .configs import ContainerConfigs
     from .depends import Depends
     from .deploy import Deploy
+    from .device_cgroup_rules import DeviceCGroupRules
     from .devices import Devices
     from .dns import Dns
     from .environment import Environment
     from .error import RenderError
     from .expose import Expose
+    from .extra_hosts import ExtraHosts
     from .formatter import escape_dollar, get_image_with_hashed_data
     from .healthcheck import Healthcheck
     from .labels import Labels
     from .ports import Ports
     from .restart import RestartPolicy
     from .validations import (
-        valid_network_mode_or_raise,
         valid_cap_or_raise,
-        valid_pull_policy_or_raise,
+        valid_ipc_mode_or_raise,
+        valid_network_mode_or_raise,
         valid_port_bind_mode_or_raise,
+        valid_pull_policy_or_raise,
     )
     from .storage import Storage
     from .sysctls import Sysctls
 except ImportError:
     from configs import ContainerConfigs
     from depends import Depends
     from deploy import Deploy
+    from device_cgroup_rules import DeviceCGroupRules
     from devices import Devices
     from dns import Dns
     from environment import Environment
     from error import RenderError
     from expose import Expose
+    from extra_hosts import ExtraHosts
     from formatter import escape_dollar, get_image_with_hashed_data
     from healthcheck import Healthcheck
     from labels import Labels
     from ports import Ports
     from restart import RestartPolicy
     from validations import (
-        valid_network_mode_or_raise,
         valid_cap_or_raise,
-        valid_pull_policy_or_raise,
+        valid_ipc_mode_or_raise,
+        valid_network_mode_or_raise,
         valid_port_bind_mode_or_raise,
+        valid_pull_policy_or_raise,
     )
     from storage import Storage
     from sysctls import Sysctls
@@ -63,6 +69,7 @@ def __init__(self, render_instance: "Render", name: str, image: str):
         self._stdin_open: bool = False
         self._init: bool | None = None
         self._read_only: bool | None = None
+        self._extra_hosts: ExtraHosts = ExtraHosts(self._render_instance)
         self._hostname: str = ""
         self._cap_drop: set[str] = set(["ALL"])  # Drop all capabilities by default and add caps granularly
         self._cap_add: set[str] = set()
@@ -75,6 +82,8 @@ def __init__(self, render_instance: "Render", name: str, image: str):
         self._grace_period: int | None = None
         self._shm_size: int | None = None
         self._storage: Storage = Storage(self._render_instance)
+        self._ipc_mode: str | None = None
+        self._device_cgroup_rules: DeviceCGroupRules = DeviceCGroupRules(self._render_instance)
         self.sysctls: Sysctls = Sysctls(self._render_instance, self)
         self.configs: ContainerConfigs = ContainerConfigs(self._render_instance, self._render_instance.configs)
         self.deploy: Deploy = Deploy(self._render_instance)
@@ -154,6 +163,9 @@ def set_user(self, user: int, group: int):
                 raise RenderError(f"User/Group [{i}] is not valid")
         self._user = f"{user}:{group}"
 
+    def add_extra_host(self, host: str, ip: str):
+        self._extra_hosts.add_host(host, ip)
+
     def add_group(self, group: int | str):
         if isinstance(group, str):
             group = str(group).strip()
@@ -182,6 +194,12 @@ def set_tty(self, enabled: bool = False):
     def set_stdin(self, enabled: bool = False):
         self._stdin_open = enabled
 
+    def set_ipc_mode(self, ipc_mode: str):
+        self._ipc_mode = valid_ipc_mode_or_raise(ipc_mode, self._render_instance.container_names())
+
+    def add_device_cgroup_rule(self, dev_grp_rule: str):
+        self._device_cgroup_rules.add_rule(dev_grp_rule)
+
     def set_init(self, enabled: bool = False):
         self._init = enabled
 
@@ -309,6 +327,15 @@ def render(self) -> dict[str, Any]:
         if self.configs.has_configs():
             result["configs"] = self.configs.render()
 
+        if self._ipc_mode is not None:
+            result["ipc"] = self._ipc_mode
+
+        if self._device_cgroup_rules.has_rules():
+            result["device_cgroup_rules"] = self._device_cgroup_rules.render()
+
+        if self._extra_hosts.has_hosts():
+            result["extra_hosts"] = self._extra_hosts.render()
+
         if self._init is not None:
             result["init"] = self._init
 

library/2.1.13/device_cgroup_rules.py

@@ -0,0 +1,54 @@
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from render import Render
+
+try:
+    from .error import RenderError
+    from .validations import valid_device_cgroup_rule_or_raise
+except ImportError:
+    from error import RenderError
+    from validations import valid_device_cgroup_rule_or_raise
+
+
+class DeviceCGroupRule:
+    def __init__(self, rule: str):
+        rule = valid_device_cgroup_rule_or_raise(rule)
+        parts = rule.split(" ")
+        major, minor = parts[1].split(":")
+
+        self._type = parts[0]
+        self._major = major
+        self._minor = minor
+        self._permissions = parts[2]
+
+    def get_key(self):
+        return f"{self._type}_{self._major}_{self._minor}"
+
+    def render(self):
+        return f"{self._type} {self._major}:{self._minor} {self._permissions}"
+
+
+class DeviceCGroupRules:
+    def __init__(self, render_instance: "Render"):
+        self._render_instance = render_instance
+        self._rules: set[DeviceCGroupRule] = set()
+        self._track_rule_combos: set[str] = set()
+
+    def add_rule(self, rule: str):
+        dev_group_rule = DeviceCGroupRule(rule)
+        if dev_group_rule in self._rules:
+            raise RenderError(f"Device Group Rule [{rule}] already added")
+
+        rule_key = dev_group_rule.get_key()
+        if rule_key in self._track_rule_combos:
+            raise RenderError(f"Device Group Rule [{rule}] has already been added for this device group")
+
+        self._rules.add(dev_group_rule)
+        self._track_rule_combos.add(rule_key)
+
+    def has_rules(self):
+        return len(self._rules) > 0
+
+    def render(self):
+        return sorted([rule.render() for rule in self._rules])

library/2.1.13/extra_hosts.py

@@ -0,0 +1,33 @@
+import ipaddress
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from render import Render
+
+try:
+    from .error import RenderError
+except ImportError:
+    from error import RenderError
+
+
+class ExtraHosts:
+    def __init__(self, render_instance: "Render"):
+        self._render_instance = render_instance
+        self._extra_hosts: dict[str, str] = {}
+
+    def add_host(self, host: str, ip: str):
+        if not ip == "host-gateway":
+            try:
+                ipaddress.ip_address(ip)
+            except ValueError:
+                raise RenderError(f"Invalid IP address [{ip}] for host [{host}]")
+
+        if host in self._extra_hosts:
+            raise RenderError(f"Host [{host}] already added with [{self._extra_hosts[host]}]")
+        self._extra_hosts[host] = ip
+
+    def has_hosts(self):
+        return len(self._extra_hosts) > 0
+
+    def render(self):
+        return {host: ip for host, ip in self._extra_hosts.items()}

ix-dev/stable/nextcloud/templates/library/base_v2_1_9/tests/test_container.py → library/2.1.13/tests/test_container.py

@@ -367,3 +367,48 @@ def test_add_ports_with_invalid_host_ips(mock_values):
     c1.healthcheck.disable()
     with pytest.raises(Exception):
         c1.add_port({"port_number": 8081, "container_port": 8080, "bind_mode": "published", "host_ips": "invalid"})
+
+
+def test_set_ipc_mode(mock_values):
+    render = Render(mock_values)
+    c1 = render.add_container("test_container", "test_image")
+    c1.healthcheck.disable()
+    c1.set_ipc_mode("host")
+    output = render.render()
+    assert output["services"]["test_container"]["ipc"] == "host"
+
+
+def test_set_ipc_empty_mode(mock_values):
+    render = Render(mock_values)
+    c1 = render.add_container("test_container", "test_image")
+    c1.healthcheck.disable()
+    c1.set_ipc_mode("")
+    output = render.render()
+    assert output["services"]["test_container"]["ipc"] == ""
+
+
+def test_set_ipc_mode_with_invalid_ipc_mode(mock_values):
+    render = Render(mock_values)
+    c1 = render.add_container("test_container", "test_image")
+    c1.healthcheck.disable()
+    with pytest.raises(Exception):
+        c1.set_ipc_mode("invalid")
+
+
+def test_set_ipc_mode_with_container_ipc_mode(mock_values):
+    render = Render(mock_values)
+    c1 = render.add_container("test_container", "test_image")
+    c1.healthcheck.disable()
+    c2 = render.add_container("test_container2", "test_image")
+    c2.healthcheck.disable()
+    c1.set_ipc_mode("container:test_container2")
+    output = render.render()
+    assert output["services"]["test_container"]["ipc"] == "container:test_container2"
+
+
+def test_set_ipc_mode_with_container_ipc_mode_and_invalid_container(mock_values):
+    render = Render(mock_values)
+    c1 = render.add_container("test_container", "test_image")
+    c1.healthcheck.disable()
+    with pytest.raises(Exception):
+        c1.set_ipc_mode("container:invalid")

library/2.1.13/tests/test_device_cgroup_rules.py

@@ -0,0 +1,79 @@
+import pytest
+
+
+from render import Render
+
+
+@pytest.fixture
+def mock_values():
+    return {
+        "images": {
+            "test_image": {
+                "repository": "nginx",
+                "tag": "latest",
+            }
+        },
+    }
+
+
+def test_device_cgroup_rule(mock_values):
+    render = Render(mock_values)
+    c1 = render.add_container("test_container", "test_image")
+    c1.healthcheck.disable()
+    c1.add_device_cgroup_rule("c 13:* rwm")
+    c1.add_device_cgroup_rule("b 10:20 rwm")
+    output = render.render()
+    assert output["services"]["test_container"]["device_cgroup_rules"] == [
+        "b 10:20 rwm",
+        "c 13:* rwm",
+    ]
+
+
+def test_device_cgroup_rule_duplicate(mock_values):
+    render = Render(mock_values)
+    c1 = render.add_container("test_container", "test_image")
+    c1.healthcheck.disable()
+    c1.add_device_cgroup_rule("c 13:* rwm")
+    with pytest.raises(Exception):
+        c1.add_device_cgroup_rule("c 13:* rwm")
+
+
+def test_device_cgroup_rule_duplicate_group(mock_values):
+    render = Render(mock_values)
+    c1 = render.add_container("test_container", "test_image")
+    c1.healthcheck.disable()
+    c1.add_device_cgroup_rule("c 13:* rwm")
+    with pytest.raises(Exception):
+        c1.add_device_cgroup_rule("c 13:* rm")
+
+
+def test_device_cgroup_rule_invalid_device(mock_values):
+    render = Render(mock_values)
+    c1 = render.add_container("test_container", "test_image")
+    c1.healthcheck.disable()
+    with pytest.raises(Exception):
+        c1.add_device_cgroup_rule("d 10:20 rwm")
+
+
+def test_device_cgroup_rule_invalid_perm(mock_values):
+    render = Render(mock_values)
+    c1 = render.add_container("test_container", "test_image")
+    c1.healthcheck.disable()
+    with pytest.raises(Exception):
+        c1.add_device_cgroup_rule("a 10:20 rwd")
+
+
+def test_device_cgroup_rule_invalid_format(mock_values):
+    render = Render(mock_values)
+    c1 = render.add_container("test_container", "test_image")
+    c1.healthcheck.disable()
+    with pytest.raises(Exception):
+        c1.add_device_cgroup_rule("a 10 20 rwd")
+
+
+def test_device_cgroup_rule_invalid_format_missing_major(mock_values):
+    render = Render(mock_values)
+    c1 = render.add_container("test_container", "test_image")
+    c1.healthcheck.disable()
+    with pytest.raises(Exception):
+        c1.add_device_cgroup_rule("a 10 rwd")