Handle namespaced resources during datastore migration properly (#2251)

projectcalico · Jan 29, 2021 · 5eb5060 · 5eb5060
1 parent 9828614
commit 5eb5060
Show file tree

Hide file tree

Showing 5 changed files with 112 additions and 1 deletion.
diff --git a/Makefile b/Makefile
@@ -313,6 +313,14 @@ run-kubernetes-master: stop-kubernetes-master
 		gcr.io/google_containers/hyperkube-amd64:${K8S_VERSION} kubectl \
 		--server=http://127.0.0.1:8080 \
 		apply -f /manifests/tests/st/manifests/mock-node.yaml
+
+	# Create a namespace in the API for the tests to use.
+	docker run \
+	    --net=host \
+	    --rm \
+		gcr.io/google_containers/hyperkube-amd64:${K8S_VERSION} kubectl \
+		--server=http://127.0.0.1:8080 \
+		create namespace test
 
 ## Stop the local kubernetes master
 stop-kubernetes-master:

diff --git a/calicoctl/commands/datastore/migrate/export.go b/calicoctl/commands/datastore/migrate/export.go
@@ -165,6 +165,12 @@ Description:
 			"--output": "yaml",
 			"get":      true,
 		}
+
+		// Add options for pulling resources from all namespaces for namespaced resources.
+		if r == "networksets" || r == "networkpolicies" {
+			mockArgs["--all-namespaces"] = true
+		}
+
 		results := common.ExecuteConfigCommand(mockArgs, common.ActionGetOrList)
 		if len(results.ResErrs) > 0 {
 			var errStr string

diff --git a/tests/st/calicoctl/test_migrate.py b/tests/st/calicoctl/test_migrate.py
@@ -19,7 +19,7 @@
 
 from tests.st.test_base import TestBase
 from tests.st.utils.utils import log_and_run, calicoctl, \
-    API_VERSION, name, ERROR_CONFLICT, NOT_FOUND, NOT_NAMESPACED, \
+    API_VERSION, name, namespace, ERROR_CONFLICT, NOT_FOUND, NOT_NAMESPACED, \
     SET_DEFAULT, NOT_SUPPORTED, KUBERNETES_NP, NOT_LOCKED, \
     NOT_KUBERNETES, NO_IPAM, writeyaml
 from tests.st.utils.data import *
@@ -116,12 +116,25 @@ def test_datastore_migrate(self):
         rc.assert_data(networkpolicy_name1_rev1)
         rc.assert_no_error()
 
+        # Create namespaced Network policy
+        rc = calicoctl("create", data=networkpolicy_name3_rev1)
+        rc.assert_no_error()
+        rc = calicoctl("get networkpolicy %s -n %s -o yaml" % (name(networkpolicy_name3_rev1), namespace(networkpolicy_name3_rev1)))
+        rc.assert_data(networkpolicy_name3_rev1)
+        rc.assert_no_error()
+
         # Create NetworkSets
         rc = calicoctl("create", data=networkset_name1_rev1)
         rc.assert_no_error()
         rc = calicoctl("get networkset %s -o yaml" % name(networkset_name1_rev1))
         rc.assert_no_error()
 
+        # Create namespaced NetworkSet
+        rc = calicoctl("create", data=networkset_name2_rev1)
+        rc.assert_no_error()
+        rc = calicoctl("get networkset %s -n %s -o yaml" % (name(networkset_name2_rev1), namespace(networkset_name2_rev1)))
+        rc.assert_no_error()
+
         # Create a Node, this should also trigger auto-creation of a cluster info
         rc = calicoctl("create", data=node_name4_rev1)
         rc.assert_no_error()
@@ -173,8 +186,12 @@ def test_datastore_migrate(self):
         rc.assert_no_error()
         rc = calicoctl("delete networkpolicy %s" % name(networkpolicy_name1_rev1))
         rc.assert_no_error()
+        rc = calicoctl("delete networkpolicy %s -n %s" % (name(networkpolicy_name3_rev1), namespace(networkpolicy_name3_rev1)))
+        rc.assert_no_error()
         rc = calicoctl("delete networkset %s" % name(networkset_name1_rev1))
         rc.assert_no_error()
+        rc = calicoctl("delete networkset %s -n %s" % (name(networkset_name2_rev1), namespace(networkset_name2_rev1)))
+        rc.assert_no_error()
         rc = calicoctl("delete node %s" % name(node_name4_rev1))
         rc.assert_no_error()
         rc = calicoctl("delete node %s" % name(node_name5_rev1))
@@ -215,8 +232,12 @@ def test_datastore_migrate(self):
         rc.assert_data(hostendpoint_name1_rev1)
         rc = calicoctl("get networkpolicy %s -o yaml" % name(networkpolicy_name1_rev1), kdd=True)
         rc.assert_data(networkpolicy_name1_rev1)
+        rc = calicoctl("get networkpolicy %s -n %s -o yaml" % (name(networkpolicy_name3_rev1), namespace(networkpolicy_name3_rev1)), kdd=True)
+        rc.assert_data(networkpolicy_name3_rev1)
         rc = calicoctl("get networkset %s -o yaml" % name(networkset_name1_rev1), kdd=True)
         rc.assert_no_error()
+        rc = calicoctl("get networkset %s -n %s -o yaml" % (name(networkset_name2_rev1), namespace(networkset_name2_rev1)), kdd=True)
+        rc.assert_no_error()
         rc = calicoctl("get node %s -o yaml" % name(node_name4_rev1), kdd=True)
         rc.assert_no_error()
         rc = calicoctl("get node %s -o yaml" % name(node_name5_rev1), kdd=True)
@@ -249,5 +270,9 @@ def test_datastore_migrate(self):
         rc.assert_no_error()
         rc = calicoctl("delete networkpolicy %s" % name(networkpolicy_name1_rev1), kdd=True)
         rc.assert_no_error()
+        rc = calicoctl("delete networkpolicy %s -n %s" % (name(networkpolicy_name3_rev1), namespace(networkpolicy_name3_rev1)), kdd=True)
+        rc.assert_no_error()
         rc = calicoctl("delete networkset %s" % name(networkset_name1_rev1), kdd=True)
         rc.assert_no_error()
+        rc = calicoctl("delete networkset %s -n %s" % (name(networkset_name2_rev1), namespace(networkset_name2_rev1)), kdd=True)
+        rc.assert_no_error()
diff --git a/tests/st/utils/data.py b/tests/st/utils/data.py
@@ -289,6 +289,51 @@
     }
 }
 
+networkpolicy_name3_rev1 = {
+    'apiVersion': API_VERSION,
+    'kind': 'NetworkPolicy',
+    'metadata': {
+        'name': 'policy-mypolicy3',
+        'namespace': 'test',
+    },
+    'spec': {
+        'order': 100,
+        'selector': "type=='database'",
+        'types': ['Ingress', 'Egress'],
+        'egress': [
+            {
+                'action': 'Allow',
+                'source': {
+                    'selector': "type=='application'"},
+            },
+        ],
+        'ingress': [
+            {
+                'ipVersion': 4,
+                'action': 'Deny',
+                'destination': {
+                    'notNets': ['10.3.0.0/16'],
+                    'notPorts': ['110:1050'],
+                    'notSelector': "type=='apples'",
+                    'nets': ['10.2.0.0/16'],
+                    'ports': ['100:200'],
+                    'selector': "type=='application'",
+                },
+                'protocol': 'TCP',
+                'source': {
+                    'notNets': ['10.1.0.0/16'],
+                    'notPorts': [1050],
+                    'notSelector': "type=='database'",
+                    'nets': ['10.0.0.0/16'],
+                    'ports': [1234, '10:1024'],
+                    'selector': "type=='application'",
+                    'namespaceSelector': 'has(role)',
+                }
+            }
+        ],
+    }
+}
+
 #
 # Global Network Policy
 #
@@ -423,6 +468,23 @@
     }
 }
 
+networkset_name2_rev1 = {
+    'apiVersion': API_VERSION,
+    'kind': 'NetworkSet',
+    'metadata': {
+        'name': 'net-set2',
+        'namespace': 'test',
+    },
+    'spec': {
+        'nets': [
+            "10.0.0.1",
+            "11.0.0.0/16",
+            "feed:beef::1",
+            "dead:beef::96",
+        ]
+    }
+}
+
 # A network set with a large number of entries.  In prototyping this test, I found that there are
 # "upstream" limits that cap how large we can go:
 #

diff --git a/tests/st/utils/utils.py b/tests/st/utils/utils.py
@@ -496,3 +496,13 @@ def name(data):
     Returns: The resource name.
     """
     return data['metadata']['name']
+
+def namespace(data):
+    """
+    Returns the namespace of the resource in the supplied data
+    Args:
+       data: A dictionary containing the resource.
+
+    Returns: The resource name.
+    """
+    return data['metadata']['namespace']