OpenStack external CCM (#820)

* openstack terraform changes * Make cluster private (no external IPs) * Use bastion to access cluster * Set hostnames explicitly Signed-off-by: Artiom Diomin <[email protected]> * Start taking into account external flag for openstack Signed-off-by: Artiom Diomin <[email protected]> * Deploy openstack external CCM Signed-off-by: Artiom Diomin <[email protected]> * reuse genClusterRoleBinding across different external CCMs Signed-off-by: Artiom Diomin <[email protected]> * lint fixes Signed-off-by: Artiom Diomin <[email protected]> * Fix e2e tests for openstack Signed-off-by: Artiom Diomin <[email protected]> * Move openstack specific terraform settings to e2e shell runner Signed-off-by: Artiom Diomin <[email protected]> * In E2E use openstack cloudConfig from the credentials file Signed-off-by: Artiom Diomin <[email protected]> * Use typed KubeOneCluster instead of template in E2E config generation Signed-off-by: Artiom Diomin <[email protected]> * Update openstack e2e image name Signed-off-by: Artiom Diomin <[email protected]> * Update openstack upgrade e2e tests Signed-off-by: Artiom Diomin <[email protected]>
kubermatic · Mar 13, 2020 · bd5ba7f · bd5ba7f
1 parent 8a2ccf3
commit bd5ba7f
Show file tree

Hide file tree

Showing 15 changed files with 433 additions and 145 deletions.
diff --git a/examples/terraform/openstack/main.tf b/examples/terraform/openstack/main.tf
@@ -148,25 +148,10 @@ resource "openstack_networking_port_v2" "lb" {
   }
 }
 
-resource "openstack_networking_floatingip_v2" "control_plane" {
-  count = 3
-  pool  = var.external_network_name
-}
-
 resource "openstack_networking_floatingip_v2" "lb" {
   pool = var.external_network_name
 }
 
-resource "openstack_networking_floatingip_associate_v2" "control_plane" {
-  count = 3
-
-  floating_ip = element(
-    openstack_networking_floatingip_v2.control_plane.*.address,
-    count.index,
-  )
-  port_id = element(openstack_networking_port_v2.control_plane.*.id, count.index)
-}
-
 resource "openstack_networking_floatingip_associate_v2" "lb" {
   floating_ip = openstack_networking_floatingip_v2.lb.address
   port_id     = openstack_networking_port_v2.lb.id
@@ -184,6 +169,10 @@ resource "null_resource" "lb_config" {
     config               = local.rendered_lb_config
   }
 
+  depends_on = [
+    openstack_compute_instance_v2.lb
+  ]
+
   connection {
     host = openstack_networking_floatingip_v2.lb.address
     user = var.ssh_username

diff --git a/examples/terraform/openstack/output.tf b/examples/terraform/openstack/output.tf
@@ -30,11 +30,14 @@ output "kubeone_hosts" {
       cluster_name         = var.cluster_name
       cloud_provider       = "openstack"
       private_address      = openstack_compute_instance_v2.control_plane.*.access_ip_v4
-      public_address       = openstack_networking_floatingip_v2.control_plane.*.address
+      hostnames            = openstack_compute_instance_v2.control_plane.*.name
       ssh_agent_socket     = var.ssh_agent_socket
       ssh_port             = var.ssh_port
       ssh_private_key_file = var.ssh_private_key_file
       ssh_user             = var.ssh_username
+      bastion              = openstack_networking_floatingip_v2.lb.address
+      bastion_port         = var.bastion_port
+      bastion_user         = var.bastion_user
     }
   }
 }
@@ -60,7 +63,6 @@ output "kubeone_workers" {
           image          = var.image
           flavor         = var.worker_flavor
           securityGroups = [openstack_networking_secgroup_v2.securitygroup.name]
-          floatingIPPool = var.external_network_name
           network        = openstack_networking_network_v2.network.name
           subnet         = openstack_networking_subnet_v2.subnet.name
           # Optional: If set, the rootDisk will be a volume. 

diff --git a/examples/terraform/openstack/variables.tf b/examples/terraform/openstack/variables.tf
@@ -53,6 +53,16 @@ variable "ssh_agent_socket" {
   default     = "env:SSH_AUTH_SOCK"
 }
 
+variable "bastion_port" {
+  description = "Bastion SSH port"
+  default     = 22
+}
+
+variable "bastion_user" {
+  description = "Bastion SSH username"
+  default     = "ubuntu"
+}
+
 # Provider specific settings
 
 variable "control_plane_flavor" {

diff --git a/hack/run-ci-e2e-test.sh b/hack/run-ci-e2e-test.sh
@@ -39,6 +39,7 @@ export TF_VAR_cluster_name=k1-${BUILD_ID}
 export TF_VAR_subnets_cidr=28
 export SSH_PUBLIC_KEY_FILE="${SSH_PRIVATE_KEY_FILE}.pub"
 export TF_VAR_ssh_public_key_file=${SSH_PUBLIC_KEY_FILE}
+CREDENTIALS_FILE_PATH=""
 
 function cleanup() {
   set +e
@@ -95,13 +96,17 @@ function setup_ci_environment_vars() {
     ;;
   "openstack")
     export OS_AUTH_URL=${OS_AUTH_URL}
-    export OS_DOMAIN_NAME=${OS_DOMAIN_NAME}
-    export OS_REGION_NAME=${OS_REGION_NAME}
+    export OS_DOMAIN_NAME=${OS_DOMAIN}
+    export OS_REGION_NAME=${OS_REGION}
     export OS_TENANT_NAME=${OS_TENANT_NAME}
     export OS_USERNAME=${OS_USERNAME}
     export OS_PASSWORD=${OS_PASSWORD}
-    fail "openstack no implemented yet"
-    # echo ${k1_credentials} >/tmp/credentials.yaml
+    export TF_VAR_external_network_name="ext-net"
+    export TF_VAR_subnet_cidr="10.0.42.0/24"
+    export TF_VAR_image="Ubuntu Bionic 18.04 (2020-03-12)"
+    export TF_VAR_lb_flavor="m1.tiny"
+    echo "${OS_K1_CREDENTIALS}" >/tmp/credentials.yaml
+    CREDENTIALS_FILE_PATH=/tmp/credentials.yaml
     ;;
   *)
     fail "unknown provider ${PROVIDER}"
@@ -150,6 +155,7 @@ function runE2E() {
     -timeout="${timeout}" \
     -run="${test_set}" \
     ./test/e2e \
+    -credentials="${CREDENTIALS_FILE_PATH}" \
     -identifier="${BUILD_ID}" \
     -provider="${PROVIDER}" \
     -os-control-plane="${TEST_OS_CONTROL_PLANE}" \

diff --git a/pkg/apis/kubeone/helpers.go b/pkg/apis/kubeone/helpers.go
@@ -70,13 +70,13 @@ func (h *HostConfig) SetLeader(leader bool) {
 // List of in-tree provider can be found here: https://github.com/kubernetes/kubernetes/tree/master/pkg/cloudprovider
 func (p CloudProviderSpec) CloudProviderInTree() bool { //nolint:stylecheck
 	switch p.Name {
-	case CloudProviderNameAWS, CloudProviderNameGCE, CloudProviderNameOpenStack:
+	case CloudProviderNameOpenStack:
+		return !p.External
+	case CloudProviderNameAWS, CloudProviderNameGCE, CloudProviderNameVSphere, CloudProviderNameAzure:
 		return true
-	case CloudProviderNameVSphere, CloudProviderNameAzure:
-		return true
-	default:
-		return false
 	}
+
+	return false
 }
 
 // KubernetesCNIVersion returns kubernetes-cni package version

diff --git a/pkg/clusterstatus/etcdstatus/etcdstatus.go b/pkg/clusterstatus/etcdstatus/etcdstatus.go
@@ -175,7 +175,6 @@ func loadTLSConfig(s *state.State) (*tls.Config, error) {
 	caCertPool := x509.NewCertPool()
 	caCertPool.AppendCertsFromPEM(caBytes)
 	tlsConfig.RootCAs = caCertPool
-	tlsConfig.BuildNameToCertificate()
 
 	return tlsConfig, nil
 }

diff --git a/pkg/templates/externalccm/ccm.go b/pkg/templates/externalccm/ccm.go
@@ -26,6 +26,8 @@ import (
 	"github.com/kubermatic/kubeone/pkg/state"
 
 	corev1 "k8s.io/api/core/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 )
 
@@ -51,6 +53,8 @@ func Ensure(s *state.State) error {
 		err = ensureDigitalOcean(s)
 	case kubeoneapi.CloudProviderNamePacket:
 		err = ensurePacket(s)
+	case kubeoneapi.CloudProviderNameOpenStack:
+		err = ensureOpenStack(s)
 	default:
 		s.Logger.Infof("External CCM for %q not yet supported, skipping", s.Cluster.CloudProvider.Name)
 		return nil
@@ -87,3 +91,23 @@ func waitForInitializedNodes(s *state.State) error {
 		return true, nil
 	})
 }
+
+func genClusterRoleBinding(name string, crole *rbacv1.ClusterRole, subj *corev1.ServiceAccount) *rbacv1.ClusterRoleBinding {
+	return &rbacv1.ClusterRoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: name,
+		},
+		RoleRef: rbacv1.RoleRef{
+			APIGroup: rbacv1.GroupName,
+			Kind:     "ClusterRole",
+			Name:     crole.GetName(),
+		},
+		Subjects: []rbacv1.Subject{
+			{
+				Kind:      "ServiceAccount",
+				Name:      subj.GetName(),
+				Namespace: subj.GetNamespace(),
+			},
+		},
+	}
+}
diff --git a/pkg/templates/externalccm/digitalocean.go b/pkg/templates/externalccm/digitalocean.go
@@ -34,7 +34,7 @@ import (
 )
 
 const (
-	digitaloceanCCMVersion     = "v0.1.21"
+	digitaloceanImage          = "digitalocean/digitalocean-cloud-controller-manager:v0.1.23"
 	digitaloceanSAName         = "cloud-controller-manager"
 	digitaloceanDeploymentName = "digitalocean-cloud-controller-manager"
 )
@@ -45,10 +45,12 @@ func ensureDigitalOcean(s *state.State) error {
 	}
 
 	ctx := context.Background()
+	sa := doServiceAccount()
+	crole := doClusterRole()
 	k8sobject := []runtime.Object{
-		doServiceAccount(),
-		doClusterRole(),
-		doClusterRoleBinding(),
+		sa,
+		crole,
+		genClusterRoleBinding("system:cloud-controller-manager", crole, sa),
 	}
 
 	for _, obj := range k8sobject {
@@ -122,26 +124,6 @@ func doClusterRole() *rbacv1.ClusterRole {
 	}
 }
 
-func doClusterRoleBinding() *rbacv1.ClusterRoleBinding {
-	return &rbacv1.ClusterRoleBinding{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: "system:cloud-controller-manager",
-		},
-		RoleRef: rbacv1.RoleRef{
-			APIGroup: rbacv1.GroupName,
-			Name:     "system:cloud-controller-manager",
-			Kind:     "ClusterRole",
-		},
-		Subjects: []rbacv1.Subject{
-			{
-				Kind:      "ServiceAccount",
-				Name:      digitaloceanSAName,
-				Namespace: metav1.NamespaceSystem,
-			},
-		},
-	}
-}
-
 func doDeployment() *appsv1.Deployment {
 	var (
 		replicas  int32 = 1
@@ -195,7 +177,7 @@ func doDeployment() *appsv1.Deployment {
 					Containers: []corev1.Container{
 						{
 							Name:  "digitalocean-cloud-controller-manager",
-							Image: "digitalocean/digitalocean-cloud-controller-manager:" + digitaloceanCCMVersion,
+							Image: digitaloceanImage,
 							Command: []string{
 								"/bin/digitalocean-cloud-controller-manager",
 								"--leader-elect=false",

diff --git a/pkg/templates/externalccm/hetzner.go b/pkg/templates/externalccm/hetzner.go
@@ -35,7 +35,7 @@ import (
 )
 
 const (
-	hetznerCCMVersion     = "v1.5.0"
+	hetznerImage          = "hetznercloud/hcloud-cloud-controller-manager:v1.5.1"
 	hetznerSAName         = "cloud-controller-manager"
 	hetznerDeploymentName = "hcloud-cloud-controller-manager"
 )
@@ -152,7 +152,7 @@ func hetznerDeployment(networkID, podSubnet string) *appsv1.Deployment {
 					Containers: []corev1.Container{
 						{
 							Name:    "hcloud-cloud-controller-manager",
-							Image:   "hetznercloud/hcloud-cloud-controller-manager:" + hetznerCCMVersion,
+							Image:   hetznerImage,
 							Command: cmd,
 							Env: []corev1.EnvVar{
 								{