feat: RunnerSet backed by StatefulSet

.github/workflows/test.yaml

@@ -17,11 +17,15 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@v2
+    - uses: actions/setup-go@v2
+      with:
+        go-version: '^1.16.5'
+    - run: go version
     - name: Install kubebuilder
       run: |
-        curl -L -O https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.2.0/kubebuilder_2.2.0_linux_amd64.tar.gz
-        tar zxvf kubebuilder_2.2.0_linux_amd64.tar.gz
-        sudo mv kubebuilder_2.2.0_linux_amd64 /usr/local/kubebuilder
+        curl -L -O https://github.com/kubernetes-sigs/kubebuilder/releases/download/v2.3.2/kubebuilder_2.3.2_linux_amd64.tar.gz
+        tar zxvf kubebuilder_2.3.2_linux_amd64.tar.gz
+        sudo mv kubebuilder_2.3.2_linux_amd64 /usr/local/kubebuilder
     - name: Run tests
       run: make test
     - name: Verify manifests are up-to-date

Makefile

@@ -11,13 +11,19 @@ TEST_REPO ?= ${DOCKER_USER}/actions-runner-controller
 TEST_ORG ?=
 TEST_ORG_REPO ?=
 SYNC_PERIOD ?= 5m
+USE_RUNNERSET ?=
+KUBECONTEXT ?= kind-acceptance
+CLUSTER ?= acceptance
 
 # From https://github.com/VictoriaMetrics/operator/pull/44
 YAML_DROP=$(YQ) delete --inplace
-YAML_DROP_PREFIX=spec.validation.openAPIV3Schema.properties.spec.properties
+
+# If you encounter errors like the below, you are very likely to update this to follow e.g. CRD version change:
+#   CustomResourceDefinition.apiextensions.k8s.io "runners.actions.summerwind.dev" is invalid: spec.preserveUnknownFields: Invalid value: true: must be false in order to use defaults in the schema
+YAML_DROP_PREFIX=spec.versions[0].schema.openAPIV3Schema.properties.spec.properties
 
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
-CRD_OPTIONS ?= "crd:trivialVersions=true"
+CRD_OPTIONS ?= "crd:trivialVersions=true,generateEmbeddedObjectMeta=true"
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -158,20 +164,20 @@ acceptance: release/clean acceptance/pull docker-build release
 acceptance/run: acceptance/kind acceptance/load acceptance/setup acceptance/deploy acceptance/tests acceptance/teardown
 
 acceptance/kind:
-	kind create cluster --name acceptance --config acceptance/kind.yaml
+	kind create cluster --name ${CLUSTER} --config acceptance/kind.yaml
 
 # Set TMPDIR to somewhere under $HOME when you use docker installed with Ubuntu snap
 # Otherwise `load docker-image` fail while running `docker save`.
 # See https://kind.sigs.k8s.io/docs/user/known-issues/#docker-installed-with-snap
 acceptance/load:
-	kind load docker-image ${NAME}:${VERSION} --name acceptance
-	kind load docker-image quay.io/brancz/kube-rbac-proxy:v0.10.0 --name acceptance
-	kind load docker-image ${RUNNER_NAME}:${RUNNER_TAG} --name acceptance
-	kind load docker-image docker:dind --name acceptance
-	kind load docker-image quay.io/jetstack/cert-manager-controller:v1.0.4 --name acceptance
-	kind load docker-image quay.io/jetstack/cert-manager-cainjector:v1.0.4 --name acceptance
-	kind load docker-image quay.io/jetstack/cert-manager-webhook:v1.0.4 --name acceptance
-	kubectl cluster-info --context kind-acceptance
+	kind load docker-image ${NAME}:${VERSION} --name ${CLUSTER}
+	kind load docker-image quay.io/brancz/kube-rbac-proxy:v0.10.0 --name ${CLUSTER}
+	kind load docker-image ${RUNNER_NAME}:${RUNNER_TAG} --name ${CLUSTER}
+	kind load docker-image docker:dind --name ${CLUSTER}
+	kind load docker-image quay.io/jetstack/cert-manager-controller:v1.0.4 --name ${CLUSTER}
+	kind load docker-image quay.io/jetstack/cert-manager-cainjector:v1.0.4 --name ${CLUSTER}
+	kind load docker-image quay.io/jetstack/cert-manager-webhook:v1.0.4 --name ${CLUSTER}
+	kubectl cluster-info --context ${KUBECONTEXT}
 
 # Pull the docker images for acceptance
 acceptance/pull:
@@ -191,11 +197,12 @@ acceptance/setup:
 	sleep 5
 
 acceptance/teardown:
-	kind delete cluster --name acceptance
+	kind delete cluster --name ${CLUSTER}
 
 acceptance/deploy:
 	NAME=${NAME} DOCKER_USER=${DOCKER_USER} VERSION=${VERSION} RUNNER_NAME=${RUNNER_NAME} RUNNER_TAG=${RUNNER_TAG} TEST_REPO=${TEST_REPO} \
 	TEST_ORG=${TEST_ORG} TEST_ORG_REPO=${TEST_ORG_REPO} SYNC_PERIOD=${SYNC_PERIOD} \
+	USE_RUNNERSET=${USE_RUNNERSET} \
 	acceptance/deploy.sh
 
 acceptance/tests:
@@ -205,8 +212,14 @@ acceptance/tests:
 github-release: release
 	ghr ${VERSION} release/
 
-# find or download controller-gen
-# download controller-gen if necessary
+# Find or download controller-gen
+#
+# Note that controller-gen newer than 0.4.1 is needed for https://github.com/kubernetes-sigs/controller-tools/issues/444#issuecomment-680168439
+# Otherwise we get errors like the below:
+#   Error: failed to install CRD crds/actions.summerwind.dev_runnersets.yaml: CustomResourceDefinition.apiextensions.k8s.io "runnersets.actions.summerwind.dev" is invalid: [spec.validation.openAPIV3Schema.properties[spec].properties[template].properties[spec].properties[containers].items.properties[ports].items.properties[protocol].default: Required value: this property is in x-kubernetes-list-map-keys, so it must have a default or be a required property, spec.validation.openAPIV3Schema.properties[spec].properties[template].properties[spec].properties[initContainers].items.properties[ports].items.properties[protocol].default: Required value: this property is in x-kubernetes-list-map-keys, so it must have a default or be a required property]
+#
+# Note that controller-gen newer than 0.6.0 is needed due to https://github.com/kubernetes-sigs/controller-tools/issues/448
+# Otherwise ObjectMeta embedded in Spec results in empty on the storage.
 controller-gen:
 ifeq (, $(shell which controller-gen))
 ifeq (, $(wildcard $(GOBIN)/controller-gen))
@@ -215,7 +228,7 @@ ifeq (, $(wildcard $(GOBIN)/controller-gen))
 	CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\
 	cd $$CONTROLLER_GEN_TMP_DIR ;\
 	go mod init tmp ;\
-	go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.3.0 ;\
+	go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.6.0 ;\
 	rm -rf $$CONTROLLER_GEN_TMP_DIR ;\
 	}
 endif

acceptance/deploy.sh

@@ -35,6 +35,7 @@ if [ "${tool}" == "helm" ]; then
     --set image.repository=${NAME} \
     --set image.tag=${VERSION} \
     -f ${VALUES_FILE}
+  kubectl apply -f charts/actions-runner-controller/crds
   kubectl -n actions-runner-system wait deploy/actions-runner-controller --for condition=available --timeout 60s
 else
   kubectl apply \
@@ -47,8 +48,12 @@ fi
 sleep 20
 
 if [ -n "${TEST_REPO}" ]; then
-  cat acceptance/testdata/runnerdeploy.yaml | envsubst | kubectl apply -f -
-  cat acceptance/testdata/hra.yaml | envsubst | kubectl apply -f -
+  if [ -n "USE_RUNNERSET" ]; then
+      cat acceptance/testdata/runnerset.yaml | envsubst | kubectl apply -f -
+  else
+    cat acceptance/testdata/runnerdeploy.yaml | envsubst | kubectl apply -f -
+    cat acceptance/testdata/hra.yaml | envsubst | kubectl apply -f -
+  fi
 else
   echo 'Skipped deploying runnerdeployment and hra. Set TEST_REPO to "yourorg/yourrepo" to deploy.'
 fi

acceptance/testdata/runnerset.yaml

@@ -0,0 +1,51 @@
+apiVersion: actions.summerwind.dev/v1alpha1
+kind: RunnerSet
+metadata:
+  name: example-runnerset
+spec:
+  # MANDATORY because it is based on StatefulSet: Results in a below error when omitted:
+  #   missing required field "selector" in dev.summerwind.actions.v1alpha1.RunnerSet.spec
+  selector:
+    matchLabels:
+      app: example-runnerset
+
+  # MANDATORY because it is based on StatefulSet: Results in a below error when omitted:
+  # missing required field "serviceName" in dev.summerwind.actions.v1alpha1.RunnerSet.spec]
+  serviceName: example-runnerset
+
+  #replicas: 1
+  ephemeral: false
+
+  repository: ${TEST_REPO}
+  #
+  # Custom runner image
+  #
+  image: ${RUNNER_NAME}:${RUNNER_TAG}
+  #
+  # dockerd within runner container
+  #
+  ## Replace `mumoshu/actions-runner-dind:dev` with your dind image
+  #dockerdWithinRunnerContainer: true
+  #
+  # Set the MTU used by dockerd-managed network interfaces (including docker-build-ubuntu)
+  #
+  #dockerMTU: 1450
+  #Runner group
+  # labels:
+  # - "mylabel 1"
+  # - "mylabel 2"
+
+  #
+  # Non-standard working directory
+  #
+  # workDir: "/"
+  template:
+    metadata:
+      labels:
+        app: example-runnerset
+    spec:
+      containers:
+      - name: runner
+        imagePullPolicy: IfNotPresent
+      #- name: docker
+      #  #image: mumoshu/actions-runner-dind:dev

api/v1alpha1/runner_types.go

@@ -27,6 +27,11 @@ import (
 
 // RunnerSpec defines the desired state of Runner
 type RunnerSpec struct {
+	RunnerConfig  `json:",inline"`
+	RunnerPodSpec `json:",inline"`
+}
+
+type RunnerConfig struct {
 	// +optional
 	// +kubebuilder:validation:Pattern=`^[^/]+$`
 	Enterprise string `json:"enterprise,omitempty"`
@@ -49,64 +54,87 @@ type RunnerSpec struct {
 	Ephemeral *bool `json:"ephemeral,omitempty"`
 
 	// +optional
-	Containers []corev1.Container `json:"containers,omitempty"`
+	Image string `json:"image"`
+
 	// +optional
-	DockerdContainerResources corev1.ResourceRequirements `json:"dockerdContainerResources,omitempty"`
+	WorkDir string `json:"workDir,omitempty"`
+
 	// +optional
-	DockerVolumeMounts []corev1.VolumeMount `json:"dockerVolumeMounts,omitempty"`
+	DockerdWithinRunnerContainer *bool `json:"dockerdWithinRunnerContainer,omitempty"`
 	// +optional
-	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
+	DockerEnabled *bool `json:"dockerEnabled,omitempty"`
 	// +optional
-	VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"`
+	DockerMTU *int64 `json:"dockerMTU,omitempty"`
 	// +optional
-	EnvFrom []corev1.EnvFromSource `json:"envFrom,omitempty"`
+	DockerRegistryMirror *string `json:"dockerRegistryMirror,omitempty"`
+	// +optional
+	VolumeSizeLimit *resource.Quantity `json:"volumeSizeLimit,omitempty"`
+}
 
+// RunnerPodSpec defines the desired pod spec fields of the runner pod
+type RunnerPodSpec struct {
 	// +optional
-	Image string `json:"image"`
+	DockerdContainerResources corev1.ResourceRequirements `json:"dockerdContainerResources,omitempty"`
+
+	// +optional
+	DockerVolumeMounts []corev1.VolumeMount `json:"dockerVolumeMounts,omitempty"`
+
+	// +optional
+	Containers []corev1.Container `json:"containers,omitempty"`
+
 	// +optional
 	ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"`
+
 	// +optional
 	Env []corev1.EnvVar `json:"env,omitempty"`
 
 	// +optional
-	Volumes []corev1.Volume `json:"volumes,omitempty"`
+	EnvFrom []corev1.EnvFromSource `json:"envFrom,omitempty"`
+
 	// +optional
-	WorkDir string `json:"workDir,omitempty"`
+	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
 
 	// +optional
-	InitContainers []corev1.Container `json:"initContainers,omitempty"`
+	VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"`
+
 	// +optional
-	SidecarContainers []corev1.Container `json:"sidecarContainers,omitempty"`
+	Volumes []corev1.Volume `json:"volumes,omitempty"`
+
+	// +optional
+	InitContainers []corev1.Container `json:"initContainers,omitempty"`
+
 	// +optional
 	NodeSelector map[string]string `json:"nodeSelector,omitempty"`
+
 	// +optional
 	ServiceAccountName string `json:"serviceAccountName,omitempty"`
+
 	// +optional
 	AutomountServiceAccountToken *bool `json:"automountServiceAccountToken,omitempty"`
+
+	// +optional
+	SidecarContainers []corev1.Container `json:"sidecarContainers,omitempty"`
+
 	// +optional
 	SecurityContext *corev1.PodSecurityContext `json:"securityContext,omitempty"`
+
 	// +optional
 	ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
+
 	// +optional
 	Affinity *corev1.Affinity `json:"affinity,omitempty"`
+
 	// +optional
 	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
-	// +optional
-	EphemeralContainers []corev1.EphemeralContainer `json:"ephemeralContainers,omitempty"`
+
 	// +optional
 	TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"`
+
 	// +optional
-	DockerdWithinRunnerContainer *bool `json:"dockerdWithinRunnerContainer,omitempty"`
-	// +optional
-	DockerEnabled *bool `json:"dockerEnabled,omitempty"`
-	// +optional
-	DockerMTU *int64 `json:"dockerMTU,omitempty"`
-	// +optional
-	DockerRegistryMirror *string `json:"dockerRegistryMirror,omitempty"`
+	EphemeralContainers []corev1.EphemeralContainer `json:"ephemeralContainers,omitempty"`
+
 	// +optional
 	HostAliases []corev1.HostAlias `json:"hostAliases,omitempty"`
-	// +optional
-	VolumeSizeLimit *resource.Quantity `json:"volumeSizeLimit,omitempty"`
 
 	// RuntimeClassName is the container runtime configuration that containers should run under.
 	// More info: https://kubernetes.io/docs/concepts/containers/runtime-class

api/v1alpha1/runner_webhook.go

@@ -21,7 +21,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	ctrl "sigs.k8s.io/controller-runtime"
-	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 )
 
@@ -34,7 +34,7 @@ func (r *Runner) SetupWebhookWithManager(mgr ctrl.Manager) error {
 		Complete()
 }
 
-// +kubebuilder:webhook:path=/mutate-actions-summerwind-dev-v1alpha1-runner,verbs=create;update,mutating=true,failurePolicy=fail,groups=actions.summerwind.dev,resources=runners,versions=v1alpha1,name=mutate.runner.actions.summerwind.dev,sideEffects=None
+// +kubebuilder:webhook:path=/mutate-actions-summerwind-dev-v1alpha1-runner,verbs=create;update,mutating=true,failurePolicy=fail,groups=actions.summerwind.dev,resources=runners,versions=v1alpha1,name=mutate.runner.actions.summerwind.dev,sideEffects=None,webhookVersions=v1beta1
 
 var _ webhook.Defaulter = &Runner{}
 
@@ -43,7 +43,7 @@ func (r *Runner) Default() {
 	// Nothing to do.
 }
 
-// +kubebuilder:webhook:path=/validate-actions-summerwind-dev-v1alpha1-runner,verbs=create;update,mutating=false,failurePolicy=fail,groups=actions.summerwind.dev,resources=runners,versions=v1alpha1,name=validate.runner.actions.summerwind.dev,sideEffects=None
+// +kubebuilder:webhook:path=/validate-actions-summerwind-dev-v1alpha1-runner,verbs=create;update,mutating=false,failurePolicy=fail,groups=actions.summerwind.dev,resources=runners,versions=v1alpha1,name=validate.runner.actions.summerwind.dev,sideEffects=None,webhookVersions=v1beta1
 
 var _ webhook.Validator = &Runner{}
 

api/v1alpha1/runnerdeployment_webhook.go

@@ -21,7 +21,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	ctrl "sigs.k8s.io/controller-runtime"
-	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 )
 
@@ -34,7 +34,7 @@ func (r *RunnerDeployment) SetupWebhookWithManager(mgr ctrl.Manager) error {
 		Complete()
 }
 
-// +kubebuilder:webhook:path=/mutate-actions-summerwind-dev-v1alpha1-runnerdeployment,verbs=create;update,mutating=true,failurePolicy=fail,groups=actions.summerwind.dev,resources=runnerdeployments,versions=v1alpha1,name=mutate.runnerdeployment.actions.summerwind.dev
+// +kubebuilder:webhook:path=/mutate-actions-summerwind-dev-v1alpha1-runnerdeployment,verbs=create;update,mutating=true,failurePolicy=fail,groups=actions.summerwind.dev,resources=runnerdeployments,versions=v1alpha1,name=mutate.runnerdeployment.actions.summerwind.dev,webhookVersions=v1beta1
 
 var _ webhook.Defaulter = &RunnerDeployment{}
 
@@ -43,7 +43,7 @@ func (r *RunnerDeployment) Default() {
 	// Nothing to do.
 }
 
-// +kubebuilder:webhook:path=/validate-actions-summerwind-dev-v1alpha1-runnerdeployment,verbs=create;update,mutating=false,failurePolicy=fail,groups=actions.summerwind.dev,resources=runnerdeployments,versions=v1alpha1,name=validate.runnerdeployment.actions.summerwind.dev
+// +kubebuilder:webhook:path=/validate-actions-summerwind-dev-v1alpha1-runnerdeployment,verbs=create;update,mutating=false,failurePolicy=fail,groups=actions.summerwind.dev,resources=runnerdeployments,versions=v1alpha1,name=validate.runnerdeployment.actions.summerwind.dev,webhookVersions=v1beta1
 
 var _ webhook.Validator = &RunnerDeployment{}
 

api/v1alpha1/runnerreplicaset_webhook.go

@@ -21,7 +21,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	ctrl "sigs.k8s.io/controller-runtime"
-	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 )
 
@@ -34,7 +34,7 @@ func (r *RunnerReplicaSet) SetupWebhookWithManager(mgr ctrl.Manager) error {
 		Complete()
 }
 
-// +kubebuilder:webhook:path=/mutate-actions-summerwind-dev-v1alpha1-runnerreplicaset,verbs=create;update,mutating=true,failurePolicy=fail,groups=actions.summerwind.dev,resources=runnerreplicasets,versions=v1alpha1,name=mutate.runnerreplicaset.actions.summerwind.dev,sideEffects=None
+// +kubebuilder:webhook:path=/mutate-actions-summerwind-dev-v1alpha1-runnerreplicaset,verbs=create;update,mutating=true,failurePolicy=fail,groups=actions.summerwind.dev,resources=runnerreplicasets,versions=v1alpha1,name=mutate.runnerreplicaset.actions.summerwind.dev,sideEffects=None,webhookVersions=v1beta1
 
 var _ webhook.Defaulter = &RunnerReplicaSet{}
 
@@ -43,7 +43,7 @@ func (r *RunnerReplicaSet) Default() {
 	// Nothing to do.
 }
 
-// +kubebuilder:webhook:path=/validate-actions-summerwind-dev-v1alpha1-runnerreplicaset,verbs=create;update,mutating=false,failurePolicy=fail,groups=actions.summerwind.dev,resources=runnerreplicasets,versions=v1alpha1,name=validate.runnerreplicaset.actions.summerwind.dev,sideEffects=None
+// +kubebuilder:webhook:path=/validate-actions-summerwind-dev-v1alpha1-runnerreplicaset,verbs=create;update,mutating=false,failurePolicy=fail,groups=actions.summerwind.dev,resources=runnerreplicasets,versions=v1alpha1,name=validate.runnerreplicaset.actions.summerwind.dev,sideEffects=None,webhookVersions=v1beta1
 
 var _ webhook.Validator = &RunnerReplicaSet{}