[PRERELEASE] Support SSH reruns [RT-1201]

Chart.yaml

@@ -4,5 +4,5 @@ description: For deploying a CircleCI Container Agent
 icon: https://raw.githubusercontent.com/circleci/media/master/logo/build/horizontal_dark.1.png
 type: application
 
-version: "101.0.10"
+version: "101.0.11"
 appVersion: "3"

README.md

@@ -2,7 +2,7 @@
 
 For deploying a CircleCI Container Agent
 
-![Version: 101.0.10](https://img.shields.io/badge/Version-101.0.10-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3](https://img.shields.io/badge/AppVersion-3-informational?style=flat-square)
+![Version: 101.0.11](https://img.shields.io/badge/Version-101.0.11-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3](https://img.shields.io/badge/AppVersion-3-informational?style=flat-square)
 
 ## Contributing
 
@@ -82,6 +82,11 @@ The command removes all the Kubernetes objects associated with the chart and del
 | agent.resourceClasses | object | `{}` | Resource class settings. The tokens specified here will be used to claim tasks & the tasks will be launched with the configured configs Ref: https://circleci.com/docs/container-runner/#resource-class-configuration-custom-pod |
 | agent.resources | object | `{}` | Agent pod resource configuration Ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ |
 | agent.runnerAPI | string | `"https://runner.circleci.com"` | CircleCI Runner API URL |
+| agent.ssh.controllerName | string | `"gateway.envoyproxy.io/gatewayclass-controller"` | The name of the infrastructure provider for the SSH rerun Gateway (see: https://gateway-api.sigs.k8s.io/implementations/). SSH reruns depend on the TCPRoute resource, so only implementations that support it are compatible at this time. Please consult the documentation for your preferred Gateway implementation for guidance on setting it up in your cluster. The Envoy Gateway has been successfully tested for SSH reruns (see: https://gateway.envoyproxy.io/latest/user/tcp-routing/). |
+| agent.ssh.enabled | bool | `false` | Controls whether to enable SSH reruns (see: https://circleci.com/docs/ssh-access-jobs/). Note that enabling SSH reruns will install additional resources to your cluster. Notably, SSH reruns requires the Kubernetes Gateway API (see: https://gateway-api.sigs.k8s.io/). |
+| agent.ssh.numPorts | int | `20` |  |
+| agent.ssh.parametersRef | object | `{}` |  |
+| agent.ssh.startPort | int | `54782` | Specify the port range that is approved for SSH connections. Note that the number of concurrent jobs rerun with SSH is limited by the number of ports in this range. |
 | agent.terminationGracePeriodSeconds | int | `18300` | Tasks are drained during the termination grace period, so this should be sufficiently long relative to the maximum run time to ensure graceful shutdown |
 | agent.tolerations | list | `[]` | Node tolerations for agent scheduling to nodes with taints Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ |
 | logging | object | `{"image":{"registry":"","repository":"circleci/logging-collector","tag":3},"rbac":{"create":true,"role":{"name":"logging-collector","rules":[]}},"serviceAccount":{"annotations":{},"create":true,"name":"logging-collector","secret":{"name":"logging-collector-token"}}}` | Configuration values for the logging containers. These containers run alongside service containers and stream their logs to the CircleCI UI |

changelog.md

@@ -2,7 +2,12 @@
 
 This is the Container Agent Helm Chart changelog
 
+# 101.0.11
+
+- [#16](https://github.com/CircleCI-Public/container-runner-helm-chart/pull/16) [PRERELEASE] Support SSH reruns
+
 # 101.0.10
+
 - [#29](https://github.com/CircleCI-Public/container-runner-helm-chart/pull/29) Added Proxy env support
 
 # 101.0.8

templates/deployment.yaml

@@ -94,6 +94,17 @@ spec:
               value: {{ .Values.logging.serviceAccount.secret.name | quote }}
             - name: KUBE_AUTODETECT_PLATFORM
               value: {{ .Values.agent.autodetectPlatform | quote }}
+
+            {{- if .Values.agent.ssh.enabled }}
+            {{- $sshName := printf "%s-ssh" (include "container-agent.fullname" .) }}
+            - name: KUBE_SSH_IS_ENABLED
+              value: "true"
+            - name: KUBE_SSH_GATEWAY_NAME
+              value: {{ $sshName }}
+            - name: KUBE_SSH_SERVICE_NAME
+              value: {{ $sshName }}
+            {{- end }} # if .Values.agent.ssh.enabled
+
             {{- if .Values.proxy.enabled }}
             - name: PROXY__SECRETS__HTTP__USERNAME
               valueFrom:

templates/role.yaml

@@ -19,6 +19,13 @@ rules:
   - apiGroups: ["", "events.k8s.io/v1"]
     resources: ["events"]
     verbs: ["watch"]
+
+  {{- if .Values.agent.ssh.enabled }}
+  - apiGroups: ["", "gateway.networking.k8s.io"]
+    resources: ["gateways", "services"]
+    verbs: ["get"]
+  {{- end }} # if .Values.agent.ssh.enabled
+
 {{- if $role.rules }}
   {{- toYaml $role.rules | indent 2 }}
 {{- end }} # if $role.rules

templates/ssh.yaml

@@ -0,0 +1,77 @@
+{{- if .Values.agent.ssh.enabled }}
+
+{{- $startPort := .Values.agent.ssh.startPort | int }}
+{{- $ports := untilStep $startPort (add $startPort .Values.agent.ssh.numPorts | int) 1 -}}
+{{- $namespace := .Release.Namespace | quote }}
+{{- $name := printf "%s-ssh" (include "container-agent.fullname" .) }}
+
+apiVersion: gateway.networking.k8s.io/v1
+kind: GatewayClass
+metadata:
+  name: {{ $name }}
+  namespace: {{ $namespace }}
+spec:
+  controllerName: {{ .Values.agent.ssh.controllerName }}
+  {{- with .Values.agent.ssh.parametersRef }}
+  parametersRef:
+    group: {{ .group }}
+    kind: {{ .kind }}
+    name: {{ .name }}
+  {{- end }}
+
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: {{ $name }}
+  namespace: {{ $namespace }}
+spec:
+  gatewayClassName: {{ $name }}
+  listeners:
+    {{- range $i, $port := $ports }}
+    - name: ssh-{{ $port }}
+      protocol: TCP
+      port: {{ $port }}
+      allowedRoutes:
+        kinds:
+          - kind: TCPRoute
+    {{- end }}
+
+---
+{{- range $i, $port := $ports }}
+{{ $portName := printf "ssh-%d" $port }}
+---
+apiVersion: gateway.networking.k8s.io/v1alpha2
+kind: TCPRoute
+metadata:
+  name: {{ $portName }}
+  namespace: {{ $namespace }}
+spec:
+  parentRefs:
+  - name: {{ $name }}
+    sectionName: {{ $portName }}
+  rules:
+  - backendRefs:
+    - name: {{ $name }}
+      port: {{ $port }}
+{{- end }}
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ $name }}
+  namespace: {{ $namespace }}
+spec:
+  selector:
+    app.kubernetes.io/managed-by: circleci-container-agent
+  ports:
+  {{- range $i, $port := $ports }}
+  {{ $portName := printf "ssh-%d" $port }}
+  - name: {{ $portName }}
+    protocol: TCP
+    port: {{ $port }}
+    targetPort: {{ $portName }}
+  {{- end }}
+
+{{- end }} # if .Values.agent.ssh.enabled

tests/__snapshot__/ssh_test.yaml.snap

@@ -0,0 +1,100 @@
+should create Gateway and Service resources if enabled:
+  1: |
+    apiVersion: gateway.networking.k8s.io/v1
+    kind: GatewayClass
+    metadata:
+      name: RELEASE-NAME-container-agent-ssh
+      namespace: NAMESPACE
+    spec:
+      controllerName: example.net/gateway-controller
+  2: |
+    apiVersion: gateway.networking.k8s.io/v1
+    kind: Gateway
+    metadata:
+      name: RELEASE-NAME-container-agent-ssh
+      namespace: NAMESPACE
+    spec:
+      gatewayClassName: RELEASE-NAME-container-agent-ssh
+      listeners:
+        - allowedRoutes:
+            kinds:
+              - kind: TCPRoute
+          name: ssh-22
+          port: 22
+          protocol: TCP
+        - allowedRoutes:
+            kinds:
+              - kind: TCPRoute
+          name: ssh-23
+          port: 23
+          protocol: TCP
+        - allowedRoutes:
+            kinds:
+              - kind: TCPRoute
+          name: ssh-24
+          port: 24
+          protocol: TCP
+  3: |
+    apiVersion: gateway.networking.k8s.io/v1alpha2
+    kind: TCPRoute
+    metadata:
+      name: ssh-22
+      namespace: NAMESPACE
+    spec:
+      parentRefs:
+        - name: RELEASE-NAME-container-agent-ssh
+          sectionName: ssh-22
+      rules:
+        - backendRefs:
+            - name: RELEASE-NAME-container-agent-ssh
+              port: 22
+  4: |
+    apiVersion: gateway.networking.k8s.io/v1alpha2
+    kind: TCPRoute
+    metadata:
+      name: ssh-23
+      namespace: NAMESPACE
+    spec:
+      parentRefs:
+        - name: RELEASE-NAME-container-agent-ssh
+          sectionName: ssh-23
+      rules:
+        - backendRefs:
+            - name: RELEASE-NAME-container-agent-ssh
+              port: 23
+  5: |
+    apiVersion: gateway.networking.k8s.io/v1alpha2
+    kind: TCPRoute
+    metadata:
+      name: ssh-24
+      namespace: NAMESPACE
+    spec:
+      parentRefs:
+        - name: RELEASE-NAME-container-agent-ssh
+          sectionName: ssh-24
+      rules:
+        - backendRefs:
+            - name: RELEASE-NAME-container-agent-ssh
+              port: 24
+  6: |
+    apiVersion: v1
+    kind: Service
+    metadata:
+      name: RELEASE-NAME-container-agent-ssh
+      namespace: NAMESPACE
+    spec:
+      ports:
+        - name: ssh-22
+          port: 22
+          protocol: TCP
+          targetPort: ssh-22
+        - name: ssh-23
+          port: 23
+          protocol: TCP
+          targetPort: ssh-23
+        - name: ssh-24
+          port: 24
+          protocol: TCP
+          targetPort: ssh-24
+      selector:
+        app.kubernetes.io/managed-by: circleci-container-agent

tests/ssh_test.yaml

@@ -0,0 +1,87 @@
+suite: test ssh
+tests:
+  - it: should create Gateway and Service resources if enabled
+    template: templates/ssh.yaml
+    set:
+      agent.ssh.enabled: true
+      agent.ssh.controllerName: "example.net/gateway-controller"
+      agent.ssh.startPort: 22
+      agent.ssh.numPorts: 3
+    asserts:
+      - matchSnapshot: {}
+
+  - it: should update Role rules to allow get Gateway and Service resources if enabled
+    template: templates/role.yaml
+    set:
+      agent.ssh.enabled: true
+    asserts:
+      - contains:
+          path: rules
+          content:
+            apiGroups: ["", "gateway.networking.k8s.io"]
+            resources: ["gateways", "services"]
+            verbs: ["get"]
+        documentIndex: 0
+
+  - it: should set container-agent ssh configuration if enabled
+    template: templates/deployment.yaml
+    set:
+      agent.ssh.enabled: true
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: KUBE_SSH_IS_ENABLED
+            value: "true"
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: KUBE_SSH_GATEWAY_NAME
+            value: "RELEASE-NAME-container-agent-ssh"
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: KUBE_SSH_SERVICE_NAME
+            value: "RELEASE-NAME-container-agent-ssh"
+
+  - it: should not create any Gateway or Service resources if disabled
+    template: templates/ssh.yaml
+    set:
+      agent.ssh.enabled: false
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: should not update Role rules to allow get Gateway and Service resources if disabled
+    template: templates/role.yaml
+    set:
+      agent.ssh.enabled: false
+    asserts:
+      - notContains:
+          path: rules
+          content:
+            apiGroups: ["", "gateway.networking.k8s.io"]
+            resources: ["gateways", "services"]
+            verbs: ["get"]
+        documentIndex: 0
+
+  - it: should not set container-agent ssh configuration if disabled
+    template: templates/deployment.yaml
+    set:
+      agent.ssh.enabled: false
+    asserts:
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: KUBE_SSH_IS_ENABLED
+            value: "true"
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: KUBE_SSH_GATEWAY_NAME
+            value: "RELEASE-NAME-container-agent-ssh"
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: KUBE_SSH_SERVICE_NAME
+            value: "RELEASE-NAME-container-agent-ssh"

values.yaml

@@ -202,6 +202,24 @@ agent:
     # -- Check interval
     interval: 15m
 
+  ssh:
+    # -- Controls whether to enable SSH reruns (see: https://circleci.com/docs/ssh-access-jobs/).
+    # Note that enabling SSH reruns will install additional resources to your cluster. Notably,
+    # SSH reruns requires the Kubernetes Gateway API (see: https://gateway-api.sigs.k8s.io/).
+    enabled: false
+
+    # -- The name of the infrastructure provider for the SSH rerun Gateway (see: https://gateway-api.sigs.k8s.io/implementations/).
+    # SSH reruns depend on the TCPRoute resource, so only implementations that support it are compatible at this time.
+    # Please consult the documentation for your preferred Gateway implementation for guidance on setting it up in your cluster.
+    # The Envoy Gateway has been successfully tested for SSH reruns (see: https://gateway.envoyproxy.io/latest/user/tcp-routing/).
+    controllerName: "gateway.envoyproxy.io/gatewayclass-controller"
+    parametersRef: {}
+
+    # -- Specify the port range that is approved for SSH connections.
+    # Note that the number of concurrent jobs rerun with SSH is limited by the number of ports in this range.
+    startPort: 54782
+    numPorts: 20
+
 # -- Kubernetes service account settings
 serviceAccount:
   create: true