Run tests from inside the cluster

.github/kustomize/job.yaml

@@ -0,0 +1,51 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: aks-periscope
+spec:
+  backoffLimit: 0
+  template:
+    metadata:
+      labels:
+        app: aks-periscope
+    spec:
+      serviceAccountName: aks-periscope-service-account
+      restartPolicy: Never
+      containers:
+      - name: go-test-runner
+        image: go-test-runner
+        securityContext:
+          privileged: true
+        envFrom:
+        - configMapRef:
+            name: containerlogs-config
+        - configMapRef:
+            name: kubeobjects-config
+        - configMapRef:
+            name: nodelogs-config
+        env:
+        - name: IN_CLUSTER
+          value: "1" # To indicate the tests run from within the cluster
+        volumeMounts:
+        - name: varlog
+          mountPath: /var/log
+        - name: resolvlog
+          mountPath: /run/systemd/resolve
+        - name: etcvmlog
+          mountPath: /etchostlogs
+        - name: runner
+          mountPath: /runner
+      volumes:
+      - name: varlog
+        hostPath:
+          path: /var/log
+      - name: resolvlog
+        hostPath:
+          path: /run/systemd/resolve
+      - name: etcvmlog
+        hostPath:
+          path: /etc
+      - name: runner
+        hostPath: 
+          path: /runner
+          type: Directory

.github/kustomize/kustomization.yaml

@@ -0,0 +1,21 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: aks-periscope
+
+resources:
+- job.yaml
+- ../../deployment/namespace.yaml
+- ../../deployment/cluster-role.yaml
+- ../../deployment/cluster-role-binding.yaml
+- ../../deployment/config-map.yaml
+- ../../deployment/crd.yaml
+- ../../deployment/service-account.yaml
+
+patches:
+- path: patch-command.yaml
+  target:
+    kind: Job
+    name: aks-periscope
+    version: v1
+    group: batch

.github/workflows/ci-pipeline.yaml

@@ -24,6 +24,7 @@ jobs:
         uses: golangci/golangci-lint-action@v2
         with:
           version: latest
+          args: --timeout 3m0s
   build:
     runs-on: ubuntu-latest
     steps:
@@ -43,10 +44,6 @@ jobs:
         with:
             fetch-depth: 2
       - uses: azure/setup-kubectl@v1
-      - name: Set up Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: ${{ env.GO_VERSION }}
       - name: Install Helm
         uses: azure/setup-helm@v1
         with:
@@ -68,7 +65,13 @@ jobs:
           nodes:
           - role: control-plane
           - role: worker
+            extraMounts:
+              - hostPath: ${PWD}
+                containerPath: /runner
           - role: worker
+            extraMounts:
+              - hostPath: ${PWD}
+                containerPath: /runner
           containerdConfigPatches:
           - |-
             [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:5000"]
@@ -98,23 +101,31 @@ jobs:
         with:
           context: .
           push: true
-          tags: localhost:5000/periscope:foo
-          file: ./builder/Dockerfile
+          tags: localhost:5000/go-test-runner:latest
+          file: ./builder/Dockerfile.tests
       - name: Deploy dummy helm chart for tests
         run: |
           helm repo add bitnami https://charts.bitnami.com/bitnami
           helm repo update
           helm install happy-panda bitnami/wordpress --namespace default
       - name: Deploy AKS Periscope
         run: |
-          (cd ./deployment && kustomize edit set image aksrepos.azurecr.io/staging/aks-periscope=localhost:5000/periscope:foo)
-          kubectl apply -f <(kustomize build ./deployment)
-          kubectl -n aks-periscope describe ds aks-periscope
-          kubectl -n aks-periscope wait po --all --for condition=ready --timeout=60s
-      - name: Go tests
-        run: go test -v -race -coverprofile=coverage.out -covermode=atomic ./...
-      - name: Upload coverage to Codecov
-        run: bash <(curl -s https://codecov.io/bash) -C $(Build.SourceVersion)
+          cat <<EOF >>./.github/kustomize/patch-command.yaml
+          - op: add
+            path: '/spec/template/spec/containers/0/command'
+            value: [ "go", "test", "-coverprofile=/runner/coverage.out", "-covermode=atomic", "./..." ]
+          EOF
+
+          (cd ./.github/kustomize && kustomize edit set image go-test-runner=localhost:5000/go-test-runner:latest)
+
+          kubectl apply -f <(kustomize build ./.github/kustomize --load-restrictor LoadRestrictionsNone)
+          
+          if ! kubectl -n aks-periscope wait job aks-periscope --for condition=complete --timeout=300s; then
+            kubectl -n aks-periscope logs -l app=aks-periscope
+            exit 1
+          fi
+      - name: Upload coverage to codecov
+        run: bash <(curl -s https://codecov.io/bash)
       - name: Stop kind
         run: kind delete cluster
         continue-on-error: false

builder/Dockerfile.tests

@@ -0,0 +1,14 @@
+FROM golang:alpine AS builder
+
+ENV GO111MODULE=on \
+    CGO_ENABLED=0 \
+    GOOS=linux \
+    GOARCH=amd64
+
+WORKDIR /build
+
+COPY go.mod .
+COPY go.sum .
+RUN go mod download
+
+COPY . .

go.mod

@@ -6,9 +6,9 @@ require (
 	github.com/Azure/azure-storage-blob-go v0.7.0
 	github.com/Azure/go-autorest/autorest/adal v0.9.14 // indirect
 	github.com/kr/pretty v0.2.1 // indirect
-	github.com/onsi/gomega v1.13.0
+	github.com/onsi/gomega v1.13.0 // indirect
 	helm.sh/helm/v3 v3.6.3
-	k8s.io/api v0.21.3 // indirect
+	k8s.io/api v0.21.3
 	k8s.io/apimachinery v0.21.3
 	k8s.io/cli-runtime v0.21.3
 	k8s.io/client-go v0.21.3

pkg/collector/dns_collector_test.go

@@ -0,0 +1,34 @@
+package collector
+
+import (
+	"testing"
+)
+
+func TestNewDNSCollector(t *testing.T) {
+	tests := []struct {
+		name    string
+		want    int
+		wantErr bool
+	}{
+		{
+			name:    "get dns logs",
+			want:    1,
+			wantErr: false,
+		},
+	}
+
+	c := NewDNSCollector()
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := c.Collect()
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Collect() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			raw := c.GetData() // kubectl -n aks-periscope exec -it aks-periscope-h7gml -- command which returns > cat etc/resolv.conf
+			if len(raw) < tt.want {
+				t.Errorf("len(GetData()) = %v, want %v", len(raw), tt.want)
+			}
+		})
+	}
+}

pkg/collector/helm_collector_test.go

@@ -6,6 +6,7 @@ import (
 	"path"
 	"testing"
 
+	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 )
 
@@ -22,16 +23,9 @@ func TestHelmCollector(t *testing.T) {
 		},
 	}
 
-	dirname, err := os.UserHomeDir()
+	config, err := getConfig()
 	if err != nil {
-		t.Fatalf("Cannot get user home dir: %v", err)
-	}
-
-	master := ""
-	kubeconfig := path.Join(dirname, ".kube/config")
-	config, err := clientcmd.BuildConfigFromFlags(master, kubeconfig)
-	if err != nil {
-		t.Fatalf("Cannot load kube config: %v", err)
+		t.Errorf("cannot get kube config: %w", err)
 	}
 
 	c := NewHelmCollector(config)
@@ -56,3 +50,30 @@ func TestHelmCollector(t *testing.T) {
 		})
 	}
 }
+
+func getConfig() (*restclient.Config, error) {
+	inCluster := os.Getenv("IN_CLUSTER")
+
+	var config *restclient.Config
+	if inCluster == "1" {
+		var err error
+		config, err = restclient.InClusterConfig()
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		dirname, err := os.UserHomeDir()
+		if err != nil {
+			return nil, err
+		}
+
+		master := ""
+		kubeconfig := path.Join(dirname, ".kube/config")
+		config, err = clientcmd.BuildConfigFromFlags(master, kubeconfig)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return config, nil
+}

pkg/collector/nodelogs_collector.go

@@ -1,6 +1,7 @@
 package collector
 
 import (
+	"log"
 	"os"
 	"strings"
 
@@ -28,6 +29,10 @@ func (collector *NodeLogsCollector) Collect() error {
 	nodeLogs := strings.Fields(os.Getenv("DIAGNOSTIC_NODELOGS_LIST"))
 
 	for _, nodeLog := range nodeLogs {
+		if _, err := os.Stat(nodeLog); os.IsNotExist(err) {
+			log.Printf("File %v does not exist", nodeLog)
+			continue
+		}
 
 		output, err := utils.ReadFileContent(nodeLog)
 		if err != nil {

pkg/collector/pods_containerlogs_collector_test.go

@@ -2,10 +2,7 @@ package collector
 
 import (
 	"os"
-	"path"
 	"testing"
-
-	"k8s.io/client-go/tools/clientcmd"
 )
 
 func TestPodsContainerLogsCollector(t *testing.T) {
@@ -21,16 +18,9 @@ func TestPodsContainerLogsCollector(t *testing.T) {
 		},
 	}
 
-	dirname, err := os.UserHomeDir()
-	if err != nil {
-		t.Fatalf("Cannot get user home dir: %v", err)
-	}
-
-	master := ""
-	kubeconfig := path.Join(dirname, ".kube/config")
-	config, err := clientcmd.BuildConfigFromFlags(master, kubeconfig)
+	config, err := getConfig()
 	if err != nil {
-		t.Fatalf("Cannot load kube config: %v", err)
+		t.Errorf("cannot get kube config: %w", err)
 	}
 
 	c := NewPodsContainerLogs(config)

pkg/collector/systemperf_collector_test.go

@@ -3,10 +3,7 @@ package collector
 import (
 	"encoding/json"
 	"os"
-	"path"
 	"testing"
-
-	"k8s.io/client-go/tools/clientcmd"
 )
 
 func TestSystemperfCollector(t *testing.T) {
@@ -22,16 +19,9 @@ func TestSystemperfCollector(t *testing.T) {
 		},
 	}
 
-	dirname, err := os.UserHomeDir()
-	if err != nil {
-		t.Fatalf("Cannot get user home dir: %v", err)
-	}
-
-	master := ""
-	kubeconfig := path.Join(dirname, ".kube/config")
-	config, err := clientcmd.BuildConfigFromFlags(master, kubeconfig)
+	config, err := getConfig()
 	if err != nil {
-		t.Fatalf("Cannot load kube config: %v", err)
+		t.Errorf("cannot get kube config: %w", err)
 	}
 
 	c := NewSystemPerfCollector(config)