Log collection through Filebeat daemonset

logs-streaming/00namespace.yml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: logs-kafka

logs-streaming/filebeat-config.yml

@@ -0,0 +1,34 @@
+kind: ConfigMap
+metadata:
+  name: filebeat-config
+  namespace: logs-kafka
+apiVersion: v1
+data:
+  filebeat.yml: |+
+
+    filebeat.prospectors:
+    - type: log
+      enabled: true
+      paths:
+        - /var/lib/docker/containers/*/*.log
+      symlinks: true
+      # reduce the risk for aggregation recursion: match pod names to exclude own's logs + test logs
+      exclude_files: ['^(.*\/)?logs-','^(.*\/)?filebeat-','^(.*\/)?test-']
+
+    processors:
+    - add_kubernetes_metadata:
+        in_cluster: true
+        namespace: logs-kafka
+
+    output.kafka:
+      hosts: ["kafka-0.broker.kafka.svc.cluster.local:9092", "kafka-1.broker.kafka.svc.cluster.local:9092", "kafka-2.broker.kafka.svc.cluster.local:9092"]
+      topic: ${TOPIC}
+
+      partition.round_robin:
+        reachable_only: false
+
+      client_id: filebeat-kubernetes
+      version: 0.11.0.0
+      required_acks: 1
+      compression: gzip
+      max_message_bytes: 1000000

logs-streaming/filebeat-logs-kube-kafka.yml

@@ -0,0 +1,64 @@
+apiVersion: apps/v1beta2
+kind: DaemonSet
+metadata:
+  name: filebeat-kube-kafka
+  namespace: logs-kafka
+spec:
+  selector:
+    matchLabels:
+      k8s-app: filebeat-kube-kafka
+      version: v1
+      kubernetes.io/cluster-service: "true"
+  template:
+    metadata:
+      labels:
+        k8s-app: filebeat-kube-kafka
+        version: v1
+        kubernetes.io/cluster-service: "true"
+    spec:
+      serviceAccountName: filebeat
+      containers:
+      - name: filebeat
+        image: solsson/kafka-filebeat-kubernetes@sha256:c2a68766e718d354b565b20f92baf0f50047b4bdf90a5055a20107a1131bee80
+        command:
+        - filebeat
+        - -e
+        - -c
+        - /etc/filebeat/filebeat.yml
+        - -d
+        - "service,beat"
+        env:
+        - name: TOPIC
+          value: ops-kube-logs-filebeat-001
+        resources:
+          limits:
+            memory: 100Mi
+          requests:
+            cpu: 100m
+            memory: 100Mi
+        volumeMounts:
+        - name: config
+          mountPath: /etc/filebeat
+          readOnly: true
+        - name: data
+          mountPath: /data
+        - name: varlog
+          mountPath: /var/log
+          readOnly: true
+        - name: varlibdockercontainers
+          mountPath: /var/lib/docker/containers
+          readOnly: true
+      terminationGracePeriodSeconds: 10
+      volumes:
+      - name: config
+        configMap:
+          name: filebeat-config
+      - name: data
+        hostPath:
+          path: /tmp/kubernets-filebeat-state
+      - name: varlog
+        hostPath:
+          path: /var/log
+      - name: varlibdockercontainers
+        hostPath:
+          path: /var/lib/docker/containers

logs-streaming/rbac/filebeat.yml

@@ -0,0 +1,23 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: filebeat
+  namespace: logs-kafka
+  annotations:
+    manifest-origin: 'github.com/Yolean/kubernetes-kafka'
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: logs-kafka-filebeat-sa-view
+  annotations:
+    manifest-origin: 'github.com/Yolean/kubernetes-kafka'
+subjects:
+- kind: ServiceAccount
+  name: filebeat
+  namespace: logs-kafka
+roleRef:
+  kind: ClusterRole
+  name: view
+  apiGroup: rbac.authorization.k8s.io

logs-streaming/test/logs-filebeat.yml

@@ -0,0 +1,75 @@
+---
+kind: ConfigMap
+metadata:
+  name: logs-filebeat
+  namespace: test-kafka
+apiVersion: v1
+data:
+
+  setup.sh: |-
+    touch /tmp/testlog
+
+    tail -f /tmp/testlog
+
+  test.sh: |-
+    exec >> /tmp/testlog
+    exec 2>&1
+
+    echo ""
+    echo "{\"#---\":\"$(date -u --iso-8601='ns')\"}"
+    kafkacat -b $BOOTSTRAP -C -t $TOPIC -f '{"#topic":"%t","#partition":%p,"#offset":%o,"#key":"%k","=":%s}\n' -o -10 -e -q
+
+    exit 0
+
+  quit-on-nonzero-exit.sh: |-
+    exit 0
+
+---
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  name: logs-filebeat
+  namespace: test-kafka
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      test-target: logs-streaming-filebeat
+      test-type: readiness
+  template:
+    metadata:
+      labels:
+        test-target: logs-streaming-filebeat
+        test-type: readiness
+    spec:
+      containers:
+      - name: testcase
+        image: solsson/kafkacat@sha256:ebebf47061300b14a4b4c2e1e4303ab29f65e4b95d34af1b14bb8f7ec6da7cef
+        env:
+        - name: BOOTSTRAP
+          value: kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092
+        - name: TOPIC
+          value: ops-kube-logs-filebeat-001
+        command:
+        - /bin/bash
+        - -e
+        - /test/setup.sh
+        readinessProbe:
+          exec:
+            command:
+            - /bin/bash
+            - -e
+            - /test/test.sh
+        livenessProbe:
+          exec:
+            command:
+            - /bin/bash
+            - -e
+            - /test/quit-on-nonzero-exit.sh
+        volumeMounts:
+        - name: config
+          mountPath: /test
+      volumes:
+      - name: config
+        configMap:
+          name: logs-filebeat

logs-streaming/topic-ops-kube-logs-filebeat.yml

@@ -0,0 +1,31 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: topic-ops-kube-logs-filebeat
+  namespace: logs-kafka
+spec:
+  template:
+    metadata:
+      labels:
+        app: topic-create
+        topic-id: ops-kube-logs-filebeat
+        topic-gen: "001"
+    spec:
+      containers:
+      - name: kafka
+        image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d
+        command:
+        - ./bin/kafka-topics.sh
+        - --zookeeper
+        - zookeeper.kafka:2181
+        - --create
+        - --if-not-exists
+        - --topic
+        - ops-kube-logs-filebeat-001
+        - --partitions
+        - "1"
+        - --replication-factor
+        - "2"
+        - --config
+        - retention.ms=6912500000
+      restartPolicy: Never