wip(charts): add a seeder job to mapcache to generate local caches [p…

…ack mapcache]
kalisio · Feb 10, 2022 · e94bf12 · e94bf12
1 parent 483534d
commit e94bf12
Show file tree

Hide file tree

Showing 5 changed files with 119 additions and 123 deletions.
diff --git a/charts/mapcache/templates/deployment.yaml b/charts/mapcache/templates/deployment.yaml
@@ -1,4 +1,3 @@
-{{- if not (hasKey .Values "localCache") }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -26,6 +25,21 @@ spec:
         {{- if .Values.envsubstConfig }}
         {{- include "common.envsubstConfig.renderInitContainer" ( dict "args" .Values.envsubstConfig "context" $ ) | nindent 8 }}
         {{- end }}
+        {{- if .Values.localCacheSource }}
+        - name: rclone-local-cache
+          image: rclone/rclone
+          command:
+            - sh
+            - -c
+            - CACHE_SOURCE={{ include "common.tplvalues.render" ( dict "value" .Values.localCacheSource "context" $) }} && CACHE_FILE=$(basename $CACHE_SOURCE) && cd /cache && rclone copy $CACHE_SOURCE . --progress && tar xf $CACHE_FILE && rm -f $CACHE_FILE
+          volumeMounts:
+            - mountPath: /cache
+              name: mapcache-data
+            - mountPath: /config/rclone/rclone.conf
+              name: rclone-config
+              subPath: rclone.conf
+              readOnly: true
+        {{- end }}
         {{- if .Values.initContainers }}
         {{- include "common.tplvalues.render" ( dict "value" .Values.initContainers "context" $ ) | nindent 8 }}
         {{- end }}
@@ -59,13 +73,19 @@ spec:
           {{- include "common.tplvalues.render" ( dict "value" .Values.configVolume "context" $ ) | nindent 10 }}
           {{- end }}
         - name: mapcache-data
-          {{- if .Values.dataVolume }}
+          {{- if .Values.localCacheSource }}
+          emptyDir: {}
+          {{- else if .Values.dataVolume }}
           {{- include "common.tplvalues.render" ( dict "value" .Values.dataVolume "context" $ ) | nindent 10 }}
           {{- end }}
         {{- if .Values.envsubstConfig }}
         {{- include "common.envsubstConfig.renderVolume" ( dict "args" .Values.envsubstConfig "context" $ ) | nindent 8 }}
         {{- end }}
+        {{- if .Values.localCacheSource }}
+        - name: rclone-config
+          secret:
+            secretName: {{ include "common.tplvalues.render" ( dict "value" .Values.rcloneConfigSecretName "context" $) }}
+        {{- end }}
         {{- if .Values.extraVolumes }}
         {{- include "common.tplvalues.render" ( dict "value" .Values.extraVolumes "context" $ ) | nindent 8 }}
         {{- end }}
-{{- end }}
diff --git a/charts/mapcache/templates/seed-job-pvc.yaml b/charts/mapcache/templates/seed-job-pvc.yaml
@@ -0,0 +1,16 @@
+{{- if .Values.seedJob.enabled }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "common.names.fullname" . }}-seed-job
+  namespace: {{ .Release.Namespace }}
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{ required "Mapcache seed job's volume size must be defined !" .Values.seedJob.pvcSize }}
+  {{- if .Values.seedJob.pvcStorageClassName }}
+  storageClassName: {{ .Values.seedJob.pvcStorageClassName }}
+  {{- end }}
+{{- end }}
diff --git a/charts/mapcache/templates/seed-job.yaml b/charts/mapcache/templates/seed-job.yaml
@@ -0,0 +1,66 @@
+{{- if .Values.seedJob.enabled }}
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: {{ include "common.names.fullname" . }}-seed-job
+  namespace: {{ .Release.Namespace }}
+spec:
+  suspend: true
+  schedule: '0 1 * * *'
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          # ttlSecondsAfterFinished: 120
+          restartPolicy: Never
+          initContainers:
+            {{- if .Values.envsubstConfig }}
+            {{- include "common.envsubstConfig.renderInitContainer" ( dict "args" .Values.envsubstConfig "context" $ ) | nindent 12 }}
+            {{- end }}
+            - name: seed-local-cache
+              image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+              command:
+                - sh
+                - -c
+                - for LAYER in {{ .Values.seedJob.layers }}; do echo "Seeding layer $LAYER ..." && mapcache_seed -c /etc/mapcache/mapcache.xml -t $LAYER -z {{ .Values.seedJob.fromZ }},{{ .Values.seedJob.toZ }} -n {{ .Values.seedJob.threadCount }}; done
+              volumeMounts:
+                - mountPath: /etc/mapcache
+                  readOnly: true
+                  name: mapcache-config
+                - mountPath: /mnt/data
+                  name: mapcache-cache
+          containers:
+            - name: rclone-local-cache
+              image: rclone/rclone
+              command:
+                - sh
+                - -c
+                - CACHE_SOURCE={{ include "common.tplvalues.render" ( dict "value" .Values.localCacheSource "context" $) }} && CACHE_FILE=$(basename $CACHE_SOURCE) && cd /cache && tar -cf /scratch/$CACHE_FILE . && rclone copyto /scratch/$CACHE_FILE $CACHE_SOURCE --progress
+              volumeMounts:
+                - mountPath: /cache
+                  name: mapcache-cache
+                - mountPath: /scratch
+                  name: scratch-space
+                - mountPath: /config/rclone/rclone.conf
+                  name: rclone-config
+                  subPath: rclone.conf
+                  readOnly: true
+          volumes:
+            - name: mapcache-config
+              {{- if .Values.envsubstConfig }}
+              emptyDir: {}
+              {{- else if .Values.configVolume }}
+              {{- include "common.tplvalues.render" ( dict "value" .Values.configVolume "context" $ ) | nindent 14 }}
+              {{- end }}
+            {{- if .Values.envsubstConfig }}
+            {{- include "common.envsubstConfig.renderVolume" ( dict "args" .Values.envsubstConfig "context" $ ) | nindent 12 }}
+            {{- end }}
+            - name: mapcache-cache
+              persistentVolumeClaim:
+                claimName: {{ include "common.names.fullname" . }}-seed-job
+            - name: scratch-space
+              emptyDir: {}
+            - name: rclone-config
+              secret:
+                secretName: {{ include "common.tplvalues.render" ( dict "value" .Values.rcloneConfigSecretName "context" $) }}
+{{- end }}
diff --git a/charts/mapcache/templates/statefulset.yaml b/charts/mapcache/templates/statefulset.yaml
diff --git a/charts/mapcache/values.yaml b/charts/mapcache/values.yaml
@@ -23,6 +23,19 @@ service:
   port: 80 # port on which we expose service
   annotations: {}
 
+seedJob:
+  enabled: false
+  # pvcSize: 10Gi
+  # pvcStorageClassName: ""
+  # layers: imagery hybrid
+  # fromZ: 0
+  # toZ: 8
+  # threadCount: 16
+
+# localCacheSource: ""
+
+# rcloneConfigSecretName: ""
+
 # configVolume is the specification for the volume where the config lies
 configVolume: {}
 #  configMap:
@@ -39,4 +52,4 @@ dataVolume: {}
 #    readOnly: true
 
 # resources are resource requests and limits to be applied
-resources: {}
+resources: {}