feat: add batch cacher (#822)

Signed-off-by: Alessandro Yuichi Okimoto <[email protected]>
bucketeer-io · Feb 22, 2024 · da18b3b · da18b3b
1 parent 3d2b159
commit da18b3b
Show file tree

Hide file tree

Showing 19 changed files with 1,147 additions and 63 deletions.
diff --git a/manifests/bucketeer/charts/batch/templates/deployment.yaml b/manifests/bucketeer/charts/batch/templates/deployment.yaml
@@ -50,6 +50,8 @@ spec:
           env:
             - name: BUCKETEER_BATCH_PROJECT
               value: "{{ .Values.env.project }}"
+            - name: BUCKETEER_BATCH_ACCOUNT_SERVICE
+              value: "{{ .Values.env.accountService }}"
             - name: BUCKETEER_BATCH_NOTIFICATION_SERVICE
               value: "{{ .Values.env.notificationService }}"
             - name: BUCKETEER_BATCH_ENVIRONMENT_SERVICE

diff --git a/manifests/bucketeer/charts/batch/values.yaml b/manifests/bucketeer/charts/batch/values.yaml
@@ -13,6 +13,7 @@ env:
   mysqlHost:
   mysqlPort:
   mysqlDBName:
+  accountService: localhost:9001
   notificationService: localhost:9001
   environmentService: localhost:9001
   autoOpsService: localhost:9001
@@ -95,42 +96,42 @@ cronjob:
   failedJobsHistoryLimit: 1
   webGatewayAddress: https://web-gateway.default.svc.cluster.local:9000
   jobs:
-  - name: experiment-status-updater
-    jobId: ExperimentStatusUpdater
-    schedule: "*/30 * * * *"
-  - name: experiment-running-watcher
-    jobId: ExperimentRunningWatcher
-    schedule: "0 1 * * *"
-  - name: feature-stale-watcher
-    jobId: FeatureStaleWatcher
-    schedule: "0 1 * * MON"
-  - name: mau-count-watcher
-    jobId: MauCountWatcher
-    schedule: "0 1 1 * *"
-  - name: ops-datetime-watcher
-    jobId: DatetimeWatcher
-    schedule: "* * * * *"
-  - name: ops-event-count-watcher
-    jobId: EventCountWatcher
-    schedule: "* * * * *"
-  - name: ops-progressive-rollout-watcher
-    jobId: ProgressiveRolloutWatcher
-    schedule: "* * * * *"
-  - name: domain-event-informer
-    jobId: DomainEventInformer
-    schedule: "* * * * *"
-  - name: redis-counter-deleter
-    jobId: RedisCounterDeleter
-    schedule: "0 0 * * *"
-  - name: experiment-calculator
-    jobId: ExperimentCalculator
-    schedule: "*/30 * * * *"
-  - name: mau-summarizer
-    jobId: MauSummarizer
-    schedule: "30 0 * * *"
-  - name: mau-partition-deleter
-    jobId: MauPartitionDeleter
-    schedule: "0 2 1 * *"
-  - name: mau-partition-creator
-    jobId: MauPartitionCreator
-    schedule: "0 2 2 * *"
+    - name: experiment-status-updater
+      jobId: ExperimentStatusUpdater
+      schedule: "*/30 * * * *"
+    - name: experiment-running-watcher
+      jobId: ExperimentRunningWatcher
+      schedule: "0 1 * * *"
+    - name: feature-stale-watcher
+      jobId: FeatureStaleWatcher
+      schedule: "0 1 * * MON"
+    - name: mau-count-watcher
+      jobId: MauCountWatcher
+      schedule: "0 1 1 * *"
+    - name: ops-datetime-watcher
+      jobId: DatetimeWatcher
+      schedule: "* * * * *"
+    - name: ops-event-count-watcher
+      jobId: EventCountWatcher
+      schedule: "* * * * *"
+    - name: ops-progressive-rollout-watcher
+      jobId: ProgressiveRolloutWatcher
+      schedule: "* * * * *"
+    - name: domain-event-informer
+      jobId: DomainEventInformer
+      schedule: "* * * * *"
+    - name: redis-counter-deleter
+      jobId: RedisCounterDeleter
+      schedule: "0 0 * * *"
+    - name: experiment-calculator
+      jobId: ExperimentCalculator
+      schedule: "*/30 * * * *"
+    - name: mau-summarizer
+      jobId: MauSummarizer
+      schedule: "30 0 * * *"
+    - name: mau-partition-deleter
+      jobId: MauPartitionDeleter
+      schedule: "0 2 1 * *"
+    - name: mau-partition-creator
+      jobId: MauPartitionCreator
+      schedule: "0 2 2 * *"
diff --git a/manifests/bucketeer/charts/web-gateway/values.yaml b/manifests/bucketeer/charts/web-gateway/values.yaml
diff --git a/pkg/batch/api/api.go b/pkg/batch/api/api.go
@@ -45,6 +45,11 @@ type batchService struct {
 	mauPartitionDeleter       jobs.Job
 	mauPartitionCreator       jobs.Job
 	domainEventInformer       jobs.Job
+	featureFlagCacher         jobs.Job
+	segmentUserCacher         jobs.Job
+	apiKeyCacher              jobs.Job
+	experimentCacher          jobs.Job
+	autoOpsRulesCacher        jobs.Job
 	logger                    *zap.Logger
 }
 
@@ -54,7 +59,9 @@ func NewBatchService(
 	eventCountWatcher, progressiveRolloutWatcher,
 	redisCounterDeleter, experimentCalculator,
 	mauSummarizer, mauPartitionDeleter, mauPartitionCreator,
-	domainEventInformer jobs.Job,
+	domainEventInformer, featureFlagCacher,
+	segmentUserCacher, apiKeyCacher,
+	experimentCacher, autoOpsRulesCacher jobs.Job,
 	logger *zap.Logger,
 ) *batchService {
 	return &batchService{
@@ -71,6 +78,11 @@ func NewBatchService(
 		mauPartitionDeleter:       mauPartitionDeleter,
 		mauPartitionCreator:       mauPartitionCreator,
 		domainEventInformer:       domainEventInformer,
+		featureFlagCacher:         featureFlagCacher,
+		segmentUserCacher:         segmentUserCacher,
+		apiKeyCacher:              apiKeyCacher,
+		experimentCacher:          experimentCacher,
+		autoOpsRulesCacher:        autoOpsRulesCacher,
 		logger:                    logger.Named("batch-service"),
 	}
 }
@@ -105,6 +117,16 @@ func (s *batchService) ExecuteBatchJob(
 		err = s.mauPartitionDeleter.Run(ctx)
 	case batch.BatchJob_MauPartitionCreator:
 		err = s.mauPartitionCreator.Run(ctx)
+	case batch.BatchJob_FeatureFlagCacher:
+		err = s.featureFlagCacher.Run(ctx)
+	case batch.BatchJob_SegmentUserCacher:
+		err = s.segmentUserCacher.Run(ctx)
+	case batch.BatchJob_ApiKeyCacher:
+		err = s.apiKeyCacher.Run(ctx)
+	case batch.BatchJob_ExperimentCacher:
+		err = s.experimentCacher.Run(ctx)
+	case batch.BatchJob_AutoOpsRulesCacher:
+		err = s.autoOpsRulesCacher.Run(ctx)
 	default:
 		s.logger.Error("Unknown job",
 			log.FieldsFromImcomingContext(ctx).AddFields(