Simplify hive

go/tasks/plugins/hive/config/config.go

@@ -5,6 +5,9 @@ package config
 import (
 	"context"
 	"net/url"
+	"time"
+
+	"github.com/lyft/flyteplugins/go/tasks/pluginmachinery/remote"
 
 	"github.com/lyft/flytestdlib/config"
 	"github.com/lyft/flytestdlib/logger"
@@ -45,27 +48,39 @@ var (
 		CommandAPIPath:            MustParse("/api/v1.2/commands/"),
 		AnalyzeLinkPath:           MustParse("/v2/analyze"),
 		TokenKey:                  "FLYTE_QUBOLE_CLIENT_TOKEN",
-		LruCacheSize:              2000,
-		Workers:                   15,
 		DefaultClusterLabel:       "default",
 		ClusterConfigs:            []ClusterConfig{{PrimaryLabel: "default", Labels: []string{"default"}, Limit: 100, ProjectScopeQuotaProportionCap: 0.7, NamespaceScopeQuotaProportionCap: 0.7}},
 		DestinationClusterConfigs: []DestinationClusterConfig{},
+		Caching: remote.CachingProperties{
+			Size:           2000,
+			ResyncInterval: config.Duration{Duration: 20 * time.Second},
+			Workers:        15,
+		},
+		WriteRateLimiter: remote.RateLimiterProperties{
+			QPS:   100,
+			Burst: 200,
+		},
+		ReadRateLimiter: remote.RateLimiterProperties{
+			QPS:   100,
+			Burst: 200,
+		},
 	}
 
 	quboleConfigSection = pluginsConfig.MustRegisterSubSection(quboleConfigSectionKey, &defaultConfig)
 )
 
 // Qubole plugin configs
 type Config struct {
-	Endpoint                  config.URL                 `json:"endpoint" pflag:",Endpoint for qubole to use"`
-	CommandAPIPath            config.URL                 `json:"commandApiPath" pflag:",API Path where commands can be launched on Qubole. Should be a valid url."`
-	AnalyzeLinkPath           config.URL                 `json:"analyzeLinkPath" pflag:",URL path where queries can be visualized on qubole website. Should be a valid url."`
-	TokenKey                  string                     `json:"quboleTokenKey" pflag:",Name of the key where to find Qubole token in the secret manager."`
-	LruCacheSize              int                        `json:"lruCacheSize" pflag:",Size of the AutoRefreshCache"`
-	Workers                   int                        `json:"workers" pflag:",Number of parallel workers to refresh the cache"`
-	DefaultClusterLabel       string                     `json:"defaultClusterLabel" pflag:",The default cluster label. This will be used if label is not specified on the hive job."`
-	ClusterConfigs            []ClusterConfig            `json:"clusterConfigs" pflag:"-,A list of cluster configs. Each of the configs corresponds to a service cluster"`
-	DestinationClusterConfigs []DestinationClusterConfig `json:"destinationClusterConfigs" pflag:"-,A list configs specifying the destination service cluster for (project, domain)"`
+	Endpoint                  config.URL                   `json:"endpoint" pflag:",Endpoint for qubole to use"`
+	CommandAPIPath            config.URL                   `json:"commandApiPath" pflag:",API Path where commands can be launched on Qubole. Should be a valid url."`
+	AnalyzeLinkPath           config.URL                   `json:"analyzeLinkPath" pflag:",URL path where queries can be visualized on qubole website. Should be a valid url."`
+	TokenKey                  string                       `json:"quboleTokenKey" pflag:",Name of the key where to find Qubole token in the secret manager."`
+	ClusterConfigs            []ClusterConfig              `json:"clusterConfigs" pflag:"-,A list of cluster configs. Each of the configs corresponds to a service cluster"`
+	DefaultClusterLabel       string                       `json:"defaultClusterLabel" pflag:",The default cluster label. This will be used if label is not specified on the hive job."`
+	DestinationClusterConfigs []DestinationClusterConfig   `json:"destinationClusterConfigs" pflag:"-,A list configs specifying the destination service cluster for (project, domain)"`
+	ReadRateLimiter           remote.RateLimiterProperties `json:"readRateLimiter" pflag:",Defines rate limiter properties for read actions (e.g. retrieve status)."`
+	WriteRateLimiter          remote.RateLimiterProperties `json:"writeRateLimiter" pflag:",Defines rate limiter properties for write actions."`
+	Caching                   remote.CachingProperties     `json:"caching" pflag:",Defines caching characteristics."`
 }
 
 // Retrieves the current config value or default.