Generic StowStore configuration (#29)

* Work in progress: Generic Stowstore * Simplifying to use only stowstore * added missing functions * removing deleted reference * Updated pflags * Fixed
eapolinario · Jul 27, 2019 · 0ff342b · 0ff342b
1 parent a8b264f
commit 0ff342b
Show file tree

Hide file tree

Showing 17 changed files with 618 additions and 207 deletions.
diff --git a/flytestdlib/Gopkg.lock b/flytestdlib/Gopkg.lock
diff --git a/flytestdlib/Gopkg.toml b/flytestdlib/Gopkg.toml
@@ -55,6 +55,10 @@
   branch = "master"
   name = "golang.org/x/time"
 
+[[override]]
+  branch = "master"
+  name = "golang.org/x/net"
+
 [[constraint]]
   name = "k8s.io/apimachinery"
   version = "kubernetes-1.13.1"
@@ -65,7 +69,7 @@
 
 [[constraint]]
   name = "github.com/graymeta/stow"
-  revision = "77c84b1dd69c41b74fe0a94ca8ee257d85947327"
+  revision = "903027f87de7054953efcdb8ba70d5dc02df38c7"
 
 [prune]
   go-tests = true

diff --git a/flytestdlib/cli/pflags/api/generator.go b/flytestdlib/cli/pflags/api/generator.go
@@ -145,6 +145,11 @@ func discoverFieldsRecursive(ctx context.Context, typ *types.Named, defaultValue
 			tag.Name = v.Name()
 		}
 
+		if tag.DefaultValue == "-" {
+			logger.Infof(ctx, "Skipping field [%s], as '-' value detected", tag.Name)
+			continue
+		}
+
 		typ := v.Type()
 		ptr, isPtr := typ.(*types.Pointer)
 		if isPtr {

diff --git a/flytestdlib/cli/pflags/api/sample.go b/flytestdlib/cli/pflags/api/sample.go
@@ -19,6 +19,7 @@ type TestType struct {
 	StringArray      []string          `json:"strs" pflag:"[]string{\"12\"%2C\"1\"}"`
 	ComplexJSONArray []ComplexJSONType `json:"complexArr"`
 	StringToJSON     ComplexJSONType   `json:"c" pflag:",I'm a complex type but can be converted from string."`
+	IgnoredMap       map[string]string `json:"ignored-map" pflag:"-,"`
 	StorageConfig    storage.Config    `json:"storage"`
 	IntValue         *int              `json:"i"`
 }

diff --git a/flytestdlib/cli/pflags/api/testdata/testtype.go b/flytestdlib/cli/pflags/api/testdata/testtype.go
diff --git a/flytestdlib/logger/config_flags.go b/flytestdlib/logger/config_flags.go
diff --git a/flytestdlib/logger/config_flags_test.go b/flytestdlib/logger/config_flags_test.go
diff --git a/flytestdlib/logger/config_test.go b/flytestdlib/logger/config_test.go
@@ -1,6 +1,7 @@
 package logger
 
 import (
+	"reflect"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -22,3 +23,19 @@ func TestSetConfig(t *testing.T) {
 		})
 	}
 }
+
+func TestGetConfig(t *testing.T) {
+	tests := []struct {
+		name string
+		want *Config
+	}{
+		// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := GetConfig(); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("GetConfig() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
diff --git a/flytestdlib/storage/config.go b/flytestdlib/storage/config.go
@@ -20,6 +20,7 @@ const (
 	TypeS3     Type = "s3"
 	TypeLocal  Type = "local"
 	TypeMinio  Type = "minio"
+	TypeStow   Type = "stow"
 )
 
 const (
@@ -43,8 +44,9 @@ var (
 
 // A common storage config.
 type Config struct {
-	Type          Type             `json:"type" pflag:",Sets the type of storage to configure [s3/minio/local/mem]."`
+	Type          Type             `json:"type" pflag:",Sets the type of storage to configure [s3/minio/local/mem/stow]."`
 	Connection    ConnectionConfig `json:"connection"`
+	Stow          *StowConfig      `json:"stow,omitempty"`
 	InitContainer string           `json:"container" pflag:",Initial container to create -if it doesn't exist-.'"`
 	// Caching is recommended to improve the performance of underlying systems. It caches the metadata and resolving
 	// inputs is accelerated. The size of the cache is large so understand how to configure the cache.
@@ -64,6 +66,11 @@ type ConnectionConfig struct {
 	DisableSSL bool       `json:"disable-ssl" pflag:",Disables SSL connection. Should only be used for development."`
 }
 
+type StowConfig struct {
+	Kind   string            `json:"kind,omitempty" pflag:"-,Kind of Stow backend to use. Refer to github/graymeta/stow"`
+	Config map[string]string `json:"config,omitempty" pflag:"-,Configuration for stow backend. Refer to github/graymeta/stow"`
+}
+
 type CachingConfig struct {
 	// Maximum size of the cache where the Blob store data is cached in-memory
 	// Refer to https://github.com/coocood/freecache to understand how to set the value

diff --git a/flytestdlib/storage/config_flags.go b/flytestdlib/storage/config_flags.go
diff --git a/flytestdlib/storage/localstore_test.go b/flytestdlib/storage/localstore_test.go
@@ -7,14 +7,17 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/lyft/flytestdlib/contextutils"
 	"github.com/lyft/flytestdlib/promutils"
+	"github.com/lyft/flytestdlib/promutils/labeled"
 
 	"github.com/lyft/flytestdlib/config"
 	"github.com/lyft/flytestdlib/internal/utils"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestNewLocalStore(t *testing.T) {
+	labeled.SetMetricKeys(contextutils.ProjectKey, contextutils.DomainKey, contextutils.WorkflowIDKey, contextutils.TaskIDKey)
 	t.Run("Valid config", func(t *testing.T) {
 		testScope := promutils.NewTestScope()
 		store, err := newLocalRawStore(&Config{
@@ -28,7 +31,7 @@ func TestNewLocalStore(t *testing.T) {
 		assert.NotNil(t, store)
 
 		// Stow local store expects the full path after the container portion (looks like a bug to me)
-		rc, err := store.ReadRaw(context.TODO(), DataReference("file://testdata/testdata/config.yaml"))
+		rc, err := store.ReadRaw(context.TODO(), DataReference("file://testdata/config.yaml"))
 		assert.NoError(t, err)
 		assert.NotNil(t, rc)
 		assert.NoError(t, rc.Close())

diff --git a/flytestdlib/storage/rawstores.go b/flytestdlib/storage/rawstores.go
@@ -11,8 +11,9 @@ type dataStoreCreateFn func(cfg *Config, metricsScope promutils.Scope) (RawStore
 var stores = map[string]dataStoreCreateFn{
 	TypeMemory: NewInMemoryRawStore,
 	TypeLocal:  newLocalRawStore,
-	TypeMinio:  newS3RawStore,
-	TypeS3:     newS3RawStore,
+	TypeMinio:  newStowRawStore,
+	TypeS3:     newStowRawStore,
+	TypeStow:   newStowRawStore,
 }
 
 // Creates a new Data Store with the supplied config.

diff --git a/flytestdlib/storage/s3store.go b/flytestdlib/storage/s3store.go
diff --git a/flytestdlib/storage/s3stsore_test.go b/flytestdlib/storage/s3stsore_test.go
diff --git a/flytestdlib/storage/stow_store_test.go b/flytestdlib/storage/stow_store_test.go
@@ -102,7 +102,8 @@ func (mockStowItem) Metadata() (map[string]interface{}, error) {
 func TestStowStore_ReadRaw(t *testing.T) {
 	t.Run("Happy Path", func(t *testing.T) {
 		testScope := promutils.NewTestScope()
-		s, err := NewStowRawStore(s3FQN("container"), newMockStowContainer("container"), testScope)
+		fn := fQNFn["s3"]
+		s, err := NewStowRawStore(fn("container"), newMockStowContainer("container"), testScope)
 		assert.NoError(t, err)
 		err = s.WriteRaw(context.TODO(), DataReference("s3://container/path"), 0, Options{}, bytes.NewReader([]byte{}))
 		assert.NoError(t, err)
@@ -119,7 +120,8 @@ func TestStowStore_ReadRaw(t *testing.T) {
 
 	t.Run("Exceeds limit", func(t *testing.T) {
 		testScope := promutils.NewTestScope()
-		s, err := NewStowRawStore(s3FQN("container"), newMockStowContainer("container"), testScope)
+		fn := fQNFn["s3"]
+		s, err := NewStowRawStore(fn("container"), newMockStowContainer("container"), testScope)
 		assert.NoError(t, err)
 		err = s.WriteRaw(context.TODO(), DataReference("s3://container/path"), 3*MiB, Options{}, bytes.NewReader([]byte{}))
 		assert.NoError(t, err)