Migrate Lyft -> Flyteorg (flyteorg#33)

* Migrate Lyft -> Flyteorg

 - Also removes uneeded dependencies
 - Cleans up makefile

Signed-off-by: Ketan Umare <[email protected]>

* All linter errors and logical errors fixed

Signed-off-by: Ketan Umare <[email protected]>

Makefile

@@ -17,10 +17,11 @@ linux_compile:
 
 .PHONY: generate_idl
 generate_idl:
-	protoc -I ./vendor/github.com/lyft/flyteidl/protos/ -I ./protos/idl/. --go_out=plugins=grpc:protos/gen ./protos/idl/service.proto
+	# TODO, move the protos to flyteidl. Currently to generate protos it is
+	# assumed that the flyteidl repo is checked out in an adjoining directory.
+	# We could use vendoring - but that causes problems when compiling
+	protoc -I ../flyteidl/protos/ -I ./protos/idl/. --go_out=plugins=grpc:protos/gen ./protos/idl/service.proto
 
 .PHONY: generate
 generate:
-	which pflags || (go get github.com/lyft/flytestdlib/cli/pflags)
-	which mockery || (go get github.com/enghabu/mockery/cmd/mockery)
 	@go generate ./...

cmd/entrypoints/migrate.go

@@ -1,11 +1,11 @@
 package entrypoints
 
 import (
+	"github.com/flyteorg/datacatalog/pkg/repositories"
+	"github.com/flyteorg/datacatalog/pkg/runtime"
+	"github.com/flyteorg/flytestdlib/logger"
+	"github.com/flyteorg/flytestdlib/promutils"
 	"github.com/lib/pq"
-	"github.com/lyft/datacatalog/pkg/repositories"
-	"github.com/lyft/datacatalog/pkg/runtime"
-	"github.com/lyft/flytestdlib/logger"
-	"github.com/lyft/flytestdlib/promutils"
 
 	"context"
 

cmd/entrypoints/root.go

@@ -6,8 +6,8 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/lyft/flytestdlib/config"
-	"github.com/lyft/flytestdlib/config/viper"
+	"github.com/flyteorg/flytestdlib/config"
+	"github.com/flyteorg/flytestdlib/config/viper"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 )
@@ -64,7 +64,7 @@ To get started run the serve subcommand which will start a server on localhost:8
 
 func initConfig(flags *pflag.FlagSet) error {
 	configAccessor = viper.NewAccessor(config.Options{
-		SearchPaths: []string{cfgFile, ".", "/etc/flyte/config", "$GOPATH/src/github.com/lyft/datacatalog"},
+		SearchPaths: []string{cfgFile, ".", "/etc/flyte/config", "$GOPATH/src/github.com/flyteorg/datacatalog"},
 		StrictMode:  false,
 	})
 

cmd/entrypoints/serve.go

@@ -5,10 +5,10 @@ import (
 	"net"
 	"net/http"
 
-	"github.com/lyft/datacatalog/pkg/config"
-	"github.com/lyft/datacatalog/pkg/rpc/datacatalogservice"
-	datacatalog "github.com/lyft/datacatalog/protos/gen"
-	"github.com/lyft/flytestdlib/logger"
+	"github.com/flyteorg/datacatalog/pkg/config"
+	"github.com/flyteorg/datacatalog/pkg/rpc/datacatalogservice"
+	datacatalog "github.com/flyteorg/datacatalog/protos/gen"
+	"github.com/flyteorg/flytestdlib/logger"
 	"github.com/spf13/cobra"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/health"

cmd/entrypoints/serve_dummy.go

@@ -4,10 +4,10 @@ import (
 	"context"
 	"net"
 
-	"github.com/lyft/datacatalog/pkg/config"
-	"github.com/lyft/datacatalog/pkg/rpc/datacatalogservice"
-	datacatalog "github.com/lyft/datacatalog/protos/gen"
-	"github.com/lyft/flytestdlib/logger"
+	"github.com/flyteorg/datacatalog/pkg/config"
+	"github.com/flyteorg/datacatalog/pkg/rpc/datacatalogservice"
+	datacatalog "github.com/flyteorg/datacatalog/protos/gen"
+	"github.com/flyteorg/flytestdlib/logger"
 	"github.com/spf13/cobra"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/reflection"

cmd/main.go

@@ -1,8 +1,8 @@
 package main
 
 import (
+	"github.com/flyteorg/datacatalog/cmd/entrypoints"
 	"github.com/golang/glog"
-	"github.com/lyft/datacatalog/cmd/entrypoints"
 )
 
 func main() {

go.mod

@@ -1,18 +1,19 @@
-module github.com/lyft/datacatalog
+module github.com/flyteorg/datacatalog
 
 go 1.13
 
 require (
 	github.com/Selvatico/go-mocket v1.0.7
+	github.com/flyteorg/flyteidl v0.18.15
+	github.com/flyteorg/flytestdlib v0.3.13
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
-	github.com/golang/protobuf v1.3.2
+	github.com/golang/protobuf v1.4.3
 	github.com/jinzhu/gorm v1.9.11
 	github.com/lib/pq v1.2.0
-	github.com/lyft/flyteidl v0.17.0
-	github.com/lyft/flytestdlib v0.3.0
-	github.com/mitchellh/mapstructure v1.1.2
-	github.com/spf13/cobra v0.0.5
+	github.com/mitchellh/mapstructure v1.4.1
+	github.com/spf13/cobra v1.1.1
 	github.com/spf13/pflag v1.0.5
-	github.com/stretchr/testify v1.4.0
-	google.golang.org/grpc v1.26.0
+	github.com/stretchr/testify v1.7.0
+	google.golang.org/grpc v1.36.0
+	google.golang.org/protobuf v1.25.0
 )

pkg/config/config.go

@@ -3,7 +3,7 @@ package config
 import (
 	"fmt"
 
-	"github.com/lyft/flytestdlib/config"
+	"github.com/flyteorg/flytestdlib/config"
 )
 
 const SectionKey = "application"

pkg/manager/impl/artifact_data_store.go

@@ -3,19 +3,19 @@ package impl
 import (
 	"context"
 
-	"github.com/lyft/datacatalog/pkg/errors"
-	"github.com/lyft/datacatalog/pkg/repositories/models"
-	datacatalog "github.com/lyft/datacatalog/protos/gen"
-	"github.com/lyft/flyteidl/gen/pb-go/flyteidl/core"
-	"github.com/lyft/flytestdlib/storage"
+	"github.com/flyteorg/datacatalog/pkg/errors"
+	"github.com/flyteorg/datacatalog/pkg/repositories/models"
+	datacatalog "github.com/flyteorg/datacatalog/protos/gen"
+	"github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core"
+	"github.com/flyteorg/flytestdlib/storage"
 	"google.golang.org/grpc/codes"
 )
 
 const artifactDataFile = "data.pb"
 
 // ArtifactDataStore stores and retrieves ArtifactData values in a data.pb
 type ArtifactDataStore interface {
-	PutData(ctx context.Context, artifact datacatalog.Artifact, data datacatalog.ArtifactData) (storage.DataReference, error)
+	PutData(ctx context.Context, artifact *datacatalog.Artifact, data *datacatalog.ArtifactData) (storage.DataReference, error)
 	GetData(ctx context.Context, dataModel models.ArtifactData) (*core.Literal, error)
 }
 
@@ -24,13 +24,13 @@ type artifactDataStore struct {
 	storagePrefix storage.DataReference
 }
 
-func (m *artifactDataStore) getDataLocation(ctx context.Context, artifact datacatalog.Artifact, data datacatalog.ArtifactData) (storage.DataReference, error) {
+func (m *artifactDataStore) getDataLocation(ctx context.Context, artifact *datacatalog.Artifact, data *datacatalog.ArtifactData) (storage.DataReference, error) {
 	dataset := artifact.Dataset
 	return m.store.ConstructReference(ctx, m.storagePrefix, dataset.Project, dataset.Domain, dataset.Name, dataset.Version, artifact.Id, data.Name, artifactDataFile)
 }
 
 // Store marshalled data in data.pb under the storage prefix
-func (m *artifactDataStore) PutData(ctx context.Context, artifact datacatalog.Artifact, data datacatalog.ArtifactData) (storage.DataReference, error) {
+func (m *artifactDataStore) PutData(ctx context.Context, artifact *datacatalog.Artifact, data *datacatalog.ArtifactData) (storage.DataReference, error) {
 	dataLocation, err := m.getDataLocation(ctx, artifact, data)
 	if err != nil {
 		return "", errors.NewDataCatalogErrorf(codes.Internal, "Unable to generate data location %s, err %v", dataLocation.String(), err)

pkg/manager/impl/artifact_manager.go

@@ -5,21 +5,21 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/lyft/datacatalog/pkg/errors"
-	"github.com/lyft/datacatalog/pkg/manager/impl/validators"
-	"github.com/lyft/datacatalog/pkg/manager/interfaces"
-	"github.com/lyft/datacatalog/pkg/repositories"
-	datacatalog "github.com/lyft/datacatalog/protos/gen"
-
-	"github.com/lyft/datacatalog/pkg/repositories/models"
-	"github.com/lyft/datacatalog/pkg/repositories/transformers"
-
-	"github.com/lyft/datacatalog/pkg/common"
-	"github.com/lyft/flytestdlib/contextutils"
-	"github.com/lyft/flytestdlib/logger"
-	"github.com/lyft/flytestdlib/promutils"
-	"github.com/lyft/flytestdlib/promutils/labeled"
-	"github.com/lyft/flytestdlib/storage"
+	"github.com/flyteorg/datacatalog/pkg/errors"
+	"github.com/flyteorg/datacatalog/pkg/manager/impl/validators"
+	"github.com/flyteorg/datacatalog/pkg/manager/interfaces"
+	"github.com/flyteorg/datacatalog/pkg/repositories"
+	datacatalog "github.com/flyteorg/datacatalog/protos/gen"
+
+	"github.com/flyteorg/datacatalog/pkg/repositories/models"
+	"github.com/flyteorg/datacatalog/pkg/repositories/transformers"
+
+	"github.com/flyteorg/datacatalog/pkg/common"
+	"github.com/flyteorg/flytestdlib/contextutils"
+	"github.com/flyteorg/flytestdlib/logger"
+	"github.com/flyteorg/flytestdlib/promutils"
+	"github.com/flyteorg/flytestdlib/promutils/labeled"
+	"github.com/flyteorg/flytestdlib/storage"
 	"google.golang.org/grpc/codes"
 )
 
@@ -48,7 +48,7 @@ type artifactManager struct {
 }
 
 // Create an Artifact along with the associated ArtifactData. The ArtifactData will be stored in an offloaded location.
-func (m *artifactManager) CreateArtifact(ctx context.Context, request datacatalog.CreateArtifactRequest) (*datacatalog.CreateArtifactResponse, error) {
+func (m *artifactManager) CreateArtifact(ctx context.Context, request *datacatalog.CreateArtifactRequest) (*datacatalog.CreateArtifactResponse, error) {
 	timer := m.systemMetrics.createResponseTime.Start(ctx)
 	defer timer.Stop()
 
@@ -61,7 +61,7 @@ func (m *artifactManager) CreateArtifact(ctx context.Context, request datacatalo
 	}
 
 	ctx = contextutils.WithProjectDomain(ctx, artifact.Dataset.Project, artifact.Dataset.Domain)
-	datasetKey := transformers.FromDatasetID(*artifact.Dataset)
+	datasetKey := transformers.FromDatasetID(artifact.Dataset)
 
 	// The dataset must exist for the artifact, let's verify that first
 	dataset, err := m.repo.DatasetRepo().Get(ctx, datasetKey)
@@ -84,7 +84,7 @@ func (m *artifactManager) CreateArtifact(ctx context.Context, request datacatalo
 	// create Artifact Data offloaded storage files
 	artifactDataModels := make([]models.ArtifactData, len(request.Artifact.Data))
 	for i, artifactData := range request.Artifact.Data {
-		dataLocation, err := m.artifactStore.PutData(ctx, *artifact, *artifactData)
+		dataLocation, err := m.artifactStore.PutData(ctx, artifact, artifactData)
 		if err != nil {
 			logger.Errorf(ctx, "Failed to store artifact data err: %v", err)
 			m.systemMetrics.createDataFailureCounter.Inc(ctx)
@@ -124,7 +124,7 @@ func (m *artifactManager) CreateArtifact(ctx context.Context, request datacatalo
 }
 
 // Get the Artifact and its associated ArtifactData. The request can query by ArtifactID or TagName.
-func (m *artifactManager) GetArtifact(ctx context.Context, request datacatalog.GetArtifactRequest) (*datacatalog.GetArtifactResponse, error) {
+func (m *artifactManager) GetArtifact(ctx context.Context, request *datacatalog.GetArtifactRequest) (*datacatalog.GetArtifactResponse, error) {
 	timer := m.systemMetrics.getResponseTime.Start(ctx)
 	defer timer.Stop()
 
@@ -156,7 +156,7 @@ func (m *artifactManager) GetArtifact(ctx context.Context, request datacatalog.G
 		}
 	case *datacatalog.GetArtifactRequest_TagName:
 		logger.Debugf(ctx, "Get artifact by tag %v", request.GetTagName())
-		tagKey := transformers.ToTagKey(*datasetID, request.GetTagName())
+		tagKey := transformers.ToTagKey(datasetID, request.GetTagName())
 		tag, err := m.repo.TagRepo().Get(ctx, tagKey)
 
 		if err != nil {
@@ -194,7 +194,7 @@ func (m *artifactManager) GetArtifact(ctx context.Context, request datacatalog.G
 	logger.Debugf(ctx, "Retrieved artifact dataset %v, id: %v", artifact.Dataset, artifact.Id)
 	m.systemMetrics.getSuccessCounter.Inc(ctx)
 	return &datacatalog.GetArtifactResponse{
-		Artifact: &artifact,
+		Artifact: artifact,
 	}, nil
 }
 
@@ -216,16 +216,16 @@ func (m *artifactManager) getArtifactDataList(ctx context.Context, artifactDataM
 	return artifactDataList, nil
 }
 
-func (m *artifactManager) ListArtifacts(ctx context.Context, request datacatalog.ListArtifactsRequest) (*datacatalog.ListArtifactsResponse, error) {
-	err := validators.ValidateListArtifactRequest(&request)
+func (m *artifactManager) ListArtifacts(ctx context.Context, request *datacatalog.ListArtifactsRequest) (*datacatalog.ListArtifactsResponse, error) {
+	err := validators.ValidateListArtifactRequest(request)
 	if err != nil {
 		logger.Warningf(ctx, "Invalid list artifact request %v, err: %v", request, err)
 		m.systemMetrics.validationErrorCounter.Inc(ctx)
 		return nil, err
 	}
 
 	// Verify the dataset exists before listing artifacts
-	datasetKey := transformers.FromDatasetID(*request.Dataset)
+	datasetKey := transformers.FromDatasetID(request.Dataset)
 	dataset, err := m.repo.DatasetRepo().Get(ctx, datasetKey)
 	if err != nil {
 		logger.Warnf(ctx, "Failed to get dataset for listing artifacts %v, err: %v", datasetKey, err)