Kenny/type metadata (flyteorg#370)

* feat: strip annotation from node template Signed-off-by: Kenny Workman <[email protected]> * feat: strip annotation tests Signed-off-by: Kenny Workman <[email protected]> * fix: strip outputs + test Signed-off-by: Kenny Workman <[email protected]> * fix: temp flyteidl for tests Signed-off-by: Kenny Workman <[email protected]> * fix: tmp update flyteidl Signed-off-by: Kenny Workman <[email protected]> * fix: range nil ok Signed-off-by: Kenny Workman <[email protected]> * fix: strip annotation in type compare Signed-off-by: Kenny Workman <[email protected]> * fix: rm .swp Signed-off-by: Kenny Workman <[email protected]> * fix: new flyteidl Signed-off-by: Kenny Workman <[email protected]>
eapolinario · Feb 8, 2022 · 7e8bc5f · 7e8bc5f
1 parent 6f4475e
commit 7e8bc5f
Show file tree

Hide file tree

Showing 5 changed files with 112 additions and 1 deletion.
diff --git a/go.mod b/go.mod
@@ -6,7 +6,7 @@ require (
 	github.com/DiSiqueira/GoTree v1.0.1-0.20180907134536-53a8e837f295
 	github.com/benlaurie/objecthash v0.0.0-20180202135721-d1e3d6079fc1
 	github.com/fatih/color v1.10.0
-	github.com/flyteorg/flyteidl v0.21.25
+	github.com/flyteorg/flyteidl v0.22.0
 	github.com/flyteorg/flyteplugins v0.10.3
 	github.com/flyteorg/flytestdlib v0.4.7
 	github.com/ghodss/yaml v1.0.0

diff --git a/go.sum b/go.sum
@@ -241,6 +241,8 @@ github.com/flyteorg/flyteidl v0.21.24 h1:e2wPBK4aiLE+fw2zmhUDNg39QoJk6Lf5lQRvj8X
 github.com/flyteorg/flyteidl v0.21.24/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U=
 github.com/flyteorg/flyteidl v0.21.25 h1:8Mc7QiaKQ1SCIQw8V1rXqoOMxX/5xq5tSEb/ZhABtHk=
 github.com/flyteorg/flyteidl v0.21.25/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U=
+github.com/flyteorg/flyteidl v0.22.0 h1:lVAXyacTCIX6Fl0qWoFzyto9Hfx0ADlyPhHPmOMiuIY=
+github.com/flyteorg/flyteidl v0.22.0/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U=
 github.com/flyteorg/flyteplugins v0.10.3 h1:FFtWZNZ3SnhGHImzQzl5B6KMMkgj60LSzD90F6DYAgA=
 github.com/flyteorg/flyteplugins v0.10.3/go.mod h1:RXgHGGUGC1akEnAd0yi4cLuYP1BF1rVkxhGjzIrm6VU=
 github.com/flyteorg/flytestdlib v0.3.13/go.mod h1:Tz8JCECAbX6VWGwFT6cmEQ+RJpZ/6L9pswu3fzWs220=
@@ -506,6 +508,8 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/kennyworkman/flyteidl v0.21.12-0.20211210005601-8c4bb8cda5fb h1:ArMy8u6WCFlc2VNBb0yIzn+U6/z49hQGYoJQvW+6YpM=
+github.com/kennyworkman/flyteidl v0.21.12-0.20211210005601-8c4bb8cda5fb/go.mod h1:vHSugApgS3hRITIafzQDU8DZD/W8wFRfFcgaFU35Dww=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=

diff --git a/pkg/compiler/transformers/k8s/node.go b/pkg/compiler/transformers/k8s/node.go
@@ -221,6 +221,28 @@ func buildTasks(tasks []*core.CompiledTask, errs errors.CompileErrors) map[commo
 				errs.Collect(errors.NewValueCollisionError(taskID, "Id", taskID))
 			}
 
+			// We don't want Annotation data available at task runtime for performance.
+			if flyteTask.Template != nil &&
+				flyteTask.Template.Interface != nil {
+
+				if flyteTask.Template.Interface.Inputs != nil {
+					for _, v := range flyteTask.Template.Interface.Inputs.Variables {
+						if v.Type != nil && v.Type.Annotation != nil {
+							v.Type.Annotation = nil
+						}
+					}
+				}
+
+				if flyteTask.Template.Interface.Outputs != nil {
+					for _, v := range flyteTask.Template.Interface.Outputs.Variables {
+						if v.Type != nil && v.Type.Annotation != nil {
+							v.Type.Annotation = nil
+						}
+					}
+				}
+
+			}
+
 			res[taskID] = &v1alpha1.TaskSpec{TaskTemplate: flyteTask.Template}
 		}
 	}

diff --git a/pkg/compiler/transformers/k8s/node_test.go b/pkg/compiler/transformers/k8s/node_test.go
@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"github.com/flyteorg/flytepropeller/pkg/apis/flyteworkflow/v1alpha1"
+	"google.golang.org/protobuf/types/known/structpb"
 	"k8s.io/apimachinery/pkg/api/resource"
 
 	"github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core"
@@ -209,3 +210,85 @@ func TestBuildNodeSpec(t *testing.T) {
 	})
 
 }
+
+func TestBuildTasks(t *testing.T) {
+
+	withoutAnnotations := make(map[string]*core.Variable)
+	withoutAnnotations["a"] = &core.Variable{
+		Type: &core.LiteralType{
+			Annotation: &core.TypeAnnotation{},
+		},
+	}
+
+	randomData, _ := structpb.NewStruct(map[string]interface{}{
+		"foo": "bar",
+	})
+
+	withAnnotations := make(map[string]*core.Variable)
+	withAnnotations["a"] = &core.Variable{
+		Type: &core.LiteralType{
+			Annotation: &core.TypeAnnotation{Annotations: randomData},
+		},
+	}
+
+	tasks := []*core.CompiledTask{
+		{
+			Template: &core.TaskTemplate{
+				Id: &core.Identifier{Name: "annotatedInput"},
+				Interface: &core.TypedInterface{
+					Inputs: &core.VariableMap{
+						Variables: withAnnotations,
+					},
+				},
+			},
+		},
+		{
+			Template: &core.TaskTemplate{
+				Id: &core.Identifier{Name: "unannotatedInput"},
+				Interface: &core.TypedInterface{
+					Inputs: &core.VariableMap{
+						Variables: withoutAnnotations,
+					},
+				},
+			},
+		},
+		{
+			Template: &core.TaskTemplate{
+				Id: &core.Identifier{Name: "annotatedOutput"},
+				Interface: &core.TypedInterface{
+					Outputs: &core.VariableMap{
+						Variables: withAnnotations,
+					},
+				},
+			},
+		},
+		{
+			Template: &core.TaskTemplate{
+				Id: &core.Identifier{Name: "unannotatedOutput"},
+				Interface: &core.TypedInterface{
+					Outputs: &core.VariableMap{
+						Variables: withoutAnnotations,
+					},
+				},
+			},
+		},
+	}
+
+	errs := errors.NewCompileErrors()
+
+	t.Run("Tasks with annotations", func(t *testing.T) {
+		taskMap := buildTasks(tasks, errs)
+
+		annInputTask := taskMap[(&core.Identifier{Name: "annotatedInput"}).String()]
+		assert.Nil(t, annInputTask.Interface.Inputs.Variables["a"].Type.Annotation)
+
+		unAnnInputTask := taskMap[(&core.Identifier{Name: "unannotatedInput"}).String()]
+		assert.Nil(t, unAnnInputTask.Interface.Inputs.Variables["a"].Type.Annotation)
+
+		annOutputTask := taskMap[(&core.Identifier{Name: "annotatedOutput"}).String()]
+		assert.Nil(t, annOutputTask.Interface.Outputs.Variables["a"].Type.Annotation)
+
+		unAnnOutputTask := taskMap[(&core.Identifier{Name: "unannotatedOutput"}).String()]
+		assert.Nil(t, unAnnOutputTask.Interface.Outputs.Variables["a"].Type.Annotation)
+	})
+}
diff --git a/pkg/compiler/validators/typing.go b/pkg/compiler/validators/typing.go
@@ -58,6 +58,8 @@ func (t trivialChecker) CastsFrom(upstreamType *flyte.LiteralType) bool {
 	downstreamTypeCopy := *t.literalType
 	upstreamTypeCopy.Metadata = &structpb.Struct{}
 	downstreamTypeCopy.Metadata = &structpb.Struct{}
+	upstreamTypeCopy.Annotation = &flyte.TypeAnnotation{}
+	downstreamTypeCopy.Annotation = &flyte.TypeAnnotation{}
 	return upstreamTypeCopy.String() == downstreamTypeCopy.String()
 }