From 7e8bc5f196f1ea131e16e2d63ba2540aebb3235c Mon Sep 17 00:00:00 2001 From: Kenny Workman <31255434+kennyworkman@users.noreply.github.com> Date: Tue, 8 Feb 2022 14:36:58 -0800 Subject: [PATCH] Kenny/type metadata (#370) * feat: strip annotation from node template Signed-off-by: Kenny Workman * feat: strip annotation tests Signed-off-by: Kenny Workman * fix: strip outputs + test Signed-off-by: Kenny Workman * fix: temp flyteidl for tests Signed-off-by: Kenny Workman * fix: tmp update flyteidl Signed-off-by: Kenny Workman * fix: range nil ok Signed-off-by: Kenny Workman * fix: strip annotation in type compare Signed-off-by: Kenny Workman * fix: rm .swp Signed-off-by: Kenny Workman * fix: new flyteidl Signed-off-by: Kenny Workman --- go.mod | 2 +- go.sum | 4 ++ pkg/compiler/transformers/k8s/node.go | 22 ++++++ pkg/compiler/transformers/k8s/node_test.go | 83 ++++++++++++++++++++++ pkg/compiler/validators/typing.go | 2 + 5 files changed, 112 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 547e902d77..8146bcbfdf 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/DiSiqueira/GoTree v1.0.1-0.20180907134536-53a8e837f295 github.com/benlaurie/objecthash v0.0.0-20180202135721-d1e3d6079fc1 github.com/fatih/color v1.10.0 - github.com/flyteorg/flyteidl v0.21.25 + github.com/flyteorg/flyteidl v0.22.0 github.com/flyteorg/flyteplugins v0.10.3 github.com/flyteorg/flytestdlib v0.4.7 github.com/ghodss/yaml v1.0.0 diff --git a/go.sum b/go.sum index 864a1037b1..723e93ea61 100644 --- a/go.sum +++ b/go.sum @@ -241,6 +241,8 @@ github.com/flyteorg/flyteidl v0.21.24 h1:e2wPBK4aiLE+fw2zmhUDNg39QoJk6Lf5lQRvj8X github.com/flyteorg/flyteidl v0.21.24/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U= github.com/flyteorg/flyteidl v0.21.25 h1:8Mc7QiaKQ1SCIQw8V1rXqoOMxX/5xq5tSEb/ZhABtHk= github.com/flyteorg/flyteidl v0.21.25/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U= +github.com/flyteorg/flyteidl v0.22.0 h1:lVAXyacTCIX6Fl0qWoFzyto9Hfx0ADlyPhHPmOMiuIY= +github.com/flyteorg/flyteidl v0.22.0/go.mod h1:576W2ViEyjTpT+kEVHAGbrTP3HARNUZ/eCwrNPmdx9U= github.com/flyteorg/flyteplugins v0.10.3 h1:FFtWZNZ3SnhGHImzQzl5B6KMMkgj60LSzD90F6DYAgA= github.com/flyteorg/flyteplugins v0.10.3/go.mod h1:RXgHGGUGC1akEnAd0yi4cLuYP1BF1rVkxhGjzIrm6VU= github.com/flyteorg/flytestdlib v0.3.13/go.mod h1:Tz8JCECAbX6VWGwFT6cmEQ+RJpZ/6L9pswu3fzWs220= @@ -506,6 +508,8 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kennyworkman/flyteidl v0.21.12-0.20211210005601-8c4bb8cda5fb h1:ArMy8u6WCFlc2VNBb0yIzn+U6/z49hQGYoJQvW+6YpM= +github.com/kennyworkman/flyteidl v0.21.12-0.20211210005601-8c4bb8cda5fb/go.mod h1:vHSugApgS3hRITIafzQDU8DZD/W8wFRfFcgaFU35Dww= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= diff --git a/pkg/compiler/transformers/k8s/node.go b/pkg/compiler/transformers/k8s/node.go index 1409545394..7ecd80b8d0 100644 --- a/pkg/compiler/transformers/k8s/node.go +++ b/pkg/compiler/transformers/k8s/node.go @@ -221,6 +221,28 @@ func buildTasks(tasks []*core.CompiledTask, errs errors.CompileErrors) map[commo errs.Collect(errors.NewValueCollisionError(taskID, "Id", taskID)) } + // We don't want Annotation data available at task runtime for performance. + if flyteTask.Template != nil && + flyteTask.Template.Interface != nil { + + if flyteTask.Template.Interface.Inputs != nil { + for _, v := range flyteTask.Template.Interface.Inputs.Variables { + if v.Type != nil && v.Type.Annotation != nil { + v.Type.Annotation = nil + } + } + } + + if flyteTask.Template.Interface.Outputs != nil { + for _, v := range flyteTask.Template.Interface.Outputs.Variables { + if v.Type != nil && v.Type.Annotation != nil { + v.Type.Annotation = nil + } + } + } + + } + res[taskID] = &v1alpha1.TaskSpec{TaskTemplate: flyteTask.Template} } } diff --git a/pkg/compiler/transformers/k8s/node_test.go b/pkg/compiler/transformers/k8s/node_test.go index 480948726d..7b87ea3950 100644 --- a/pkg/compiler/transformers/k8s/node_test.go +++ b/pkg/compiler/transformers/k8s/node_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/flyteorg/flytepropeller/pkg/apis/flyteworkflow/v1alpha1" + "google.golang.org/protobuf/types/known/structpb" "k8s.io/apimachinery/pkg/api/resource" "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" @@ -209,3 +210,85 @@ func TestBuildNodeSpec(t *testing.T) { }) } + +func TestBuildTasks(t *testing.T) { + + withoutAnnotations := make(map[string]*core.Variable) + withoutAnnotations["a"] = &core.Variable{ + Type: &core.LiteralType{ + Annotation: &core.TypeAnnotation{}, + }, + } + + randomData, _ := structpb.NewStruct(map[string]interface{}{ + "foo": "bar", + }) + + withAnnotations := make(map[string]*core.Variable) + withAnnotations["a"] = &core.Variable{ + Type: &core.LiteralType{ + Annotation: &core.TypeAnnotation{Annotations: randomData}, + }, + } + + tasks := []*core.CompiledTask{ + { + Template: &core.TaskTemplate{ + Id: &core.Identifier{Name: "annotatedInput"}, + Interface: &core.TypedInterface{ + Inputs: &core.VariableMap{ + Variables: withAnnotations, + }, + }, + }, + }, + { + Template: &core.TaskTemplate{ + Id: &core.Identifier{Name: "unannotatedInput"}, + Interface: &core.TypedInterface{ + Inputs: &core.VariableMap{ + Variables: withoutAnnotations, + }, + }, + }, + }, + { + Template: &core.TaskTemplate{ + Id: &core.Identifier{Name: "annotatedOutput"}, + Interface: &core.TypedInterface{ + Outputs: &core.VariableMap{ + Variables: withAnnotations, + }, + }, + }, + }, + { + Template: &core.TaskTemplate{ + Id: &core.Identifier{Name: "unannotatedOutput"}, + Interface: &core.TypedInterface{ + Outputs: &core.VariableMap{ + Variables: withoutAnnotations, + }, + }, + }, + }, + } + + errs := errors.NewCompileErrors() + + t.Run("Tasks with annotations", func(t *testing.T) { + taskMap := buildTasks(tasks, errs) + + annInputTask := taskMap[(&core.Identifier{Name: "annotatedInput"}).String()] + assert.Nil(t, annInputTask.Interface.Inputs.Variables["a"].Type.Annotation) + + unAnnInputTask := taskMap[(&core.Identifier{Name: "unannotatedInput"}).String()] + assert.Nil(t, unAnnInputTask.Interface.Inputs.Variables["a"].Type.Annotation) + + annOutputTask := taskMap[(&core.Identifier{Name: "annotatedOutput"}).String()] + assert.Nil(t, annOutputTask.Interface.Outputs.Variables["a"].Type.Annotation) + + unAnnOutputTask := taskMap[(&core.Identifier{Name: "unannotatedOutput"}).String()] + assert.Nil(t, unAnnOutputTask.Interface.Outputs.Variables["a"].Type.Annotation) + }) +} diff --git a/pkg/compiler/validators/typing.go b/pkg/compiler/validators/typing.go index 369c820eca..ec7a672b5f 100644 --- a/pkg/compiler/validators/typing.go +++ b/pkg/compiler/validators/typing.go @@ -58,6 +58,8 @@ func (t trivialChecker) CastsFrom(upstreamType *flyte.LiteralType) bool { downstreamTypeCopy := *t.literalType upstreamTypeCopy.Metadata = &structpb.Struct{} downstreamTypeCopy.Metadata = &structpb.Struct{} + upstreamTypeCopy.Annotation = &flyte.TypeAnnotation{} + downstreamTypeCopy.Annotation = &flyte.TypeAnnotation{} return upstreamTypeCopy.String() == downstreamTypeCopy.String() }