From 11c885b521bc2c938a65628e699086d9d671985b Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Fri, 3 Mar 2023 22:26:17 -0800 Subject: [PATCH 1/3] Structured Dataset with generic format should be castable to Schema Signed-off-by: Kevin Su --- pkg/compiler/validators/typing.go | 2 +- pkg/compiler/validators/typing_test.go | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/pkg/compiler/validators/typing.go b/pkg/compiler/validators/typing.go index 9f9c1321c..de43916f8 100644 --- a/pkg/compiler/validators/typing.go +++ b/pkg/compiler/validators/typing.go @@ -115,7 +115,7 @@ func (t schemaTypeChecker) CastsFrom(upstreamType *flyte.LiteralType) bool { } // Flyte Schema can only be serialized to parquet - if !strings.EqualFold(structuredDatasetType.Format, "parquet") { + if len(structuredDatasetType.Format) != 0 && !strings.EqualFold(structuredDatasetType.Format, "parquet") { return false } diff --git a/pkg/compiler/validators/typing_test.go b/pkg/compiler/validators/typing_test.go index a4b6a7a3a..97dcdd9df 100644 --- a/pkg/compiler/validators/typing_test.go +++ b/pkg/compiler/validators/typing_test.go @@ -579,6 +579,14 @@ func TestSchemaCasting(t *testing.T) { }, }, } + genericStructuredDataset := &core.LiteralType{ + Type: &core.LiteralType_StructuredDatasetType{ + StructuredDatasetType: &core.StructuredDatasetType{ + Columns: []*core.StructuredDatasetType_DatasetColumn{}, + Format: "", + }, + }, + } subsetIntegerSchema := &core.LiteralType{ Type: &core.LiteralType_Schema{ Schema: &core.SchemaType{ @@ -657,6 +665,11 @@ func TestSchemaCasting(t *testing.T) { assert.True(t, castable, "Schema(a=Integer, b=Float) should be castable to Schema(a=Integer)") }) + t.Run("GenericToSubsetTypedSchema", func(t *testing.T) { + castable := AreTypesCastable(genericStructuredDataset, subsetIntegerSchema) + assert.True(t, castable, "StructuredDataset() with generic format should be castable toSchema(a=Integer)") + }) + t.Run("SupersetStructuredToSubsetTypedSchema", func(t *testing.T) { castable := AreTypesCastable(supersetStructuredDataset, subsetIntegerSchema) assert.True(t, castable, "StructuredDataset(a=Integer, b=Float) should be castable to Schema(a=Integer)") From 1778dfbdc8c6047a9b8af77d74824cab0e7d0048 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Fri, 3 Mar 2023 22:26:30 -0800 Subject: [PATCH 2/3] Structured Dataset with generic format should be castable to Schema Signed-off-by: Kevin Su --- pkg/compiler/validators/typing_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/compiler/validators/typing_test.go b/pkg/compiler/validators/typing_test.go index 97dcdd9df..b035e7578 100644 --- a/pkg/compiler/validators/typing_test.go +++ b/pkg/compiler/validators/typing_test.go @@ -667,7 +667,7 @@ func TestSchemaCasting(t *testing.T) { t.Run("GenericToSubsetTypedSchema", func(t *testing.T) { castable := AreTypesCastable(genericStructuredDataset, subsetIntegerSchema) - assert.True(t, castable, "StructuredDataset() with generic format should be castable toSchema(a=Integer)") + assert.True(t, castable, "StructuredDataset() with generic format should be castable to Schema(a=Integer)") }) t.Run("SupersetStructuredToSubsetTypedSchema", func(t *testing.T) { From 913ac5dc549e59082fe9906fcff2bcafa9a9b997 Mon Sep 17 00:00:00 2001 From: Kevin Su Date: Sat, 4 Mar 2023 01:25:34 -0800 Subject: [PATCH 3/3] update Signed-off-by: Kevin Su --- pkg/compiler/validators/typing.go | 3 ++- pkg/compiler/validators/typing_test.go | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/compiler/validators/typing.go b/pkg/compiler/validators/typing.go index de43916f8..09268359d 100644 --- a/pkg/compiler/validators/typing.go +++ b/pkg/compiler/validators/typing.go @@ -147,7 +147,8 @@ func (t structuredDatasetChecker) CastsFrom(upstreamType *flyte.LiteralType) boo } if schemaType != nil { // Flyte Schema can only be serialized to parquet - if !strings.EqualFold(t.literalType.GetStructuredDatasetType().Format, "parquet") { + format := t.literalType.GetStructuredDatasetType().Format + if len(format) != 0 && !strings.EqualFold(format, "parquet") { return false } return structuredDatasetCastFromSchema(schemaType, t.literalType.GetStructuredDatasetType()) diff --git a/pkg/compiler/validators/typing_test.go b/pkg/compiler/validators/typing_test.go index b035e7578..8344339f0 100644 --- a/pkg/compiler/validators/typing_test.go +++ b/pkg/compiler/validators/typing_test.go @@ -670,6 +670,11 @@ func TestSchemaCasting(t *testing.T) { assert.True(t, castable, "StructuredDataset() with generic format should be castable to Schema(a=Integer)") }) + t.Run("SubsetTypedSchemaToGeneric", func(t *testing.T) { + castable := AreTypesCastable(subsetIntegerSchema, genericStructuredDataset) + assert.True(t, castable, "Schema(a=Integer) should be castable to StructuredDataset() with generic format") + }) + t.Run("SupersetStructuredToSubsetTypedSchema", func(t *testing.T) { castable := AreTypesCastable(supersetStructuredDataset, subsetIntegerSchema) assert.True(t, castable, "StructuredDataset(a=Integer, b=Float) should be castable to Schema(a=Integer)")