This repository has been archived by the owner on Dec 20, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Retrieve launch plan interfaces from admin (#103)
# TL;DR When Propeller comes across a launch plan node in a `DynamicJobSpec` it will hit Admin to retrieve the interface for the LP to do the interface check. ## Type - [ ] Bug Fix - [x] Feature - [ ] Plugin ## Are all requirements met? - [x] Code completed - [x] Smoke tested - [x] Unit tests added - [x] Code documentation added - [x] Any pending items have an associated Issue ## Complete description Please see the issue linked below and also the SDK PR flyteorg/flytekit#92 for more information. A sample dynamic job spec object has been uploaded here as well. Please see the text file for the type of dynamic job spec this PR is meant to support. [dynamic_job_spec.txt](https://github.com/lyft/flytepropeller/files/4430252/dynamic_job_spec.txt) * Added a `GetLaunchPlan` function to a `launchplan/Reader` interface which sits alongside the `launchplan/Executor` interface. Admin client wrapper now satisfies both interfaces. * Added a call to that function in the dynamic job handler `buildContextualDynamicWorkflow` function. ## Tracking Issue flyteorg/flyte#139 ## Follow-up issue flyteorg/flyte#246 This PR will be deprecated upon completion of this issue.
- Loading branch information
1 parent
61ee3c8
commit 41a756f
Showing
18 changed files
with
647 additions
and
83 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| package compiler | ||
|
|
||
| import ( | ||
| "github.com/lyft/flyteidl/gen/pb-go/flyteidl/admin" | ||
| "github.com/lyft/flyteidl/gen/pb-go/flyteidl/core" | ||
| ) | ||
|
|
||
| // This object is meant to satisfy github.com/lyft/flytepropeller/pkg/compiler/common.InterfaceProvider | ||
| // This file is pretty much copied from Admin, (sorry for the link, a real link made go mod import admin) | ||
| // github-dot-com/lyft/flyteadmin/blob/1acce744b8c7839ab77a0eb1ed922905af15baa5/pkg/workflowengine/impl/interface_provider.go | ||
| // but that implementation relies on the internal Admin Gorm model. We should consider deprecating that one in favor | ||
| // of this one as Admin already has a dependency on the Propeller compiler. | ||
| type LaunchPlanInterfaceProvider struct { | ||
| expectedInputs core.ParameterMap | ||
| expectedOutputs core.VariableMap | ||
| identifier *core.Identifier | ||
| } | ||
|
|
||
| func (p *LaunchPlanInterfaceProvider) GetID() *core.Identifier { | ||
| return p.identifier | ||
| } | ||
| func (p *LaunchPlanInterfaceProvider) GetExpectedInputs() *core.ParameterMap { | ||
| return &p.expectedInputs | ||
|
|
||
| } | ||
| func (p *LaunchPlanInterfaceProvider) GetExpectedOutputs() *core.VariableMap { | ||
| return &p.expectedOutputs | ||
| } | ||
|
|
||
| func NewLaunchPlanInterfaceProvider(launchPlan admin.LaunchPlan) *LaunchPlanInterfaceProvider { | ||
| return &LaunchPlanInterfaceProvider{ | ||
| expectedInputs: *launchPlan.Closure.ExpectedInputs, | ||
| expectedOutputs: *launchPlan.Closure.ExpectedOutputs, | ||
| identifier: launchPlan.Id, | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,72 @@ | ||
| package compiler | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| "github.com/lyft/flyteidl/gen/pb-go/flyteidl/admin" | ||
| "github.com/lyft/flyteidl/gen/pb-go/flyteidl/core" | ||
| "github.com/lyft/flytepropeller/pkg/utils" | ||
| "github.com/stretchr/testify/assert" | ||
| ) | ||
|
|
||
| var launchPlanIdentifier = core.Identifier{ | ||
| ResourceType: core.ResourceType_LAUNCH_PLAN, | ||
| Project: "project", | ||
| Domain: "domain", | ||
| Name: "name", | ||
| Version: "version", | ||
| } | ||
|
|
||
| var inputs = core.ParameterMap{ | ||
| Parameters: map[string]*core.Parameter{ | ||
| "foo": { | ||
| Var: &core.Variable{ | ||
| Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}}, | ||
| }, | ||
| Behavior: &core.Parameter_Default{ | ||
| Default: utils.MustMakeLiteral("foo-value"), | ||
| }, | ||
| }, | ||
| }, | ||
| } | ||
| var outputs = core.VariableMap{ | ||
| Variables: map[string]*core.Variable{ | ||
| "foo": { | ||
| Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_STRING}}, | ||
| }, | ||
| }, | ||
| } | ||
|
|
||
| func getDummyLaunchPlan() admin.LaunchPlan { | ||
| launchPlanClosure := admin.LaunchPlanClosure{ | ||
| ExpectedInputs: &inputs, | ||
| ExpectedOutputs: &outputs, | ||
| } | ||
| return admin.LaunchPlan{ | ||
| Id: &launchPlanIdentifier, | ||
| Spec: nil, | ||
| Closure: &launchPlanClosure, | ||
| } | ||
| } | ||
|
|
||
| func TestGetId(t *testing.T) { | ||
| launchPlan := getDummyLaunchPlan() | ||
| provider := NewLaunchPlanInterfaceProvider(launchPlan) | ||
| assert.Equal(t, &core.Identifier{ResourceType: 3, Project: "project", Domain: "domain", Name: "name", Version: "version"}, provider.GetID()) | ||
| } | ||
|
|
||
| func TestGetExpectedInputs(t *testing.T) { | ||
| launchPlan := getDummyLaunchPlan() | ||
| provider := NewLaunchPlanInterfaceProvider(launchPlan) | ||
| assert.Contains(t, (*provider.GetExpectedInputs()).Parameters, "foo") | ||
| assert.NotNil(t, (*provider.GetExpectedInputs()).Parameters["foo"].Var.Type.GetSimple()) | ||
| assert.EqualValues(t, "STRING", (*provider.GetExpectedInputs()).Parameters["foo"].Var.Type.GetSimple().String()) | ||
| assert.NotNil(t, (*provider.GetExpectedInputs()).Parameters["foo"].GetDefault()) | ||
| } | ||
|
|
||
| func TestGetExpectedOutputs(t *testing.T) { | ||
| launchPlan := getDummyLaunchPlan() | ||
| provider := NewLaunchPlanInterfaceProvider(launchPlan) | ||
| assert.EqualValues(t, outputs.Variables["foo"].GetType().GetType(), | ||
| provider.GetExpectedOutputs().Variables["foo"].GetType().GetType()) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.