Skip to content

Commit

Permalink
Hash entire PipelineContainerSpec
Browse files Browse the repository at this point in the history 
Signed-off-by: droctothorpe <[email protected]>
Co-authored-by: zazulam <[email protected]>
  • Loading branch information
@droctothorpe @zazulam
droctothorpe and zazulam committed Aug 7, 2024
1 parent 68d4c0b commit 5e88d75
Show file tree
Hide file tree
Showing 4 changed files with 41 additions and 37 deletions.
34 changes: 19 additions & 15 deletions backend/src/v2/compiler/argocompiler/argo.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package argocompiler
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"strings"

Expand Down Expand Up @@ -285,19 +286,17 @@ func (c *workflowCompiler) hashComponentCommandAndArgs(componentName string) str
log.Debug("executorName: ", executorName)
// If one of them matches the executorLabel we extracted earlier...
if executorName == executorLabel {
// Get the corresponding list of commands.
commandList := executorValue.GetContainer().GetCommand()
// And the list of args.
argsList := executorValue.GetContainer().GetArgs()
if !(commandList == nil && argsList == nil) { // Some components have neither command nor args.
// Convert the command list into a hash string.
stringToHash := strings.Join(commandList, " ")
if argsList != nil {
// Convert the args list into a hash string and append it to the command list hash string.
stringToHash += strings.Join(argsList, " ")

return hashString(stringToHash)
// Get the corresponding container.
container := executorValue.GetContainer()
if container != nil {
containerHash, err := hashValue(container)
if err != nil {
// Do not bubble up since this is not a breaking error
// and we can just return the componentName in full.
log.Debug("Error hashing container: ", err)
}

return containerHash
}
}
}
Expand All @@ -306,11 +305,16 @@ func (c *workflowCompiler) hashComponentCommandAndArgs(componentName string) str
return componentName
}

func hashString(s string) string {
// hashValue serializes and hashes a provided value.
func hashValue(value interface{}) (string, error) {
bytes, err := json.Marshal(value)
if err != nil {
return "", err
}
h := sha256.New()
h.Write([]byte(s))
h.Write([]byte(bytes))

return hex.EncodeToString(h.Sum(nil))
return hex.EncodeToString(h.Sum(nil)), nil
}

const (
Expand Down
28 changes: 14 additions & 14 deletions backend/src/v2/compiler/argocompiler/testdata/create_mount_delete_dynamic_pvc.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@ spec:
parameters:
- name: kubernetes-comp-comp
value: '{"pvcMount":[{"mountPath":"/data","taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}]}'
- name: components-db3197b81c484742322b041296c85f7998bd2442d4ffa0459f23dc73c57b9ef1
- name: components-95f802401136aebf1bf728a6675d7adba5513b53673a3698e00a6d8744638080
value: '{"executorLabel":"exec-comp"}'
- name: implementations-db3197b81c484742322b041296c85f7998bd2442d4ffa0459f23dc73c57b9ef1
- name: implementations-95f802401136aebf1bf728a6675d7adba5513b53673a3698e00a6d8744638080
value: '{"args":["--executor_input","{{$}}","--function_to_execute","comp"],"command":["sh","-c","\nif
! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3
-m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1
Expand All @@ -21,13 +21,13 @@ spec:
*\n\ndef comp():\n pass\n\n"],"image":"python:3.7"}'
- name: kubernetes-comp-comp-2
value: '{"pvcMount":[{"mountPath":"/reused_data","taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}]}'
- name: components-comp-createpvc
- name: components-98f254581598234b59377784d6cbf209de79e0bcda8013fe4c4397b5d3a26767
value: '{"executorLabel":"exec-createpvc","inputDefinitions":{"parameters":{"access_modes":{"parameterType":"LIST"},"annotations":{"isOptional":true,"parameterType":"STRUCT"},"pvc_name":{"isOptional":true,"parameterType":"STRING"},"pvc_name_suffix":{"isOptional":true,"parameterType":"STRING"},"size":{"parameterType":"STRING"},"storage_class_name":{"defaultValue":"","isOptional":true,"parameterType":"STRING"},"volume_name":{"isOptional":true,"parameterType":"STRING"}}},"outputDefinitions":{"parameters":{"name":{"parameterType":"STRING"}}}}'
- name: implementations-comp-createpvc
- name: implementations-98f254581598234b59377784d6cbf209de79e0bcda8013fe4c4397b5d3a26767
value: '{"image":"argostub/createpvc"}'
- name: components-comp-deletepvc
- name: components-ecfc655dce17b0d317707d37fc226fb7de858cc93d45916945122484a13ef725
value: '{"executorLabel":"exec-deletepvc","inputDefinitions":{"parameters":{"pvc_name":{"parameterType":"STRING"}}}}'
- name: implementations-comp-deletepvc
- name: implementations-ecfc655dce17b0d317707d37fc226fb7de858cc93d45916945122484a13ef725
value: '{"image":"argostub/deletepvc"}'
- name: components-root
value: '{"dag":{"tasks":{"comp":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp"},"dependentTasks":["createpvc"],"taskInfo":{"name":"comp"}},"comp-2":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp-2"},"dependentTasks":["comp","createpvc"],"taskInfo":{"name":"comp-2"}},"createpvc":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-createpvc"},"inputs":{"parameters":{"access_modes":{"runtimeValue":{"constant":["ReadWriteOnce"]}},"pvc_name_suffix":{"runtimeValue":{"constant":"-my-pvc"}},"size":{"runtimeValue":{"constant":"5Gi"}},"storage_class_name":{"runtimeValue":{"constant":"standard"}}}},"taskInfo":{"name":"createpvc"}},"deletepvc":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-deletepvc"},"dependentTasks":["comp-2","createpvc"],"inputs":{"parameters":{"pvc_name":{"taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}}},"taskInfo":{"name":"deletepvc"}}}}}'
Expand Down Expand Up @@ -197,11 +197,11 @@ spec:
- arguments:
parameters:
- name: component
value: '{{workflow.parameters.components-db3197b81c484742322b041296c85f7998bd2442d4ffa0459f23dc73c57b9ef1}}'
value: '{{workflow.parameters.components-95f802401136aebf1bf728a6675d7adba5513b53673a3698e00a6d8744638080}}'
- name: task
value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp"},"dependentTasks":["createpvc"],"taskInfo":{"name":"comp"}}'
- name: container
value: '{{workflow.parameters.implementations-db3197b81c484742322b041296c85f7998bd2442d4ffa0459f23dc73c57b9ef1}}'
value: '{{workflow.parameters.implementations-95f802401136aebf1bf728a6675d7adba5513b53673a3698e00a6d8744638080}}'
- name: parent-dag-id
value: '{{inputs.parameters.parent-dag-id}}'
- name: kubernetes-config
Expand All @@ -222,11 +222,11 @@ spec:
- arguments:
parameters:
- name: component
value: '{{workflow.parameters.components-db3197b81c484742322b041296c85f7998bd2442d4ffa0459f23dc73c57b9ef1}}'
value: '{{workflow.parameters.components-95f802401136aebf1bf728a6675d7adba5513b53673a3698e00a6d8744638080}}'
- name: task
value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-comp-2"},"dependentTasks":["comp","createpvc"],"taskInfo":{"name":"comp-2"}}'
- name: container
value: '{{workflow.parameters.implementations-db3197b81c484742322b041296c85f7998bd2442d4ffa0459f23dc73c57b9ef1}}'
value: '{{workflow.parameters.implementations-95f802401136aebf1bf728a6675d7adba5513b53673a3698e00a6d8744638080}}'
- name: parent-dag-id
value: '{{inputs.parameters.parent-dag-id}}'
- name: kubernetes-config
Expand All @@ -247,23 +247,23 @@ spec:
- arguments:
parameters:
- name: component
value: '{{workflow.parameters.components-comp-createpvc}}'
value: '{{workflow.parameters.components-98f254581598234b59377784d6cbf209de79e0bcda8013fe4c4397b5d3a26767}}'
- name: task
value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-createpvc"},"inputs":{"parameters":{"access_modes":{"runtimeValue":{"constant":["ReadWriteOnce"]}},"pvc_name_suffix":{"runtimeValue":{"constant":"-my-pvc"}},"size":{"runtimeValue":{"constant":"5Gi"}},"storage_class_name":{"runtimeValue":{"constant":"standard"}}}},"taskInfo":{"name":"createpvc"}}'
- name: container
value: '{{workflow.parameters.implementations-comp-createpvc}}'
value: '{{workflow.parameters.implementations-98f254581598234b59377784d6cbf209de79e0bcda8013fe4c4397b5d3a26767}}'
- name: parent-dag-id
value: '{{inputs.parameters.parent-dag-id}}'
name: createpvc
template: system-container-driver
- arguments:
parameters:
- name: component
value: '{{workflow.parameters.components-comp-deletepvc}}'
value: '{{workflow.parameters.components-ecfc655dce17b0d317707d37fc226fb7de858cc93d45916945122484a13ef725}}'
- name: task
value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-deletepvc"},"dependentTasks":["comp-2","createpvc"],"inputs":{"parameters":{"pvc_name":{"taskOutputParameter":{"outputParameterKey":"name","producerTask":"createpvc"}}}},"taskInfo":{"name":"deletepvc"}}'
- name: container
value: '{{workflow.parameters.implementations-comp-deletepvc}}'
value: '{{workflow.parameters.implementations-ecfc655dce17b0d317707d37fc226fb7de858cc93d45916945122484a13ef725}}'
- name: parent-dag-id
value: '{{inputs.parameters.parent-dag-id}}'
depends: comp-2.Succeeded && createpvc.Succeeded
Expand Down
8 changes: 4 additions & 4 deletions backend/src/v2/compiler/argocompiler/testdata/create_pod_metadata.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@ spec:
parameters:
- name: kubernetes-comp-hello-world
value: '{"podMetadata":{"annotations":{"experiment_id":"234567","run_id":"123456"},"labels":{"kubeflow.com/common":"test","kubeflow.com/kfp":"pipeline-node"}}}'
- name: components-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b
- name: components-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390
value: '{"executorLabel":"exec-hello-world","inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}'
- name: implementations-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b
- name: implementations-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390
value: '{"args":["--text","{{$.inputs.parameters[''text'']}}"],"command":["sh","-ec","program_path=$(mktemp)\nprintf
\"%s\" \"$0\" \u003e \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n","def
hello_world(text):\n print(text)\n return text\n\nimport argparse\n_parser
Expand Down Expand Up @@ -191,11 +191,11 @@ spec:
- arguments:
parameters:
- name: component
value: '{{workflow.parameters.components-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b}}'
value: '{{workflow.parameters.components-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390}}'
- name: task
value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}'
- name: container
value: '{{workflow.parameters.implementations-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b}}'
value: '{{workflow.parameters.implementations-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390}}'
- name: parent-dag-id
value: '{{inputs.parameters.parent-dag-id}}'
- name: kubernetes-config
Expand Down
8 changes: 4 additions & 4 deletions backend/src/v2/compiler/argocompiler/testdata/hello_world.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ metadata:
spec:
arguments:
parameters:
- name: components-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b
- name: components-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390
value: '{"executorLabel":"exec-hello-world","inputDefinitions":{"parameters":{"text":{"type":"STRING"}}}}'
- name: implementations-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b
- name: implementations-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390
value: '{"args":["--text","{{$.inputs.parameters[''text'']}}"],"command":["sh","-ec","program_path=$(mktemp)\nprintf
\"%s\" \"$0\" \u003e \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n","def
hello_world(text):\n print(text)\n return text\n\nimport argparse\n_parser
Expand Down Expand Up @@ -183,11 +183,11 @@ spec:
- arguments:
parameters:
- name: component
value: '{{workflow.parameters.components-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b}}'
value: '{{workflow.parameters.components-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390}}'
- name: task
value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-hello-world"},"inputs":{"parameters":{"text":{"componentInputParameter":"text"}}},"taskInfo":{"name":"hello-world"}}'
- name: container
value: '{{workflow.parameters.implementations-285f834445312aa2162cf75d155ac477a6149126369484809c864b8d1943533b}}'
value: '{{workflow.parameters.implementations-34e222d692a0573c88000b8cb02ad24423491a53e061e9bba36d3718dd4c3390}}'
- name: parent-dag-id
value: '{{inputs.parameters.parent-dag-id}}'
name: hello-world-driver
Expand Down

0 comments on commit 5e88d75

Please sign in to comment.