Merge branch 'master' into feature/open-telemetry

Signed-off-by: Daniel Rammer <[email protected]>

.github/workflows/master.yml

@@ -35,7 +35,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v2
         with:
-          go-version: '1.18'
+          go-version: '1.19'
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v2
         with:
@@ -54,7 +54,7 @@ jobs:
           fetch-depth: "0"
       - uses: actions/setup-go@v2
         with:
-          go-version: '1.18'
+          go-version: '1.19'
       - name: Unit Tests
         run: make mod_download && make test_unit_codecov
       - name: Push CodeCov
@@ -65,7 +65,7 @@ jobs:
           fail_ci_if_error: false
       - uses: actions/setup-go@v2
         with:
-          go-version: '1.18'
+          go-version: '1.19'
       - name: Lint
         run: make install && make lint
       - name: Bench tests

.github/workflows/pull_request.yml

@@ -13,7 +13,7 @@ jobs:
           fetch-depth: "0"
       - uses: actions/setup-go@v2
         with:
-          go-version: '1.18'
+          go-version: '1.19'
       - name: Unit Tests
         run: make mod_download && make test_unit_codecov
       - name: Push CodeCov
@@ -28,7 +28,7 @@ jobs:
     name: Lint
     uses: flyteorg/flytetools/.github/workflows/lint.yml@master
     with:
-      go-version: '1.18'
+      go-version: '1.19'
   test-generate-integrity:
     name: Ensure go generate has run
     runs-on: ubuntu-latest
@@ -39,7 +39,7 @@ jobs:
           fetch-depth: "0"
       - uses: actions/setup-go@v2
         with:
-          go-version: '1.18'
+          go-version: '1.19'
       - name: Go generate and diff
         run: DELTA_CHECK=true make generate
 

contextutils/context.go

@@ -1,10 +1,12 @@
-// Contains common flyte context utils.
+// Package contextutils contains common flyte context utils.
 package contextutils
 
 import (
 	"context"
 	"fmt"
 	"runtime/pprof"
+
+	"google.golang.org/grpc/metadata"
 )
 
 type Key string
@@ -25,6 +27,7 @@ const (
 	LaunchPlanIDKey    Key = "lp"
 	ResourceVersionKey Key = "res_ver"
 	SignalIDKey        Key = "signal"
+	RequestIDKey       Key = "x-request-id"
 )
 
 func (k Key) String() string {
@@ -43,6 +46,7 @@ var logKeys = []Key{
 	RoutineLabelKey,
 	LaunchPlanIDKey,
 	ResourceVersionKey,
+	RequestIDKey,
 }
 
 // MetricKeysFromStrings is a convenience method to convert a slice of strings into a slice of Keys
@@ -56,17 +60,17 @@ func MetricKeysFromStrings(keys []string) []Key {
 	return res
 }
 
-// Gets a new context with the resource version set.
+// WithResourceVersion gets a new context with the resource version set.
 func WithResourceVersion(ctx context.Context, resourceVersion string) context.Context {
 	return context.WithValue(ctx, ResourceVersionKey, resourceVersion)
 }
 
-// Gets a new context with namespace set.
+// WithNamespace gets a new context with namespace set.
 func WithNamespace(ctx context.Context, namespace string) context.Context {
 	return context.WithValue(ctx, NamespaceKey, namespace)
 }
 
-// Gets a new context with JobId set. If the existing context already has a job id, the new context will have
+// WithJobID gets a new context with JobId set. If the existing context already has a job id, the new context will have
 // <old_jobID>/<new_jobID> set as the job id.
 func WithJobID(ctx context.Context, jobID string) context.Context {
 	existingJobID := ctx.Value(JobIDKey)
@@ -77,22 +81,22 @@ func WithJobID(ctx context.Context, jobID string) context.Context {
 	return context.WithValue(ctx, JobIDKey, jobID)
 }
 
-// Gets a new context with AppName set.
+// WithAppName gets a new context with AppName set.
 func WithAppName(ctx context.Context, appName string) context.Context {
 	return context.WithValue(ctx, AppNameKey, appName)
 }
 
-// Gets a new context with Phase set.
+// WithPhase gets a new context with Phase set.
 func WithPhase(ctx context.Context, phase string) context.Context {
 	return context.WithValue(ctx, PhaseKey, phase)
 }
 
-// Gets a new context with ExecutionID set.
+// WithExecutionID gets a new context with ExecutionID set.
 func WithExecutionID(ctx context.Context, execID string) context.Context {
 	return context.WithValue(ctx, ExecIDKey, execID)
 }
 
-// Gets a new context with NodeID (nested) set.
+// WithNodeID gets a new context with NodeID (nested) set.
 func WithNodeID(ctx context.Context, nodeID string) context.Context {
 	existingNodeID := ctx.Value(NodeIDKey)
 	if existingNodeID != nil {
@@ -101,38 +105,44 @@ func WithNodeID(ctx context.Context, nodeID string) context.Context {
 	return context.WithValue(ctx, NodeIDKey, nodeID)
 }
 
-// Gets a new context with WorkflowName set.
+// WithWorkflowID gets a new context with WorkflowName set.
 func WithWorkflowID(ctx context.Context, workflow string) context.Context {
 	return context.WithValue(ctx, WorkflowIDKey, workflow)
 }
 
-// Gets a new context with a launch plan ID set.
+// WithLaunchPlanID gets a new context with a launch plan ID set.
 func WithLaunchPlanID(ctx context.Context, launchPlan string) context.Context {
 	return context.WithValue(ctx, LaunchPlanIDKey, launchPlan)
 }
 
-// Get new context with Project and Domain values set
+// WithProjectDomain gets new context with Project and Domain values set
 func WithProjectDomain(ctx context.Context, project, domain string) context.Context {
 	c := context.WithValue(ctx, ProjectKey, project)
 	return context.WithValue(c, DomainKey, domain)
 }
 
-// Gets a new context with WorkflowName set.
+// WithTaskID gets a new context with WorkflowName set.
 func WithTaskID(ctx context.Context, taskID string) context.Context {
 	return context.WithValue(ctx, TaskIDKey, taskID)
 }
 
-// Gets a new context with TaskType set.
+// WithTaskType gets a new context with TaskType set.
 func WithTaskType(ctx context.Context, taskType string) context.Context {
 	return context.WithValue(ctx, TaskTypeKey, taskType)
 }
 
-// Gets a new context with SignalID set.
+// WithSignalID gets a new context with SignalID set.
 func WithSignalID(ctx context.Context, signalID string) context.Context {
 	return context.WithValue(ctx, SignalIDKey, signalID)
 }
 
-// Gets a new context with Go Routine label key set and a label assigned to the context using pprof.Labels.
+// WithRequestID gets a new context with RequestID set.
+func WithRequestID(ctx context.Context, requestID string) context.Context {
+	return metadata.AppendToOutgoingContext(context.WithValue(ctx, RequestIDKey, requestID), RequestIDKey.String(), requestID)
+}
+
+// WithGoroutineLabel gets a new context with Go Routine label key set and a label assigned to the context using
+// pprof.Labels.
 // You can then call pprof.SetGoroutineLabels(ctx) to annotate the current go-routine and have that show up in
 // pprof analysis.
 func WithGoroutineLabel(ctx context.Context, routineLabel string) context.Context {
@@ -156,8 +166,8 @@ func addStringFieldWithDefaults(ctx context.Context, m map[string]string, fieldK
 	m[fieldKey.String()] = val.(string)
 }
 
-// Gets a map of all known logKeys set on the context. logKeys are special and should be used incase, context fields
-// are to be added to the log lines.
+// GetLogFields gets a map of all known logKeys set on the context. logKeys are special and should be used incase,
+// context fields are to be added to the log lines.
 func GetLogFields(ctx context.Context) map[string]interface{} {
 	res := map[string]interface{}{}
 	for _, k := range logKeys {

contextutils/context_test.go

@@ -111,10 +111,11 @@ func TestWithSignalID(t *testing.T) {
 
 func TestGetFields(t *testing.T) {
 	ctx := context.Background()
-	ctx = WithJobID(WithNamespace(ctx, "ns123"), "job123")
+	ctx = WithRequestID(WithJobID(WithNamespace(ctx, "ns123"), "job123"), "req123")
 	m := GetLogFields(ctx)
 	assert.Equal(t, "ns123", m[NamespaceKey.String()])
 	assert.Equal(t, "job123", m[JobIDKey.String()])
+	assert.Equal(t, "req123", m[RequestIDKey.String()])
 }
 
 func TestValues(t *testing.T) {

go.mod

@@ -1,6 +1,6 @@
 module github.com/flyteorg/flytestdlib
 
-go 1.18
+go 1.19
 
 require (
 	github.com/aws/aws-sdk-go v1.44.2

version/version.go

@@ -1,6 +1,7 @@
 package version
 
 import (
+	"fmt"
 	"time"
 
 	"github.com/sirupsen/logrus"
@@ -18,12 +19,18 @@ var (
 	Version = "unknown"
 	// Build timestamp
 	BuildTime = time.Now().String()
+	// Git branch that was used to build the binary
+	GitBranch = ""
 )
 
 // Use this method to log the build information for the current app. The app name should be provided. To inject the build
 // and version information refer to the top-level comment in this file
 func LogBuildInformation(appName string) {
 	logrus.Info("------------------------------------------------------------------------")
-	logrus.Infof("App [%s], Version [%s], BuildSHA [%s], BuildTS [%s]", appName, Version, Build, BuildTime)
+	msg := fmt.Sprintf("App [%s], Version [%s], BuildSHA [%s], BuildTS [%s]", appName, Version, Build, BuildTime)
+	if GitBranch != "" {
+		msg += fmt.Sprintf(", Git Branch [%s]", GitBranch)
+	}
+	logrus.Info(msg)
 	logrus.Info("------------------------------------------------------------------------")
 }

version/version_test.go

@@ -21,9 +21,10 @@ func TestLogBuildInformation(t *testing.T) {
 
 	n := time.Now()
 	BuildTime = n.String()
+	GitBranch = "main"
 	buf := bytes.NewBufferString("")
 	logrus.SetFormatter(dFormat{})
 	logrus.SetOutput(buf)
 	LogBuildInformation("hello")
-	assert.Equal(t, buf.String(), fmt.Sprintf("------------------------------------------------------------------------App [hello], Version [unknown], BuildSHA [unknown], BuildTS [%s]------------------------------------------------------------------------", n.String()))
+	assert.Equal(t, buf.String(), fmt.Sprintf("------------------------------------------------------------------------App [hello], Version [unknown], BuildSHA [unknown], BuildTS [%s], Git Branch [main]------------------------------------------------------------------------", n.String()))
 }