Skip to content
This repository has been archived by the owner on Oct 9, 2023. It is now read-only.

Commit

Permalink
Add RFC3339 time format log link vars for start and finish time
Browse files Browse the repository at this point in the history
  • Loading branch information
@fg91
fg91 committed Jun 13, 2023
1 parent 2a7a6f9 commit b4e2372
Show file tree
Hide file tree
Showing 3 changed files with 69 additions and 44 deletions.
21 changes: 13 additions & 8 deletions go/tasks/logs/logging_utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,16 +38,21 @@ func GetLogsForContainerInPod(ctx context.Context, logPlugin tasklog.Plugin, pod
return nil, nil
}

startTime := pod.CreationTimestamp.Unix()
finishTime := time.Now().Unix()

logs, err := logPlugin.GetTaskLogs(
tasklog.Input{
PodName: pod.Name,
PodUID: string(pod.GetUID()),
Namespace: pod.Namespace,
ContainerName: pod.Spec.Containers[index].Name,
ContainerID: pod.Status.ContainerStatuses[index].ContainerID,
LogName: nameSuffix,
PodUnixStartTime: pod.CreationTimestamp.Unix(),
PodUnixFinishTime: time.Now().Unix(),
PodName: pod.Name,
PodUID: string(pod.GetUID()),
Namespace: pod.Namespace,
ContainerName: pod.Spec.Containers[index].Name,
ContainerID: pod.Status.ContainerStatuses[index].ContainerID,
LogName: nameSuffix,
PodRFC3339StartTime: time.Unix(startTime, 0).Format(time.RFC3339),
PodRFC3339FinishTime: time.Unix(finishTime, 0).Format(time.RFC3339),
PodUnixStartTime: startTime,
PodUnixFinishTime: finishTime,
},
)

Expand Down
20 changes: 11 additions & 9 deletions go/tasks/pluginmachinery/tasklog/plugin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,17 @@ import "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core"
// Input contains all available information about task's execution that a log plugin can use to construct task's
// log links.
type Input struct {
HostName string `json:"hostname"`
PodName string `json:"podName"`
Namespace string `json:"namespace"`
ContainerName string `json:"containerName"`
ContainerID string `json:"containerId"`
LogName string `json:"logName"`
PodUnixStartTime int64 `json:"podUnixStartTime"`
PodUnixFinishTime int64 `json:"podUnixFinishTime"`
PodUID string `json:"podUID"`
HostName string `json:"hostname"`
PodName string `json:"podName"`
Namespace string `json:"namespace"`
ContainerName string `json:"containerName"`
ContainerID string `json:"containerId"`
LogName string `json:"logName"`
PodRFC3339StartTime string `json:"podRFC3339StartTime"`
PodRFC3339FinishTime string `json:"podRFC3339FinishTime"`
PodUnixStartTime int64 `json:"podUnixStartTime"`
PodUnixFinishTime int64 `json:"podUnixFinishTime"`
PodUID string `json:"podUID"`
}

// Output contains all task logs a plugin generates for a given Input.
Expand Down
72 changes: 45 additions & 27 deletions go/tasks/pluginmachinery/tasklog/template.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ import (
// {{ .containerId }}: The container id docker/crio generated at run time,
// {{ .logName }}: A deployment specific name where to expect the logs to be.
// {{ .hostname }}: The hostname where the pod is running and where logs reside.
// {{ .PodRFC3339StartTime }}: The pod creation time in RFC3339 format
// {{ .PodRFC3339FinishTime }}: Don't have a good mechanism for this yet, but approximating with time.Now for now
// {{ .podUnixStartTime }}: The pod creation time (in unix seconds, not millis)
// {{ .podUnixFinishTime }}: Don't have a good mechanism for this yet, but approximating with time.Now for now
type TemplateLogPlugin struct {
Expand All @@ -30,28 +32,32 @@ type regexValPair struct {
}

type templateRegexes struct {
PodName *regexp.Regexp
PodUID *regexp.Regexp
Namespace *regexp.Regexp
ContainerName *regexp.Regexp
ContainerID *regexp.Regexp
LogName *regexp.Regexp
Hostname *regexp.Regexp
PodUnixStartTime *regexp.Regexp
PodUnixFinishTime *regexp.Regexp
PodName *regexp.Regexp
PodUID *regexp.Regexp
Namespace *regexp.Regexp
ContainerName *regexp.Regexp
ContainerID *regexp.Regexp
LogName *regexp.Regexp
Hostname *regexp.Regexp
PodRFC3339StartTime *regexp.Regexp
PodRFC3339FinishTime *regexp.Regexp
PodUnixStartTime *regexp.Regexp
PodUnixFinishTime *regexp.Regexp
}

func mustInitTemplateRegexes() templateRegexes {
return templateRegexes{
PodName: mustCreateRegex("podName"),
PodUID: mustCreateRegex("podUID"),
Namespace: mustCreateRegex("namespace"),
ContainerName: mustCreateRegex("containerName"),
ContainerID: mustCreateRegex("containerID"),
LogName: mustCreateRegex("logName"),
Hostname: mustCreateRegex("hostname"),
PodUnixStartTime: mustCreateRegex("podUnixStartTime"),
PodUnixFinishTime: mustCreateRegex("podUnixFinishTime"),
PodName: mustCreateRegex("podName"),
PodUID: mustCreateRegex("podUID"),
Namespace: mustCreateRegex("namespace"),
ContainerName: mustCreateRegex("containerName"),
ContainerID: mustCreateRegex("containerID"),
LogName: mustCreateRegex("logName"),
Hostname: mustCreateRegex("hostname"),
PodRFC3339StartTime: mustCreateRegex("podRFC3339StartTime"),
PodRFC3339FinishTime: mustCreateRegex("podRFC3339FinishTime"),
PodUnixStartTime: mustCreateRegex("podUnixStartTime"),
PodUnixFinishTime: mustCreateRegex("podUnixFinishTime"),
}
}

Expand All @@ -69,16 +75,18 @@ func replaceAll(template string, values []regexValPair) string {
return template
}

func (s TemplateLogPlugin) GetTaskLog(podName, podUID, namespace, containerName, containerID, logName string, podUnixStartTime, podUnixFinishTime int64) (core.TaskLog, error) {
func (s TemplateLogPlugin) GetTaskLog(podName, podUID, namespace, containerName, containerID, logName string, podRFC3339StartTime string, podRFC3339FinishTime string, podUnixStartTime, podUnixFinishTime int64) (core.TaskLog, error) {
o, err := s.GetTaskLogs(Input{
LogName: logName,
Namespace: namespace,
PodName: podName,
PodUID: podUID,
ContainerName: containerName,
ContainerID: containerID,
PodUnixStartTime: podUnixStartTime,
PodUnixFinishTime: podUnixFinishTime,
LogName: logName,
Namespace: namespace,
PodName: podName,
PodUID: podUID,
ContainerName: containerName,
ContainerID: containerID,
PodRFC3339StartTime: podRFC3339StartTime,
PodRFC3339FinishTime: podRFC3339FinishTime,
PodUnixStartTime: podUnixStartTime,
PodUnixFinishTime: podUnixFinishTime,
})

if err != nil || len(o.TaskLogs) == 0 {
Expand Down Expand Up @@ -131,6 +139,14 @@ func (s TemplateLogPlugin) GetTaskLogs(input Input) (Output, error) {
regex: regexes.Hostname,
val: input.HostName,
},
{
regex: regexes.PodRFC3339StartTime,
val: input.PodRFC3339StartTime,
},
{
regex: regexes.PodRFC3339FinishTime,
val: input.PodRFC3339FinishTime,
},
{
regex: regexes.PodUnixStartTime,
val: strconv.FormatInt(input.PodUnixStartTime, 10),
Expand Down Expand Up @@ -160,6 +176,8 @@ func (s TemplateLogPlugin) GetTaskLogs(input Input) (Output, error) {
// {{ .containerId }}: The container id docker/crio generated at run time,
// {{ .logName }}: A deployment specific name where to expect the logs to be.
// {{ .hostname }}: The hostname where the pod is running and where logs reside.
// {{ .PodRFC3339StartTime }}: The pod creation time in RFC3339 format
// {{ .PodRFC3339FinishTime }}: Don't have a good mechanism for this yet, but approximating with time.Now for now
// {{ .podUnixStartTime }}: The pod creation time (in unix seconds, not millis)
// {{ .podUnixFinishTime }}: Don't have a good mechanism for this yet, but approximating with time.Now for now
func NewTemplateLogPlugin(templateUris []string, messageFormat core.TaskLog_MessageFormat) TemplateLogPlugin {
Expand Down

0 comments on commit b4e2372

Please sign in to comment.