From 72d03aa85418a7b8094f12b5758db6990780a0b5 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Thu, 7 Nov 2024 18:54:37 -0500 Subject: [PATCH 1/2] Publish job telemetry to run-service. --- src/Runner.Common/RunServer.cs | 4 +- src/Runner.Listener/JobDispatcher.cs | 2 +- src/Runner.Worker/JobRunner.cs | 108 ++++++++++-------- .../RSWebApi/Contracts/CompleteJobRequest.cs | 3 + src/Sdk/RSWebApi/Contracts/Telemetry.cs | 20 ++++ src/Sdk/RSWebApi/RunServiceHttpClient.cs | 3 + 6 files changed, 93 insertions(+), 47 deletions(-) create mode 100644 src/Sdk/RSWebApi/Contracts/Telemetry.cs diff --git a/src/Runner.Common/RunServer.cs b/src/Runner.Common/RunServer.cs index 50ad0556018..8a7f4d5b4fa 100644 --- a/src/Runner.Common/RunServer.cs +++ b/src/Runner.Common/RunServer.cs @@ -28,6 +28,7 @@ Task CompleteJobAsync( IList stepResults, IList jobAnnotations, string environmentUrl, + IList telemetry, CancellationToken token); Task RenewJobAsync(Guid planId, Guid jobId, CancellationToken token); @@ -76,11 +77,12 @@ public Task CompleteJobAsync( IList stepResults, IList jobAnnotations, string environmentUrl, + IList telemetry, CancellationToken cancellationToken) { CheckConnection(); return RetryRequest( - async () => await _runServiceHttpClient.CompleteJobAsync(requestUri, planId, jobId, result, outputs, stepResults, jobAnnotations, environmentUrl, cancellationToken), cancellationToken); + async () => await _runServiceHttpClient.CompleteJobAsync(requestUri, planId, jobId, result, outputs, stepResults, jobAnnotations, environmentUrl, telemetry, cancellationToken), cancellationToken); } public Task RenewJobAsync(Guid planId, Guid jobId, CancellationToken cancellationToken) diff --git a/src/Runner.Listener/JobDispatcher.cs b/src/Runner.Listener/JobDispatcher.cs index ede26db1810..e58e1ea7b83 100644 --- a/src/Runner.Listener/JobDispatcher.cs +++ b/src/Runner.Listener/JobDispatcher.cs @@ -1198,7 +1198,7 @@ private async Task ForceFailJob(IRunServer runServer, Pipelines.AgentJobRequestM jobAnnotations.Add(annotation.Value); } - await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, TaskResult.Failed, outputs: null, stepResults: null, jobAnnotations: jobAnnotations, environmentUrl: null, CancellationToken.None); + await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, TaskResult.Failed, outputs: null, stepResults: null, jobAnnotations: jobAnnotations, environmentUrl: null, telemetry: null, CancellationToken.None); } catch (Exception ex) { diff --git a/src/Runner.Worker/JobRunner.cs b/src/Runner.Worker/JobRunner.cs index c52dc29512d..5e46069fa13 100644 --- a/src/Runner.Worker/JobRunner.cs +++ b/src/Runner.Worker/JobRunner.cs @@ -15,6 +15,7 @@ using GitHub.Runner.Sdk; using GitHub.Services.Common; using GitHub.Services.WebApi; +using Sdk.RSWebApi.Contracts; using Pipelines = GitHub.DistributedTask.Pipelines; namespace GitHub.Runner.Worker @@ -279,7 +280,12 @@ private async Task CompleteJobAsync(IRunServer runServer, IExecution jobContext.Debug($"Finishing: {message.JobDisplayName}"); TaskResult result = jobContext.Complete(taskResult); - await ShutdownQueue(throwOnFailure: false); + var jobQueueTelemetry = await ShutdownQueue(throwOnFailure: false); + // include any job telemetry from the background upload process. + if (jobQueueTelemetry.Count > 0) + { + jobContext.Global.JobTelemetry.AddRange(jobQueueTelemetry); + } // Make sure to clean temp after file upload since they may be pending fileupload still use the TEMP dir. _tempDirectoryManager?.CleanupTempDirectory(); @@ -297,6 +303,13 @@ private async Task CompleteJobAsync(IRunServer runServer, IExecution environmentUrl = urlStringToken.Value; } + // Get telemetry + IList telemetry = null; + if (jobContext.Global.JobTelemetry.Count > 0) + { + telemetry = jobContext.Global.JobTelemetry.Select(x => new Telemetry { Type = x.Type.ToString(), Message = x.Message, }).ToList(); + } + Trace.Info($"Raising job completed against run service"); var completeJobRetryLimit = 5; var exceptions = new List(); @@ -304,7 +317,7 @@ private async Task CompleteJobAsync(IRunServer runServer, IExecution { try { - await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, result, jobContext.JobOutputs, jobContext.Global.StepsResult, jobContext.Global.JobAnnotations, environmentUrl, default); + await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, result, jobContext.JobOutputs, jobContext.Global.StepsResult, jobContext.Global.JobAnnotations, environmentUrl, telemetry, default); return result; } catch (Exception ex) @@ -329,49 +342,7 @@ private async Task CompleteJobAsync(IJobServer jobServer, IExecution if (_runnerSettings.DisableUpdate == true) { - try - { - var currentVersion = new PackageVersion(BuildConstants.RunnerPackage.Version); - ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); - VssCredentials serverCredential = VssUtil.GetVssCredential(systemConnection); - - var runnerServer = HostContext.GetService(); - await runnerServer.ConnectAsync(systemConnection.Url, serverCredential); - var serverPackages = await runnerServer.GetPackagesAsync("agent", BuildConstants.RunnerPackage.PackageName, 5, includeToken: false, cancellationToken: CancellationToken.None); - if (serverPackages.Count > 0) - { - serverPackages = serverPackages.OrderByDescending(x => x.Version).ToList(); - Trace.Info($"Newer packages {StringUtil.ConvertToJson(serverPackages.Select(x => x.Version.ToString()))}"); - - var warnOnFailedJob = false; // any minor/patch version behind. - var warnOnOldRunnerVersion = false; // >= 2 minor version behind - if (serverPackages.Any(x => x.Version.CompareTo(currentVersion) > 0)) - { - Trace.Info($"Current runner version {currentVersion} is behind the latest runner version {serverPackages[0].Version}."); - warnOnFailedJob = true; - } - - if (serverPackages.Where(x => x.Version.Major == currentVersion.Major && x.Version.Minor > currentVersion.Minor).Count() > 1) - { - Trace.Info($"Current runner version {currentVersion} is way behind the latest runner version {serverPackages[0].Version}."); - warnOnOldRunnerVersion = true; - } - - if (result == TaskResult.Failed && warnOnFailedJob) - { - jobContext.Warning($"This job failure may be caused by using an out of date self-hosted runner. You are currently using runner version {currentVersion}. Please update to the latest version {serverPackages[0].Version}"); - } - else if (warnOnOldRunnerVersion) - { - jobContext.Warning($"This self-hosted runner is currently using runner version {currentVersion}. This version is out of date. Please update to the latest version {serverPackages[0].Version}"); - } - } - } - catch (Exception ex) - { - // Ignore any error since suggest runner update is best effort. - Trace.Error($"Caught exception during runner version check: {ex}"); - } + await WarningOutdatedRunnerAsync(jobContext, message, result); } try @@ -506,5 +477,52 @@ private async Task> ShutdownQueue(bool throwOnFailure) return Array.Empty(); } + + private async Task WarningOutdatedRunnerAsync(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, TaskResult result) + { + try + { + var currentVersion = new PackageVersion(BuildConstants.RunnerPackage.Version); + ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); + VssCredentials serverCredential = VssUtil.GetVssCredential(systemConnection); + + var runnerServer = HostContext.GetService(); + await runnerServer.ConnectAsync(systemConnection.Url, serverCredential); + var serverPackages = await runnerServer.GetPackagesAsync("agent", BuildConstants.RunnerPackage.PackageName, 5, includeToken: false, cancellationToken: CancellationToken.None); + if (serverPackages.Count > 0) + { + serverPackages = serverPackages.OrderByDescending(x => x.Version).ToList(); + Trace.Info($"Newer packages {StringUtil.ConvertToJson(serverPackages.Select(x => x.Version.ToString()))}"); + + var warnOnFailedJob = false; // any minor/patch version behind. + var warnOnOldRunnerVersion = false; // >= 2 minor version behind + if (serverPackages.Any(x => x.Version.CompareTo(currentVersion) > 0)) + { + Trace.Info($"Current runner version {currentVersion} is behind the latest runner version {serverPackages[0].Version}."); + warnOnFailedJob = true; + } + + if (serverPackages.Where(x => x.Version.Major == currentVersion.Major && x.Version.Minor > currentVersion.Minor).Count() > 1) + { + Trace.Info($"Current runner version {currentVersion} is way behind the latest runner version {serverPackages[0].Version}."); + warnOnOldRunnerVersion = true; + } + + if (result == TaskResult.Failed && warnOnFailedJob) + { + jobContext.Warning($"This job failure may be caused by using an out of date self-hosted runner. You are currently using runner version {currentVersion}. Please update to the latest version {serverPackages[0].Version}"); + } + else if (warnOnOldRunnerVersion) + { + jobContext.Warning($"This self-hosted runner is currently using runner version {currentVersion}. This version is out of date. Please update to the latest version {serverPackages[0].Version}"); + } + } + } + catch (Exception ex) + { + // Ignore any error since suggest runner update is best effort. + Trace.Error($"Caught exception during runner version check: {ex}"); + } + } } } diff --git a/src/Sdk/RSWebApi/Contracts/CompleteJobRequest.cs b/src/Sdk/RSWebApi/Contracts/CompleteJobRequest.cs index fff5156a40d..ff02ab6586c 100644 --- a/src/Sdk/RSWebApi/Contracts/CompleteJobRequest.cs +++ b/src/Sdk/RSWebApi/Contracts/CompleteJobRequest.cs @@ -27,6 +27,9 @@ public class CompleteJobRequest [DataMember(Name = "annotations", EmitDefaultValue = false)] public IList Annotations { get; set; } + [DataMember(Name = "telemetry", EmitDefaultValue = false)] + public IList Telemetry { get; set; } + [DataMember(Name = "environmentUrl", EmitDefaultValue = false)] public string EnvironmentUrl { get; set; } } diff --git a/src/Sdk/RSWebApi/Contracts/Telemetry.cs b/src/Sdk/RSWebApi/Contracts/Telemetry.cs new file mode 100644 index 00000000000..9dda8aa130c --- /dev/null +++ b/src/Sdk/RSWebApi/Contracts/Telemetry.cs @@ -0,0 +1,20 @@ +using System.Runtime.Serialization; + +namespace Sdk.RSWebApi.Contracts +{ + [DataContract] + public struct Telemetry + { + public Telemetry(string message, string type) + { + Message = message; + Type = type; + } + + [DataMember(Name = "message", EmitDefaultValue = false)] + public string Message { get; set; } + + [DataMember(Name = "type", EmitDefaultValue = false)] + public string Type { get; set; } + } +} diff --git a/src/Sdk/RSWebApi/RunServiceHttpClient.cs b/src/Sdk/RSWebApi/RunServiceHttpClient.cs index 2a1b0b998ad..f4d0c539f0d 100644 --- a/src/Sdk/RSWebApi/RunServiceHttpClient.cs +++ b/src/Sdk/RSWebApi/RunServiceHttpClient.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Net; using System.Net.Http; using System.Threading; @@ -126,6 +127,7 @@ public async Task CompleteJobAsync( IList stepResults, IList jobAnnotations, string environmentUrl, + IList telemetry, CancellationToken cancellationToken = default) { HttpMethod httpMethod = new HttpMethod("POST"); @@ -138,6 +140,7 @@ public async Task CompleteJobAsync( StepResults = stepResults, Annotations = jobAnnotations, EnvironmentUrl = environmentUrl, + Telemetry = telemetry, }; requestUri = new Uri(requestUri, "completejob"); From 86df8345f9095e0c45c5bdea1b2a1ddf2a6e12cd Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Thu, 7 Nov 2024 18:59:26 -0500 Subject: [PATCH 2/2] . --- src/Runner.Worker/JobRunner.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Runner.Worker/JobRunner.cs b/src/Runner.Worker/JobRunner.cs index 5e46069fa13..f1f9a72f1f4 100644 --- a/src/Runner.Worker/JobRunner.cs +++ b/src/Runner.Worker/JobRunner.cs @@ -282,7 +282,7 @@ private async Task CompleteJobAsync(IRunServer runServer, IExecution var jobQueueTelemetry = await ShutdownQueue(throwOnFailure: false); // include any job telemetry from the background upload process. - if (jobQueueTelemetry.Count > 0) + if (jobQueueTelemetry?.Count > 0) { jobContext.Global.JobTelemetry.AddRange(jobQueueTelemetry); } @@ -349,7 +349,7 @@ private async Task CompleteJobAsync(IJobServer jobServer, IExecution { var jobQueueTelemetry = await ShutdownQueue(throwOnFailure: true); // include any job telemetry from the background upload process. - if (jobQueueTelemetry.Count > 0) + if (jobQueueTelemetry?.Count > 0) { jobContext.Global.JobTelemetry.AddRange(jobQueueTelemetry); }