diff --git a/src/Runner.Common/RunServer.cs b/src/Runner.Common/RunServer.cs index 50ad0556018..8a7f4d5b4fa 100644 --- a/src/Runner.Common/RunServer.cs +++ b/src/Runner.Common/RunServer.cs @@ -28,6 +28,7 @@ Task CompleteJobAsync( IList stepResults, IList jobAnnotations, string environmentUrl, + IList telemetry, CancellationToken token); Task RenewJobAsync(Guid planId, Guid jobId, CancellationToken token); @@ -76,11 +77,12 @@ public Task CompleteJobAsync( IList stepResults, IList jobAnnotations, string environmentUrl, + IList telemetry, CancellationToken cancellationToken) { CheckConnection(); return RetryRequest( - async () => await _runServiceHttpClient.CompleteJobAsync(requestUri, planId, jobId, result, outputs, stepResults, jobAnnotations, environmentUrl, cancellationToken), cancellationToken); + async () => await _runServiceHttpClient.CompleteJobAsync(requestUri, planId, jobId, result, outputs, stepResults, jobAnnotations, environmentUrl, telemetry, cancellationToken), cancellationToken); } public Task RenewJobAsync(Guid planId, Guid jobId, CancellationToken cancellationToken) diff --git a/src/Runner.Listener/JobDispatcher.cs b/src/Runner.Listener/JobDispatcher.cs index ede26db1810..e58e1ea7b83 100644 --- a/src/Runner.Listener/JobDispatcher.cs +++ b/src/Runner.Listener/JobDispatcher.cs @@ -1198,7 +1198,7 @@ private async Task ForceFailJob(IRunServer runServer, Pipelines.AgentJobRequestM jobAnnotations.Add(annotation.Value); } - await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, TaskResult.Failed, outputs: null, stepResults: null, jobAnnotations: jobAnnotations, environmentUrl: null, CancellationToken.None); + await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, TaskResult.Failed, outputs: null, stepResults: null, jobAnnotations: jobAnnotations, environmentUrl: null, telemetry: null, CancellationToken.None); } catch (Exception ex) { diff --git a/src/Runner.Worker/JobRunner.cs b/src/Runner.Worker/JobRunner.cs index c52dc29512d..f1f9a72f1f4 100644 --- a/src/Runner.Worker/JobRunner.cs +++ b/src/Runner.Worker/JobRunner.cs @@ -15,6 +15,7 @@ using GitHub.Runner.Sdk; using GitHub.Services.Common; using GitHub.Services.WebApi; +using Sdk.RSWebApi.Contracts; using Pipelines = GitHub.DistributedTask.Pipelines; namespace GitHub.Runner.Worker @@ -279,7 +280,12 @@ private async Task CompleteJobAsync(IRunServer runServer, IExecution jobContext.Debug($"Finishing: {message.JobDisplayName}"); TaskResult result = jobContext.Complete(taskResult); - await ShutdownQueue(throwOnFailure: false); + var jobQueueTelemetry = await ShutdownQueue(throwOnFailure: false); + // include any job telemetry from the background upload process. + if (jobQueueTelemetry?.Count > 0) + { + jobContext.Global.JobTelemetry.AddRange(jobQueueTelemetry); + } // Make sure to clean temp after file upload since they may be pending fileupload still use the TEMP dir. _tempDirectoryManager?.CleanupTempDirectory(); @@ -297,6 +303,13 @@ private async Task CompleteJobAsync(IRunServer runServer, IExecution environmentUrl = urlStringToken.Value; } + // Get telemetry + IList telemetry = null; + if (jobContext.Global.JobTelemetry.Count > 0) + { + telemetry = jobContext.Global.JobTelemetry.Select(x => new Telemetry { Type = x.Type.ToString(), Message = x.Message, }).ToList(); + } + Trace.Info($"Raising job completed against run service"); var completeJobRetryLimit = 5; var exceptions = new List(); @@ -304,7 +317,7 @@ private async Task CompleteJobAsync(IRunServer runServer, IExecution { try { - await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, result, jobContext.JobOutputs, jobContext.Global.StepsResult, jobContext.Global.JobAnnotations, environmentUrl, default); + await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, result, jobContext.JobOutputs, jobContext.Global.StepsResult, jobContext.Global.JobAnnotations, environmentUrl, telemetry, default); return result; } catch (Exception ex) @@ -329,56 +342,14 @@ private async Task CompleteJobAsync(IJobServer jobServer, IExecution if (_runnerSettings.DisableUpdate == true) { - try - { - var currentVersion = new PackageVersion(BuildConstants.RunnerPackage.Version); - ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); - VssCredentials serverCredential = VssUtil.GetVssCredential(systemConnection); - - var runnerServer = HostContext.GetService(); - await runnerServer.ConnectAsync(systemConnection.Url, serverCredential); - var serverPackages = await runnerServer.GetPackagesAsync("agent", BuildConstants.RunnerPackage.PackageName, 5, includeToken: false, cancellationToken: CancellationToken.None); - if (serverPackages.Count > 0) - { - serverPackages = serverPackages.OrderByDescending(x => x.Version).ToList(); - Trace.Info($"Newer packages {StringUtil.ConvertToJson(serverPackages.Select(x => x.Version.ToString()))}"); - - var warnOnFailedJob = false; // any minor/patch version behind. - var warnOnOldRunnerVersion = false; // >= 2 minor version behind - if (serverPackages.Any(x => x.Version.CompareTo(currentVersion) > 0)) - { - Trace.Info($"Current runner version {currentVersion} is behind the latest runner version {serverPackages[0].Version}."); - warnOnFailedJob = true; - } - - if (serverPackages.Where(x => x.Version.Major == currentVersion.Major && x.Version.Minor > currentVersion.Minor).Count() > 1) - { - Trace.Info($"Current runner version {currentVersion} is way behind the latest runner version {serverPackages[0].Version}."); - warnOnOldRunnerVersion = true; - } - - if (result == TaskResult.Failed && warnOnFailedJob) - { - jobContext.Warning($"This job failure may be caused by using an out of date self-hosted runner. You are currently using runner version {currentVersion}. Please update to the latest version {serverPackages[0].Version}"); - } - else if (warnOnOldRunnerVersion) - { - jobContext.Warning($"This self-hosted runner is currently using runner version {currentVersion}. This version is out of date. Please update to the latest version {serverPackages[0].Version}"); - } - } - } - catch (Exception ex) - { - // Ignore any error since suggest runner update is best effort. - Trace.Error($"Caught exception during runner version check: {ex}"); - } + await WarningOutdatedRunnerAsync(jobContext, message, result); } try { var jobQueueTelemetry = await ShutdownQueue(throwOnFailure: true); // include any job telemetry from the background upload process. - if (jobQueueTelemetry.Count > 0) + if (jobQueueTelemetry?.Count > 0) { jobContext.Global.JobTelemetry.AddRange(jobQueueTelemetry); } @@ -506,5 +477,52 @@ private async Task> ShutdownQueue(bool throwOnFailure) return Array.Empty(); } + + private async Task WarningOutdatedRunnerAsync(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, TaskResult result) + { + try + { + var currentVersion = new PackageVersion(BuildConstants.RunnerPackage.Version); + ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); + VssCredentials serverCredential = VssUtil.GetVssCredential(systemConnection); + + var runnerServer = HostContext.GetService(); + await runnerServer.ConnectAsync(systemConnection.Url, serverCredential); + var serverPackages = await runnerServer.GetPackagesAsync("agent", BuildConstants.RunnerPackage.PackageName, 5, includeToken: false, cancellationToken: CancellationToken.None); + if (serverPackages.Count > 0) + { + serverPackages = serverPackages.OrderByDescending(x => x.Version).ToList(); + Trace.Info($"Newer packages {StringUtil.ConvertToJson(serverPackages.Select(x => x.Version.ToString()))}"); + + var warnOnFailedJob = false; // any minor/patch version behind. + var warnOnOldRunnerVersion = false; // >= 2 minor version behind + if (serverPackages.Any(x => x.Version.CompareTo(currentVersion) > 0)) + { + Trace.Info($"Current runner version {currentVersion} is behind the latest runner version {serverPackages[0].Version}."); + warnOnFailedJob = true; + } + + if (serverPackages.Where(x => x.Version.Major == currentVersion.Major && x.Version.Minor > currentVersion.Minor).Count() > 1) + { + Trace.Info($"Current runner version {currentVersion} is way behind the latest runner version {serverPackages[0].Version}."); + warnOnOldRunnerVersion = true; + } + + if (result == TaskResult.Failed && warnOnFailedJob) + { + jobContext.Warning($"This job failure may be caused by using an out of date self-hosted runner. You are currently using runner version {currentVersion}. Please update to the latest version {serverPackages[0].Version}"); + } + else if (warnOnOldRunnerVersion) + { + jobContext.Warning($"This self-hosted runner is currently using runner version {currentVersion}. This version is out of date. Please update to the latest version {serverPackages[0].Version}"); + } + } + } + catch (Exception ex) + { + // Ignore any error since suggest runner update is best effort. + Trace.Error($"Caught exception during runner version check: {ex}"); + } + } } } diff --git a/src/Sdk/RSWebApi/Contracts/CompleteJobRequest.cs b/src/Sdk/RSWebApi/Contracts/CompleteJobRequest.cs index fff5156a40d..ff02ab6586c 100644 --- a/src/Sdk/RSWebApi/Contracts/CompleteJobRequest.cs +++ b/src/Sdk/RSWebApi/Contracts/CompleteJobRequest.cs @@ -27,6 +27,9 @@ public class CompleteJobRequest [DataMember(Name = "annotations", EmitDefaultValue = false)] public IList Annotations { get; set; } + [DataMember(Name = "telemetry", EmitDefaultValue = false)] + public IList Telemetry { get; set; } + [DataMember(Name = "environmentUrl", EmitDefaultValue = false)] public string EnvironmentUrl { get; set; } } diff --git a/src/Sdk/RSWebApi/Contracts/Telemetry.cs b/src/Sdk/RSWebApi/Contracts/Telemetry.cs new file mode 100644 index 00000000000..9dda8aa130c --- /dev/null +++ b/src/Sdk/RSWebApi/Contracts/Telemetry.cs @@ -0,0 +1,20 @@ +using System.Runtime.Serialization; + +namespace Sdk.RSWebApi.Contracts +{ + [DataContract] + public struct Telemetry + { + public Telemetry(string message, string type) + { + Message = message; + Type = type; + } + + [DataMember(Name = "message", EmitDefaultValue = false)] + public string Message { get; set; } + + [DataMember(Name = "type", EmitDefaultValue = false)] + public string Type { get; set; } + } +} diff --git a/src/Sdk/RSWebApi/RunServiceHttpClient.cs b/src/Sdk/RSWebApi/RunServiceHttpClient.cs index 2a1b0b998ad..f4d0c539f0d 100644 --- a/src/Sdk/RSWebApi/RunServiceHttpClient.cs +++ b/src/Sdk/RSWebApi/RunServiceHttpClient.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Net; using System.Net.Http; using System.Threading; @@ -126,6 +127,7 @@ public async Task CompleteJobAsync( IList stepResults, IList jobAnnotations, string environmentUrl, + IList telemetry, CancellationToken cancellationToken = default) { HttpMethod httpMethod = new HttpMethod("POST"); @@ -138,6 +140,7 @@ public async Task CompleteJobAsync( StepResults = stepResults, Annotations = jobAnnotations, EnvironmentUrl = environmentUrl, + Telemetry = telemetry, }; requestUri = new Uri(requestUri, "completejob");