DebugInformation property on responses and exceptions

src/Elasticsearch.Net/Exceptions/ElasticsearchClientException.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Text;
 
 namespace Elasticsearch.Net
 {
@@ -30,7 +31,32 @@ public ElasticsearchClientException(PipelineFailure failure, string message, IAp
 			Response = apiCall;
 			FailureReason = failure;
 			AuditTrail = apiCall?.AuditTrail;
+		}
 
+		public string DebugInformation
+		{
+			get
+			{
+				var sb = new StringBuilder();
+				sb.AppendLine($"# FailureReason: {FailureReason.GetStringValue()} when trying to {Request.Method.GetStringValue()} {Request.Uri}");
+				if (this.Response != null)
+					ResponseStatics.DebugInformationBuilder(this.Response, sb);
+				else
+				{
+					ResponseStatics.DebugAuditTrail(this.AuditTrail, sb);
+					ResponseStatics.DebugAuditTrailExceptions(this.AuditTrail, sb);
+				}
+				if (InnerException != null)
+				{
+					sb.AppendLine($"# Inner Exception: {InnerException.Message}");
+					sb.AppendLine(InnerException.ToString());
+				}
+				sb.AppendLine($"# Exception:");
+				sb.AppendLine(this.ToString());
+
+				return sb.ToString();
+			}
 		}
+
 	}
 }

src/Elasticsearch.Net/Responses/ElasticsearchResponse.cs

@@ -3,14 +3,49 @@
 using System.ComponentModel;
 using System.Globalization;
 using System.Linq;
+using System.Text;
 
 namespace Elasticsearch.Net
 {
-	internal static class ResponseStatics
+	public static class ResponseStatics
 	{
-		public static readonly string PrintFormat = "StatusCode: {1}, {0}\tMethod: {2}, {0}\tUrl: {3}, {0}\tRequest: {4}, {0}\tResponse: {5}";
-		public static readonly string ErrorFormat = "{0}\tExceptionMessage: {1}{0}\t StackTrace: {2}";
-		public static readonly string AlreadyCaptured = "<Response stream not captured or already read to completion by serializer. Set DisableDirectStreaming() on ConnectionSettings to force it to be set on the response.>";
+		private static readonly string ResponseAlreadyCaptured = "<Response stream not captured or already read to completion by serializer. Set DisableDirectStreaming() on ConnectionSettings to force it to be set on the response.>";
+		private static readonly string RequestAlreadyCaptured = "<Request stream not captured or already read to completion by serializer. Set DisableDirectStreaming() on ConnectionSettings to force it to be set on the response.>";
+		public static string DebugInformationBuilder(IApiCallDetails r, StringBuilder sb)
+		{
+			sb.AppendLine($"# Audit trail of this API call:");
+			var auditTrail = (r.AuditTrail ?? Enumerable.Empty<Audit>()).ToList();
+			DebugAuditTrail(auditTrail, sb);
+			if (r.ServerError != null) sb.AppendLine($"# ServerError: {r.ServerError}");
+			if (r.OriginalException != null) sb.AppendLine($"# OriginalException: {r.OriginalException}");
+			DebugAuditTrailExceptions(auditTrail, sb);
+
+			var response = r.ResponseBodyInBytes?.Utf8String() ?? ResponseStatics.ResponseAlreadyCaptured;
+			var request = r.RequestBodyInBytes?.Utf8String() ?? ResponseStatics.RequestAlreadyCaptured;
+			sb.AppendLine($"# Request:\r\n{request}");
+			sb.AppendLine($"# Response:\r\n{response}");
+
+			return sb.ToString();
+		}
+
+		public static void DebugAuditTrailExceptions(List<Audit> auditTrail, StringBuilder sb)
+		{
+			var auditExceptions = auditTrail.Select((audit, i) => new {audit, i}).Where(a => a.audit.Exception != null);
+			foreach (var a in auditExceptions)
+				sb.AppendLine($"# Audit exception in step {a.i} {a.audit.Event.GetStringValue()}:\r\n{a.audit.Exception}");
+		}
+
+		public static void DebugAuditTrail(List<Audit> auditTrail, StringBuilder sb)
+		{
+			if (auditTrail == null) return;
+			foreach (var audit in auditTrail)
+			{
+				sb.Append($" - {audit.Event.GetStringValue()}:");
+				if (audit.Node?.Uri != null) sb.Append($" Node: {audit.Node.Uri}");
+				if (audit.Exception != null) sb.Append($" Exception: {audit.Exception.GetType().Name}");
+				sb.AppendLine($" Took: {(audit.Ended - audit.Started)}");
+			}
+		}
 	}
 
 	public class ElasticsearchResponse<T> : IApiCallDetails
@@ -59,25 +94,16 @@ public ElasticsearchResponse(int statusCode, IEnumerable<int> allowedStatusCodes
 			this.HttpStatusCode = statusCode;
 		}
 
-		public override string ToString()
+		public string DebugInformation
 		{
-			var r = this;
-			var e = r.OriginalException;
-			var response = this.ResponseBodyInBytes?.Utf8String() ?? ResponseStatics.AlreadyCaptured;
-
-			var requestJson = r.RequestBodyInBytes?.Utf8String();
-
-			var print = string.Format(ResponseStatics.PrintFormat,
-				Environment.NewLine,
-				r.HttpStatusCode.HasValue ? r.HttpStatusCode.Value.ToString(CultureInfo.InvariantCulture) : "-1",
-				r.HttpMethod,
-				r.Uri,
-				requestJson,
-				response
-			);
-			if (!this.Success && e != null)
-				print += string.Format(ResponseStatics.ErrorFormat,Environment.NewLine, e.Message, e.StackTrace);
-			return print;
+			get
+			{
+				var sb = new StringBuilder();
+				sb.AppendLine(this.ToString());
+				return ResponseStatics.DebugInformationBuilder(this, sb);
+			}
 		}
+
+		public override string ToString() =>  $"{(Success ? "S" : "Uns")}uccesful low level call on {HttpMethod.GetStringValue()}: {Uri.PathAndQuery}";
 	}
 }

src/Elasticsearch.Net/Responses/IApiCallDetails.cs

@@ -47,5 +47,7 @@ public interface IApiCallDetails
 		byte[] RequestBodyInBytes { get; }
 
 		List<Audit> AuditTrail { get; }
+
+		string DebugInformation { get; }
 	}
 }

src/Elasticsearch.Net/Responses/ServerError.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -53,6 +54,15 @@ internal static ServerError Create(IDictionary<string, object> dict, IJsonSerial
 				Error = (Error)strategy.DeserializeObject(error, typeof(Error))
 			};
 		}
+
+		public override string ToString() 
+		{
+			var sb = new StringBuilder();
+			sb.Append($"ServerError: {Status}");
+			if (Error != null)
+				sb.Append(Error);
+			return sb.ToString();
+		}
 	}
 
 	public interface IRootCause
@@ -85,6 +95,8 @@ internal static Error Create(IDictionary<string, object> dict, IJsonSerializerSt
 			error.RootCause = os.Select(o => (RootCause)strategy.DeserializeObject(o, typeof(RootCause))).ToList();
 			return error;
 		}
+
+		public override string ToString() => $"Type: {this.Type} Reason: \"{this.Reason}\"";
 	}
 
 	public class RootCause : IRootCause
@@ -100,7 +112,9 @@ internal static RootCause Create(IDictionary<string, object> dict, IJsonSerializ
 			var rootCause = new RootCause();
 			rootCause.FillValues(dict);
 			return rootCause;
-		}
+		}
+
+		public override string ToString() => $"Type: {this.Type} Reason: \"{this.Reason}\"";
 	}
 
 	internal static class RootCauseExtensions

src/Nest/Aggregations/Bucket/SignificantTerms/SignificantTermsAggregation.cs

@@ -110,6 +110,14 @@ public class SignificantTermsAggregationDescriptor<T>
 
 		public SignificantTermsAggregationDescriptor<T> Size(int size) => Assign(a => a.Size = size);
 
+		public SignificantTermsAggregationDescriptor<T> ExecutionHint(TermsAggregationExecutionHint? hint) => Assign(a => a.ExecutionHint = hint);
+
+		public SignificantTermsAggregationDescriptor<T> Include(Func<FluentDictionary<string, string>, FluentDictionary<string, string>> include) => 
+			Assign(a => a.Include = include?.Invoke(new FluentDictionary<string, string>()));
+
+		public SignificantTermsAggregationDescriptor<T> Exclude(Func<FluentDictionary<string, string>, FluentDictionary<string, string>> exclude) => 
+			Assign(a => a.Exclude = exclude?.Invoke(new FluentDictionary<string, string>()));
+
 		public SignificantTermsAggregationDescriptor<T> ShardSize(int shardSize) => Assign(a => a.ShardSize = shardSize);
 
 		public SignificantTermsAggregationDescriptor<T> MinimumDocumentCount(int minimumDocumentCount) =>

src/Nest/Cluster/NodesStats/NodesStatsRequest.cs

@@ -3,6 +3,7 @@
 	public partial interface INodesStatsRequest { }
 
 	public partial class NodesStatsRequest { }
+
 	[DescriptorFor("NodesStats")]
 	public partial class NodesStatsDescriptor { }
 }

src/Nest/CommonAbstractions/Response/ApiCallDetailsOverride.cs

@@ -23,6 +23,7 @@ internal class ApiCallDetailsOverride : IApiCallDetails
 		public byte[] ResponseBodyInBytes => this._original.ResponseBodyInBytes;
 		public byte[] RequestBodyInBytes => this._original.RequestBodyInBytes;
 		public List<Audit> AuditTrail => this._original.AuditTrail;
+		public string DebugInformation => this._original.DebugInformation;
 
 		public ApiCallDetailsOverride(IApiCallDetails original, bool isValid)
 		{

src/Nest/CommonAbstractions/Response/ResponseBase.cs

@@ -1,4 +1,7 @@
 using System;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
 using Elasticsearch.Net;
 using Newtonsoft.Json;
 
@@ -18,6 +21,9 @@ public interface IResponse : IBodyWithApiCallDetails
 		[JsonIgnore]
 		Exception OriginalException { get; }
 
+		[JsonIgnore]
+		string DebugInformation { get; }
+
 	}
 
 	public abstract class ResponseBase : IResponse
@@ -27,9 +33,26 @@ public abstract class ResponseBase : IResponse
 		IApiCallDetails IBodyWithApiCallDetails.CallDetails { get; set; }
 
 		public virtual IApiCallDetails ApiCall => ((IBodyWithApiCallDetails)this).CallDetails;
-
-		public virtual ServerError ServerError  => this.ApiCall?.ServerError;
 
-		public Exception OriginalException  => this.ApiCall?.OriginalException;
+		public virtual ServerError ServerError => this.ApiCall?.ServerError;
+
+		public Exception OriginalException => this.ApiCall?.OriginalException;
+
+		public string DebugInformation
+		{
+			get
+			{
+				var sb = new StringBuilder();
+				sb.Append($"{(!IsValid ? "Inv" : "V")}alid NEST response built from a ");
+				sb.AppendLine(ApiCall?.ToString().ToCamelCase() ?? "null ApiCall which is highly exceptional, please open a bug if you see this");
+				if (!this.IsValid) this.DebugIsValid(sb);
+				ResponseStatics.DebugInformationBuilder(ApiCall, sb);
+				return sb.ToString();
+			}
+		}
+		protected virtual void DebugIsValid(StringBuilder sb) { }
+
+		public override string ToString() =>  $"{(!IsValid ? "Inv" : "V")}alid NEST response built from a {this.ApiCall?.ToString().ToCamelCase()}";
+
 	}
 }

src/Nest/CommonOptions/Failures/BulkError.cs

@@ -17,5 +17,6 @@ public class BulkError
 		[JsonProperty("reason")]
 		public string Reason { get; internal set; }
 
+		public override string ToString() => $"Type: {Type} Reason: \"{Reason}\"";
 	}
 }

src/Nest/Document/Multiple/Bulk/BulkResponse.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using System.Linq;
+using System.Text;
 using Newtonsoft.Json;
 
 namespace Nest
@@ -16,6 +17,12 @@ public interface IBulkResponse : IResponse
 	public class BulkResponse : ResponseBase, IBulkResponse
 	{
 		public override bool IsValid => base.IsValid && !this.Errors && !this.ItemsWithErrors.HasAny();
+		protected override void DebugIsValid(StringBuilder sb)
+		{
+			sb.AppendLine($"# Invalid Bulk items:");
+			foreach(var i in Items.Select((item, i) => new { item, i}).Where(i=>!i.item.IsValid))
+				sb.AppendLine($"  operation[{i.i}]: {i.item}");
+		}
 
 		[JsonProperty("took")]
 		public int Took { get; internal set; }

src/Nest/Document/Multiple/Bulk/BulkResponseItem/BulkResponseItemBase.cs

@@ -43,5 +43,7 @@ public bool IsValid
 				}
 			}
 		}
+
+		public override string ToString() => $"{Operation} returned {Status} _index: {Index} _type: {Type} _id: {Id} _version: {Version} error: {Error}";
 	}
 }

src/Nest/Search/MultiSearch/MultiSearchResponse.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using System.Linq;
+using System.Text;
 using Elasticsearch.Net;
 using Newtonsoft.Json;
 
@@ -15,6 +16,13 @@ public MultiSearchResponse()
 		}
 
 		public override bool IsValid => base.IsValid && this.AllResponses.All(b => b.IsValid);
+
+		protected override void DebugIsValid(StringBuilder sb)
+		{
+			sb.AppendLine($"# Invalid searches (inspect individual response.DebugInformation for more detail):");
+			foreach(var i in AllResponses.Select((item, i) => new { item, i}).Where(i=>!i.item.IsValid))
+				sb.AppendLine($"  search[{i.i}]: {i.item}");
+		}
 
 		[JsonConverter(typeof(VerbatimDictionaryKeysJsonConverter))]	
 		internal IDictionary<string, object> Responses { get; set; }

src/Nest/Search/Percolator/MultiPercolate/MultiPercolateResponse.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using System.Linq;
+using System.Text;
 using Elasticsearch.Net;
 using Newtonsoft.Json;
 
@@ -16,6 +17,13 @@ public class MultiPercolateResponse : ResponseBase, IMultiPercolateResponse
 	{
 		public override bool IsValid => base.IsValid && this.Responses.All(r => r.IsValid);
 
+		protected override void DebugIsValid(StringBuilder sb)
+		{
+			sb.AppendLine($"# Invalid percolations (inspect individual response.DebugInformation for more detail):");
+			foreach(var i in AllResponses.Select((item, i) => new { item, i}).Where(i=>!i.item.IsValid))
+				sb.AppendLine($"  search[{i.i}]: {i.item}");
+		}
+
 		[JsonProperty("responses")]
 		internal IEnumerable<PercolateResponse> AllResponses { get; set; }
 

src/Tests/ClientConcepts/Exceptions/ExceptionTests.cs

@@ -10,10 +10,10 @@ namespace Tests.ClientConcepts.Exceptions
 {
 	public class ExceptionTests
 	{
-		//[I]
+		[I]
 		public void ServerTestWhenThrowExceptionsEnabled()
 		{
-			var settings = new ConnectionSettings(new Uri("http://ipv4.fiddler:9200"))
+			var settings = new ConnectionSettings(new Uri($"http://{TestClient.Host}:9200"))
 				.ThrowExceptions();
 			var client = new ElasticClient(settings);
 			var exception = Assert.Throws<ElasticsearchClientException>(() => client.GetMapping<Project>(s => s.Index("doesntexist")));
@@ -23,7 +23,7 @@ public void ServerTestWhenThrowExceptionsEnabled()
 			exception.Response.ServerError.Status.Should().BeGreaterThan(0);
 		}
 
-		//[I]
+		[I]
 		public void ClientTestWhenThrowExceptionsEnabled()
 		{
 			var settings = new ConnectionSettings(new Uri("http://doesntexist:9200"))
@@ -34,18 +34,18 @@ public void ClientTestWhenThrowExceptionsEnabled()
 			inner.Should().NotBeNull();
 		}
 
-		//[I]
+		[I]
 		public void ServerTestWhenThrowExceptionsDisabled()
 		{
-			var settings = new ConnectionSettings(new Uri("http://ipv4.fiddler:9200"));
+			var settings = new ConnectionSettings(new Uri($"http://{TestClient.Host}:9200"));
 			var client = new ElasticClient(settings);
 			var response = client.GetMapping<Project>(s => s.Index("doesntexist"));
 			response.CallDetails.OriginalException.Should().NotBeNull();
 			response.CallDetails.ServerError.Should().NotBeNull();
 			response.CallDetails.ServerError.Status.Should().BeGreaterThan(0);
 		}
 
-		//[I]
+		[I]
 		public void ClientTestWhenThrowExceptionsDisabled()
 		{
 			var settings = new ConnectionSettings(new Uri("http://doesntexist:9200"));