feat: durable sink using file size rolled buffers (#68)

Add extension method `DurableHttpUsingFileSizeRolledBuffers`, creating a
durable sink using buffer files with a file size based rolling behavior.

Closes #67

CHANGELOG.md

@@ -6,6 +6,14 @@ This project adheres to [Semantic Versioning](http://semver.org/) and is followi
 
 ## Unreleased
 
+### :zap: Added
+
+- Extension method `DurableHttpUsingFileSizeRolledBuffers`, creating a durable sink using buffer files with a file size based rolling behavior
+
+### :zzz: Deprecated
+
+- Extension method `DurableHttp` has been renamed to `DurableHttpUsingTimeRolledBuffers`, providing clarification between the two durable sink types
+
 ## [5.1.0] - 2019-01-07
 
 ### :zap: Added

Directory.Build.props

@@ -5,6 +5,7 @@
     <Authors>Mattias Kindborg</Authors>
     <Company />
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
+    <NoWarn>1591</NoWarn>
   </PropertyGroup>
 
 </Project>

serilog-sinks-http.sln.DotSettings

@@ -271,4 +271,5 @@ protected virtual void Dispose(bool disposing)
 	<s:Boolean x:Key="/Default/PatternsAndTemplates/LiveTemplates/Template/=A9A0CC3540FBAA41A26223D5BE7478F6/ShortenQualifiedReferences/@EntryValue">True</s:Boolean>
 	<s:String x:Key="/Default/PatternsAndTemplates/LiveTemplates/Template/=A9A0CC3540FBAA41A26223D5BE7478F6/Text/@EntryValue">$FIELD$ = MockRepository.GenerateMock&lt;$INTERFACE$&gt;();</s:String>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Serilog/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=Xunit/@EntryIndexedValue">True</s:Boolean>
 	</wpf:ResourceDictionary>

src/Serilog.Sinks.Http/LoggerSinkConfigurationExtensions.cs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 using System;
+using System.ComponentModel;
 using System.Net.Http;
 using Serilog.Configuration;
 using Serilog.Events;
@@ -26,8 +27,8 @@
 namespace Serilog
 {
     /// <summary>
-    /// Adds the WriteTo.Http() and WriteTo.DurableHttp() extension method to
-    /// <see cref="LoggerConfiguration"/>.
+    /// Class containing extension methods to <see cref="LoggerConfiguration"/>, configuring sinks
+    /// sending log events over the network using HTTP.
     /// </summary>
     public static class LoggerSinkConfigurationExtensions
     {
@@ -90,21 +91,55 @@ public static LoggerConfiguration Http(
             return sinkConfiguration.Sink(sink, restrictedToMinimumLevel);
         }
 
+        [Obsolete("Use DurableHttpUsingTimeRolledBuffers instead of this sink provider")]
+        [EditorBrowsable(EditorBrowsableState.Never)]
+        public static LoggerConfiguration DurableHttp(
+            this LoggerSinkConfiguration sinkConfiguration,
+            string requestUri,
+            string bufferPathFormat = "Buffer-{Date}.json",
+            long? bufferFileSizeLimitBytes = null,
+            int? retainedBufferFileCountLimit = 31,
+            int batchPostingLimit = 1000,
+            TimeSpan? period = null,
+            ITextFormatter textFormatter = null,
+            IBatchFormatter batchFormatter = null,
+            LogEventLevel restrictedToMinimumLevel = LevelAlias.Minimum,
+            IHttpClient httpClient = null)
+        {
+            return DurableHttpUsingTimeRolledBuffers(
+                sinkConfiguration,
+                requestUri,
+                bufferPathFormat,
+                bufferFileSizeLimitBytes,
+                retainedBufferFileCountLimit,
+                batchPostingLimit,
+                period,
+                textFormatter,
+                batchFormatter,
+                restrictedToMinimumLevel,
+                httpClient);
+        }
+
         /// <summary>
         /// Adds a durable sink that sends log events using HTTP POST over the network. A durable
-        /// sink will persist log events on disk before sending them over the network, thus
-        /// protecting against data loss after a system or process restart.
+        /// sink will persist log events on disk in buffer files before sending them over the
+        /// network, thus protecting against data loss after a system or process restart. The
+        /// buffer files will use a rolling behavior defined by the time interval specified in
+        /// <paramref name="bufferPathFormat"/>, i.e. a new buffer file is created every time a new
+        /// interval is started. The maximum size of a file is defined by
+        /// <paramref name="bufferFileSizeLimitBytes"/>, and when that limit is reached all
+        /// incoming log events will be dropped until a new interval is started.
         /// </summary>
         /// <param name="sinkConfiguration">The logger configuration.</param>
         /// <param name="requestUri">The URI the request is sent to.</param>
         /// <param name="bufferPathFormat">
-        /// The path format for a set of files that will be used to buffer events until they can be
-        /// successfully sent over the network. Default value is "Buffer-{Date}.json". To use file
-        /// rotation that is on an 30 or 60 minute interval pass "Buffer-{HalfHour}.json" or
-        /// "Buffer-{Hour}.json".
+        /// The relative or absolute path format for a set of files that will be used to buffer
+        /// events until they can be successfully sent over the network. Default value is
+        /// "Buffer-{Date}.json". To use file rotation that is on an 30 or 60 minute interval pass
+        /// "Buffer-{HalfHour}.json" or "Buffer-{Hour}.json".
         /// </param>
         /// <param name="bufferFileSizeLimitBytes">
-        /// The maximum size, in bytes, to which the buffer log file for a specific date will be
+        /// The approximate maximum size, in bytes, to which a buffer file for a specific time interval will be
         /// allowed to grow. By default no limit will be applied.
         /// </param>
         /// <param name="retainedBufferFileCountLimit">
@@ -136,7 +171,7 @@ public static LoggerConfiguration Http(
         /// <see cref="HttpClient"/>.
         /// </param>
         /// <returns>Logger configuration, allowing configuration to continue.</returns>
-        public static LoggerConfiguration DurableHttp(
+        public static LoggerConfiguration DurableHttpUsingTimeRolledBuffers(
             this LoggerSinkConfiguration sinkConfiguration,
             string requestUri,
             string bufferPathFormat = "Buffer-{Date}.json",
@@ -157,7 +192,7 @@ public static LoggerConfiguration DurableHttp(
             batchFormatter = batchFormatter ?? new DefaultBatchFormatter();
             httpClient = httpClient ?? new DefaultHttpClient();
 
-            var sink = new DurableHttpSink(
+            var sink = new TimeRolledDurableHttpSink(
                 requestUri,
                 bufferPathFormat,
                 bufferFileSizeLimitBytes,
@@ -170,5 +205,93 @@ public static LoggerConfiguration DurableHttp(
 
             return sinkConfiguration.Sink(sink, restrictedToMinimumLevel);
         }
+
+        /// <summary>
+        /// Adds a durable sink that sends log events using HTTP POST over the network. A durable
+        /// sink will persist log events on disk in buffer files before sending them over the
+        /// network, thus protecting against data loss after a system or process restart. The
+        /// buffer files will use a rolling behavior defined by the file size specified in
+        /// <paramref name="bufferFileSizeLimitBytes"/>, i.e. a new buffer file is created when
+        /// current has passed its limit. The maximum number of retained files is defined by
+        /// <paramref name="retainedBufferFileCountLimit"/>, and when that limit is reached the
+        /// oldest file is dropped to make room for a new.
+        /// </summary>
+        /// <param name="sinkConfiguration">The logger configuration.</param>
+        /// <param name="requestUri">The URI the request is sent to.</param>
+        /// <param name="bufferBaseFileName">
+        /// The relative or absolute path for a set of files that will be used to buffer events
+        /// until they can be successfully transmitted across the network. Individual files will be
+        /// created using the pattern "<paramref name="bufferBaseFileName"/>*.json", which should
+        /// not clash with any other file names in the same directory. Default value is "Buffer".
+        /// </param>
+        /// <param name="bufferFileSizeLimitBytes">
+        /// The approximate maximum size, in bytes, to which a buffer file will be allowed to grow.
+        /// For unrestricted growth, pass null. The default is 1 GB. To avoid writing partial
+        /// events, the last event within the limit will be written in full even if it exceeds the
+        /// limit.
+        /// </param>
+        /// <param name="retainedBufferFileCountLimit">
+        /// The maximum number of buffer files that will be retained, including the current buffer
+        /// file. Under normal operation only 2 files will be kept, however if the log server is
+        /// unreachable, the number of files specified by <paramref name="retainedBufferFileCountLimit"/>
+        /// will be kept on the file system. For unlimited retention, pass null. Default value is 31.
+        /// </param>
+        /// <param name="batchPostingLimit">
+        /// The maximum number of events to post in a single batch. Default value is 1000.
+        /// </param>
+        /// <param name="period">
+        /// The time to wait between checking for event batches. Default value is 2 seconds.
+        /// </param>
+        /// <param name="textFormatter">
+        /// The formatter rendering individual log events into text, for example JSON. Default
+        /// value is <see cref="NormalRenderedTextFormatter"/>.
+        /// </param>
+        /// <param name="batchFormatter">
+        /// The formatter batching multiple log events into a payload that can be sent over the
+        /// network. Default value is <see cref="DefaultBatchFormatter"/>.
+        /// </param>
+        /// <param name="restrictedToMinimumLevel">
+        /// The minimum level for events passed through the sink. Default value is
+        /// <see cref="LevelAlias.Minimum"/>.
+        /// </param>
+        /// <param name="httpClient">
+        /// A custom <see cref="IHttpClient"/> implementation. Default value is
+        /// <see cref="HttpClient"/>.
+        /// </param>
+        /// <returns>Logger configuration, allowing configuration to continue.</returns>
+        public static LoggerConfiguration DurableHttpUsingFileSizeRolledBuffers(
+            this LoggerSinkConfiguration sinkConfiguration,
+            string requestUri,
+            string bufferBaseFileName = "Buffer",
+            long? bufferFileSizeLimitBytes = 1024 * 1024 * 1024,
+            int? retainedBufferFileCountLimit = 31,
+            int batchPostingLimit = 1000,
+            TimeSpan? period = null,
+            ITextFormatter textFormatter = null,
+            IBatchFormatter batchFormatter = null,
+            LogEventLevel restrictedToMinimumLevel = LevelAlias.Minimum,
+            IHttpClient httpClient = null)
+        {
+            if (sinkConfiguration == null) throw new ArgumentNullException(nameof(sinkConfiguration));
+
+            // Default values
+            period = period ?? TimeSpan.FromSeconds(2);
+            textFormatter = textFormatter ?? new NormalRenderedTextFormatter();
+            batchFormatter = batchFormatter ?? new DefaultBatchFormatter();
+            httpClient = httpClient ?? new DefaultHttpClient();
+
+            var sink = new FileSizeRolledDurableHttpSink(
+                requestUri,
+                bufferBaseFileName,
+                bufferFileSizeLimitBytes,
+                retainedBufferFileCountLimit,
+                batchPostingLimit,
+                period.Value,
+                textFormatter,
+                batchFormatter,
+                httpClient);
+
+            return sinkConfiguration.Sink(sink, restrictedToMinimumLevel);
+        }
     }
 }

src/Serilog.Sinks.Http/Serilog.Sinks.Http.csproj

@@ -24,8 +24,9 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Serilog.Sinks.PeriodicBatching" Version="2.*" />
-    <PackageReference Include="Serilog.Sinks.RollingFile" Version="3.*" />
+    <PackageReference Include="Serilog.Sinks.File" Version="4.0.0" />
+    <PackageReference Include="Serilog.Sinks.PeriodicBatching" Version="2.1.1" />
+    <PackageReference Include="Serilog.Sinks.RollingFile" Version="3.3.0" />
     <!-- SourceLink -->
     <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0-beta2-18618-05" PrivateAssets="All"/>
   </ItemGroup>

src/Serilog.Sinks.Http/Sinks/Http/Private/IO/DirectoryService.cs

@@ -0,0 +1,25 @@
+// Copyright 2015-2018 Serilog Contributors
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System.IO;
+
+namespace Serilog.Sinks.Http.Private.IO
+{
+    public class DirectoryService : IDirectoryService
+    {
+        public string[] GetFiles(string path, string searchPattern) =>
+            Directory.GetFiles(path, searchPattern);
+    }
+}
+

src/Serilog.Sinks.Http/Sinks/Http/Private/IO/IDirectoryService.cs

@@ -0,0 +1,38 @@
+// Copyright 2015-2018 Serilog Contributors
+// 
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// 
+//     http://www.apache.org/licenses/LICENSE-2.0
+// 
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+namespace Serilog.Sinks.Http.Private.IO
+{
+    public interface IDirectoryService
+    {
+        /// <summary>
+        /// Returns the names of files (including their paths) that match the specified search
+        /// pattern in the specified directory.
+        /// </summary>
+        /// <param name="path">
+        /// The relative or absolute path to the directory to search. This string is not
+        /// case-sensitive.
+        /// </param>
+        /// <param name="searchPattern">
+        /// The search string to match against the names of files in <paramref name="path" />. This
+        /// parameter can contain a combination of valid literal path and wildcard (* and ?)
+        /// characters, but doesn't support regular expressions.
+        /// </param>
+        /// <returns>
+        /// An array of the full names (including paths) for the files in the specified directory
+        /// that match the specified search pattern, or an empty array if no files are found.
+        /// </returns>
+        string[] GetFiles(string path, string searchPattern);
+    }
+}

src/Serilog.Sinks.Http/Sinks/Http/Private/Network/BookmarkFile.cs

@@ -11,7 +11,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member
+
 using System;
 using System.IO;
 using System.Text;
@@ -22,12 +22,12 @@ public class BookmarkFile : IDisposable
     {
         private readonly FileStream fileStream;
 
-        public BookmarkFile(string bookmarkFilename)
+        public BookmarkFile(string bookmarkFileName)
         {
-            if (bookmarkFilename == null) throw new ArgumentNullException(nameof(bookmarkFilename));
+            if (bookmarkFileName == null) throw new ArgumentNullException(nameof(bookmarkFileName));
 
             fileStream = System.IO.File.Open(
-                bookmarkFilename,
+                bookmarkFileName,
                 FileMode.OpenOrCreate,
                 FileAccess.ReadWrite,
                 FileShare.Read);
@@ -68,7 +68,7 @@ public void WriteBookmark(long nextLineBeginsAtOffset, string currentFile)
             }
         }
 
-        public void Dispose() => fileStream?.Dispose();
+        public void Dispose() =>
+            fileStream?.Dispose();
     }
 }
-#pragma warning restore CS1591 // Missing XML comment for publicly visible type or member

src/Serilog.Sinks.Http/Sinks/Http/Private/Network/DateFormats.cs

@@ -11,7 +11,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member
+
 namespace Serilog.Sinks.Http.Private.Network
 {
     public enum DateFormats
@@ -21,4 +21,3 @@ public enum DateFormats
         HalfHour
     }
 }
-#pragma warning restore CS1591 // Missing XML comment for publicly visible type or member

src/Serilog.Sinks.Http/Sinks/Http/Private/Network/DefaultHttpClient.cs

@@ -11,7 +11,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member
+
 using System.Net.Http;
 using System.Threading.Tasks;
 
@@ -31,4 +31,3 @@ public void Dispose() =>
             client.Dispose();
     }
 }
-#pragma warning restore CS1591 // Missing XML comment for publicly visible type or member