Roll buffer files on size in durable mode (#92)

* Fixes #52 - roll buffer file on size (uses pre-release Serilog.Sinks.File)

Changes the semantics of the parameter previously called bufferFileSizeLimitBytes - this is now bufferSizeLimitBytes and is applied to the whole file set.

* Make sure debug/verbose logs make it through

* Boost durable mode test coverage

* Abandon resx; fix test

* Trying to figure out encoding differences causing CI failure

* Update to VS2017 build image

*  Jump through some hoops to try to get consistent test results

* Revert broken byte count

* Might as well make text x-plat

Build.ps1

@@ -11,7 +11,11 @@ $suffix = @{ $true = ""; $false = "$branch-$revision"}[$branch -eq "master" -and
 foreach ($src in ls src/Serilog.*) {
     Push-Location $src
 
-    & dotnet pack -c Release -o ..\..\.\artifacts --version-suffix=$suffix --include-source
+	if ($suffix) {
+		& dotnet pack -c Release -o ..\..\.\artifacts --version-suffix=$suffix --include-source
+	} else {
+	    & dotnet pack -c Release -o ..\..\.\artifacts --include-source
+	}
     if($LASTEXITCODE -ne 0) { exit 1 }    
 
     Pop-Location

appveyor.yml

@@ -1,13 +1,9 @@
 version: '{build}'
 skip_tags: true
-image: Visual Studio 2015
+image: Visual Studio 2017
 configuration: Release
 install:
   - ps: mkdir -Force ".\build\" | Out-Null
-  - ps: Invoke-WebRequest "https://raw.githubusercontent.com/dotnet/cli/rel/1.0.0/scripts/obtain/dotnet-install.ps1" -OutFile ".\build\installcli.ps1"
-  - ps: $env:DOTNET_INSTALL_DIR = "$pwd\.dotnetcli"
-  - ps: '& .\build\installcli.ps1 -InstallDir "$env:DOTNET_INSTALL_DIR" -NoPath -Version 1.0.0'
-  - ps: $env:Path = "$env:DOTNET_INSTALL_DIR;$env:Path"
 build_script:
 - ps: ./Build.ps1
 test: off

sample/Sample/Sample.csproj

@@ -32,4 +32,8 @@
     <ProjectReference Include="..\..\src\Serilog.Sinks.Seq\Serilog.Sinks.Seq.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Folder Include="logs\" />
+  </ItemGroup>
+
 </Project>

src/Serilog.Sinks.Seq/SeqLoggerConfigurationExtensions.cs

@@ -42,9 +42,10 @@ public static class SeqLoggerConfigurationExtensions
         /// <param name="period">The time to wait between checking for event batches.</param>
         /// <param name="bufferBaseFilename">Path for a set of files that will be used to buffer events until they
         /// can be successfully transmitted across the network. Individual files will be created using the
-        /// pattern <paramref name="bufferBaseFilename"/>-{Date}.json.</param>
+        /// pattern <paramref name="bufferBaseFilename"/>*.json, which should not clash with any other filenames
+        /// in the same directory.</param>
         /// <param name="apiKey">A Seq <i>API key</i> that authenticates the client to the Seq server.</param>
-        /// <param name="bufferFileSizeLimitBytes">The maximum size, in bytes, to which the buffer
+        /// <param name="bufferSizeLimitBytes">The maximum amount of data, in bytes, to which the buffer
         /// log file for a specific date will be allowed to grow. By default no limit will be applied.</param>
         /// <param name="eventBodyLimitBytes">The maximum size, in bytes, that the JSON representation of
         /// an event may take before it is dropped rather than being sent to the Seq server. Specify null for no limit.
@@ -72,7 +73,7 @@ public static LoggerConfiguration Seq(
             TimeSpan? period = null,
             string apiKey = null,
             string bufferBaseFilename = null,
-            long? bufferFileSizeLimitBytes = null,
+            long? bufferSizeLimitBytes = null,
             long? eventBodyLimitBytes = 256*1024,
             LoggingLevelSwitch controlLevelSwitch = null,
             HttpMessageHandler messageHandler = null,
@@ -82,8 +83,8 @@ public static LoggerConfiguration Seq(
         {
             if (loggerSinkConfiguration == null) throw new ArgumentNullException(nameof(loggerSinkConfiguration));
             if (serverUrl == null) throw new ArgumentNullException(nameof(serverUrl));
-            if (bufferFileSizeLimitBytes.HasValue && bufferFileSizeLimitBytes < 0)
-                throw new ArgumentOutOfRangeException(nameof(bufferFileSizeLimitBytes), "Negative value provided; file size limit must be non-negative.");
+            if (bufferSizeLimitBytes.HasValue && bufferSizeLimitBytes < 0)
+                throw new ArgumentOutOfRangeException(nameof(bufferSizeLimitBytes), "Negative value provided; buffer size limit must be non-negative.");
             if (queueSizeLimit < 0)
                 throw new ArgumentOutOfRangeException(nameof(queueSizeLimit), "Queue size limit must be non-zero.");
 
@@ -113,7 +114,7 @@ public static LoggerConfiguration Seq(
                     apiKey,
                     batchPostingLimit,
                     defaultedPeriod,
-                    bufferFileSizeLimitBytes,
+                    bufferSizeLimitBytes,
                     eventBodyLimitBytes,
                     controlLevelSwitch,
                     messageHandler,

src/Serilog.Sinks.Seq/Serilog.Sinks.Seq.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <Description>Serilog sink that writes to the Seq log server over HTTP/HTTPS.</Description>
-    <VersionPrefix>3.4.1</VersionPrefix>
+    <VersionPrefix>4.0.0</VersionPrefix>
     <Authors>Serilog Contributors</Authors>
     <Copyright>Copyright © Serilog Contributors 2013-2017</Copyright>
     <TargetFrameworks>netstandard1.1;netstandard1.3;net45</TargetFrameworks>
@@ -44,8 +44,7 @@
   </ItemGroup>
 
   <ItemGroup Condition=" '$(TargetFramework)' != 'netstandard1.1' ">
-    <PackageReference Include="Serilog.Sinks.File" Version="3.2.0" />
-    <PackageReference Include="Serilog.Sinks.RollingFile" Version="3.3.0" />
+    <PackageReference Include="Serilog.Sinks.File" Version="4.0.0-dev-00788" />
   </ItemGroup>
 
 </Project>

src/Serilog.Sinks.Seq/Sinks/Seq/Durable/BookmarkFile.cs

@@ -33,13 +33,14 @@ public FileSetPosition TryReadBookmark()
         {
             if (_bookmark.Length != 0)
             {
+                _bookmark.Position = 0;
+
                 // Important not to dispose this StreamReader as the stream must remain open.
                 var reader = new StreamReader(_bookmark, Encoding.UTF8, false, 128);
                 var current = reader.ReadLine();
 
                 if (current != null)
                 {
-                    _bookmark.Position = 0;
                     var parts = current.Split(new[] { ":::" }, StringSplitOptions.RemoveEmptyEntries);
                     if (parts.Length == 2)
                     {
@@ -56,10 +57,14 @@ public void WriteBookmark(FileSetPosition bookmark)
             if (bookmark.File == null)
                 return;
 
-            using (var writer = new StreamWriter(_bookmark))
-            {
-                writer.WriteLine("{0}:::{1}", bookmark.NextLineStart, bookmark.File);
-            }
+            // Don't need to truncate, since we only ever read a single line and
+            // writes are always newline-terminated
+            _bookmark.Position = 0;
+
+            // Cannot dispose, as `leaveOpen` is not available on all target platforms
+            var writer = new StreamWriter(_bookmark);
+            writer.WriteLine("{0}:::{1}", bookmark.NextLineStart, bookmark.File);
+            writer.Flush();
         }
 
         public void Dispose()

src/Serilog.Sinks.Seq/Sinks/Seq/Durable/DurableSeqSink.cs

@@ -17,7 +17,6 @@
 using System;
 using Serilog.Core;
 using Serilog.Events;
-using Serilog.Sinks.RollingFile;
 using System.Net.Http;
 using System.Text;
 
@@ -26,15 +25,15 @@ namespace Serilog.Sinks.Seq.Durable
     class DurableSeqSink : ILogEventSink, IDisposable
     {
         readonly HttpLogShipper _shipper;
-        readonly RollingFileSink _sink;
+        readonly Logger _sink;
 
         public DurableSeqSink(
             string serverUrl,
             string bufferBaseFilename,
             string apiKey,
             int batchPostingLimit,
             TimeSpan period,
-            long? bufferFileSizeLimitBytes,
+            long? bufferSizeLimitBytes,
             long? eventBodyLimitBytes,
             LoggingLevelSwitch levelControlSwitch,
             HttpMessageHandler messageHandler,
@@ -43,23 +42,30 @@ public DurableSeqSink(
             if (serverUrl == null) throw new ArgumentNullException(nameof(serverUrl));
             if (bufferBaseFilename == null) throw new ArgumentNullException(nameof(bufferBaseFilename));
 
+
             _shipper = new HttpLogShipper(
                 serverUrl, 
                 bufferBaseFilename, 
                 apiKey, 
                 batchPostingLimit, 
                 period, 
-                eventBodyLimitBytes, 
+                eventBodyLimitBytes,
                 levelControlSwitch,
                 messageHandler,
-                retainedInvalidPayloadsLimitBytes);
+                retainedInvalidPayloadsLimitBytes,
+                bufferSizeLimitBytes);
 
-            _sink = new RollingFileSink(
-                bufferBaseFilename + "-{Date}.json",
-                new RawJsonFormatter(),
-                bufferFileSizeLimitBytes,
-                null,
-                encoding: Encoding.UTF8);
+            const long individualFileSizeLimitBytes = 100L * 1024 * 1024;
+            _sink = new LoggerConfiguration()
+                .MinimumLevel.Verbose()
+                .WriteTo.File(new RawJsonFormatter(),
+                        bufferBaseFilename + "-.json",
+                        rollingInterval: RollingInterval.Day,
+                        fileSizeLimitBytes: individualFileSizeLimitBytes,
+                        rollOnFileSizeLimit: true,
+                        retainedFileCountLimit: null,
+                        encoding: Encoding.UTF8)
+                .CreateLogger();
         }
 
         public void Dispose()
@@ -74,7 +80,7 @@ public void Emit(LogEvent logEvent)
             // are worth the ambiguity.
             if (_shipper.IsIncluded(logEvent))
             {
-                _sink.Emit(logEvent);
+                _sink.Write(logEvent);
             }
         }
     }

src/Serilog.Sinks.Seq/Sinks/Seq/Durable/FileSet.cs

@@ -19,6 +19,7 @@
 using System.IO;
 using System.Linq;
 using System.Net;
+using System.Text.RegularExpressions;
 using Serilog.Debugging;
 
 namespace Serilog.Sinks.Seq.Durable
@@ -28,6 +29,7 @@ class FileSet
         readonly string _bookmarkFilename;
         readonly string _candidateSearchPath;
         readonly string _logFolder;
+        readonly Regex _filenameMatcher;
 
         const string InvalidPayloadFilePrefix = "invalid-";
 
@@ -37,75 +39,87 @@ public FileSet(string bufferBaseFilename)
 
             _bookmarkFilename = Path.GetFullPath(bufferBaseFilename + ".bookmark");
             _logFolder = Path.GetDirectoryName(_bookmarkFilename);
-            _candidateSearchPath = Path.GetFileName(bufferBaseFilename) + "*.json";
+            _candidateSearchPath = Path.GetFileName(bufferBaseFilename) + "-*.json";
+            _filenameMatcher = new Regex("^" + Regex.Escape(Path.GetFileName(bufferBaseFilename)) + "-(?<date>\\d{8})(?<sequence>_[0-9]{3,}){0,1}\\.json$");
         }
 
         public BookmarkFile OpenBookmarkFile()
         {
             return new BookmarkFile(_bookmarkFilename);
         }
 
-        public string MakeInvalidPayloadFilename(HttpStatusCode statusCode)
+        public string[] GetBufferFiles()
         {
-            var invalidPayloadFilename = $"{InvalidPayloadFilePrefix}{statusCode}-{Guid.NewGuid():n}.json";
-            return Path.Combine(_logFolder, invalidPayloadFilename);
+            return Directory.GetFiles(_logFolder, _candidateSearchPath)
+                .Select(n => new KeyValuePair<string, Match>(n, _filenameMatcher.Match(Path.GetFileName(n))))
+                .Where(nm => nm.Value.Success)
+                .OrderBy(nm => nm.Value.Groups["date"].Value, StringComparer.OrdinalIgnoreCase)
+                .ThenBy(nm => int.Parse("0" + nm.Value.Groups["sequence"].Value.Replace("_", "")))
+                .Select(nm => nm.Key)
+                .ToArray();
         }
 
-        public void CleanUpInvalidPayloadFiles(long maxNumberOfBytesToRetain)
+        public void CleanUpBufferFiles(long bufferSizeLimitBytes, int alwaysRetainCount)
         {
             try
             {
-                var candiateFiles = Directory.EnumerateFiles(_logFolder, $"{InvalidPayloadFilePrefix}*.json");
-                DeleteOldFiles(maxNumberOfBytesToRetain, candiateFiles);
+                var bufferFiles = GetBufferFiles();
+                Array.Reverse(bufferFiles);
+                DeleteExceedingCumulativeSize(bufferFiles.Select(f => new FileInfo(f)), bufferSizeLimitBytes, 2);
             }
             catch (Exception ex)
             {
-                SelfLog.WriteLine("Exception thrown while trying to clean up invalid payload files: {0}", ex);
+                SelfLog.WriteLine("Exception thrown while cleaning up buffer files: {0}", ex);
             }
         }
 
-        public string[] GetFiles()
+        public string MakeInvalidPayloadFilename(HttpStatusCode statusCode)
         {
-            return Directory.GetFiles(_logFolder, _candidateSearchPath)
-                .OrderBy(n => n)
-                .ToArray();
+            var invalidPayloadFilename = $"{InvalidPayloadFilePrefix}{statusCode}-{Guid.NewGuid():n}.json";
+            return Path.Combine(_logFolder, invalidPayloadFilename);
         }
 
-        static void DeleteOldFiles(long maxNumberOfBytesToRetain, IEnumerable<string> files)
+        public void CleanUpInvalidPayloadFiles(long maxNumberOfBytesToRetain)
         {
-            var orderedFileInfos = from candiateFile in files
-                let candiateFileInfo = new FileInfo(candiateFile)
-                orderby candiateFileInfo.LastAccessTimeUtc descending
-                select candiateFileInfo;
-
-            var invalidPayloadFilesToDelete = WhereCumulativeSizeGreaterThan(orderedFileInfos, maxNumberOfBytesToRetain);
+            try
+            {
+                var candidateFiles = from file in Directory.EnumerateFiles(_logFolder, $"{InvalidPayloadFilePrefix}*.json")
+                                     let candiateFileInfo = new FileInfo(file)
+                                     orderby candiateFileInfo.LastWriteTimeUtc descending
+                                     select candiateFileInfo;
 
-            foreach (var fileToDelete in invalidPayloadFilesToDelete)
+                DeleteExceedingCumulativeSize(candidateFiles, maxNumberOfBytesToRetain, 0);
+            }
+            catch (Exception ex)
             {
-                try
-                {
-                    fileToDelete.Delete();
-                }
-                catch (Exception ex)
-                {
-                    SelfLog.WriteLine("Exception '{0}' thrown while trying to delete file {1}", ex.Message, fileToDelete.FullName);
-                }
+                SelfLog.WriteLine("Exception thrown while cleaning up invalid payload files: {0}", ex);
             }
         }
-        
-        static IEnumerable<FileInfo> WhereCumulativeSizeGreaterThan(IEnumerable<FileInfo> files, long maxCumulativeSize)
+
+        static void DeleteExceedingCumulativeSize(IEnumerable<FileInfo> files, long maxNumberOfBytesToRetain, int alwaysRetainCount)
         {
             long cumulative = 0;
+            var i = 0;
             foreach (var file in files)
             {
                 cumulative += file.Length;
-                if (cumulative > maxCumulativeSize)
+
+                if (i++ < alwaysRetainCount)
+                    continue;
+
+                if (cumulative <= maxNumberOfBytesToRetain)
+                    continue;
+
+                try
                 {
-                    yield return file;
+                    file.Delete();
+                }
+                catch (Exception ex)
+                {
+                    SelfLog.WriteLine("Exception thrown while trying to delete file {0}: {1}", file.FullName, ex);
                 }
             }
         }
-
     }
 }
 

src/Serilog.Sinks.Seq/Sinks/Seq/Durable/FileSetPosition.cs

@@ -18,16 +18,14 @@ namespace Serilog.Sinks.Seq.Durable
 {
     struct FileSetPosition
     {
-        readonly string _file;
-        readonly long _nextLineStart;
+        public string File { get; }
 
-        public string File => _file;
-        public long NextLineStart => _nextLineStart;
+        public long NextLineStart { get; }
 
         public FileSetPosition(long nextLineStart, string file)
         {
-            _nextLineStart = nextLineStart;
-            _file = file;
+            NextLineStart = nextLineStart;
+            File = file;
         }
 
         public static readonly FileSetPosition None = default(FileSetPosition);