Skip to content

Commit

Permalink
#9 Extended token information & removed scanner argument checks
Browse files Browse the repository at this point in the history
  • Loading branch information
@Andrei15193
Andrei15193 committed May 30, 2020
1 parent e4772c1 commit 3092f45
Show file tree
Hide file tree
Showing 8 changed files with 51 additions and 100 deletions.
15 changes: 0 additions & 15 deletions Mup.Tests/CreoleScannerTests.cs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
using Mup.Creole;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
Expand All @@ -12,20 +11,6 @@ public class CreoleScannerTests
{
private readonly CreoleScanner _scanner = new CreoleScanner();

[Fact]
public void CannotScanFromNullTextReaderWithBuffer()
{
Assert.Throws<ArgumentNullException>(() => _scanner.Scan(null, 0));
}

[Theory, MemberData(nameof(ScannerTestData.InvalidBufferSizeTestData), MemberType = typeof(ScannerTestData))]
public void CannotScanWithNegativeOrZeroBufferSize(int bufferSize)
{
using var reader = new StringReader(string.Empty);

Assert.Throws<ArgumentException>(() => _scanner.Scan(reader, bufferSize));
}

[Theory, MemberData(nameof(CreoleScannerTestData.TextToTokensTestData), MemberType = typeof(CreoleScannerTestData))]
public void ScanTextUsingReaderWithBuffer(string text, IEnumerable<object> tokens)
{
Expand Down
16 changes: 0 additions & 16 deletions Mup.Tests/ScannerTestData.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,22 +24,6 @@ public static class ScannerTestData
}
};

public static IEnumerable<object[]> InvalidBufferSizeTestData { get; } = new List<object[]>
{
new object[]
{
-1
},
new object[]
{
-2
},
new object[]
{
0
}
};

public static IEnumerable<object[]> UnrecognizedCharacterTestData { get; } = new List<object[]>
{
new object[]
Expand Down
43 changes: 21 additions & 22 deletions Mup/Creole/CreoleScanner.cs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
using Mup.Scanner;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Text;
using static Mup.Creole.CreoleTokenCode;
using static System.Char;
Expand All @@ -20,9 +19,9 @@ internal class CreoleScanner : CharacterScanner

private readonly StringBuilder _textBuilder = new StringBuilder();

private IList<CreoleToken> _tokens;
private List<CreoleToken> _tokens;

internal ReadOnlyCollection<CreoleToken> Result { get; private set; }
internal IReadOnlyList<CreoleToken> Result { get; private set; }

protected override void Reset()
{
Expand All @@ -46,7 +45,7 @@ protected override void Process(char character)
protected override void ScanCompleted()
{
_AddToken();
Result = new ReadOnlyCollection<CreoleToken>(_tokens);
Result = _tokens;
}

private void _ProcessCharacter(char character)
Expand All @@ -59,74 +58,74 @@ private void _ProcessCharacter(char character)

case '*':
_AddToken();
_tokens.Add(new CreoleToken(Asterisk, "*"));
_tokens.Add(new CreoleToken(Asterisk, "*", Line, Column));
break;

case '/':
_AddToken();
_tokens.Add(new CreoleToken(Slash, "/"));
_tokens.Add(new CreoleToken(Slash, "/", Line, Column));
break;

case '\\':
_AddToken();
_tokens.Add(new CreoleToken(BackSlash, "\\"));
_tokens.Add(new CreoleToken(BackSlash, "\\", Line, Column));
break;

case '[':
_AddToken();
_tokens.Add(new CreoleToken(BracketOpen, "["));
_tokens.Add(new CreoleToken(BracketOpen, "[", Line, Column));
break;

case ']':
_AddToken();
_tokens.Add(new CreoleToken(BracketClose, "]"));
_tokens.Add(new CreoleToken(BracketClose, "]", Line, Column));
break;

case '{':
_AddToken();
_tokens.Add(new CreoleToken(BraceOpen, "{"));
_tokens.Add(new CreoleToken(BraceOpen, "{", Line, Column));
break;

case '}':
_AddToken();
_tokens.Add(new CreoleToken(BraceClose, "}"));
_tokens.Add(new CreoleToken(BraceClose, "}", Line, Column));
break;

case '<':
_AddToken();
_tokens.Add(new CreoleToken(AngleOpen, "<"));
_tokens.Add(new CreoleToken(AngleOpen, "<", Line, Column));
break;

case '>':
_AddToken();
_tokens.Add(new CreoleToken(AngleClose, ">"));
_tokens.Add(new CreoleToken(AngleClose, ">", Line, Column));
break;

case '=':
_AddToken();
_tokens.Add(new CreoleToken(Equal, "="));
_tokens.Add(new CreoleToken(Equal, "=", Line, Column));
break;

case '-':
_AddToken();
_tokens.Add(new CreoleToken(Dash, "-"));
_tokens.Add(new CreoleToken(Dash, "-", Line, Column));
break;

case '#':
_AddToken();
_tokens.Add(new CreoleToken(Hash, "#"));
_tokens.Add(new CreoleToken(Hash, "#", Line, Column));
break;

case '|':
_AddToken();
_tokens.Add(new CreoleToken(Pipe, "|"));
_tokens.Add(new CreoleToken(Pipe, "|", Line, Column));
break;

default:
if (IsPunctuation(character))
{
_AddToken();
_tokens.Add(new CreoleToken(Punctuation, character.ToString()));
_tokens.Add(new CreoleToken(Punctuation, character.ToString(), Line, Column));
}
else
{
Expand All @@ -151,7 +150,7 @@ private void _ProcessEscapedCharacter(char character)
{
_AddToken();

_tokens.Add(new CreoleToken(Tilde, _escapeCahracterString));
_tokens.Add(new CreoleToken(Tilde, _escapeCahracterString, Line, Column));

_tokenCode = Text;
_textBuilder.Append(character);
Expand All @@ -160,7 +159,7 @@ private void _ProcessEscapedCharacter(char character)
{
_AddToken();

_tokens.Add(new CreoleToken(Text, _escapeCahracterString));
_tokens.Add(new CreoleToken(Text, _escapeCahracterString, Line, Column));

_tokenCode = Text;
}
Expand Down Expand Up @@ -216,11 +215,11 @@ private void _AddToken()
_lineFeedCount = 0;
}

_tokens.Add(new CreoleToken(_tokenCode, _textBuilder.ToString()));
_tokens.Add(new CreoleToken(_tokenCode, _textBuilder.ToString(), Line, Column));
_textBuilder.Length = 0;
}
else if (_isEscaped)
_tokens.Add(new CreoleToken(Text, _escapeCahracterString));
_tokens.Add(new CreoleToken(Text, _escapeCahracterString, Line, Column));
}
}
}
31 changes: 4 additions & 27 deletions Mup/Creole/CreoleToken.cs
View file
Original file line number Diff line number Diff line change
@@ -1,35 +1,12 @@
using System;
using Mup.Scanner;

namespace Mup.Creole
{
internal class CreoleToken : IEquatable<CreoleToken>
internal class CreoleToken : Token<CreoleTokenCode>
{
public static bool operator ==(CreoleToken left, CreoleToken right)
=> left.Equals(right);

public static bool operator !=(CreoleToken left, CreoleToken right)
=> !left.Equals(right);

internal CreoleToken(CreoleTokenCode code, string text)
internal CreoleToken(CreoleTokenCode code, string text, int line, int column)
: base(code, text, line, column)
{
Code = code;
Text = text;
}

internal CreoleTokenCode Code { get; }

internal string Text { get; }

public bool Equals(CreoleToken other)
=> (Code == other.Code && Text == other.Text);

public override bool Equals(object obj)
=> (obj != null && ((obj as CreoleToken)?.Equals(this) ?? false));

public override int GetHashCode()
=> (new { Code, Text }).GetHashCode();

public override string ToString()
=> $"{Code}: {Text}";
}
}
7 changes: 3 additions & 4 deletions Mup/Creole/CreoleTokenRange.cs
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,16 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Collections.ObjectModel;

namespace Mup.Creole
{
internal class CreoleTokenRange : IEnumerable<CreoleToken>
{
private readonly ReadOnlyCollection<CreoleToken> _tokens;
private readonly IReadOnlyList<CreoleToken> _tokens;
private readonly int _startIndex;
private readonly int _count;

internal CreoleTokenRange(ReadOnlyCollection<CreoleToken> tokens)
internal CreoleTokenRange(IReadOnlyList<CreoleToken> tokens)
{
if (tokens == null)
throw new ArgumentNullException(nameof(tokens));
Expand All @@ -21,7 +20,7 @@ internal CreoleTokenRange(ReadOnlyCollection<CreoleToken> tokens)
_count = _tokens.Count;
}

internal CreoleTokenRange(ReadOnlyCollection<CreoleToken> tokens, int rangeStartIndex, int rangeLength)
internal CreoleTokenRange(IReadOnlyList<CreoleToken> tokens, int rangeStartIndex, int rangeLength)
{
if (tokens == null)
throw new ArgumentNullException(nameof(tokens));
Expand Down
2 changes: 1 addition & 1 deletion Mup/CreoleParser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ public IParseTree Parse(TextReader reader, int bufferSize)
return parseTree;
}

private IParseTree _Parse(ReadOnlyCollection<CreoleToken> tokens)
private IParseTree _Parse(IReadOnlyList<CreoleToken> tokens)
{
var context = new CreoleParserContext(Options.InlineHyperlinkProtocols);
var tokenRange = new CreoleTokenRange(tokens);
Expand Down
16 changes: 1 addition & 15 deletions Mup/Scanner/CharacterScanner.cs
View file
Original file line number Diff line number Diff line change
@@ -1,18 +1,11 @@
using System;
using System.IO;
using System.IO;

namespace Mup.Scanner
{
internal abstract class CharacterScanner
{
private const int _defaultBufferSize = 2048;

internal void Scan(TextReader reader, int bufferSize)
{
if (reader == null)
throw new ArgumentNullException(nameof(reader));
if (bufferSize <= 0)
throw new ArgumentException("The buffer size must be greater than zero.", nameof(bufferSize));
_Reset();

int bufferLength;
Expand All @@ -33,11 +26,6 @@ internal void Scan(TextReader reader, int bufferSize)

protected int Column { get; private set; }

protected int Index { get; private set; }

protected virtual int DefaultBuffer
=> _defaultBufferSize;

protected virtual void Reset()
{
}
Expand All @@ -50,15 +38,13 @@ private void _Reset()
{
Line = 1;
Column = 1;
Index = 0;
Reset();
}

private void _Process(char character)
{
Process(character);

Index++;
if (character == '\n')
{
Line++;
Expand Down
21 changes: 21 additions & 0 deletions Mup/Scanner/Token.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
namespace Mup.Scanner
{
internal class Token<TCode>
{
public Token(TCode code, string text, int line, int column)
{
Code = code;
Text = text;
Line = line;
Column = column;
}

public TCode Code { get; }

public string Text { get; }

public int Line { get; }

public int Column { get; }
}
}

0 comments on commit 3092f45

Please sign in to comment.