Skip to content

Commit

Permalink
#9 Refactored to generic Token
Browse files Browse the repository at this point in the history
  • Loading branch information
@Andrei15193
Andrei15193 committed May 30, 2020
1 parent 3092f45 commit 124a92f
Show file tree
Hide file tree
Showing 26 changed files with 155 additions and 187 deletions.
42 changes: 21 additions & 21 deletions Mup/Creole/CreoleScanner.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@ internal class CreoleScanner : CharacterScanner

private readonly StringBuilder _textBuilder = new StringBuilder();

private List<CreoleToken> _tokens;
private List<Token<CreoleTokenCode>> _tokens;

internal IReadOnlyList<CreoleToken> Result { get; private set; }
internal IReadOnlyList<Token<CreoleTokenCode>> Result { get; private set; }

protected override void Reset()
{
Expand All @@ -31,7 +31,7 @@ protected override void Reset()
_tokenCode = Text;
_lineFeedCount = 0;
_textBuilder.Length = 0;
_tokens = new List<CreoleToken>();
_tokens = new List<Token<CreoleTokenCode>>();
}

protected override void Process(char character)
Expand All @@ -58,74 +58,74 @@ private void _ProcessCharacter(char character)

case '*':
_AddToken();
_tokens.Add(new CreoleToken(Asterisk, "*", Line, Column));
_tokens.Add(new Token<CreoleTokenCode>(Asterisk, "*", Line, Column));
break;

case '/':
_AddToken();
_tokens.Add(new CreoleToken(Slash, "/", Line, Column));
_tokens.Add(new Token<CreoleTokenCode>(Slash, "/", Line, Column));
break;

case '\\':
_AddToken();
_tokens.Add(new CreoleToken(BackSlash, "\\", Line, Column));
_tokens.Add(new Token<CreoleTokenCode>(BackSlash, "\\", Line, Column));
break;

case '[':
_AddToken();
_tokens.Add(new CreoleToken(BracketOpen, "[", Line, Column));
_tokens.Add(new Token<CreoleTokenCode>(BracketOpen, "[", Line, Column));
break;

case ']':
_AddToken();
_tokens.Add(new CreoleToken(BracketClose, "]", Line, Column));
_tokens.Add(new Token<CreoleTokenCode>(BracketClose, "]", Line, Column));
break;

case '{':
_AddToken();
_tokens.Add(new CreoleToken(BraceOpen, "{", Line, Column));
_tokens.Add(new Token<CreoleTokenCode>(BraceOpen, "{", Line, Column));
break;

case '}':
_AddToken();
_tokens.Add(new CreoleToken(BraceClose, "}", Line, Column));
_tokens.Add(new Token<CreoleTokenCode>(BraceClose, "}", Line, Column));
break;

case '<':
_AddToken();
_tokens.Add(new CreoleToken(AngleOpen, "<", Line, Column));
_tokens.Add(new Token<CreoleTokenCode>(AngleOpen, "<", Line, Column));
break;

case '>':
_AddToken();
_tokens.Add(new CreoleToken(AngleClose, ">", Line, Column));
_tokens.Add(new Token<CreoleTokenCode>(AngleClose, ">", Line, Column));
break;

case '=':
_AddToken();
_tokens.Add(new CreoleToken(Equal, "=", Line, Column));
_tokens.Add(new Token<CreoleTokenCode>(Equal, "=", Line, Column));
break;

case '-':
_AddToken();
_tokens.Add(new CreoleToken(Dash, "-", Line, Column));
_tokens.Add(new Token<CreoleTokenCode>(Dash, "-", Line, Column));
break;

case '#':
_AddToken();
_tokens.Add(new CreoleToken(Hash, "#", Line, Column));
_tokens.Add(new Token<CreoleTokenCode>(Hash, "#", Line, Column));
break;

case '|':
_AddToken();
_tokens.Add(new CreoleToken(Pipe, "|", Line, Column));
_tokens.Add(new Token<CreoleTokenCode>(Pipe, "|", Line, Column));
break;

default:
if (IsPunctuation(character))
{
_AddToken();
_tokens.Add(new CreoleToken(Punctuation, character.ToString(), Line, Column));
_tokens.Add(new Token<CreoleTokenCode>(Punctuation, character.ToString(), Line, Column));
}
else
{
Expand All @@ -150,7 +150,7 @@ private void _ProcessEscapedCharacter(char character)
{
_AddToken();

_tokens.Add(new CreoleToken(Tilde, _escapeCahracterString, Line, Column));
_tokens.Add(new Token<CreoleTokenCode>(Tilde, _escapeCahracterString, Line, Column));

_tokenCode = Text;
_textBuilder.Append(character);
Expand All @@ -159,7 +159,7 @@ private void _ProcessEscapedCharacter(char character)
{
_AddToken();

_tokens.Add(new CreoleToken(Text, _escapeCahracterString, Line, Column));
_tokens.Add(new Token<CreoleTokenCode>(Text, _escapeCahracterString, Line, Column));

_tokenCode = Text;
}
Expand Down Expand Up @@ -215,11 +215,11 @@ private void _AddToken()
_lineFeedCount = 0;
}

_tokens.Add(new CreoleToken(_tokenCode, _textBuilder.ToString(), Line, Column));
_tokens.Add(new Token<CreoleTokenCode>(_tokenCode, _textBuilder.ToString(), Line, Column));
_textBuilder.Length = 0;
}
else if (_isEscaped)
_tokens.Add(new CreoleToken(Text, _escapeCahracterString, Line, Column));
_tokens.Add(new Token<CreoleTokenCode>(Text, _escapeCahracterString, Line, Column));
}
}
}
12 changes: 0 additions & 12 deletions Mup/Creole/CreoleToken.cs
View file

This file was deleted.

89 changes: 0 additions & 89 deletions Mup/Creole/CreoleTokenRange.cs
View file

This file was deleted.

5 changes: 3 additions & 2 deletions Mup/Creole/ElementProcessors/CreoleCodeBlockElementProcessor.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using Mup.Creole.Elements;
using Mup.Scanner;
using System.Text;
using static Mup.Creole.CreoleTokenCode;

Expand All @@ -25,7 +26,7 @@ private enum State : byte

private bool _isAtBeginningOfLine = true;

internal CreoleCodeBlockElementProcessor(CreoleParserContext context, CreoleTokenRange tokens)
internal CreoleCodeBlockElementProcessor(CreoleParserContext context, TokenRange<CreoleTokenCode> tokens)
: base(context, tokens)
{
}
Expand Down Expand Up @@ -114,7 +115,7 @@ private CreoleElement _GetCodeBlockElement()
return new CreolePreformattedBlockElement(codeBlockText);
}

private string _GetCodeBlockTextFrom(CreoleTokenRange tokens)
private string _GetCodeBlockTextFrom(TokenRange<CreoleTokenCode> tokens)
{
var builder = new StringBuilder();

Expand Down
13 changes: 7 additions & 6 deletions Mup/Creole/ElementProcessors/CreoleElementProcessor.cs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using Mup.Creole.ElementProcessors.RichText;
using Mup.Creole.Elements;
using Mup.Scanner;
using System;
using System.Collections.Generic;
using static Mup.Creole.CreoleTokenCode;
Expand All @@ -9,17 +10,17 @@ namespace Mup.Creole.ElementProcessors
internal abstract class CreoleElementProcessor : IDisposable
{
private static readonly IEnumerable<CreoleElement> _emptyCreoleElements = new CreoleElement[0];
private readonly CreoleTokenRange _tokens;
private readonly TokenRange<CreoleTokenCode> _tokens;
private int _index = 0;
private bool _isOnNewLine = true;
private int _elementStartIndex = 0;
private int _elementEndIndex = 0;
private readonly IEnumerator<CreoleToken> _token;
private readonly IEnumerator<Token<CreoleTokenCode>> _token;
private bool _tokenHasValue;
private CreoleElementInfo _result = null;
private bool _isCompleted = false;

protected CreoleElementProcessor(CreoleParserContext context, CreoleTokenRange tokens)
protected CreoleElementProcessor(CreoleParserContext context, TokenRange<CreoleTokenCode> tokens)
{
Context = context;
_tokens = tokens;
Expand Down Expand Up @@ -83,7 +84,7 @@ internal bool MoveNext()

protected CreoleParserContext Context { get; }

protected CreoleToken Token
protected Token<CreoleTokenCode> Token
=> _token.Current;

protected int Index
Expand Down Expand Up @@ -119,7 +120,7 @@ protected string GetPlainText()
=> GetPlainText(_elementStartIndex, _elementEndIndex);

protected string GetPlainText(int startIndex, int endIndex)
=> CreoleTokenRangeHelper.GetPlainText(GetTokens(startIndex, endIndex));
=> TokenRangeHelper.GetPlainText(GetTokens(startIndex, endIndex));

protected IEnumerable<CreoleElement> GetRichText()
=> GetRichText(_elementStartIndex, _elementEndIndex);
Expand All @@ -134,7 +135,7 @@ protected IEnumerable<CreoleElement> GetRichText(int startIndex, int endIndex)
return richTextProcessor.Process(tokens);
}

protected CreoleTokenRange GetTokens(int startIndex, int endIndex)
protected TokenRange<CreoleTokenCode> GetTokens(int startIndex, int endIndex)
{
if (startIndex < _elementStartIndex)
throw new ArgumentException("Content cannot begin before the element starts.", nameof(startIndex));
Expand Down
3 changes: 2 additions & 1 deletion Mup/Creole/ElementProcessors/CreoleHeadingElementProcessor.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using Mup.Creole.Elements;
using Mup.Scanner;
using static Mup.Creole.CreoleTokenCode;

namespace Mup.Creole.ElementProcessors
Expand All @@ -20,7 +21,7 @@ private enum State : byte
private State _state = State.NotInHeading;
private byte _equalTokensCount = 0;

internal CreoleHeadingElementProcessor(CreoleParserContext context, CreoleTokenRange tokens)
internal CreoleHeadingElementProcessor(CreoleParserContext context, TokenRange<CreoleTokenCode> tokens)
: base(context, tokens)
{
}
Expand Down
3 changes: 2 additions & 1 deletion Mup/Creole/ElementProcessors/CreoleHorizontalRuleElementProcessor.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using Mup.Creole.Elements;
using Mup.Scanner;
using static Mup.Creole.CreoleTokenCode;

namespace Mup.Creole.ElementProcessors
Expand All @@ -16,7 +17,7 @@ private enum State : byte
private int dashCount = 0;
private State _state = State.NotInHorizontalRule;

internal CreoleHorizontalRuleElementProcessor(CreoleParserContext context, CreoleTokenRange tokens)
internal CreoleHorizontalRuleElementProcessor(CreoleParserContext context, TokenRange<CreoleTokenCode> tokens)
: base(context, tokens)
{
}
Expand Down
3 changes: 2 additions & 1 deletion Mup/Creole/ElementProcessors/CreoleListElementProcessor.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using Mup.Creole.Elements;
using Mup.Scanner;
using System.Collections.Generic;
using static Mup.Creole.CreoleTokenCode;

Expand All @@ -25,7 +26,7 @@ private enum State
private int _listItemLevel;
private Stack<CreoleListInfo> _listInfos = new Stack<CreoleListInfo>();

public CreoleListElementProcessor(CreoleParserContext context, CreoleTokenRange tokens)
public CreoleListElementProcessor(CreoleParserContext context, TokenRange<CreoleTokenCode> tokens)
: base(context, tokens)
{
}
Expand Down
3 changes: 2 additions & 1 deletion Mup/Creole/ElementProcessors/CreoleParagraphElementProcessor.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using Mup.Creole.Elements;
using Mup.Scanner;
using static Mup.Creole.CreoleTokenCode;

namespace Mup.Creole.ElementProcessors
Expand All @@ -14,7 +15,7 @@ private enum State : byte

private State _state = State.NotInParagraph;

internal CreoleParagraphElementProcessor(CreoleParserContext context, CreoleTokenRange tokens)
internal CreoleParagraphElementProcessor(CreoleParserContext context, TokenRange<CreoleTokenCode> tokens)
: base(context, tokens)
{
}
Expand Down
Loading

0 comments on commit 124a92f

Please sign in to comment.