feat(lexer): comment token support (#261)

* feat(lexer): return comment tokens * fix(testrunner): fix error comparison in testrunner * test(lexer): add position test for comments * test(parser): add position tests * feat(parser): skip comments in parsing
vektah · Jun 23, 2023 · 5e5180f · 5e5180f
1 parent d9c7581
commit 5e5180f
Show file tree

Hide file tree

Showing 6 changed files with 115 additions and 19 deletions.
diff --git a/lexer/lexer.go b/lexer/lexer.go
@@ -121,10 +121,7 @@ func (s *Lexer) ReadToken() (token Token, err error) {
 	case '|':
 		return s.makeValueToken(Pipe, "")
 	case '#':
-		if comment, err := s.readComment(); err != nil {
-			return comment, err
-		}
-		return s.ReadToken()
+		return s.readComment()
 
 	case '_', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z':
 		return s.readName()

diff --git a/lexer/lexer_test.yml b/lexer/lexer_test.yml
@@ -26,22 +26,38 @@ simple tokens:
         column: 3
         value: 'foo'
 
-  - name: skips whitespace
-    input: "\n\n    foo\n\n\n"
+  - name: records line and column with comments
+    input: "\n\n\n#foo\n  #bar\n  foo\n"
     tokens:
+      -
+        kind: COMMENT
+        start: 3
+        end: 7
+        line: 4
+        column: 0
+        value: '#foo'
+      -
+        kind: COMMENT
+        start: 10
+        end: 14
+        line: 5
+        column: 3
+        value: '#bar'
       -
         kind: NAME
-        start: 6
-        end: 9
+        start: 17
+        end: 20
+        line: 6
+        column: 3
         value: 'foo'
 
-  - name: skips comments
-    input: "\n    #comment\n    foo#comment\n"
+  - name: skips whitespace
+    input: "\n\n    foo\n\n\n"
     tokens:
       -
         kind: NAME
-        start: 18
-        end: 21
+        start: 6
+        end: 9
         value: 'foo'
 
   - name: skips commas
@@ -78,6 +94,57 @@ simple tokens:
         end: 1
         value: a
 
+lexes comments:
+  - name: basic
+    input: '#simple'
+    tokens:
+      -
+        kind: COMMENT
+        start: 0
+        end: 7
+        value: '#simple'
+
+  - name: two lines
+    input: "#first\n#second"
+    tokens:
+      -
+        kind: COMMENT
+        start: 0
+        end: 6
+        value: "#first"
+      -
+        kind: COMMENT
+        start: 7
+        end: 14
+        value: "#second"
+
+  - name: whitespace
+    input: '# white space '
+    tokens:
+      -
+        kind: COMMENT
+        start: 0
+        end: 14
+        value: '# white space '
+
+  - name: not escaped
+    input: '#not escaped \n\r\b\t\f'
+    tokens:
+      -
+        kind: COMMENT
+        start: 0
+        end: 23
+        value: '#not escaped \n\r\b\t\f'
+
+  - name: slashes
+    input: '#slashes \\ \/'
+    tokens:
+      -
+        kind: COMMENT
+        start: 0
+        end: 14
+        value: '#slashes \\ \/'
+
 lexes strings:
   - name: basic
     input: '"simple"'

diff --git a/parser/parser.go b/parser/parser.go
@@ -35,6 +35,9 @@ func (p *parser) peek() lexer.Token {
 
 	if !p.peeked {
 		p.peekToken, p.peekError = p.lexer.ReadToken()
+		if p.peekToken.Kind == lexer.Comment {
+			return p.peek()
+		}
 		p.peeked = true
 	}
 
@@ -57,6 +60,9 @@ func (p *parser) next() lexer.Token {
 		p.prev, p.err = p.peekToken, p.peekError
 	} else {
 		p.prev, p.err = p.lexer.ReadToken()
+		if p.prev.Kind == lexer.Comment {
+			return p.next()
+		}
 	}
 	return p.prev
 }

diff --git a/parser/query_test.go b/parser/query_test.go
@@ -3,9 +3,10 @@ package parser
 import (
 	"testing"
 
-	"github.com/vektah/gqlparser/v2/gqlerror"
+	"github.com/stretchr/testify/assert"
 
 	"github.com/vektah/gqlparser/v2/ast"
+	"github.com/vektah/gqlparser/v2/gqlerror"
 	"github.com/vektah/gqlparser/v2/parser/testrunner"
 )
 
@@ -24,3 +25,22 @@ func TestQueryDocument(t *testing.T) {
 		}
 	})
 }
+
+func TestQueryPosition(t *testing.T) {
+	t.Run("query line number with comments", func(t *testing.T) {
+		query, err := ParseQuery(&ast.Source{
+			Input: `
+	# comment 1
+query SomeOperation {
+	# comment 2
+	myAction {
+		id
+	}
+}
+      `,
+		})
+		assert.Nil(t, err)
+		assert.Equal(t, 3, query.Operations.ForName("SomeOperation").Position.Line)
+		assert.Equal(t, 5, query.Operations.ForName("SomeOperation").SelectionSet[0].GetPosition().Line)
+	})
+}
diff --git a/parser/schema_test.go b/parser/schema_test.go
@@ -46,4 +46,15 @@ func TestTypePosition(t *testing.T) {
 		assert.Nil(t, parseErr)
 		assert.Equal(t, 2, schema.Definitions.ForName("query").Fields.ForName("me").Type.Position.Line)
 	})
+	t.Run("type line number with comments", func(t *testing.T) {
+		schema, parseErr := ParseSchema(&ast.Source{
+			Input: `type query {
+            # comment
+						me: User
+					}
+			`,
+		})
+		assert.Nil(t, parseErr)
+		assert.Equal(t, 3, schema.Definitions.ForName("query").Fields.ForName("me").Type.Position.Line)
+	})
 }
diff --git a/parser/testrunner/runner.go b/parser/testrunner/runner.go
@@ -55,12 +55,7 @@ func Test(t *testing.T, filename string, f func(t *testing.T, input string) Spec
 
 					if spec.Error == nil {
 						if result.Error != nil {
-							gqlErr, ok := err.(*gqlerror.Error)
-							if ok {
-								t.Errorf("unexpected error %s", gqlErr.Message)
-							} else {
-								t.Errorf("unexpected error %+v", gqlerror.Wrap(err))
-							}
+							t.Errorf("unexpected error %s", result.Error.Message)
 						}
 					} else if result.Error == nil {
 						t.Errorf("expected error but got none")