04_lexer_comments_as_tokens.patch

commit 17aa7e51f4a3bf12d4662720ef39f6f60336dc83
Author: Lukas Fittl <lukas@fittl.com>
Date:   Sun Jan 3 16:01:44 2021 -0800

    pg_query: Track comments separately from whitespace in lexer/parser
    
    For syntax highlighting and extracting comments from a query, its very
    helpful to know the exact locations of a comment in the query string.
    
    Previously the lexer discarded all comments as whitespace, making it
    impossible to determine where they are located in the query string. With
    this change, the lexer returns them as SQL_COMMENT/C_COMMENT tokens.

diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y
index 164b9586c1..7dd5505cd1 100644
--- a/src/backend/parser/gram.y
+++ b/src/backend/parser/gram.y
@@ -678,6 +678,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query);
 %token <ival>	ICONST PARAM
 %token			TYPECAST DOT_DOT COLON_EQUALS EQUALS_GREATER
 %token			LESS_EQUALS GREATER_EQUALS NOT_EQUALS
+%token			SQL_COMMENT C_COMMENT
 
 /*
  * If you want to make any keyword changes, update the keyword table in
diff --git a/src/backend/parser/parser.c b/src/backend/parser/parser.c
index e17c310cc1..9b2926878a 100644
--- a/src/backend/parser/parser.c
+++ b/src/backend/parser/parser.c
@@ -156,6 +156,9 @@ base_yylex(YYSTYPE *lvalp, YYLTYPE *llocp, core_yyscan_t yyscanner)
 		case WITHOUT:
 			cur_token_length = 7;
 			break;
+		case SQL_COMMENT:
+		case C_COMMENT:
+			return base_yylex(lvalp, llocp, yyscanner);
 		default:
 			return cur_token;
 	}
diff --git a/src/backend/parser/scan.l b/src/backend/parser/scan.l
index a62798cf98..154defc6b8 100644
--- a/src/backend/parser/scan.l
+++ b/src/backend/parser/scan.l
@@ -226,7 +226,7 @@ non_newline		[^\n\r]
 
 comment			("--"{non_newline}*)
 
-whitespace		({space}+|{comment})
+whitespace		({space}+)
 
 /*
  * SQL requires at least one newline in the whitespace separating
@@ -235,8 +235,8 @@ whitespace		({space}+|{comment})
  * it, whereas {whitespace} should generally have a * after it...
  */
 
-special_whitespace		({space}+|{comment}{newline})
-horiz_whitespace		({horiz_space}|{comment})
+special_whitespace		({space}+)
+horiz_whitespace		({horiz_space})
 whitespace_with_newline	({horiz_whitespace}*{newline}{special_whitespace}*)
 
 quote			'
@@ -443,6 +443,11 @@ other			.
 					/* ignore */
 				}
 
+{comment}	{
+					SET_YYLLOC();
+					return SQL_COMMENT;
+				}
+
 {xcstart}		{
 					/* Set location in case of syntax error in comment */
 					SET_YYLLOC();
@@ -461,7 +466,11 @@ other			.
 
 {xcstop}		{
 					if (yyextra->xcdepth <= 0)
+					{
 						BEGIN(INITIAL);
+						yyextra->yyllocend = yytext - yyextra->scanbuf + yyleng;
+						return C_COMMENT;
+					}
 					else
 						(yyextra->xcdepth)--;
 				}
diff --git a/src/interfaces/ecpg/preproc/parser.c b/src/interfaces/ecpg/preproc/parser.c
index 38e7acb680..4814158aaf 100644
--- a/src/interfaces/ecpg/preproc/parser.c
+++ b/src/interfaces/ecpg/preproc/parser.c
@@ -86,6 +86,9 @@ filtered_base_yylex(void)
 		case UIDENT:
 		case USCONST:
 			break;
+		case SQL_COMMENT:
+		case C_COMMENT:
+			return filtered_base_yylex();
 		default:
 			return cur_token;
 	}
diff --git a/src/pl/plpgsql/src/pl_gram.y b/src/pl/plpgsql/src/pl_gram.y
index 6a09bfdd67..fb1b519b27 100644
--- a/src/pl/plpgsql/src/pl_gram.y
+++ b/src/pl/plpgsql/src/pl_gram.y
@@ -232,6 +232,7 @@ static	void			check_raise_parameters(PLpgSQL_stmt_raise *stmt);
 %token <ival>	ICONST PARAM
 %token			TYPECAST DOT_DOT COLON_EQUALS EQUALS_GREATER
 %token			LESS_EQUALS GREATER_EQUALS NOT_EQUALS
+%token			SQL_COMMENT C_COMMENT
 
 /*
  * Other tokens recognized by plpgsql's lexer interface layer (pl_scanner.c).
diff --git a/src/pl/plpgsql/src/pl_scanner.c b/src/pl/plpgsql/src/pl_scanner.c
index 4e98af2395..40a17caaf9 100644
--- a/src/pl/plpgsql/src/pl_scanner.c
+++ b/src/pl/plpgsql/src/pl_scanner.c
@@ -342,6 +342,11 @@ internal_yylex(TokenAuxData *auxdata)
 		{
 			auxdata->lval.str = pstrdup(yytext);
 		}
+
+		else if (token == SQL_COMMENT || token == C_COMMENT)
+		{
+			token = internal_yylex(auxdata);
+		}
 	}
 
 	return token;