Core: Fixed greedy rematching reach bug (#2705)

components/prism-core.js

@@ -984,10 +984,18 @@ function matchGrammar(text, tokenList, grammar, startNode, startPos, rematch) {
 				if (removeCount > 1) {
 					// at least one Token object was removed, so we have to do some rematching
 					// this can only happen if the current pattern is greedy
-					matchGrammar(text, tokenList, grammar, currentNode.prev, pos, {
+
+					/** @type {RematchOptions} */
+					var nestedRematch = {
 						cause: token + ',' + j,
 						reach: reach
-					});
+					};
+					matchGrammar(text, tokenList, grammar, currentNode.prev, pos, nestedRematch);
+
+					// the reach might have been extended because of the rematching
+					if (rematch && nestedRematch.reach > rematch.reach) {
+						rematch.reach = nestedRematch.reach;
+					}
 				}
 			}
 		}

docs/global.html

@@ -143,7 +143,7 @@ <h4 class="name" id="Grammar">Grammar</h4>
 
     <dt class="tag-source">Source:</dt>
     <dd class="tag-source"><ul class="dummy"><li>
-        <a href="prism-core.js.html">prism-core.js</a>, <a href="prism-core.js.html#line1171">line 1171</a>
+        <a href="prism-core.js.html">prism-core.js</a>, <a href="prism-core.js.html#line1179">line 1179</a>
     </li></ul></dd>
 
 
@@ -274,7 +274,7 @@ <h4 class="name" id="GrammarToken">GrammarToken</h4>
 
     <dt class="tag-source">Source:</dt>
     <dd class="tag-source"><ul class="dummy"><li>
-        <a href="prism-core.js.html">prism-core.js</a>, <a href="prism-core.js.html#line1150">line 1150</a>
+        <a href="prism-core.js.html">prism-core.js</a>, <a href="prism-core.js.html#line1158">line 1158</a>
     </li></ul></dd>
 
 
@@ -559,7 +559,7 @@ <h4 class="name" id="HighlightCallback"><span class="type-signature"></span>High
 
     <dt class="tag-source">Source:</dt>
     <dd class="tag-source"><ul class="dummy"><li>
-        <a href="prism-core.js.html">prism-core.js</a>, <a href="prism-core.js.html#line1179">line 1179</a>
+        <a href="prism-core.js.html">prism-core.js</a>, <a href="prism-core.js.html#line1187">line 1187</a>
     </li></ul></dd>
 
 
@@ -713,7 +713,7 @@ <h4 class="name" id="HookCallback"><span class="type-signature"></span>HookCallb
 
     <dt class="tag-source">Source:</dt>
     <dd class="tag-source"><ul class="dummy"><li>
-        <a href="prism-core.js.html">prism-core.js</a>, <a href="prism-core.js.html#line1189">line 1189</a>
+        <a href="prism-core.js.html">prism-core.js</a>, <a href="prism-core.js.html#line1197">line 1197</a>
     </li></ul></dd>
 
 

docs/prism-core.js.html

@@ -1037,10 +1037,18 @@ <h1 class="page-title">prism-core.js</h1>
 				if (removeCount > 1) {
 					// at least one Token object was removed, so we have to do some rematching
 					// this can only happen if the current pattern is greedy
-					matchGrammar(text, tokenList, grammar, currentNode.prev, pos, {
+
+					/** @type {RematchOptions} */
+					var nestedRematch = {
 						cause: token + ',' + j,
 						reach: reach
-					});
+					};
+					matchGrammar(text, tokenList, grammar, currentNode.prev, pos, nestedRematch);
+
+					// the reach might have been extended because of the rematching
+					if (rematch &amp;&amp; nestedRematch.reach > rematch.reach) {
+						rematch.reach = nestedRematch.reach;
+					}
 				}
 			}
 		}

prism.js

@@ -989,10 +989,18 @@ function matchGrammar(text, tokenList, grammar, startNode, startPos, rematch) {
 				if (removeCount > 1) {
 					// at least one Token object was removed, so we have to do some rematching
 					// this can only happen if the current pattern is greedy
-					matchGrammar(text, tokenList, grammar, currentNode.prev, pos, {
+
+					/** @type {RematchOptions} */
+					var nestedRematch = {
 						cause: token + ',' + j,
 						reach: reach
-					});
+					};
+					matchGrammar(text, tokenList, grammar, currentNode.prev, pos, nestedRematch);
+
+					// the reach might have been extended because of the rematching
+					if (rematch && nestedRematch.reach > rematch.reach) {
+						rematch.reach = nestedRematch.reach;
+					}
 				}
 			}
 		}

tests/languages/javascript/issue2694.test

@@ -0,0 +1,70 @@
+replace(/'/, `'`)
+
+const var1 = `this is fine`;
+const var2 = `this is fine`;
+
+// `load bearing comment`
+
+const var3 = `break starts here`;
+const var4 = `break ends here`;
+
+----------------------------------------------------
+
+[
+	["function", "replace"],
+	["punctuation", "("],
+	["regex", [
+		["regex-delimiter", "/"],
+		["regex-source", "'"],
+		["regex-delimiter", "/"]
+	]],
+	["punctuation", ","],
+	["template-string", [
+		["template-punctuation", "`"],
+		["string", "'"],
+		["template-punctuation", "`"]
+	]],
+	["punctuation", ")"],
+
+	["keyword", "const"],
+	" var1 ",
+	["operator", "="],
+	["template-string", [
+		["template-punctuation", "`"],
+		["string", "this is fine"],
+		["template-punctuation", "`"]
+	]],
+	["punctuation", ";"],
+
+	["keyword", "const"],
+	" var2 ",
+	["operator", "="],
+	["template-string", [
+		["template-punctuation", "`"],
+		["string", "this is fine"],
+		["template-punctuation", "`"]
+	]],
+	["punctuation", ";"],
+
+	["comment", "// `load bearing comment`"],
+
+	["keyword", "const"],
+	" var3 ",
+	["operator", "="],
+	["template-string", [
+		["template-punctuation", "`"],
+		["string", "break starts here"],
+		["template-punctuation", "`"]
+	]],
+	["punctuation", ";"],
+
+	["keyword", "const"],
+	" var4 ",
+	["operator", "="],
+	["template-string", [
+		["template-punctuation", "`"],
+		["string", "break ends here"],
+		["template-punctuation", "`"]
+	]],
+	["punctuation", ";"]
+]