Remove old test format parser and simplify yaml frontmatter parsing

tools/lint/lib/checks/license.py

@@ -6,18 +6,16 @@
 _MAX_YEAR = 2030
 
 _LICENSE_PATTERN = re.compile(
-    r'\/\/ Copyright( \([cC]\))? (\w+) .+\. {1,2}All rights reserved\.[\r\n]{1,2}' +
+    r'// Copyright( \([C]\))? (\w+) .+\. {1,2}All rights reserved\.[\r\n]{1,2}' +
     r'(' +
-        r'\/\/ (' +
-            r'This code is governed by the( BSD)? license found in the LICENSE file\.' +
-            r'|' +
-            r'See LICENSE for details' +
-        r')' +
+        r'// This code is governed by the( BSD)? license found in the LICENSE file\.' +
         r'|' +
-        r'\/\/ Use of this source code is governed by a BSD-style license that can be[\r\n]{1,2}' +
-        r'\/\/ found in the LICENSE file\.' +
+        r'// See LICENSE for details.' +
         r'|' +
-        r'\/\/ See LICENSE or https://github\.com/tc39/test262/blob/master/LICENSE' +
+        r'// Use of this source code is governed by a BSD-style license that can be[\r\n]{1,2}' +
+        r'// found in the LICENSE file\.' +
+        r'|' +
+        r'// See LICENSE or https://github\.com/tc39/test262/blob/master/LICENSE' +
     r')', re.IGNORECASE)
 
 class CheckLicense(Check):

tools/packaging/parseTestRecord.py

@@ -3,84 +3,40 @@
 # Copyright 2011 by Google, Inc.  All rights reserved.
 # This code is governed by the BSD license found in the LICENSE file.
 
-# TODO: resolve differences with common.py and unify into one file.
+from __future__ import print_function
 
-import logging
-import optparse
 import os
-from os import path
-import platform
 import re
-import subprocess
-import sys
-import tempfile
-import time
 import imp
 
-# from TestCasePackagerConfig import *
-
-headerPatternStr = r"(?:(?:\s*\/\/.*)?\s*\n)*"
-captureCommentPatternStr = r"\/\*\*?((?:\s|\S)*?)\*\/\s*\n"
-anyPatternStr = r"(?:\s|\S)*"
-
-headerPattern = re.compile("^" + headerPatternStr)
-
-# Should match anything
-testRecordPattern = re.compile(r"^(" + headerPatternStr +
-                               r")(?:" + captureCommentPatternStr +
-                               r")?(" + anyPatternStr +
-                               r")$")
-
-stars = re.compile(r"\s*\n\s*\*\s?")
-atattrs = re.compile(r"\s*\n\s*\*\s*@")
-
-yamlPattern = re.compile(r"---((?:\s|\S)*)---")
-newlinePattern = re.compile(r"\n")
+# Matches trailing whitespace and any following blank lines.
+_BLANK_LINES = r"([ \t]*[\r\n]{1,2})*"
+
+# Matches the YAML frontmatter block.
+_YAML_PATTERN = re.compile(r"/\*---(.*)---\*/" + _BLANK_LINES, re.DOTALL)
+
+# Matches all known variants for the license block.
+_LICENSE_PATTERN = re.compile(
+   r'// Copyright( \(C\))? (\w+) .+\. {1,2}All rights reserved\.[\r\n]{1,2}' +
+   r'(' +
+       r'// This code is governed by the( BSD)? license found in the LICENSE file\.' +
+       r'|' +
+       r'// See LICENSE for details\.' +
+       r'|' +
+       r'// Use of this source code is governed by a BSD-style license that can be[\r\n]{1,2}' +
+       r'// found in the LICENSE file\.' +
+       r'|' +
+       r'// See LICENSE or https://github\.com/tc39/test262/blob/master/LICENSE' +
+   r')[\r\n]{1,2}' + _BLANK_LINES, re.IGNORECASE)
 
 yamlLoad = None
 
-def stripStars(text):
-    return stars.sub('\n', text).strip()
-
-def stripHeader(src):
-    header = headerPattern.match(src).group(0)
-    return src[len(header):]
-
-def matchParts(src, name):
-    match = testRecordPattern.match(src)
-    if match == None:
-        raise Exception('unrecognized: ' + name)
-    return match
-
-def hasYAML(text):
-    match = yamlPattern.match(text)
-    if match == None:
-        return False
-    return True
-
-def oldAttrParser(testRecord, body, name):
-    propTexts = atattrs.split(body)
-    testRecord['commentary'] = stripStars(propTexts[0])
-    del propTexts[0]
-    for propText in propTexts:
-        propMatch = re.match(r"^\w+", propText)
-        if propMatch == None:
-            raise Exception('Malformed "@" attribute: ' + name)
-        propName = propMatch.group(0)
-        propVal = stripStars(propText[len(propName):])
-
-        if propName in testRecord:
-            raise Exception('duplicate: ' + propName)
-        testRecord[propName] = propVal;
-
-def yamlAttrParser(testRecord, attrs, name):
-    match = yamlPattern.match(attrs)
-    body = match.group(1)
+def yamlAttrParser(testRecord, attrs, name, onerror):
     importYamlLoad()
-    parsed = yamlLoad(body)
 
-    if (parsed is None):
-        print("Failed to parse yaml in name %s"%(name))
+    parsed = yamlLoad(attrs)
+    if parsed is None:
+        onerror("Failed to parse yaml in name %s" % name)
         return
 
     for key in parsed:
@@ -93,65 +49,61 @@ def yamlAttrParser(testRecord, attrs, name):
         for flag in testRecord['flags']:
             testRecord[flag] = ""
 
-def findAttrs(src):
-    match = re.search(r'\/\*---(?:[\s]*)((?:[\s\S])*)(?:[\s]*)---\*\/', src, re.DOTALL)
-    if not match:
-        return (None, None)
-
-    return (match.group(0), match.group(1).strip())
-
 def findLicense(src):
-    _LICENSE_PATTERN = re.compile(
-        r'\/\/ Copyright( \([cC]\))? (\w+) .+\. {1,2}All rights reserved\.[\r\n]{1,2}' +
-        r'(' +
-            r'\/\/ (' +
-                r'This code is governed by the( BSD)? license found in the LICENSE file\.' +
-                r'|' +
-                r'See LICENSE for details' +
-            r')' +
-            r'|' +
-            r'\/\/ Use of this source code is governed by a BSD-style license that can be[\r\n]{1,2}' +
-            r'\/\/ found in the LICENSE file\.' +
-            r'|' +
-            r'\/\/ See LICENSE or https://github\.com/tc39/test262/blob/master/LICENSE' +
-        r')', re.IGNORECASE)
-
     match = _LICENSE_PATTERN.search(src)
     if not match:
         return None
 
-    return match.group(0).strip()
+    return match.group(0)
 
-def parseTestRecord(src, name):
-    testRecord = {}
+def findAttrs(src):
+    match = _YAML_PATTERN.search(src)
+    if not match:
+        return (None, None)
+
+    return (match.group(0), match.group(1).strip())
 
-    header = ""
-    test = ""
-    attrs = ""
+def parseTestRecord(src, name, onerror = print):
+    # Find the license block.
+    header = findLicense(src)
+
+    # Find the YAML frontmatter.
+    (frontmatter, attrs) = findAttrs(src)
+
+    # YAML frontmatter is required for all tests.
+    if frontmatter is None:
+        onerror("Missing frontmatter: %s" % name)
+
+    # The license should be placed before the frontmatter and there shouldn't be
+    # be any extra content between the license and the frontmatter.
+    if header is not None and frontmatter is not None:
+        headerIdx = src.index(header)
+        frontmatterIdx = src.index(frontmatter)
+        if headerIdx > frontmatterIdx:
+            onerror("Unexpected license after frontmatter: %s" % name)
+
+        # Search for any extra test content, but ignore whitespace only or comment lines.
+        extra = src[headerIdx + len(header) : frontmatterIdx]
+        if extra and any(line.strip() and not line.lstrip().startswith("//") for line in extra.split("\n")):
+            onerror("Unexpected test content between license and frontmatter: %s" % name)
+
+    # Remove the license and YAML parts from the actual test content.
+    test = src
+    if frontmatter is not None:
+        test = test.replace(frontmatter, '')
+    if header is not None:
+        test = test.replace(header, '')
 
-    try:
-        match = matchParts(src, name)
-        header = match.group(1).strip()
-        attrs = match.group(2)
-        test = match.group(3)
-    except:
-        # match = something else that works without copyright
-        header = findLicense(src)
-        [frontmatter, attrs] = findAttrs(src)
-        test = src
-        if frontmatter:
-            test = test.replace(frontmatter, '')
-        if header:
-            test = test.replace(header, '')
-
-    testRecord['header'] = header
+    testRecord = {}
+    testRecord['header'] = header.strip() if header else ''
     testRecord['test'] = test
 
     if attrs:
-        if hasYAML(attrs):
-            yamlAttrParser(testRecord, attrs, name)
-        else:
-            oldAttrParser(testRecord, attrs, name)
+        yamlAttrParser(testRecord, attrs, name, onerror)
+
+    # Report if the license block is missing in non-generated tests.
+    if header is None and "generated" not in testRecord:
+        onerror("No license found in: %s" % name)
 
     return testRecord