Fix #263: update to unicode 8.0.0. Also remove esutils. (#342)

package.json

@@ -21,11 +21,11 @@
     "prepublish": "rm -rf dist/* && npm update && npm run build",
     "benchmark": "node benchmark",
     "profile": "node --prof profile.js && node-tick-processor",
-    "cjsify": "npm run build && cjsify dist/index.js --no-node --export Shift --output dist/shift.js"
+    "cjsify": "npm run build && cjsify dist/index.js --no-node --export Shift --output dist/shift.js",
+    "regenerate-unicode": "node scripts/generate-unicode-data.js > src/unicode.js"
   },
   "dependencies": {
     "es6-map": "^0.1.1",
-    "esutils": "^2.0.2",
     "multimap": "^0.1.1",
     "shift-ast": "^4.0.0",
     "shift-reducer": "^4.0.0"
@@ -45,11 +45,13 @@
     "microtime": "^2.0.0",
     "mocha": "2.3.4",
     "nyc": "10.1.2",
+    "regenerate": "^1.3.2",
     "shift-spec": "^2016.0.0",
     "test262-parser-tests": "0.0.1",
     "tick": "0.1.1",
     "traceur": "0.0.91",
-    "uglifyjs": "2.4.10"
+    "uglifyjs": "2.4.10",
+    "unicode-8.0.0": "^0.7.0"
   },
   "keywords": [
     "Shift",

scripts/generate-unicode-data.js

@@ -0,0 +1,41 @@
+let path = require('path');
+
+let regenerate = require('regenerate');
+let unicodeVersion = '8.0.0';
+
+let ascii = c => c < 128;
+let nonAscii = c => c >= 128;
+
+let makeCharRegex = s => `/^${regenerate().add(...s).toString()}$/`;
+let makeCharArray = s => JSON.stringify(regenerate().add(...s).toArray());
+
+let whitespace = [0x09, 0x0B, 0x0c, 0x20, 0xA0, 0xFEFF].concat(require(`unicode-${unicodeVersion}/General_Category/Space_Separator/code-points`));
+let whitespaceSmall = whitespace.filter(ascii);
+let whitespaceArray = makeCharArray(whitespace.filter(a => a >= 256));
+let whitespaceBool = Array(128).fill(0).map((v, i) => whitespace.indexOf(i) !== -1);
+// Note that we've omitted the range [0xA0, 0xFF], which must be tested in another way.
+
+
+let idStart = ['$'.charCodeAt(0), '_'.charCodeAt(0)].concat(require(`unicode-${unicodeVersion}/Binary_Property/ID_Start/code-points`));
+let idStartSmall = idStart.filter(ascii);
+let idStartLarge = idStart.filter(nonAscii);
+let idStartBool = Array(128).fill(0).map((v, i) => idStartSmall.indexOf(i) !== -1);
+
+
+let idContinue = ['$'.charCodeAt(0), '_'.charCodeAt(0), 0x200C, 0x200D].concat(require(`unicode-${unicodeVersion}/Binary_Property/ID_Continue/code-points`));
+let idContinueSmall = idContinue.filter(ascii);
+let idContinueLarge = idContinue.filter(nonAscii);
+let idContinueBool = Array(128).fill(0).map((v, i) => idContinueSmall.indexOf(i) !== -1);
+
+
+console.log(`// Generated by ${path.join(path.parse(__dirname).base, path.parse(__filename).base)}
+
+export const whitespaceArray = ${whitespaceArray};
+export const whitespaceBool = ${JSON.stringify(whitespaceBool)};
+
+export const idStartLargeRegex = ${makeCharRegex(idStartLarge)};
+export const idStartBool = ${JSON.stringify(idStartBool)};
+
+export const idContinueLargeRegex = ${makeCharRegex(idContinueLarge)};
+export const idContinueBool = ${JSON.stringify(idContinueBool)};
+`);

src/early-errors.js

@@ -15,7 +15,7 @@
  */
 
 import reduce, { MonoidalReducer } from 'shift-reducer';
-import { isRestrictedWord, isStrictModeReservedWord } from './utils';
+import { isStrictModeReservedWord } from './utils';
 
 import { EarlyErrorState, EarlyError } from './early-error-state';
 
@@ -93,7 +93,7 @@ export class EarlyErrorChecker extends MonoidalReducer {
 
   reduceAssignmentTargetIdentifier(node) {
     let s = this.identity;
-    if (isRestrictedWord(node.name) || isStrictModeReservedWord(node.name)) {
+    if (node.name === 'eval' || node.name === 'arguments' || isStrictModeReservedWord(node.name)) {
       s = s.addStrictError(new EarlyError(node, `The identifier ${JSON.stringify(node.name)} must not be in binding position in strict mode`));
     }
     return s;
@@ -123,7 +123,7 @@ export class EarlyErrorChecker extends MonoidalReducer {
 
   reduceBindingIdentifier(node) {
     let s = this.identity;
-    if (isRestrictedWord(node.name) || isStrictModeReservedWord(node.name)) {
+    if (node.name === 'eval' || node.name === 'arguments' || isStrictModeReservedWord(node.name)) {
       s = s.addStrictError(new EarlyError(node, `The identifier ${JSON.stringify(node.name)} must not be in binding position in strict mode`));
     }
     s = s.bindName(node.name, node);