From 037c43293e0eaf3a565a187e72258ace13abdc48 Mon Sep 17 00:00:00 2001 From: David Barbet Date: Tue, 5 Sep 2023 17:23:34 -0700 Subject: [PATCH] Switch to jsonc-parser for tolerant json parsing --- l10n/bundle.l10n.json | 2 +- package-lock.json | 24 +- src/json.ts | 208 +----------------- src/shared/assets.ts | 3 +- .../unitTests}/json.test.ts | 105 ++++++--- 5 files changed, 86 insertions(+), 256 deletions(-) rename {omnisharptest/omnisharpUnitTests => test/unitTests}/json.test.ts (58%) diff --git a/l10n/bundle.l10n.json b/l10n/bundle.l10n.json index bfdf042d50..e7ee67d469 100644 --- a/l10n/bundle.l10n.json +++ b/l10n/bundle.l10n.json @@ -30,7 +30,7 @@ "For more information about the 'console' field, see {0}": "For more information about the 'console' field, see {0}", "WARNING": "WARNING", "The C# extension was unable to automatically decode projects in the current workspace to create a runnable launch.json file. A template launch.json file has been created as a placeholder.\n\nIf the server is currently unable to load your project, you can attempt to resolve this by restoring any missing project dependencies (example: run 'dotnet restore') and by fixing any reported errors from building the projects in your workspace.\nIf this allows the server to now load your project then --\n * Delete this file\n * Open the Visual Studio Code command palette (View->Command Palette)\n * run the command: '.NET: Generate Assets for Build and Debug'.\n\nIf your project requires a more complex launch configuration, you may wish to delete this configuration and pick a different template using the 'Add Configuration...' button at the bottom of this file.": "The C# extension was unable to automatically decode projects in the current workspace to create a runnable launch.json file. A template launch.json file has been created as a placeholder.\n\nIf the server is currently unable to load your project, you can attempt to resolve this by restoring any missing project dependencies (example: run 'dotnet restore') and by fixing any reported errors from building the projects in your workspace.\nIf this allows the server to now load your project then --\n * Delete this file\n * Open the Visual Studio Code command palette (View->Command Palette)\n * run the command: '.NET: Generate Assets for Build and Debug'.\n\nIf your project requires a more complex launch configuration, you may wish to delete this configuration and pick a different template using the 'Add Configuration...' button at the bottom of this file.", - "Failed to parse tasks.json file": "Failed to parse tasks.json file", + "Failed to parse tasks.json file: {0}": "Failed to parse tasks.json file: {0}", "Don't Ask Again": "Don't Ask Again", "Required assets to build and debug are missing from '{0}'. Add them?": "Required assets to build and debug are missing from '{0}'. Add them?", "Cancel": "Cancel", diff --git a/package-lock.json b/package-lock.json index d6e8cd7f6c..cea73180ec 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2497,12 +2497,6 @@ "integrity": "sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==", "dev": true }, - "node_modules/@types/json5": { - "version": "0.0.29", - "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz", - "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==", - "dev": true - }, "node_modules/@types/jsonwebtoken": { "version": "9.0.2", "resolved": "https://registry.npmjs.org/@types/jsonwebtoken/-/jsonwebtoken-9.0.2.tgz", @@ -15800,6 +15794,12 @@ "strip-bom": "^3.0.0" } }, + "node_modules/tsconfig-paths/node_modules/@types/json5": { + "version": "0.0.29", + "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz", + "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==", + "dev": true + }, "node_modules/tsconfig-paths/node_modules/json5": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz", @@ -19103,12 +19103,6 @@ "integrity": "sha512-wOuvG1SN4Us4rez+tylwwwCV1psiNVOkJeM3AUWUNWg/jDQY2+HE/444y5gc+jBmRqASOm2Oeh5c1axHobwRKQ==", "dev": true }, - "@types/json5": { - "version": "0.0.29", - "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz", - "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==", - "dev": true - }, "@types/jsonwebtoken": { "version": "9.0.2", "resolved": "https://registry.npmjs.org/@types/jsonwebtoken/-/jsonwebtoken-9.0.2.tgz", @@ -29255,6 +29249,12 @@ "strip-bom": "^3.0.0" }, "dependencies": { + "@types/json5": { + "version": "0.0.29", + "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz", + "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==", + "dev": true + }, "json5": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz", diff --git a/src/json.ts b/src/json.ts index 93804fb6d3..6081c187e4 100644 --- a/src/json.ts +++ b/src/json.ts @@ -3,212 +3,8 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -const enum CharCode { - asterisk = 0x2a, // * - backSlash = 0x5c, // \ - closeBrace = 0x7d, // } - closeBracket = 0x5d, // ] - comma = 0x2c, // , - doubleQuote = 0x22, // " - slash = 0x2f, // / - - byteOrderMark = 0xfeff, - - // line terminator characters (see https://en.wikipedia.org/wiki/Newline#Unicode) - carriageReturn = 0x0d, - formFeed = 0x0c, - lineFeed = 0x0a, - lineSeparator = 0x2028, - nextLine = 0x85, - paragraphSeparator = 0x2029, - verticalTab = 0x0b, - - // whitespace characters (see https://en.wikipedia.org/wiki/Whitespace_character#Unicode) - tab = 0x09, - space = 0x20, - nonBreakingSpace = 0xa0, - ogham = 0x1680, - enQuad = 0x2000, - emQuad = 0x2001, - enSpace = 0x2002, - emSpace = 0x2003, - threePerEmSpace = 0x2004, - fourPerEmSpace = 0x2005, - sixPerEmSpace = 0x2006, - figureSpace = 0x2007, - punctuationSpace = 0x2008, - thinSpace = 0x2009, - hairSpace = 0x200a, - zeroWidthSpace = 0x200b, - narrowNoBreakSpace = 0x202f, - mathematicalSpace = 0x205f, - ideographicSpace = 0x3000, -} - -function isLineBreak(code: number) { - return ( - code === CharCode.lineFeed || - code === CharCode.carriageReturn || - code === CharCode.verticalTab || - code === CharCode.formFeed || - code === CharCode.lineSeparator || - code === CharCode.paragraphSeparator - ); -} - -function isWhitespace(code: number) { - return ( - code === CharCode.space || - code === CharCode.tab || - code === CharCode.lineFeed || - code === CharCode.verticalTab || - code === CharCode.formFeed || - code === CharCode.carriageReturn || - code === CharCode.nextLine || - code === CharCode.nonBreakingSpace || - code === CharCode.ogham || - (code >= CharCode.enQuad && code <= CharCode.zeroWidthSpace) || - code === CharCode.lineSeparator || - code === CharCode.paragraphSeparator || - code === CharCode.narrowNoBreakSpace || - code === CharCode.mathematicalSpace || - code === CharCode.ideographicSpace || - code === CharCode.byteOrderMark - ); -} - -function cleanJsonText(text: string) { - const parts: string[] = []; - let partStart = 0; - - let index = 0; - const length = text.length; - - function next(): number { - const result = peek(); - index++; - return result; - } - - function peek(offset = 0): number { - return text.charCodeAt(index + offset); - } - - function peekPastWhitespace(): number { - let pos = index; - let code = NaN; - - do { - code = text.charCodeAt(pos); - pos++; - } while (isWhitespace(code)); - - return code; - } - - function scanString() { - while (index < length) { - const code = next(); - - if (code === CharCode.doubleQuote) { - // End of string. We're done - break; - } - - if (code === CharCode.backSlash) { - // Skip escaped character. We don't care about verifying the escape sequence. - // We just don't want to accidentally scan an escaped double-quote as the end of the string. - index++; - } - - if (isLineBreak(code)) { - // string ended unexpectedly - break; - } - } - } - - // eslint-disable-next-line no-constant-condition - while (true) { - const code = next(); - - switch (code) { - // byte-order mark - case CharCode.byteOrderMark: - // We just skip the byte-order mark - parts.push(text.substring(partStart, index - 1)); - partStart = index; - break; - - // strings - case CharCode.doubleQuote: - scanString(); - break; - - // comments - case CharCode.slash: - // Single-line comment - if (peek() === CharCode.slash) { - // Be careful not to include the first slash in the text part. - parts.push(text.substring(partStart, index - 1)); - - // Start after the second slash and scan until a line-break character is encountered. - index++; - while (index < length) { - if (isLineBreak(peek())) { - break; - } - - index++; - } - - partStart = index; - } - - // Multi-line comment - if (peek() === CharCode.asterisk) { - // Be careful not to include the first slash in the text part. - parts.push(text.substring(partStart, index - 1)); - - // Start after the asterisk and scan until a */ is encountered. - index++; - while (index < length) { - if (peek() === CharCode.asterisk && peek(1) === CharCode.slash) { - index += 2; - break; - } - - index++; - } - - partStart = index; - } - - break; - - case CharCode.comma: { - // Ignore trailing commas in object member lists and array element lists - const nextCode = peekPastWhitespace(); - if (nextCode === CharCode.closeBrace || nextCode === CharCode.closeBracket) { - parts.push(text.substring(partStart, index - 1)); - partStart = index; - } - - break; - } - default: - } - - if (index >= length && index > partStart) { - parts.push(text.substring(partStart, length)); - break; - } - } - - return parts.join(''); -} +import { parse } from 'jsonc-parser'; export function tolerantParse(text: string) { - text = cleanJsonText(text); - return JSON.parse(text); + return parse(text); } diff --git a/src/shared/assets.ts b/src/shared/assets.ts index 164fc6aa16..098c88b474 100644 --- a/src/shared/assets.ts +++ b/src/shared/assets.ts @@ -571,7 +571,8 @@ export async function getBuildOperations(generator: AssetGenerator): Promise { - suiteSetup(() => should()); - - test('no comments', () => { +jestLib.describe('JSON', () => { + jestLib.test('no comments', () => { const text = `{ "hello": "world" }`; @@ -17,10 +15,10 @@ suite('JSON', () => { const json = tolerantParse(text); const result = JSON.stringify(json, null, 4); - result.should.equal(text); + jestLib.expect(result).toEqual(text); }); - test('no comments (minified)', () => { + jestLib.test('no comments (minified)', () => { const text = `{"hello":"world","from":"json"}`; const expected = `{ @@ -31,10 +29,10 @@ suite('JSON', () => { const json = tolerantParse(text); const result = JSON.stringify(json, null, 4); - result.should.equal(expected); + jestLib.expect(result).toEqual(expected); }); - test('single-line comment before JSON', () => { + jestLib.test('single-line comment before JSON', () => { const text = `// comment { "hello": "world\\"" // comment @@ -47,10 +45,10 @@ suite('JSON', () => { const json = tolerantParse(text); const result = JSON.stringify(json, null, 4); - result.should.equal(expected); + jestLib.expect(result).toEqual(expected); }); - test('single-line comment on separate line', () => { + jestLib.test('single-line comment on separate line', () => { const text = `{ // comment "hello": "world" @@ -63,10 +61,10 @@ suite('JSON', () => { const json = tolerantParse(text); const result = JSON.stringify(json, null, 4); - result.should.equal(expected); + jestLib.expect(result).toEqual(expected); }); - test('single-line comment at end of line', () => { + jestLib.test('single-line comment at end of line', () => { const text = `{ "hello": "world" // comment }`; @@ -78,10 +76,10 @@ suite('JSON', () => { const json = tolerantParse(text); const result = JSON.stringify(json, null, 4); - result.should.equal(expected); + jestLib.expect(result).toEqual(expected); }); - test('single-line comment at end of text', () => { + jestLib.test('single-line comment at end of text', () => { const text = `{ "hello": "world" } // comment`; @@ -93,10 +91,10 @@ suite('JSON', () => { const json = tolerantParse(text); const result = JSON.stringify(json, null, 4); - result.should.equal(expected); + jestLib.expect(result).toEqual(expected); }); - test('ignore single-line comment inside string', () => { + jestLib.test('ignore single-line comment inside string', () => { const text = `{ "hello": "world // comment" }`; @@ -104,10 +102,10 @@ suite('JSON', () => { const json = tolerantParse(text); const result = JSON.stringify(json, null, 4); - result.should.equal(text); + jestLib.expect(result).toEqual(text); }); - test('single-line comment after string with escaped double quote', () => { + jestLib.test('single-line comment after string with escaped double quote', () => { const text = `{ "hello": "world\\"" // comment }`; @@ -119,10 +117,10 @@ suite('JSON', () => { const json = tolerantParse(text); const result = JSON.stringify(json, null, 4); - result.should.equal(expected); + jestLib.expect(result).toEqual(expected); }); - test('multi-line comment at start of text', () => { + jestLib.test('multi-line comment at start of text', () => { const text = `/**/{ "hello": "world" }`; @@ -134,10 +132,10 @@ suite('JSON', () => { const json = tolerantParse(text); const result = JSON.stringify(json, null, 4); - result.should.equal(expected); + jestLib.expect(result).toEqual(expected); }); - test('comment out key/value pair', () => { + jestLib.test('comment out key/value pair', () => { const text = `{ /*"hello": "world"*/ "from": "json" @@ -150,10 +148,10 @@ suite('JSON', () => { const json = tolerantParse(text); const result = JSON.stringify(json, null, 4); - result.should.equal(expected); + jestLib.expect(result).toEqual(expected); }); - test('multi-line comment at end of text', () => { + jestLib.test('multi-line comment at end of text', () => { const text = `{ "hello": "world" }/**/`; @@ -165,10 +163,10 @@ suite('JSON', () => { const json = tolerantParse(text); const result = JSON.stringify(json, null, 4); - result.should.equal(expected); + jestLib.expect(result).toEqual(expected); }); - test('ignore multi-line comment inside string', () => { + jestLib.test('ignore multi-line comment inside string', () => { const text = `{ "hello": "wo/**/rld" }`; @@ -180,10 +178,10 @@ suite('JSON', () => { const json = tolerantParse(text); const result = JSON.stringify(json, null, 4); - result.should.equal(expected); + jestLib.expect(result).toEqual(expected); }); - test('ignore BOM', () => { + jestLib.test('ignore BOM', () => { const text = `\uFEFF{ "hello": "world" }`; @@ -195,10 +193,10 @@ suite('JSON', () => { const json = tolerantParse(text); const result = JSON.stringify(json, null, 4); - result.should.equal(expected); + jestLib.expect(result).toEqual(expected); }); - test('ignore trailing comma in object member list', () => { + jestLib.test('ignore trailing comma in object member list', () => { const text = `{ "obj": { "hello": "world", @@ -216,10 +214,10 @@ suite('JSON', () => { const json = tolerantParse(text); const result = JSON.stringify(json, null, 4); - result.should.equal(expected); + jestLib.expect(result).toEqual(expected); }); - test('ignore trailing comma in array element list', () => { + jestLib.test('ignore trailing comma in array element list', () => { const text = `{ "array": [ "element1", @@ -237,10 +235,10 @@ suite('JSON', () => { const json = tolerantParse(text); const result = JSON.stringify(json, null, 4); - result.should.equal(expected); + jestLib.expect(result).toEqual(expected); }); - test('ignore trailing comma in object member list with leading and trailing whitespace', () => { + jestLib.test('ignore trailing comma in object member list with leading and trailing whitespace', () => { const text = `{ "obj": { "a" : 1 , } }`; @@ -254,6 +252,41 @@ suite('JSON', () => { const json = tolerantParse(text); const result = JSON.stringify(json, null, 4); - result.should.equal(expected); + jestLib.expect(result).toEqual(expected); + }); + + jestLib.test('single-line comments in multiple locations', () => { + const text = ` +// This comment should be allowed. +{ + // This comment should be allowed. + "version": "2.0.0", // This comment should be allowed. + "tasks": [ + // This comment should be allowed. + { + "label": "foo", // This comment should be allowed. + "type": "shell", + "command": "true", + // This comment should be allowed. + }, + ], +} +// This comment should be allowed.`; + + const expected = `{ + "version": "2.0.0", + "tasks": [ + { + "label": "foo", + "type": "shell", + "command": "true" + } + ] +}`; + + const json = tolerantParse(text); + const result = JSON.stringify(json, null, 4); + + jestLib.expect(result).toEqual(expected); }); });