diff --git a/parser.js b/parser.js index d3b706b..f1295ca 100644 --- a/parser.js +++ b/parser.js @@ -13,7 +13,7 @@ module.exports = class Parser { this.prevIndent = undefined this.step = undefined - this.root.source = { input, start: { column: 1, line: 1 } } + this.root.source = { input, start: { column: 1, line: 1, offset: 0 } } } atrule(part) { @@ -41,13 +41,15 @@ module.exports = class Parser { } badProp(token) { - this.error('Unexpected separator in property', token[2], token[3]) + let pos = this.getPosition(token[2]) + this.error('Unexpected separator in property', pos.offset) } checkCurly(tokens) { for (let token of tokens) { if (token[0] === '{') { - this.error('Unnecessary curly bracket', token[2], token[3]) + let pos = this.getPosition(token[2]) + this.error('Unnecessary curly bracket', pos.offset) } } } @@ -55,7 +57,8 @@ module.exports = class Parser { checkSemicolon(tokens) { for (let token of tokens) { if (token[0] === ';') { - this.error('Unnecessary semicolon', token[2], token[3]) + let pos = this.getPosition(token[2]) + this.error('Unnecessary semicolon', pos.offset) } } } @@ -64,7 +67,7 @@ module.exports = class Parser { let token = part.tokens[0] let node = new Comment() this.init(node, part) - node.source.end = { column: token[5], line: token[4] } + node.source.end = this.getPosition(token[3]) this.commentText(node, token) } @@ -72,7 +75,7 @@ module.exports = class Parser { commentText(node, token) { let text = token[1] - if (token[6] === 'inline') { + if (token[4] === 'inline') { node.raws.inline = true text = text.slice(2) } else { @@ -80,6 +83,7 @@ module.exports = class Parser { } let match = text.match(/^(\s*)([^]*\S)(\s*)\n?$/) + if (match) { node.text = match[2] node.raws.left = match[1] @@ -138,9 +142,9 @@ module.exports = class Parser { let comment = new Comment() this.current.push(comment) comment.source = { - end: { column: last[5], line: last[4] }, + end: this.getPosition(last[3]), input: this.input, - start: { column: last[3], line: last[2] } + start: this.getPosition(last[2]) } let prev = value[value.length - 1] if (prev && prev[0] === 'space') { @@ -172,8 +176,9 @@ module.exports = class Parser { this.raw(node, 'value', value, colon) } - error(msg, line, column) { - throw this.input.error(msg, line, column) + error(msg, offset) { + let pos = this.getPosition(offset) + throw this.input.error(msg, pos.line, pos.column) } firstSpaces(tokens) { @@ -189,6 +194,15 @@ module.exports = class Parser { return result } + getPosition(offset) { + let pos = this.input.fromOffset(offset) + return { + column: pos.col, + line: pos.line, + offset + } + } + indent(part) { let indent = part.indent.length let isPrev = typeof this.prevIndent !== 'undefined' @@ -227,7 +241,8 @@ module.exports = class Parser { } indentedFirstLine(part) { - this.error('First line should not have indent', part.number, 1) + let pos = this.getPosition(part.tokens[0][2]) + this.error('First line should not have indent', pos.offset) } init(node, part) { @@ -243,7 +258,7 @@ module.exports = class Parser { } node.source = { input: this.input, - start: { column: part.tokens[0][3], line: part.tokens[0][2] } + start: this.getPosition(part.tokens[0][2]) } } @@ -291,10 +306,7 @@ module.exports = class Parser { for (let i = this.tokens.length - 1; i >= 0; i--) { if (this.tokens[i].length > 3) { let last = this.tokens[i] - this.root.source.end = { - column: last[5] || last[3], - line: last[4] || last[2] - } + this.root.source.end = this.getPosition(last[3]) break } } @@ -330,7 +342,7 @@ module.exports = class Parser { if (!clean) { let sss = tokens.reduce((all, i) => all + i[1], '') let raw = tokens.reduce((all, i) => { - if (i[0] === 'comment' && i[6] === 'inline') { + if (i[0] === 'comment' && i[4] === 'inline') { return all + '/* ' + i[1].slice(2).trim() + ' */' } else { return all + i[1] @@ -350,10 +362,7 @@ module.exports = class Parser { } if (!last) last = altLast - node.source.end = { - column: last[5] || last[3], - line: last[4] || last[2] - } + node.source.end = this.getPosition(last[3]) } rule(part) { @@ -376,15 +385,18 @@ module.exports = class Parser { } unnamedAtrule(token) { - this.error('At-rule without name', token[2], token[3]) + let pos = this.getPosition(token[2]) + this.error('At-rule without name', pos.offset) } unnamedDecl(token) { - this.error('Declaration without name', token[2], token[3]) + let pos = this.getPosition(token[2]) + this.error('Declaration without name', pos.offset) } wrongIndent(expected, real, part) { + let pos = this.getPosition(part.tokens[0][2]) let msg = `Expected ${expected} indent, but get ${real}` - this.error(msg, part.number, 1) + this.error(msg, pos.offset) } } diff --git a/test/cases/all.json b/test/cases/all.json index 303fca6..252e41e 100644 --- a/test/cases/all.json +++ b/test/cases/all.json @@ -20,11 +20,13 @@ "source": { "start": { "line": 2, - "column": 3 + "column": 3, + "offset": 4 }, "end": { "line": 2, - "column": 13 + "column": 14, + "offset": 15 } }, "prop": "color", @@ -34,11 +36,13 @@ "source": { "start": { "line": 1, - "column": 1 + "column": 1, + "offset": 0 }, "end": { "line": 1, - "column": 1 + "column": 2, + "offset": 1 } }, "selector": "a" @@ -53,11 +57,13 @@ "source": { "start": { "line": 4, - "column": 1 + "column": 1, + "offset": 17 }, "end": { "line": 4, - "column": 25 + "column": 26, + "offset": 42 } }, "params": "(max-width: 400px)", @@ -77,11 +83,13 @@ "source": { "start": { "line": 6, - "column": 5 + "column": 5, + "offset": 55 }, "end": { "line": 6, - "column": 19 + "column": 20, + "offset": 70 } }, "prop": "padding", @@ -91,11 +99,13 @@ "source": { "start": { "line": 5, - "column": 3 + "column": 3, + "offset": 45 }, "end": { "line": 5, - "column": 7 + "column": 8, + "offset": 50 } }, "selector": ".body" @@ -117,11 +127,13 @@ "source": { "start": { "line": 9, - "column": 3 + "column": 3, + "offset": 80 }, "end": { "line": 9, - "column": 40 + "column": 41, + "offset": 118 } }, "prop": "font-family", @@ -137,11 +149,13 @@ "source": { "start": { "line": 10, - "column": 3 + "column": 3, + "offset": 121 }, "end": { "line": 10, - "column": 32 + "column": 33, + "offset": 151 } }, "text": "transform: rotate(90deg)" @@ -155,11 +169,13 @@ "source": { "start": { "line": 11, - "column": 3 + "column": 3, + "offset": 154 }, "end": { "line": 11, - "column": 19 + "column": 20, + "offset": 171 } }, "prop": "background", @@ -169,11 +185,13 @@ "source": { "start": { "line": 8, - "column": 1 + "column": 1, + "offset": 72 }, "end": { "line": 8, - "column": 5 + "column": 6, + "offset": 77 } }, "selector": ".page" @@ -193,11 +211,13 @@ "source": { "start": { "line": 14, - "column": 3 + "column": 3, + "offset": 182 }, "end": { "line": 14, - "column": 14 + "column": 15, + "offset": 194 } }, "prop": "width", @@ -218,11 +238,13 @@ "source": { "start": { "line": 16, - "column": 5 + "column": 5, + "offset": 209 }, "end": { "line": 16, - "column": 16 + "column": 17, + "offset": 221 } }, "prop": "width", @@ -232,11 +254,13 @@ "source": { "start": { "line": 15, - "column": 3 + "column": 3, + "offset": 197 }, "end": { "line": 15, - "column": 9 + "column": 10, + "offset": 204 } }, "selector": "&_title" @@ -245,11 +269,13 @@ "source": { "start": { "line": 13, - "column": 1 + "column": 1, + "offset": 173 }, "end": { "line": 13, - "column": 6 + "column": 7, + "offset": 179 } }, "selector": ".phone" @@ -275,11 +301,13 @@ "source": { "start": { "line": 20, - "column": 5 + "column": 5, + "offset": 244 }, "end": { "line": 20, - "column": 16 + "column": 17, + "offset": 256 } }, "prop": "color", @@ -294,11 +322,13 @@ "source": { "start": { "line": 21, - "column": 5 + "column": 5, + "offset": 261 }, "end": { "line": 21, - "column": 21 + "column": 22, + "offset": 278 } }, "prop": "background", @@ -308,11 +338,13 @@ "source": { "start": { "line": 19, - "column": 3 + "column": 3, + "offset": 231 }, "end": { "line": 19, - "column": 10 + "column": 11, + "offset": 239 } }, "selector": "--black:" @@ -321,11 +353,13 @@ "source": { "start": { "line": 18, - "column": 1 + "column": 1, + "offset": 223 }, "end": { "line": 18, - "column": 5 + "column": 6, + "offset": 228 } }, "selector": ":root" @@ -334,11 +368,13 @@ "source": { "start": { "line": 1, - "column": 1 + "column": 1, + "offset": 0 }, "end": { - "line": 21, - "column": 21 + "line": 22, + "column": 1, + "offset": 279 } } } diff --git a/test/cases/atrules.json b/test/cases/atrules.json index 58921c3..b335b22 100644 --- a/test/cases/atrules.json +++ b/test/cases/atrules.json @@ -15,11 +15,13 @@ "source": { "start": { "line": 1, - "column": 1 + "column": 1, + "offset": 0 }, "end": { "line": 1, - "column": 5 + "column": 6, + "offset": 5 } }, "params": "" @@ -34,11 +36,13 @@ "source": { "start": { "line": 3, - "column": 1 + "column": 1, + "offset": 7 }, "end": { "line": 3, - "column": 16 + "column": 17, + "offset": 23 } }, "params": "\"UTF-8\"" @@ -53,11 +57,13 @@ "source": { "start": { "line": 5, - "column": 1 + "column": 1, + "offset": 25 }, "end": { "line": 6, - "column": 29 + "column": 30, + "offset": 80 } }, "params": "(color: black),\n (background: black)", @@ -77,11 +83,13 @@ "source": { "start": { "line": 7, - "column": 3 + "column": 3, + "offset": 83 }, "end": { "line": 7, - "column": 44 + "column": 45, + "offset": 125 } }, "params": "(max-width: 400px) ", @@ -97,11 +105,13 @@ "source": { "start": { "line": 8, - "column": 5 + "column": 5, + "offset": 130 }, "end": { "line": 8, - "column": 22 + "column": 23, + "offset": 148 } }, "text": "write code here" @@ -114,11 +124,13 @@ "source": { "start": { "line": 1, - "column": 1 + "column": 1, + "offset": 0 }, "end": { - "line": 8, - "column": 22 + "line": 9, + "column": 1, + "offset": 149 } } } diff --git a/test/cases/colons.json b/test/cases/colons.json index f0de613..55ec383 100644 --- a/test/cases/colons.json +++ b/test/cases/colons.json @@ -15,11 +15,13 @@ "source": { "start": { "line": 1, - "column": 1 + "column": 1, + "offset": 0 }, "end": { "line": 1, - "column": 34 + "column": 35, + "offset": 34 } }, "params": ":--heading h1, h2" @@ -45,11 +47,13 @@ "source": { "start": { "line": 5, - "column": 5 + "column": 5, + "offset": 56 }, "end": { "line": 5, - "column": 10 + "column": 11, + "offset": 62 } }, "prop": "top", @@ -64,11 +68,13 @@ "source": { "start": { "line": 6, - "column": 5 + "column": 5, + "offset": 67 }, "end": { "line": 6, - "column": 11 + "column": 12, + "offset": 74 } }, "prop": "left", @@ -78,11 +84,13 @@ "source": { "start": { "line": 4, - "column": 3 + "column": 3, + "offset": 44 }, "end": { "line": 4, - "column": 9 + "column": 10, + "offset": 51 } }, "selector": "margin:" @@ -102,11 +110,13 @@ "source": { "start": { "line": 8, - "column": 5 + "column": 5, + "offset": 87 }, "end": { "line": 8, - "column": 14 + "column": 15, + "offset": 97 } }, "prop": "size", @@ -121,11 +131,13 @@ "source": { "start": { "line": 9, - "column": 5 + "column": 5, + "offset": 102 }, "end": { "line": 9, - "column": 15 + "column": 16, + "offset": 113 } }, "prop": "weight", @@ -135,11 +147,13 @@ "source": { "start": { "line": 7, - "column": 3 + "column": 3, + "offset": 77 }, "end": { "line": 7, - "column": 7 + "column": 8, + "offset": 82 } }, "selector": "font:" @@ -148,11 +162,13 @@ "source": { "start": { "line": 3, - "column": 1 + "column": 1, + "offset": 36 }, "end": { "line": 3, - "column": 5 + "column": 6, + "offset": 41 } }, "selector": ".test" @@ -172,11 +188,13 @@ "source": { "start": { "line": 12, - "column": 3 + "column": 3, + "offset": 130 }, "end": { "line": 12, - "column": 13 + "column": 14, + "offset": 141 } }, "prop": "color", @@ -186,11 +204,13 @@ "source": { "start": { "line": 11, - "column": 1 + "column": 1, + "offset": 115 }, "end": { "line": 11, - "column": 12 + "column": 13, + "offset": 127 } }, "selector": "a:--any-link" @@ -216,11 +236,13 @@ "source": { "start": { "line": 15, - "column": 5 + "column": 5, + "offset": 175 }, "end": { "line": 15, - "column": 15 + "column": 16, + "offset": 186 } }, "prop": "color", @@ -230,11 +252,13 @@ "source": { "start": { "line": 14, - "column": 3 + "column": 3, + "offset": 157 }, "end": { "line": 14, - "column": 15 + "column": 16, + "offset": 170 } }, "selector": "&.is-disabled" @@ -243,11 +267,13 @@ "source": { "start": { "line": 13, - "column": 1 + "column": 1, + "offset": 142 }, "end": { "line": 13, - "column": 12 + "column": 13, + "offset": 154 } }, "selector": "a:--any-link" @@ -267,11 +293,13 @@ "source": { "start": { "line": 18, - "column": 3 + "column": 3, + "offset": 209 }, "end": { "line": 18, - "column": 13 + "column": 14, + "offset": 220 } }, "prop": "color", @@ -281,11 +309,13 @@ "source": { "start": { "line": 17, - "column": 1 + "column": 1, + "offset": 188 }, "end": { "line": 17, - "column": 18 + "column": 19, + "offset": 206 } }, "selector": "article :--heading" @@ -305,11 +335,13 @@ "source": { "start": { "line": 21, - "column": 3 + "column": 3, + "offset": 230 }, "end": { "line": 21, - "column": 13 + "column": 14, + "offset": 241 } }, "prop": "--red", @@ -319,11 +351,13 @@ "source": { "start": { "line": 20, - "column": 1 + "column": 1, + "offset": 222 }, "end": { "line": 20, - "column": 5 + "column": 6, + "offset": 227 } }, "selector": ":root" @@ -343,11 +377,13 @@ "source": { "start": { "line": 24, - "column": 3 + "column": 3, + "offset": 250 }, "end": { "line": 24, - "column": 19 + "column": 20, + "offset": 267 } }, "prop": "color", @@ -357,11 +393,13 @@ "source": { "start": { "line": 23, - "column": 1 + "column": 1, + "offset": 243 }, "end": { "line": 23, - "column": 4 + "column": 5, + "offset": 247 } }, "selector": "body" @@ -370,11 +408,13 @@ "source": { "start": { "line": 1, - "column": 1 + "column": 1, + "offset": 0 }, "end": { - "line": 24, - "column": 19 + "line": 25, + "column": 1, + "offset": 268 } } } diff --git a/test/cases/comments.json b/test/cases/comments.json index df54375..2c13b3c 100644 --- a/test/cases/comments.json +++ b/test/cases/comments.json @@ -15,11 +15,13 @@ "source": { "start": { "line": 1, - "column": 1 + "column": 1, + "offset": 0 }, "end": { "line": 2, - "column": 10 + "column": 11, + "offset": 19 } }, "text": "multi\n line" @@ -35,11 +37,13 @@ "source": { "start": { "line": 3, - "column": 1 + "column": 1, + "offset": 20 }, "end": { "line": 3, - "column": 9 + "column": 10, + "offset": 29 } }, "text": "inline" @@ -59,11 +63,13 @@ "source": { "start": { "line": 5, - "column": 3 + "column": 3, + "offset": 34 }, "end": { "line": 5, - "column": 14 + "column": 15, + "offset": 46 } }, "prop": "color", @@ -80,11 +86,13 @@ "source": { "start": { "line": 5, - "column": 16 + "column": 16, + "offset": 47 }, "end": { "line": 5, - "column": 22 + "column": 23, + "offset": 54 } }, "text": "last" @@ -98,11 +106,13 @@ "source": { "start": { "line": 6, - "column": 3 + "column": 3, + "offset": 57 }, "end": { "line": 6, - "column": 12 + "column": 13, + "offset": 67 } }, "prop": "z-index", @@ -112,11 +122,13 @@ "source": { "start": { "line": 4, - "column": 1 + "column": 1, + "offset": 30 }, "end": { "line": 4, - "column": 1 + "column": 2, + "offset": 31 } }, "selector": "a" @@ -132,11 +144,13 @@ "source": { "start": { "line": 8, - "column": 1 + "column": 1, + "offset": 69 }, "end": { "line": 8, - "column": 4 + "column": 5, + "offset": 73 } }, "text": "a" @@ -153,11 +167,13 @@ "source": { "start": { "line": 9, - "column": 3 + "column": 3, + "offset": 76 }, "end": { "line": 9, - "column": 17 + "column": 18, + "offset": 91 } }, "text": "color: black" @@ -166,11 +182,13 @@ "source": { "start": { "line": 1, - "column": 1 + "column": 1, + "offset": 0 }, "end": { - "line": 9, - "column": 17 + "line": 10, + "column": 1, + "offset": 92 } } } diff --git a/test/cases/decls.json b/test/cases/decls.json index 0c784e4..402973e 100644 --- a/test/cases/decls.json +++ b/test/cases/decls.json @@ -13,11 +13,13 @@ "source": { "start": { "line": 1, - "column": 1 + "column": 1, + "offset": 0 }, "end": { "line": 1, - "column": 12 + "column": 13, + "offset": 12 } }, "prop": "color", @@ -32,11 +34,13 @@ "source": { "start": { "line": 3, - "column": 1 + "column": 1, + "offset": 14 }, "end": { "line": 4, - "column": 41 + "column": 42, + "offset": 98 } }, "prop": "background", @@ -51,11 +55,13 @@ "source": { "start": { "line": 6, - "column": 1 + "column": 1, + "offset": 100 }, "end": { "line": 8, - "column": 16 + "column": 17, + "offset": 147 } }, "prop": "box-shadow", @@ -70,11 +76,13 @@ "source": { "start": { "line": 10, - "column": 1 + "column": 1, + "offset": 149 }, "end": { "line": 10, - "column": 19 + "column": 20, + "offset": 168 } }, "prop": "$(var)-color", @@ -84,11 +92,13 @@ "source": { "start": { "line": 1, - "column": 1 + "column": 1, + "offset": 0 }, "end": { - "line": 10, - "column": 19 + "line": 11, + "column": 1, + "offset": 169 } } } diff --git a/test/cases/empty.json b/test/cases/empty.json index 960e17e..d00e248 100644 --- a/test/cases/empty.json +++ b/test/cases/empty.json @@ -15,11 +15,13 @@ "source": { "start": { "line": 1, - "column": 1 + "column": 1, + "offset": 0 }, "end": { "line": 1, - "column": 2 + "column": 3, + "offset": 2 } }, "text": "" @@ -28,11 +30,13 @@ "source": { "start": { "line": 1, - "column": 1 + "column": 1, + "offset": 0 }, "end": { - "line": 1, - "column": 2 + "line": 2, + "column": 1, + "offset": 3 } } } diff --git a/test/cases/important.json b/test/cases/important.json index 0cd71e1..33be04f 100644 --- a/test/cases/important.json +++ b/test/cases/important.json @@ -14,11 +14,13 @@ "source": { "start": { "line": 1, - "column": 1 + "column": 1, + "offset": 0 }, "end": { "line": 1, - "column": 11 + "column": 12, + "offset": 11 } }, "prop": "long", @@ -35,11 +37,13 @@ "source": { "start": { "line": 2, - "column": 1 + "column": 1, + "offset": 23 }, "end": { "line": 2, - "column": 12 + "column": 13, + "offset": 35 } }, "prop": "short", @@ -50,11 +54,13 @@ "source": { "start": { "line": 1, - "column": 1 + "column": 1, + "offset": 0 }, "end": { - "line": 2, - "column": 22 + "line": 3, + "column": 1, + "offset": 46 } } } diff --git a/test/cases/rules.json b/test/cases/rules.json index 0536993..99de8e5 100644 --- a/test/cases/rules.json +++ b/test/cases/rules.json @@ -26,11 +26,13 @@ "source": { "start": { "line": 3, - "column": 5 + "column": 5, + "offset": 10 }, "end": { "line": 3, - "column": 16 + "column": 17, + "offset": 22 } }, "prop": "color", @@ -40,11 +42,13 @@ "source": { "start": { "line": 2, - "column": 3 + "column": 3, + "offset": 4 }, "end": { "line": 2, - "column": 3 + "column": 4, + "offset": 5 } }, "selector": "b" @@ -53,11 +57,13 @@ "source": { "start": { "line": 1, - "column": 1 + "column": 1, + "offset": 0 }, "end": { "line": 1, - "column": 1 + "column": 2, + "offset": 1 } }, "selector": "a" @@ -77,11 +83,13 @@ "source": { "start": { "line": 7, - "column": 3 + "column": 3, + "offset": 31 }, "end": { "line": 7, - "column": 14 + "column": 15, + "offset": 43 } }, "prop": "color", @@ -91,11 +99,13 @@ "source": { "start": { "line": 5, - "column": 1 + "column": 1, + "offset": 24 }, "end": { "line": 6, - "column": 1 + "column": 2, + "offset": 28 } }, "selector": "a,\nb" @@ -120,11 +130,13 @@ "source": { "start": { "line": 11, - "column": 3 + "column": 3, + "offset": 74 }, "end": { "line": 11, - "column": 14 + "column": 15, + "offset": 86 } }, "prop": "color", @@ -134,11 +146,13 @@ "source": { "start": { "line": 9, - "column": 1 + "column": 1, + "offset": 45 }, "end": { "line": 10, - "column": 2 + "column": 3, + "offset": 71 } }, "selector": "a b \nem" @@ -158,11 +172,13 @@ "source": { "start": { "line": 14, - "column": 3 + "column": 3, + "offset": 96 }, "end": { "line": 14, - "column": 15 + "column": 16, + "offset": 109 } }, "prop": "family", @@ -172,11 +188,13 @@ "source": { "start": { "line": 13, - "column": 1 + "column": 1, + "offset": 88 }, "end": { "line": 13, - "column": 5 + "column": 6, + "offset": 93 } }, "selector": "font:" @@ -196,11 +214,13 @@ "source": { "start": { "line": 17, - "column": 3 + "column": 3, + "offset": 122 }, "end": { "line": 17, - "column": 14 + "column": 15, + "offset": 134 } }, "prop": "color", @@ -210,11 +230,13 @@ "source": { "start": { "line": 16, - "column": 1 + "column": 1, + "offset": 111 }, "end": { "line": 16, - "column": 8 + "column": 9, + "offset": 119 } }, "selector": "[attr=;]" @@ -223,11 +245,13 @@ "source": { "start": { "line": 1, - "column": 1 + "column": 1, + "offset": 0 }, "end": { - "line": 17, - "column": 14 + "line": 18, + "column": 1, + "offset": 135 } } } diff --git a/test/parse.test.js b/test/parse.test.js index 9e742a7..025a28c 100644 --- a/test/parse.test.js +++ b/test/parse.test.js @@ -14,19 +14,19 @@ test('detects indent', () => { test('throws on first indent', () => { throws(() => { parse(' @charset "UTF-8"') - }, /:1:1: First line should not have indent/) + }, /:1:3: First line should not have indent/) }) test('throws on too big indent', () => { throws(() => { parse('@supports\n @media\n // test') - }, /:3:1: Expected 4 indent, but get 6/) + }, /:3:7: Expected 4 indent, but get 6/) }) test('throws on wrong indent step', () => { throws(() => { parse('@supports\n @media\n @media') - }, /:3:1: Expected 0 or 2 indent, but get 1/) + }, /:3:2: Expected 0 or 2 indent, but get 1/) }) test('throws on decl without property', () => { @@ -90,7 +90,11 @@ test('generates correct source maps on trailing spaces', () => { }) test('sets end position for root', () => { - deepStrictEqual(parse('a\n b: 1\n').source.end, { column: 6, line: 2 }) + deepStrictEqual(parse('a\n b: 1\n').source.end, { + column: 1, + line: 3, + offset: 9 + }) }) let tests = readdirSync(join(__dirname, 'cases')).filter( diff --git a/test/tokenize.test.js b/test/tokenize.test.js index 18dcebd..313d5ed 100644 --- a/test/tokenize.test.js +++ b/test/tokenize.test.js @@ -4,71 +4,88 @@ let { Input } = require('postcss') let tokenize = require('../tokenize') -function run(css, tokens) { - deepStrictEqual(tokenize(new Input(css)), tokens) +function run(css, expectedTokens) { + let tokens = tokenize(new Input(css)) + deepStrictEqual(tokens, expectedTokens) } -test('tokenizes inine comments', () => { - run('// a', [['comment', '// a', 1, 1, 1, 4, 'inline']]) +test('tokenizes inline comments', () => { + run('// a', [['comment', '// a', 0, 4, 'inline']]) }) -test('tokenizes inine comments and new lines', () => { +test('tokenizes inline comments and new lines', () => { run('// a\r\n', [ - ['comment', '// a', 1, 1, 1, 4, 'inline'], - ['newline', '\r\n', 1] + ['comment', '// a', 0, 4, 'inline'], + ['newline', '\r\n', 4, 6] ]) }) -test('tokenizes new lines arround spaces', () => { +test('tokenizes new lines around spaces', () => { run(' \n ', [ - ['space', ' '], - ['newline', '\n', 1], - ['space', ' '] + ['space', ' ', 0, 1], + ['newline', '\n', 1, 2], + ['space', ' ', 2, 3] ]) }) test('tokenizes Windows new lines', () => { - run('\r\n', [['newline', '\r\n', 1]]) + run('\r\n', [['newline', '\r\n', 0, 2]]) }) test('tokenizes single carriage return', () => { run('\ra', [ - ['newline', '\r', 1], - ['word', 'a', 2, 1, 2, 1] + ['newline', '\r', 0, 1], + ['word', 'a', 1, 2] ]) }) test('tokenizes last carriage return', () => { - run('\r', [['newline', '\r', 1]]) + run('\r', [['newline', '\r', 0, 1]]) }) test('tokenizes comma', () => { run('a,b', [ - ['word', 'a', 1, 1, 1, 1], + ['word', 'a', 0, 1], [',', ',', 1, 2], - ['word', 'b', 1, 3, 1, 3] + ['word', 'b', 2, 3] ]) }) test('escapes control symbols', () => { run('\\(\\{\\"\\@\\\\""', [ - ['word', '\\(', 1, 1, 1, 2], - ['word', '\\{', 1, 3, 1, 4], - ['word', '\\"', 1, 5, 1, 6], - ['word', '\\@', 1, 7, 1, 8], - ['word', '\\\\', 1, 9, 1, 10], - ['string', '""', 1, 11, 1, 12] + ['word', '\\(', 0, 2], + ['word', '\\{', 2, 4], + ['word', '\\"', 4, 6], + ['word', '\\@', 6, 8], + ['word', '\\\\', 8, 10], + ['string', '""', 10, 12] ]) }) test('escapes new line', () => { run('\\\n\\\r\\\r\n\\\f\\\\\n', [ - ['word', '\\\n', 1, 1, 1, 2], - ['word', '\\\r', 2, 1, 2, 2], - ['word', '\\\r\n', 3, 1, 3, 3], - ['word', '\\\f', 4, 1, 4, 2], - ['word', '\\\\', 5, 1, 5, 2], - ['newline', '\n', 5] + ['word', '\\\n', 0, 2], + ['word', '\\\r', 2, 4], + ['word', '\\\r\n', 4, 7], + ['word', '\\\f', 7, 9], + ['word', '\\\\', 9, 11], + ['newline', '\n', 11, 12] + ]) +}) + +test('tokenizes close curly brace', () => { + run('a { color: black; }', [ + ['word', 'a', 0, 1], + ['space', ' ', 1, 2], + ['{', '{', 2, 3], + ['space', ' ', 3, 4], + ['word', 'color', 4, 9], + [':', ':', 9, 10], + ['space', ' ', 10, 11], + ['word', 'black', 11, 16], + [';', ';', 16, 17], + ['space', ' ', 17, 18], + ['}', '}', 18, 19] ]) }) diff --git a/tokenize.js b/tokenize.js index 8f94aac..5cb5648 100644 --- a/tokenize.js +++ b/tokenize.js @@ -26,56 +26,35 @@ module.exports = function tokenize(input) { let tokens = [] let css = input.css.valueOf() - let code, - content, - escape, - escaped, - escapePos, - last, - lines, - n, - next, - nextLine, - nextOffset, - prev, - quote + let code, content, escape, escaped, escapePos, n, next, prev, quote let length = css.length - let offset = -1 - let line = 1 + let offset = 0 let pos = 0 function unclosed(what) { - throw input.error('Unclosed ' + what, line, pos - offset) + throw input.error('Unclosed ' + what, pos) } while (pos < length) { code = css.charCodeAt(pos) - if ( - code === NEWLINE || - code === FEED || - (code === CR && css.charCodeAt(pos + 1) !== NEWLINE) - ) { - offset = pos - line += 1 - } - switch (code) { case CR: if (css.charCodeAt(pos + 1) === NEWLINE) { - offset = pos - line += 1 + tokens.push(['newline', '\r\n', offset, offset + 2]) + offset += 2 pos += 1 - tokens.push(['newline', '\r\n', line - 1]) } else { - tokens.push(['newline', '\r', line - 1]) + tokens.push(['newline', '\r', offset, offset + 1]) + offset++ } break - case FEED: case NEWLINE: - tokens.push(['newline', css.slice(pos, pos + 1), line - 1]) + case FEED: + tokens.push(['newline', css.slice(pos, pos + 1), offset, offset + 1]) + offset++ break case SPACE: @@ -86,28 +65,39 @@ module.exports = function tokenize(input) { code = css.charCodeAt(next) } while (code === SPACE || code === TAB) - tokens.push(['space', css.slice(pos, next)]) + tokens.push([ + 'space', + css.slice(pos, next), + offset, + offset + (next - pos) + ]) + offset += next - pos pos = next - 1 break case OPEN_CURLY: - tokens.push(['{', '{', line, pos - offset]) + tokens.push(['{', '{', offset, offset + 1]) + offset++ break case CLOSE_CURLY: - tokens.push(['}', '}', line, pos - offset]) + tokens.push(['}', '}', offset, offset + 1]) + offset++ break case COLON: - tokens.push([':', ':', line, pos - offset]) + tokens.push([':', ':', offset, offset + 1]) + offset++ break case SEMICOLON: - tokens.push([';', ';', line, pos - offset]) + tokens.push([';', ';', offset, offset + 1]) + offset++ break case COMMA: - tokens.push([',', ',', line, pos - offset]) + tokens.push([',', ',', offset, offset + 1]) + offset++ break case OPEN_PARENTHESES: @@ -135,38 +125,27 @@ module.exports = function tokenize(input) { } } while (escaped) - tokens.push([ - 'brackets', - css.slice(pos, next + 1), - line, - pos - offset, - line, - next - offset - ]) + tokens.push(['brackets', css.slice(pos, next + 1), offset, next + 1]) + offset = next + 1 pos = next } else { next = css.indexOf(')', pos + 1) content = css.slice(pos, next + 1) if (next === -1 || RE_BAD_BRACKET.test(content)) { - tokens.push(['(', '(', line, pos - offset]) + tokens.push(['(', '(', offset]) + offset++ } else { - tokens.push([ - 'brackets', - content, - line, - pos - offset, - line, - next - offset - ]) + tokens.push(['brackets', content, offset, next + 1]) + offset = next + 1 pos = next } } - break case CLOSE_PARENTHESES: - tokens.push([')', ')', line, pos - offset]) + tokens.push([')', ')', offset]) + offset++ break case SINGLE_QUOTE: @@ -185,28 +164,10 @@ module.exports = function tokenize(input) { } while (escaped) content = css.slice(pos, next + 1) - lines = content.split('\n') - last = lines.length - 1 - if (last > 0) { - nextLine = line + last - nextOffset = next - lines[last].length - } else { - nextLine = line - nextOffset = offset - } - - tokens.push([ - 'string', - css.slice(pos, next + 1), - line, - pos - offset, - nextLine, - next - nextOffset - ]) + tokens.push(['string', content, offset, offset + content.length]) - offset = nextOffset - line = nextLine + offset += content.length pos = next break @@ -218,14 +179,15 @@ module.exports = function tokenize(input) { } else { next = RE_AT_END.lastIndex - 2 } + tokens.push([ 'at-word', css.slice(pos, next + 1), - line, - pos - offset, - line, - next - offset + offset, + offset + (next - pos + 1) ]) + + offset += next - pos + 1 pos = next break @@ -233,8 +195,6 @@ module.exports = function tokenize(input) { next = pos escape = true - nextLine = line - while (css.charCodeAt(next + 1) === BACKSLASH) { next += 1 escape = !escape @@ -243,28 +203,21 @@ module.exports = function tokenize(input) { if (escape) { if (code === CR && css.charCodeAt(next + 2) === NEWLINE) { next += 2 - nextLine += 1 - nextOffset = next } else if (code === CR || code === NEWLINE || code === FEED) { next += 1 - nextLine += 1 - nextOffset = next } else { next += 1 } } + tokens.push([ 'word', css.slice(pos, next + 1), - line, - pos - offset, - line, - next - offset + offset, + offset + (next - pos + 1) ]) - if (nextLine !== line) { - line = nextLine - offset = nextOffset - } + + offset += next - pos + 1 pos = next break @@ -276,28 +229,10 @@ module.exports = function tokenize(input) { if (next === 0) unclosed('comment') content = css.slice(pos, next + 1) - lines = content.split('\n') - last = lines.length - 1 - if (last > 0) { - nextLine = line + last - nextOffset = next - lines[last].length - } else { - nextLine = line - nextOffset = offset - } - - tokens.push([ - 'comment', - content, - line, - pos - offset, - nextLine, - next - nextOffset - ]) + tokens.push(['comment', content, offset, offset + content.length]) - offset = nextOffset - line = nextLine + offset += content.length pos = next } else if (code === SLASH && n === SLASH) { RE_NEW_LINE.lastIndex = pos + 1 @@ -313,13 +248,12 @@ module.exports = function tokenize(input) { tokens.push([ 'comment', content, - line, - pos - offset, - line, - next - offset, + offset, + offset + content.length, 'inline' ]) + offset += content.length pos = next } else { RE_WORD_END.lastIndex = pos + 1 @@ -330,14 +264,11 @@ module.exports = function tokenize(input) { next = RE_WORD_END.lastIndex - 2 } - tokens.push([ - 'word', - css.slice(pos, next + 1), - line, - pos - offset, - line, - next - offset - ]) + content = css.slice(pos, next + 1) + + tokens.push(['word', content, offset, offset + content.length]) + + offset += content.length pos = next }