Skip to content

Commit

Permalink
Merge pull request #320 from VirtusLab/issue-60
Browse files Browse the repository at this point in the history 
fixed parsing of folded blocks, hopefully, yaml is insane
  • Loading branch information
@lbialy
lbialy authored Jul 21, 2024
2 parents 5a3f9c8 + 40f0cf0 commit a3aa5b9
Show file tree
Hide file tree
Showing 2 changed files with 300 additions and 8 deletions.
66 changes: 58 additions & 8 deletions core/shared/src/main/scala/org/virtuslab/yaml/internal/load/reader/Tokenizer.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -472,7 +472,10 @@ private class StringTokenizer(str: String) extends Tokenizer {
}

@tailrec
def readFolded(): String =
def readFolded(
prevCharWasNewline: Boolean = false,
thisLineIsIndented: Boolean = false
): String = {
in.peek() match {
case Reader.nullTerminator => sb.result()
case _ if in.isNewline =>
Expand All @@ -483,22 +486,69 @@ private class StringTokenizer(str: String) extends Tokenizer {
in.skipCharacter()
skipUntilNextIndent(foldedIndent)
}
readFolded()

if (in.column != foldedIndent || in.peek() == Reader.nullTerminator) {
if (chompingIndicator == BlockChompingIndicator.Keep) sb.append("\n")
sb.result()
} else {
readFolded(prevCharWasNewline = true)
}
} else {
in.skipCharacter()
in.skipCharacter() // skip newline

skipUntilNextIndent(foldedIndent)

if (in.column != foldedIndent || in.peek() == Reader.nullTerminator) {
sb.append("\n")
sb.result()
chompingIndicator match {
case Keep => // if keep, strip all trailing newlines and spaces but count them and append counted amount of newlines
var count = 1
var lastChar = sb.apply(sb.length - 1)
while (lastChar == '\n' || lastChar == ' ') {
sb.deleteCharAt(sb.length - 1)
lastChar = sb.apply(sb.length - 1)
count += 1
}
sb.append("\n" * count)
case Strip => // if strip, strip all trailing newlines and spaces
var lastChar = sb.apply(sb.length - 1)
while (lastChar == '\n' || lastChar == ' ') {
sb.deleteCharAt(sb.length - 1)
lastChar = sb.apply(sb.length - 1)
}
case Clip => // if clip, strip all trailing newlines and spaces and append a single newline
var lastChar = sb.apply(sb.length - 1)
while (lastChar == '\n' || lastChar == ' ') {
sb.deleteCharAt(sb.length - 1)
lastChar = sb.apply(sb.length - 1)
}
sb.append('\n')
}

sb.result() // final result
} else {
sb.append(" ")
readFolded()
if (prevCharWasNewline || thisLineIsIndented) {
sb.append("\n")
} else {
sb.append(" ")
}

readFolded(prevCharWasNewline = true)
}
}

case ' ' if in.column == foldedIndent => // beginning of a line that is indented
if (prevCharWasNewline) { // we are at the beginning of a line that is indented
sb.update(sb.size - 1, '\n') // replace last space with a newline
}

sb.append(in.read())
readFolded(thisLineIsIndented = true)

case char =>
sb.append(in.read())
readFolded()
readFolded(thisLineIsIndented = thisLineIsIndented)
}
}

val scalar = readFolded()
val chompedScalar = chompingIndicator.removeBlankLinesAtEnd(scalar)
Expand Down
242 changes: 242 additions & 0 deletions core/shared/src/test/scala/org/virtuslab/yaml/parser/ScalarSuite.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -326,6 +326,36 @@ class ScalarSpec extends BaseYamlSuite {
assertEquals(yaml.events, Right(expectedEvents))
}

test("issue 60 - parsing final break style in folded scalar") {
val yaml = """|certificate: >+
| -----BEGIN CERTIFICATE-----
| 0MTk0MVoXDenkKThvP7IS9q
| +Dzv5hG392KWh5f8xJNs4LbZyl901MeReiLrPH3w=
| -----END CERTIFICATE----
|
|
|kind: v1""".stripMargin

val expectedEvents = List(
StreamStart,
DocumentStart(),
MappingStart(),
Scalar("certificate"),
Scalar(
// should preserve line breaks and not consume next mapping (kind: v1)
"-----BEGIN CERTIFICATE----- 0MTk0MVoXDenkKThvP7IS9q +Dzv5hG392KWh5f8xJNs4LbZyl901MeReiLrPH3w= -----END CERTIFICATE----\\n\\n\\n",
ScalarStyle.Folded
),
Scalar("kind"),
Scalar("v1"),
MappingEnd,
DocumentEnd(),
StreamEnd
)

assertEquals(yaml.events, Right(expectedEvents))
}

test("folded indent scalar") {
val yaml = s"""|--- >
|line1
Expand Down Expand Up @@ -508,4 +538,216 @@ class ScalarSpec extends BaseYamlSuite {
}
assertEquals(isStringType, Seq.fill(isStringType.length)(true))
}

// from https://yaml-multiline.info/

/**
* example: >\n
* ··Several lines of text,\n
* ··with some "quotes" of various 'types',\n
* ··and also a blank line:\n
* ··\n
* ··and two blank lines:\n
* ··\n
* ··\n
* ··and some text with\n
* ····extra indentation\n
* ··on the next line,\n
* ··plus another line at the end.\n
* ··\n
* ··\n
*/
def yamlInput(indicator: String) = s"""example: $indicator
| Several lines of text,
| with some "quotes" of various 'types',
| and also a blank line:
|
| and two blank lines:
|
|
| and some text with
| extra indentation
| on the next line,
| plus another line at the end.
|
|
|""".stripMargin

test("block scalars: folded style with clip indicator") {
val yaml = yamlInput(">")

// expected:
// Several lines of text, with some "quotes" of various 'types', and also a blank line:\n
// and two blank lines:\n\n
// and some text with\n
// extra indentation\n
// on the next line, plus another line at the end.\n

val expectedEvents = List(
StreamStart,
DocumentStart(),
MappingStart(),
Scalar("example"),
Scalar(
"Several lines of text, with some \"quotes\" of various 'types', and also a blank line:\\nand two blank lines:\\n\\nand some text with\\n extra indentation\\non the next line, plus another line at the end.\\n",
ScalarStyle.Folded
),
MappingEnd,
DocumentEnd(),
StreamEnd
)
assertEquals(yaml.events, Right(expectedEvents))
}

test("block scalars: folded style with strip indicator") {
val yaml = yamlInput(">-")

// expected:
// Several lines of text, with some "quotes" of various 'types', and also a blank line:\n
// and two blank lines:\n\n
// and some text with\n
// extra indentation\n
// on the next line, plus another line at the end.

val expectedEvents = List(
StreamStart,
DocumentStart(),
MappingStart(),
Scalar("example"),
Scalar(
"Several lines of text, with some \"quotes\" of various 'types', and also a blank line:\\nand two blank lines:\\n\\nand some text with\\n extra indentation\\non the next line, plus another line at the end.",
ScalarStyle.Folded
),
MappingEnd,
DocumentEnd(),
StreamEnd
)
assertEquals(yaml.events, Right(expectedEvents))
}

test("block scalars: folded style with keep indicator") {
val yaml = yamlInput(">+")

// expected:
// Several lines of text, with some "quotes" of various 'types', and also a blank line:\n
// and two blank lines:\n\n
// and some text with\n
// extra indentation\n
// on the next line, plus another line at the end.\n
// \n
// \n

val expectedEvents = List(
StreamStart,
DocumentStart(),
MappingStart(),
Scalar("example"),
Scalar(
"Several lines of text, with some \"quotes\" of various 'types', and also a blank line:\\nand two blank lines:\\n\\nand some text with\\n extra indentation\\non the next line, plus another line at the end.\\n\\n\\n",
ScalarStyle.Folded
),
MappingEnd,
DocumentEnd(),
StreamEnd
)
assertEquals(yaml.events, Right(expectedEvents))
}

test("block scalars: literal style with clip indicator") {
val yaml = yamlInput("|")

// expected:
// Several lines of text,\n
// with some "quotes" of various 'types',\n
// and also a blank line:\n
// \n
// and two blank lines:\n
// \n
// \n
// and some text with\n
// extra indentation\n
// on the next line,\n
// plus another line at the end.\n

val expectedEvents = List(
StreamStart,
DocumentStart(),
MappingStart(),
Scalar("example"),
Scalar(
"Several lines of text,\nwith some \"quotes\" of various 'types',\nand also a blank line:\n\nand two blank lines:\n\n\nand some text with\n extra indentation\non the next line,\nplus another line at the end.\n",
ScalarStyle.Literal
),
MappingEnd,
DocumentEnd(),
StreamEnd
)
assertEquals(yaml.events, Right(expectedEvents))
}

test("block scalars: literal style with strip indicator") {
val yaml = yamlInput("|-")

// expected:
// Several lines of text,\n
// with some "quotes" of various 'types',\n
// and also a blank line:\n
// \n
// and two blank lines:\n
// \n
// \n
// and some text with\n
// extra indentation\n
// on the next line,\n
// plus another line at the end.

val expectedEvents = List(
StreamStart,
DocumentStart(),
MappingStart(),
Scalar("example"),
Scalar(
"Several lines of text,\nwith some \"quotes\" of various 'types',\nand also a blank line:\n\nand two blank lines:\n\n\nand some text with\n extra indentation\non the next line,\nplus another line at the end.",
ScalarStyle.Literal
),
MappingEnd,
DocumentEnd(),
StreamEnd
)
assertEquals(yaml.events, Right(expectedEvents))
}

test("block scalars: literal style with keep indicator") {
val yaml = yamlInput("|+")

// expected:
// Several lines of text,\n
// with some "quotes" of various 'types',\n
// and also a blank line:\n
// \n
// and two blank lines:\n
// \n
// \n
// and some text with\n
// extra indentation\n
// on the next line,\n
// plus another line at the end.\n
// \n
// \n

val expectedEvents = List(
StreamStart,
DocumentStart(),
MappingStart(),
Scalar("example"),
Scalar(
"Several lines of text,\nwith some \"quotes\" of various 'types',\nand also a blank line:\n\nand two blank lines:\n\n\nand some text with\n extra indentation\non the next line,\nplus another line at the end.\n\n\n",
ScalarStyle.Literal
),
MappingEnd,
DocumentEnd(),
StreamEnd
)
assertEquals(yaml.events, Right(expectedEvents))
}
}

0 comments on commit a3aa5b9

Please sign in to comment.