Skip to content
This repository has been archived by the owner on Jun 4, 2024. It is now read-only.

Commit

Permalink
Allow preservation of existing newlines in comment-delimited blocks.
Browse files Browse the repository at this point in the history 
This allows the user to instruct the formatter to preserve newlines in a block of code by adding a directive to a comment. The original behaviour can be restored with another comment.

This feature is currently experimental and will not work with all types of formatting.

Fixes #11
  • Loading branch information
@hovinen
hovinen committed Feb 1, 2021
1 parent ccc3461 commit 89f8581
Show file tree
Hide file tree
Showing 6 changed files with 165 additions and 13 deletions.
17 changes: 17 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,23 @@ documentation.
Currently, it is only possible to configure the formatter when it is embedded as a library. See
[KotlinFormatter](formatter/src/main/kotlin/org/kotlin/formatter/KotlinFormatter.kt) for details.

## Preserving newlines

The formatter contains experimental support for preserving newlines in blocks of code when
requested. The formatter normally aggressively removes syntactically unnecessary newlines when no
line break is needed to maintain the 100 column limit. This can cause some DSLs to be rendered in a
less readable manner. To instruct the formatter to preserve newlines, add the following comment:

```kotlin
// ktformat: start-preserve-newlines
```

To restore default behaviour, add the following comment:

```kotlin
// ktformat: end-preserve-newlines
```

## Comparison with other tools

### Kotlin autoformatter vs. [ktlint](https://github.com/pinterest/ktlint)
Expand Down
27 changes: 25 additions & 2 deletions formatter/src/main/kotlin/org/kotlin/formatter/Token.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,10 @@ object ClosingForcedBreakToken : Token()
* @property whitespaceLength the number of spaces of whitespace to be output by this token if no
* line break is introduced; normally either zero or one
*/
data class SynchronizedBreakToken(internal val whitespaceLength: Int) : Token()
data class SynchronizedBreakToken(
internal val whitespaceLength: Int,
internal val content: String = " ".repeat(whitespaceLength)
) : Token()

/**
* A directive to add a closing line break only if the containing block does not fit on one line.
Expand All @@ -127,7 +130,10 @@ data class SynchronizedBreakToken(internal val whitespaceLength: Int) : Token()
* @property whitespaceLength the number of spaces of whitespace to be output by this token if no
* line break is introduced; normally either zero or one
*/
data class ClosingSynchronizedBreakToken(internal val whitespaceLength: Int) : Token()
data class ClosingSynchronizedBreakToken(
internal val whitespaceLength: Int,
internal val content: String = " ".repeat(whitespaceLength)
) : Token()

/**
* A directive to add line break without indent only if the containing block does not fit on one
Expand Down Expand Up @@ -196,3 +202,20 @@ object MarkerToken : Token()
* printer.
*/
object BlockFromMarkerToken : Token()

/**
* A directive that, from this token on, newlines in the original source code should be preserved
* when they would ordinarily be removed as redundant by the autoformatter.
*
* If this token appears while the formatter is already in the preserve newlines mode, it has no
* effect.
*/
object BeginPreserveNewlinesToken : Token()

/**
* A directive that, from this point on, newlines in the original code should be treated normally
* and removed when they are redundant.
*
* If this token appears while the formatter is in regular mode, it has no effect.
*/
object EndPreserveNewlinesToken : Token()
26 changes: 24 additions & 2 deletions formatter/src/main/kotlin/org/kotlin/formatter/output/TokenPreprocessor.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,13 @@ package org.kotlin.formatter.output

import java.lang.Integer.min
import java.util.Stack
import org.kotlin.formatter.BeginPreserveNewlinesToken
import org.kotlin.formatter.BeginToken
import org.kotlin.formatter.BeginWeakToken
import org.kotlin.formatter.BlockFromMarkerToken
import org.kotlin.formatter.ClosingForcedBreakToken
import org.kotlin.formatter.ClosingSynchronizedBreakToken
import org.kotlin.formatter.EndPreserveNewlinesToken
import org.kotlin.formatter.EndToken
import org.kotlin.formatter.ForcedBreakToken
import org.kotlin.formatter.KDocContentToken
Expand Down Expand Up @@ -34,6 +36,7 @@ import org.kotlin.formatter.WhitespaceToken
*/
class TokenPreprocessor {
private val resultStack = Stack<StackElement>()
private var preserveNewlines = false

/**
* Returns a list of [Token] with the lengths of all [WhitespaceToken] and [BeginToken]
Expand Down Expand Up @@ -122,7 +125,13 @@ class TokenPreprocessor {
}
is SynchronizedBreakToken -> {
val lastToken = lastToken()
if (!(lastToken is ForcedBreakToken || lastToken is ClosingForcedBreakToken)) {
if (preserveNewlines && token.content.contains('\n')) {
resultStack.peek()
.tokens
.add(ForcedBreakToken(count = token.content.count { it == '\n' }))
} else if (!(lastToken is ForcedBreakToken || lastToken is
ClosingForcedBreakToken)
) {
resultStack.peek().tokens.add(token)
}
}
Expand All @@ -132,10 +141,18 @@ class TokenPreprocessor {
val lastElementTokens = lastElementWithTokens()?.tokens
lastElementTokens?.removeAt(lastElementTokens.size - 1)
resultStack.peek().tokens.add(ClosingForcedBreakToken)
} else if (preserveNewlines && token.content.contains('\n')) {
resultStack.peek().tokens.add(ClosingForcedBreakToken)
} else if (lastToken !is ClosingForcedBreakToken) {
resultStack.peek().tokens.add(token)
}
}
is BeginPreserveNewlinesToken -> {
preserveNewlines = true
}
is EndPreserveNewlinesToken -> {
preserveNewlines = false
}
else -> resultStack.peek().tokens.add(token)
}
}
Expand Down Expand Up @@ -231,7 +248,9 @@ class TokenPreprocessor {
private fun appendTokensInWhitespaceElement(element: WhitespaceStackElement) {
val firstToken = element.tokens.firstOrNull()
val tokens = resultStack.peek().tokens
if (followingBlockIsCommentWithNewlines(firstToken, element)) {
if (followingBlockIsCommentWithNewlines(firstToken, element) ||
hasNewlineToBePreserved(element)
) {
tokens.add(ForcedBreakToken(count = min(element.content.countNewlines(), 2)))
} else {
tokens.add(WhitespaceToken(length = element.totalLength, content = element.content))
Expand All @@ -244,6 +263,9 @@ class TokenPreprocessor {
element: WhitespaceStackElement
) = firstToken is BeginToken && firstToken.state.isComment && element.content.contains('\n')

private fun hasNewlineToBePreserved(element: WhitespaceStackElement): Boolean =
preserveNewlines && element.content.contains('\n')

private fun String.countNewlines(): Int = count { it == '\n' }

private fun appendTokensInLiteralWhitespaceElement(element: LiteralWhitespaceStackElement) {
Expand Down
26 changes: 18 additions & 8 deletions formatter/src/main/kotlin/org/kotlin/formatter/scanning/FunctionLiteralScanner.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ internal class FunctionLiteralScanner(private val kotlinScanner: KotlinScanner)
nodeOfType(KtTokens.LBRACE) thenMapToTokens {
listOf(BeginWeakToken(), BeginToken(State.CODE), LeafNodeToken("{"))
}
possibleWhitespace()
zeroOrOne {
possibleWhitespace()
exactlyOne {
nodeOfType(KtNodeTypes.VALUE_PARAMETER_LIST) thenMapToTokens { nodes ->
listOf(WhitespaceToken(" "))
Expand All @@ -41,13 +41,23 @@ internal class FunctionLiteralScanner(private val kotlinScanner: KotlinScanner)
zeroOrOne { emptyBlock() thenMapToTokens { listOf(nonBreakingSpaceToken()) } }
} thenMapTokens { it.plus(EndToken) }
zeroOrOne {
nodeOfType(KtNodeTypes.BLOCK) thenMapToTokens { nodes ->
val tokens = kotlinScanner.scanNodes(nodes, ScannerState.BLOCK)
if (tokens.isNotEmpty()) {
listOf(SynchronizedBreakToken(whitespaceLength = 1)).plus(tokens)
.plus(ClosingSynchronizedBreakToken(whitespaceLength = 1))
} else {
listOf()
either {
possibleWhitespace()
emptyBlock()
possibleWhitespace()
} or {
possibleWhitespace() thenMapToTokens { nodes ->
val content = nodes.firstOrNull()?.text ?: " "
listOf(SynchronizedBreakToken(content = content, whitespaceLength = 1))
}
nodeOfType(KtNodeTypes.BLOCK) thenMapToTokens { nodes ->
kotlinScanner.scanNodes(nodes, ScannerState.BLOCK)
}
possibleWhitespace() thenMapToTokens { nodes ->
val content = nodes.firstOrNull()?.text ?: " "
listOf(
ClosingSynchronizedBreakToken(content = content, whitespaceLength = 1)
)
}
}
}
Expand Down
14 changes: 13 additions & 1 deletion formatter/src/main/kotlin/org/kotlin/formatter/scanning/LeafScanner.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ import org.jetbrains.kotlin.com.intellij.lang.ASTNode
import org.jetbrains.kotlin.com.intellij.psi.impl.source.tree.LeafPsiElement
import org.jetbrains.kotlin.kdoc.lexer.KDocTokens
import org.jetbrains.kotlin.lexer.KtTokens
import org.kotlin.formatter.BeginPreserveNewlinesToken
import org.kotlin.formatter.BeginToken
import org.kotlin.formatter.EndPreserveNewlinesToken
import org.kotlin.formatter.EndToken
import org.kotlin.formatter.LeafNodeToken
import org.kotlin.formatter.LiteralWhitespaceToken
Expand All @@ -28,7 +30,8 @@ internal class LeafScanner {
internal fun scanCommentNode(node: ASTNode): List<Token> =
when (node.elementType) {
KtTokens.EOL_COMMENT ->
listOf(BeginToken(stateBasedOnCommentContent(node.text)))
preserveNewlinesToken(node.text)
.plus(BeginToken(stateBasedOnCommentContent(node.text)))
.plus(LeafScanner().tokenizeString(node.text) { LiteralWhitespaceToken(it) })
.plus(EndToken)
KtTokens.BLOCK_COMMENT ->
Expand All @@ -42,6 +45,15 @@ internal class LeafScanner {
else -> throw IllegalArgumentException("Invalid node for comment $node")
}

private fun preserveNewlinesToken(text: String): List<Token> =
if (text.contains("ktformat: start-preserve-newlines")) {
listOf(BeginPreserveNewlinesToken)
} else if (text.contains("ktformat: end-preserve-newlines")) {
listOf(EndPreserveNewlinesToken)
} else {
listOf()
}

private fun tokenizeNodeContentInBlockComment(node: ASTNode): List<Token> {
val text =
node.text.removePrefix("/*").removeSuffix("*/").replace(Regex("\n[ ]+\\*"), "\n").trim()
Expand Down
68 changes: 68 additions & 0 deletions formatter/src/test/kotlin/org/kotlin/formatter/KotlinFormatterTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5340,6 +5340,74 @@ class KotlinFormatterTest {
assertThat(result).isEqualTo("package org.kotlin.formatter")
}

@Test
fun `disables collapse of newlines when requested`() {
val result =
KotlinFormatter()
.format(
"""
// ktformat: start-preserve-newlines
aFunction {
anotherFunction()
}
""".trimIndent()
)

assertThat(result)
.isEqualTo(
"""
// ktformat: start-preserve-newlines
aFunction {
anotherFunction()
}
""".trimIndent()
)
}

@Test
fun `does not insert unnecessary newlines when newlines are being preserved`() {
val result =
KotlinFormatter()
.format(
"""
// ktformat: start-preserve-newlines
aFunction { anotherFunction() }
""".trimIndent()
)

assertThat(result)
.isEqualTo(
"""
// ktformat: start-preserve-newlines
aFunction { anotherFunction() }
""".trimIndent()
)
}

@Test
fun `reenables collapse of newlines when requested`() {
val result =
KotlinFormatter()
.format(
"""
// ktformat: start-preserve-newlines
// ktformat: end-preserve-newlines
aFunction {
anotherFunction()
}
""".trimIndent()
)

assertThat(result)
.isEqualTo(
"""
// ktformat: start-preserve-newlines
// ktformat: end-preserve-newlines
aFunction { anotherFunction() }
""".trimIndent()
)
}

@Nested
inner class FormatFile {
private val originalOut = System.out
Expand Down

0 comments on commit 89f8581

Please sign in to comment.