fix: support here documents in command substitutions (#255)

reubeno · Nov 3, 2024 · 9e16201 · 9e16201
1 parent 7dcbe7c
commit 9e16201
Show file tree

Hide file tree

Showing 6 changed files with 527 additions and 139 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/brush-core/src/shell.rs b/brush-core/src/shell.rs
@@ -439,7 +439,7 @@ impl Shell {
             brush_parser::Parser::new(&mut reader, &self.parser_options(), source_info);
 
         tracing::debug!(target: trace_categories::PARSE, "Parsing sourced file: {}", source_info.source);
-        let parse_result = parser.parse(false);
+        let parse_result = parser.parse();
 
         let mut other_positional_parameters = args.iter().map(|s| s.as_ref().to_owned()).collect();
         let mut other_shell_name = Some(source_info.source.clone());
@@ -1180,5 +1180,5 @@ fn parse_string_impl(
         brush_parser::Parser::new(&mut reader, &parser_options, &source_info);
 
     tracing::debug!(target: trace_categories::PARSE, "Parsing string as program...");
-    parser.parse(true)
+    parser.parse()
 }
diff --git a/brush-parser/Cargo.toml b/brush-parser/Cargo.toml
@@ -30,6 +30,7 @@ utf8-chars = "3.0.5"
 anyhow = "1.0.91"
 assert_matches = "1.5.0"
 criterion = { version = "0.5.1", features = ["html_reports"] }
+pretty_assertions = { version = "1.4.1", features = ["unstable"] }
 
 [target.'cfg(unix)'.dev-dependencies]
 pprof = { version = "0.13.0", features = ["criterion", "flamegraph"] }

diff --git a/brush-parser/src/parser.rs b/brush-parser/src/parser.rs
@@ -50,15 +50,7 @@ impl<R: std::io::BufRead> Parser<R> {
     }
 
     /// Parses the input into an abstract syntax tree (AST) of a shell program.
-    ///
-    /// # Arguments
-    ///
-    /// * `stop_on_unescaped_newline` - Whether or not to stop parsing when an unescaped newline is
-    ///   encountered.
-    pub fn parse(
-        &mut self,
-        stop_on_unescaped_newline: bool,
-    ) -> Result<ast::Program, error::ParseError> {
+    pub fn parse(&mut self) -> Result<ast::Program, error::ParseError> {
         //
         // References:
         //   * https://www.gnu.org/software/bash/manual/bash.html#Shell-Syntax
@@ -90,18 +82,13 @@ impl<R: std::io::BufRead> Parser<R> {
                 }
             };
 
+            let reason = result.reason;
             if let Some(token) = result.token {
-                tracing::debug!(target: "tokenize", "TOKEN {}: {:?}", tokens.len(), token);
+                tracing::debug!(target: "tokenize", "TOKEN {}: {:?} {reason:?}", tokens.len(), token);
                 tokens.push(token);
             }
 
-            if matches!(result.reason, TokenEndReason::EndOfInput) {
-                break;
-            }
-
-            if stop_on_unescaped_newline
-                && matches!(result.reason, TokenEndReason::UnescapedNewLine)
-            {
+            if matches!(reason, TokenEndReason::EndOfInput) {
                 break;
             }
         }
@@ -646,27 +633,27 @@ peg::parser! {
         rule filename() -> &'input Token =
             word()
 
-        rule io_here() -> ast::IoHereDocument =
-           specific_operator("<<-") here_end:here_end() doc:[_] {
-                let requires_expansion = !here_end.to_str().contains(['\'', '"', '\\']);
+        pub(crate) rule io_here() -> ast::IoHereDocument =
+           specific_operator("<<-") here_tag:here_tag() doc:[_] closing_tag:here_tag() {
+                let requires_expansion = !here_tag.to_str().contains(['\'', '"', '\\']);
                 ast::IoHereDocument {
                     remove_tabs: true,
                     requires_expansion,
-                    here_end: ast::Word::from(here_end),
+                    here_end: ast::Word::from(here_tag),
                     doc: ast::Word::from(doc)
                 }
             } /
-            specific_operator("<<") here_end:here_end() doc:[_] {
-                let requires_expansion = !here_end.to_str().contains(['\'', '"', '\\']);
+            specific_operator("<<") here_tag:here_tag() doc:[_] closing_tag:here_tag() {
+                let requires_expansion = !here_tag.to_str().contains(['\'', '"', '\\']);
                 ast::IoHereDocument {
                     remove_tabs: false,
                     requires_expansion,
-                    here_end: ast::Word::from(here_end),
+                    here_end: ast::Word::from(here_tag),
                     doc: ast::Word::from(doc)
                 }
             }
 
-        rule here_end() -> &'input Token =
+        rule here_tag() -> &'input Token =
             word()
 
         rule process_substitution() -> (ast::ProcessSubstitutionKind, ast::SubshellCommand) =