Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Implement raw string literals #3556

Merged
merged 10 commits into from
Nov 28, 2023
Merged

feat: Implement raw string literals #3556

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
bbd9193
feat: Implement raw string literals
pczarn Nov 23, 2023
fb553d7
Add documentation for raw strings
pczarn Nov 24, 2023
1d5eaef
Add tests for raw strings
pczarn Nov 24, 2023
b212b26
Changes for raw strings
pczarn Nov 24, 2023
71a011b
Fixes for raw strings tests
pczarn Nov 24, 2023
4bdffc3
Add test for raw strings
pczarn Nov 24, 2023
236d71c
Add test for nested raw strings
pczarn Nov 24, 2023
825d5a9
Fix logic bug for raw strings
pczarn Nov 24, 2023
433f7bd
Merge branch 'master' into raw-str
kevaundray Nov 28, 2023
498171e
Update docs/docs/language_concepts/data_types/03_strings.md
jfecher Nov 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions compiler/noirc_frontend/src/lexer/lexer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
position: Position,
done: bool,
skip_comments: bool,
skip_whitespaces: bool,

Check warning on line 19 in compiler/noirc_frontend/src/lexer/lexer.rs

View workflow job for this annotation

GitHub Actions / Spellcheck / Spellcheck 

Unknown word (whitespaces)
}

pub type SpannedTokenResult = Result<SpannedToken, LexerErrorKind>;
Expand All @@ -43,7 +43,7 @@
position: 0,
done: false,
skip_comments: true,
skip_whitespaces: true,

Check warning on line 46 in compiler/noirc_frontend/src/lexer/lexer.rs

View workflow job for this annotation

GitHub Actions / Spellcheck / Spellcheck 

Unknown word (whitespaces)
}
}

Expand All @@ -52,8 +52,8 @@
self
}

pub fn skip_whitespaces(mut self, flag: bool) -> Self {

Check warning on line 55 in compiler/noirc_frontend/src/lexer/lexer.rs

View workflow job for this annotation

GitHub Actions / Spellcheck / Spellcheck 

Unknown word (whitespaces)
self.skip_whitespaces = flag;

Check warning on line 56 in compiler/noirc_frontend/src/lexer/lexer.rs

View workflow job for this annotation

GitHub Actions / Spellcheck / Spellcheck 

Unknown word (whitespaces)
self
}

Expand Down Expand Up @@ -96,7 +96,7 @@
match self.next_char() {
Some(x) if x.is_whitespace() => {
let spanned = self.eat_whitespace(x);
if self.skip_whitespaces {

Check warning on line 99 in compiler/noirc_frontend/src/lexer/lexer.rs

View workflow job for this annotation

GitHub Actions / Spellcheck / Spellcheck 

Unknown word (whitespaces)
self.next_token()
} else {
Ok(spanned)
Expand Down Expand Up @@ -465,13 +465,15 @@
// Problem: we commit to eating raw strings once we see one or two characters.
// This is unclean, but likely ok in all practical cases, and works with existing
// `Lexer` methods.
let peek2 = self.peek2_char();
if (self.peek_char_is('#') && (peek2 == Some('#') || peek2 == Some('"')))
|| self.peek_char_is('"')
{
self.eat_raw_string()
} else {
self.eat_alpha_numeric('r')
let peek1 = self.peek_char().unwrap_or('X');
let peek2 = self.peek2_char().unwrap_or('X');
match (peek1, peek2) {
('#', '#') | ('#', '"') | ('"', _) => {
self.eat_raw_string()
}
_ => {
self.eat_alpha_numeric('r')
}
}
}

Expand Down
46 changes: 44 additions & 2 deletions compiler/noirc_frontend/src/parser/parser.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2552,7 +2552,7 @@
}

#[test]
fn parse_raw_string() {
fn parse_raw_string_expr() {
let cases = vec![
Case { source: r##" r"foo" "##, expect: r##"r"foo""##, errors: 0 },
Case { source: r##" r#"foo"# "##, expect: r##"r#"foo"#"##, errors: 0 },
Expand All @@ -2561,12 +2561,54 @@
Case { source: r##" r#"\"# "##, expect: r##"r#"\"#"##, errors: 0 },
Case { source: r##" r#"\\"# "##, expect: r##"r#"\\"#"##, errors: 0 },
Case { source: r##" r#"\\\"# "##, expect: r##"r#"\\\"#"##, errors: 0 },
// escape sequence
Case { source: r##" r#"\t\n\\t\\n\\\t\\\n\\\\"# "##, expect: r##"r#"\t\n\\t\\n\\\t\\\n\\\\"#"##, errors: 0 },
Case { source: r##" r#"\\\\\\\\"# "##, expect: r##"r#"\\\\\\\\"#"##, errors: 0 },
// mismatch - errors:
Case { source: r###" r#"foo"## "###, expect: r###"r#"foo"#"###, errors: 1 },
Case { source: r###" r##"foo"# "###, expect: "(none)", errors: 2 },
// mismatch: short:
Case { source: r###" r"foo"# "###, expect: r###"r"foo""###, errors: 1 },
Case { source: r###" r#"foo" "###, expect: "(none)", errors: 2 },
// empty string
Case { source: r####"r"""####, expect: r####"r"""####, errors: 0 },
Case { source: r####"r###""###"####, expect: r####"r###""###"####, errors: 0 },
// miscelanneous

Check warning on line 2576 in compiler/noirc_frontend/src/parser/parser.rs

View workflow job for this annotation

GitHub Actions / Spellcheck / Spellcheck 

Unknown word (miscelanneous)
TomAFrench marked this conversation as resolved.
Show resolved Hide resolved
Case { source: r###" r#\"foo\"# "###, expect: "plain::r", errors: 2 },
Case { source: r###" r\"foo\" "###, expect: "plain::r", errors: 1 },
Case { source: r###" r##"foo"# "###, expect: "(none)", errors: 2 },
// missing 'r' letter
Case { source: r###" ##"foo"# "###, expect: r#""foo""#, errors: 2 },
Case { source: r###" #"foo" "###, expect: "plain::foo", errors: 2 },
// whitespace
Case { source: r###" r #"foo"# "###, expect: "plain::r", errors: 2 },
Case { source: r###" r# "foo"# "###, expect: "plain::r", errors: 3 },
Case { source: r###" r#"foo" # "###, expect: "(none)", errors: 2 },
// after identifier
Case { source: r###" bar#"foo"# "###, expect: "plain::bar", errors: 2 },
];

check_cases_with_errors(&cases[..], expression());
check_cases_with_errors(&cases[..], literal());
}

#[test]
fn parse_raw_string_lit() {
let lit_cases = vec![
Case { source: r##" r"foo" "##, expect: r##"r"foo""##, errors: 0 },
Case { source: r##" r#"foo"# "##, expect: r##"r#"foo"#"##, errors: 0 },
// backslash
Case { source: r##" r"\\" "##, expect: r##"r"\\""##, errors: 0 },
Case { source: r##" r#"\"# "##, expect: r##"r#"\"#"##, errors: 0 },
Case { source: r##" r#"\\"# "##, expect: r##"r#"\\"#"##, errors: 0 },
Case { source: r##" r#"\\\"# "##, expect: r##"r#"\\\"#"##, errors: 0 },
// escape sequence
Case { source: r##" r#"\t\n\\t\\n\\\t\\\n\\\\"# "##, expect: r##"r#"\t\n\\t\\n\\\t\\\n\\\\"#"##, errors: 0 },
Case { source: r##" r#"\\\\\\\\"# "##, expect: r##"r#"\\\\\\\\"#"##, errors: 0 },
// mismatch - errors:
Case { source: r###" r#"foo"## "###, expect: r###"r#"foo"#"###, errors: 1 },
Case { source: r###" r##"foo"# "###, expect: "(none)", errors: 2 },
];

check_cases_with_errors(&lit_cases[..], literal());
}
}
Loading