Fix multi-point unicode glyphs in yuck lexer
This commit is contained in:
parent
de8c79bc7d
commit
e698fd315e
5 changed files with 54 additions and 4 deletions
|
@ -97,4 +97,5 @@ impl<'input> Iterator for Lexer<'input> {
|
||||||
fn test_simplexpr_lexer() {
|
fn test_simplexpr_lexer() {
|
||||||
use itertools::Itertools;
|
use itertools::Itertools;
|
||||||
insta::assert_debug_snapshot!(Lexer::new(0, 0, r#"(foo + - "()" "a\"b" true false [] 12.2)"#).collect_vec());
|
insta::assert_debug_snapshot!(Lexer::new(0, 0, r#"(foo + - "()" "a\"b" true false [] 12.2)"#).collect_vec());
|
||||||
|
insta::assert_debug_snapshot!(Lexer::new(0, 0, r#"" " + music"#).collect_vec());
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,32 @@
|
||||||
|
---
|
||||||
|
source: crates/simplexpr/src/parser/lexer.rs
|
||||||
|
expression: "Lexer::new(0, 0, r#\"\" \" + music\"#).collect_vec()"
|
||||||
|
|
||||||
|
---
|
||||||
|
[
|
||||||
|
Ok(
|
||||||
|
(
|
||||||
|
0,
|
||||||
|
StrLit(
|
||||||
|
"\"\u{f001} \"",
|
||||||
|
),
|
||||||
|
8,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
Ok(
|
||||||
|
(
|
||||||
|
9,
|
||||||
|
Plus,
|
||||||
|
10,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
Ok(
|
||||||
|
(
|
||||||
|
11,
|
||||||
|
Ident(
|
||||||
|
"music",
|
||||||
|
),
|
||||||
|
16,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
]
|
|
@ -101,13 +101,15 @@ impl Iterator for Lexer {
|
||||||
let string = &self.source[self.pos..];
|
let string = &self.source[self.pos..];
|
||||||
|
|
||||||
if string.starts_with('{') {
|
if string.starts_with('{') {
|
||||||
// self.pos += 1;
|
|
||||||
let expr_start = self.pos;
|
let expr_start = self.pos;
|
||||||
let mut in_string = false;
|
let mut in_string = false;
|
||||||
loop {
|
loop {
|
||||||
if self.pos >= self.source.len() {
|
if self.pos >= self.source.len() {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
while !self.source.is_char_boundary(self.pos) {
|
||||||
|
self.pos += 1;
|
||||||
|
}
|
||||||
let string = &self.source[self.pos..];
|
let string = &self.source[self.pos..];
|
||||||
|
|
||||||
if string.starts_with('}') && !in_string {
|
if string.starts_with('}') && !in_string {
|
||||||
|
@ -137,7 +139,6 @@ impl Iterator for Lexer {
|
||||||
Some(x) => x,
|
Some(x) => x,
|
||||||
None => {
|
None => {
|
||||||
self.failed = true;
|
self.failed = true;
|
||||||
dbg!(&string);
|
|
||||||
return Some(Err(parse_error::ParseError::LexicalError(Span(self.pos, self.pos, self.file_id))));
|
return Some(Err(parse_error::ParseError::LexicalError(Span(self.pos, self.pos, self.file_id))));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
@ -163,4 +164,5 @@ fn test_yuck_lexer() {
|
||||||
insta::assert_debug_snapshot!(Lexer::new(0, r#"(foo + - "text" )"#.to_string()).collect_vec());
|
insta::assert_debug_snapshot!(Lexer::new(0, r#"(foo + - "text" )"#.to_string()).collect_vec());
|
||||||
insta::assert_debug_snapshot!(Lexer::new(0, r#"{ bla "} \" }" " \" "}"#.to_string()).collect_vec());
|
insta::assert_debug_snapshot!(Lexer::new(0, r#"{ bla "} \" }" " \" "}"#.to_string()).collect_vec());
|
||||||
insta::assert_debug_snapshot!(Lexer::new(0, r#""< \" >""#.to_string()).collect_vec());
|
insta::assert_debug_snapshot!(Lexer::new(0, r#""< \" >""#.to_string()).collect_vec());
|
||||||
|
insta::assert_debug_snapshot!(Lexer::new(0, r#"{ " " + music}"#.to_string()).collect_vec());
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,16 @@
|
||||||
|
---
|
||||||
|
source: crates/yuck/src/parser/lexer.rs
|
||||||
|
expression: "Lexer::new(0, r#\"{ \" \" + music}\"#.to_string()).collect_vec()"
|
||||||
|
|
||||||
|
---
|
||||||
|
[
|
||||||
|
Ok(
|
||||||
|
(
|
||||||
|
0,
|
||||||
|
SimplExpr(
|
||||||
|
"{ \"\u{f001} \" + music}",
|
||||||
|
),
|
||||||
|
18,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
]
|
|
@ -24,8 +24,7 @@
|
||||||
|
|
||||||
(defwidget music []
|
(defwidget music []
|
||||||
(box :class "music" :orientation "h" :space-evenly false :halign "center"
|
(box :class "music" :orientation "h" :space-evenly false :halign "center"
|
||||||
;{ " " + music}))
|
{ " " + music}))
|
||||||
{music}))
|
|
||||||
|
|
||||||
(defwidget slider-vol []
|
(defwidget slider-vol []
|
||||||
(box :class "slider-vol" :orientation "h" :space-evenly "false"
|
(box :class "slider-vol" :orientation "h" :space-evenly "false"
|
||||||
|
|
Loading…
Add table
Reference in a new issue