Fix lexer string escaping

Cargo.lock

@@ -1889,6 +1889,7 @@ dependencies = [
  "itertools 0.10.1",
  "lalrpop",
  "lalrpop-util",
+ "lazy_static",
  "logos",
  "maplit",
  "regex",

crates/eww/src/error_handling_ctx.rs

@@ -20,30 +20,35 @@ pub fn clear_files() {
 }
 
 pub fn print_error(err: &anyhow::Error) {
-    if let Some(err) = err.downcast_ref::<DiagError>() {
+    let result: anyhow::Result<_> = try {
-        eprintln!("{:?}\n{}", err, stringify_diagnostic(&err.diag));
+        if let Some(err) = err.downcast_ref::<DiagError>() {
-    } else if let Some(err) = err.downcast_ref::<AstError>() {
+            eprintln!("{:?}\n{}", err, stringify_diagnostic(&err.diag)?);
-        eprintln!("{:?}\n{}", err, stringify_diagnostic(&err.to_diagnostic()));
+        } else if let Some(err) = err.downcast_ref::<AstError>() {
-    } else if let Some(err) = err.downcast_ref::<EvalError>() {
+            eprintln!("{:?}\n{}", err, stringify_diagnostic(&err.to_diagnostic())?);
-        eprintln!("{:?}\n{}", err, stringify_diagnostic(&eval_error_to_diagnostic(err, err.span().unwrap_or(DUMMY_SPAN))));
+        } else if let Some(err) = err.downcast_ref::<EvalError>() {
-    } else {
+            eprintln!("{:?}\n{}", err, stringify_diagnostic(&eval_error_to_diagnostic(err, err.span().unwrap_or(DUMMY_SPAN)))?);
+        } else {
+            log::error!("{:?}", err);
+        }
+    };
+    if result.is_err() {
         log::error!("{:?}", err);
     }
 }
 
 pub fn format_error(err: &anyhow::Error) -> String {
     match err.downcast_ref::<AstError>() {
-        Some(err) => stringify_diagnostic(&err.to_diagnostic()),
+        Some(err) => stringify_diagnostic(&err.to_diagnostic()).unwrap_or_else(|_| format!("{:?}", err)),
         None => format!("{:?}", err),
     }
 }
 
-pub fn stringify_diagnostic(diagnostic: &Diagnostic<usize>) -> String {
+pub fn stringify_diagnostic(diagnostic: &Diagnostic<usize>) -> anyhow::Result<String> {
     use codespan_reporting::term;
     let config = term::Config::default();
     let mut buf = Vec::new();
     let mut writer = term::termcolor::Ansi::new(&mut buf);
     let files = ERROR_HANDLING_CTX.lock().unwrap();
-    term::emit(&mut writer, &config, &*files, &diagnostic).unwrap();
+    term::emit(&mut writer, &config, &*files, &diagnostic)?;
-    String::from_utf8(buf).unwrap()
+    Ok(String::from_utf8(buf)?)
 }

crates/simplexpr/Cargo.toml

@@ -14,6 +14,7 @@ itertools = "0.10"
 thiserror = "1.0"
 maplit = "1.0"
 logos = "0.12"
+lazy_static = "1.4"
 
 serde = {version = "1.0", features = ["derive"]}
 serde_json = "1.0"

crates/simplexpr/src/ast.rs

@@ -47,7 +47,7 @@ impl std::fmt::Display for SimplExpr {
             SimplExpr::Literal(_, x) => write!(f, "\"{}\"", x),
             SimplExpr::BinOp(_, l, op, r) => write!(f, "({} {} {})", l, op, r),
             SimplExpr::UnaryOp(_, op, x) => write!(f, "{}{}", op, x),
-            SimplExpr::IfElse(_, a, b, c) => write!(f, "(if {} then {} else {})", a, b, c),
+            SimplExpr::IfElse(_, a, b, c) => write!(f, "({} ? {} : {})", a, b, c),
             SimplExpr::JsonAccess(_, value, index) => write!(f, "{}[{}]", value, index),
             SimplExpr::FunctionCall(_, function_name, args) => {
                 write!(f, "{}({})", function_name, args.iter().join(", "))

crates/simplexpr/src/parser/lexer.rs

@@ -1,4 +1,9 @@
 use logos::Logos;
+use regex::Regex;
+
+lazy_static::lazy_static! {
+    static ref ESCAPE_REPLACE_REGEX: Regex = Regex::new(r"\\(.)").unwrap();
+}
 
 #[rustfmt::skip]
 #[derive(Logos, Debug, PartialEq, Eq, Clone, strum::Display, strum::EnumString)]
@@ -34,7 +39,7 @@ pub enum Token {
     Ident(String),
     #[regex(r"[+-]?(?:[0-9]+[.])?[0-9]+", |x| x.slice().to_string())]
     NumLit(String),
-    #[regex(r#""(?:[^"\\]|\\.)*""#, |x| x.slice().to_string())]
+    #[regex(r#""(?:[^"\\]|\\.)*""#, |x| ESCAPE_REPLACE_REGEX.replace_all(x.slice(), "$1").to_string())]
     StrLit(String),
 
 
@@ -78,3 +83,10 @@ impl<'input> Iterator for Lexer<'input> {
         }
     }
 }
+
+#[cfg(test)]
+#[test]
+fn test_simplexpr_lexer() {
+    use itertools::Itertools;
+    insta::assert_debug_snapshot!(Lexer::new(0, r#"(foo + - "()" "a\"b" true false [] 12.2)"#).collect_vec());
+}

crates/simplexpr/src/parser/snapshots/simplexpr__parser__lexer__simplexpr_lexer.snap

@@ -0,0 +1,99 @@
+---
+source: crates/simplexpr/src/parser/lexer.rs
+expression: "Lexer::new(0, r#\"(foo + - \"()\" \"a\\\"b\" true false [] 12.2)\"#).collect_vec()"
+
+---
+[
+    Ok(
+        (
+            0,
+            LPren,
+            1,
+        ),
+    ),
+    Ok(
+        (
+            1,
+            Ident(
+                "foo",
+            ),
+            4,
+        ),
+    ),
+    Ok(
+        (
+            5,
+            Plus,
+            6,
+        ),
+    ),
+    Ok(
+        (
+            7,
+            Minus,
+            8,
+        ),
+    ),
+    Ok(
+        (
+            9,
+            StrLit(
+                "\"()\"",
+            ),
+            13,
+        ),
+    ),
+    Ok(
+        (
+            14,
+            StrLit(
+                "\"a\"b\"",
+            ),
+            20,
+        ),
+    ),
+    Ok(
+        (
+            21,
+            True,
+            25,
+        ),
+    ),
+    Ok(
+        (
+            26,
+            False,
+            31,
+        ),
+    ),
+    Ok(
+        (
+            32,
+            LBrack,
+            33,
+        ),
+    ),
+    Ok(
+        (
+            33,
+            RBrack,
+            34,
+        ),
+    ),
+    Ok(
+        (
+            35,
+            NumLit(
+                "12.2",
+            ),
+            39,
+        ),
+    ),
+    Ok(
+        (
+            39,
+            RPren,
+            40,
+        ),
+    ),
+]

crates/simplexpr/src/parser/snapshots/simplexpr__parser__tests__test-10.snap

@@ -0,0 +1,8 @@
+---
+source: crates/simplexpr/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, \"\\\"foo\\\" + 12.4\"))"
+
+---
+Ok(
+    ("foo" + "12.4"),
+)

crates/simplexpr/src/parser/snapshots/simplexpr__parser__tests__test-11.snap

@@ -0,0 +1,8 @@
+---
+source: crates/simplexpr/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, \"hi[\\\"ho\\\"]\"))"
+
+---
+Ok(
+    hi["ho"],
+)

crates/simplexpr/src/parser/snapshots/simplexpr__parser__tests__test-12.snap

@@ -0,0 +1,8 @@
+---
+source: crates/simplexpr/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, \"foo.bar.baz\"))"
+
+---
+Ok(
+    foo["bar"]["baz"],
+)

crates/simplexpr/src/parser/snapshots/simplexpr__parser__tests__test-13.snap

@@ -0,0 +1,8 @@
+---
+source: crates/simplexpr/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, \"foo.bar[2 + 2] * asdf[foo.bar]\"))"
+
+---
+Ok(
+    (foo["bar"][("2" + "2")] * asdf[foo["bar"]]),
+)

crates/simplexpr/src/parser/snapshots/simplexpr__parser__tests__test-2.snap

@@ -0,0 +1,8 @@
+---
+source: crates/simplexpr/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, \"2 + 5\"))"
+
+---
+Ok(
+    ("2" + "5"),
+)

crates/simplexpr/src/parser/snapshots/simplexpr__parser__tests__test-3.snap

@@ -0,0 +1,8 @@
+---
+source: crates/simplexpr/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, \"2 * 5 + 1 * 1 + 3\"))"
+
+---
+Ok(
+    ((("2" * "5") + ("1" * "1")) + "3"),
+)

crates/simplexpr/src/parser/snapshots/simplexpr__parser__tests__test-4.snap

@@ -0,0 +1,8 @@
+---
+source: crates/simplexpr/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, \"(1 + 2) * 2\"))"
+
+---
+Ok(
+    (("1" + "2") * "2"),
+)

crates/simplexpr/src/parser/snapshots/simplexpr__parser__tests__test-5.snap

@@ -0,0 +1,8 @@
+---
+source: crates/simplexpr/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, \"1 + true ? 2 : 5\"))"
+
+---
+Ok(
+    (("1" + "true") ? "2" : "5"),
+)

crates/simplexpr/src/parser/snapshots/simplexpr__parser__tests__test-6.snap

@@ -0,0 +1,8 @@
+---
+source: crates/simplexpr/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, \"1 + true ? 2 : 5 + 2\"))"
+
+---
+Ok(
+    (("1" + "true") ? "2" : ("5" + "2")),
+)

crates/simplexpr/src/parser/snapshots/simplexpr__parser__tests__test-7.snap

@@ -0,0 +1,8 @@
+---
+source: crates/simplexpr/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, \"1 + (true ? 2 : 5) + 2\"))"
+
+---
+Ok(
+    (("1" + ("true" ? "2" : "5")) + "2"),
+)

crates/simplexpr/src/parser/snapshots/simplexpr__parser__tests__test-8.snap

@@ -0,0 +1,8 @@
+---
+source: crates/simplexpr/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, \"foo(1, 2)\"))"
+
+---
+Ok(
+    foo("1", "2"),
+)

crates/simplexpr/src/parser/snapshots/simplexpr__parser__tests__test-9.snap

@@ -0,0 +1,8 @@
+---
+source: crates/simplexpr/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, \"! false || ! true\"))"
+
+---
+Ok(
+    (!"false" || !"true"),
+)

crates/simplexpr/src/parser/snapshots/simplexpr__parser__tests__test.snap

@@ -0,0 +1,8 @@
+---
+source: crates/simplexpr/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, \"1\"))"
+
+---
+Ok(
+    "1",
+)

crates/yuck/src/parser/lexer.rs

@@ -58,6 +58,10 @@ macro_rules! regex_rules {
     }
 }
 
+lazy_static::lazy_static! {
+    static ref ESCAPE_REPLACE_REGEX: Regex = Regex::new(r"\\(.)").unwrap();
+}
+
 regex_rules! {
     r"\(" => |_| Token::LPren,
     r"\)" => |_| Token::RPren,
@@ -65,10 +69,10 @@ regex_rules! {
     r"\]" => |_| Token::RBrack,
     r"true" => |_| Token::True,
     r"false" => |_| Token::False,
-    r#""(?:[^"\\]|\\.)*""# => |x| Token::StrLit(x),
+    r#""(?:[^"\\]|\\.)*""# => |x| Token::StrLit(ESCAPE_REPLACE_REGEX.replace_all(&x, "$1").to_string()),
     r#"[+-]?(?:[0-9]+[.])?[0-9]+"# => |x| Token::NumLit(x),
     r#":[^\s\)\]}]+"# => |x| Token::Keyword(x),
-    r#"[a-zA-Z_!\?<>/\.\*-\+][^\s{}\(\)\[\](){}]*"# => |x| Token::Symbol(x),
+    r#"[a-zA-Z_!\?<>/\.\*-\+\-][^\s{}\(\)\[\](){}]*"# => |x| Token::Symbol(x),
     r#";.*"# => |_| Token::Comment,
     r"[ \t\n\f]+" => |_| Token::Skip
 }
@@ -133,6 +137,7 @@ impl Iterator for Lexer {
                     Some(x) => x,
                     None => {
                         self.failed = true;
+                        dbg!(&string);
                         return Some(Err(parse_error::ParseError::LexicalError(Span(self.pos, self.pos, self.file_id))));
                     }
                 };
@@ -150,3 +155,12 @@ impl Iterator for Lexer {
         }
     }
 }
+
+#[cfg(test)]
+#[test]
+fn test_yuck_lexer() {
+    use itertools::Itertools;
+    insta::assert_debug_snapshot!(Lexer::new(0, r#"(foo + - "text" )"#.to_string()).collect_vec());
+    insta::assert_debug_snapshot!(Lexer::new(0, r#"{ bla "} \" }" " \" "}"#.to_string()).collect_vec());
+    insta::assert_debug_snapshot!(Lexer::new(0, r#""< \" >""#.to_string()).collect_vec());
+}

crates/yuck/src/parser/snapshots/yuck__parser__lexer__yuck_lexer-2.snap

@@ -0,0 +1,16 @@
+---
+source: crates/yuck/src/parser/lexer.rs
+expression: "Lexer::new(0, r#\"{ bla \"} \\\" }\" \" \\\" \"}\"#.to_string()).collect_vec()"
+
+---
+[
+    Ok(
+        (
+            0,
+            SimplExpr(
+                "{ bla \"} \\\" }\" \" \\\" \"}",
+            ),
+            21,
+        ),
+    ),
+]

crates/yuck/src/parser/snapshots/yuck__parser__lexer__yuck_lexer-3.snap

@@ -0,0 +1,16 @@
+---
+source: crates/yuck/src/parser/lexer.rs
+expression: "Lexer::new(0, r#\"\"< \\\" >\"\"#.to_string()).collect_vec()"
+
+---
+[
+    Ok(
+        (
+            0,
+            StrLit(
+                "\"< \" >\"",
+            ),
+            8,
+        ),
+    ),
+]

crates/yuck/src/parser/snapshots/yuck__parser__lexer__yuck_lexer.snap

@@ -0,0 +1,57 @@
+---
+source: crates/yuck/src/parser/lexer.rs
+expression: "Lexer::new(0, r#\"(foo + - \"text\" )\"#.to_string()).collect_vec()"
+
+---
+[
+    Ok(
+        (
+            0,
+            LPren,
+            1,
+        ),
+    ),
+    Ok(
+        (
+            1,
+            Symbol(
+                "foo",
+            ),
+            4,
+        ),
+    ),
+    Ok(
+        (
+            5,
+            Symbol(
+                "+",
+            ),
+            6,
+        ),
+    ),
+    Ok(
+        (
+            7,
+            Symbol(
+                "-",
+            ),
+            8,
+        ),
+    ),
+    Ok(
+        (
+            9,
+            StrLit(
+                "\"text\"",
+            ),
+            15,
+        ),
+    ),
+    Ok(
+        (
+            16,
+            RPren,
+            17,
+        ),
+    ),
+]

crates/yuck/src/parser/snapshots/yuck__parser__test-10.snap

@@ -0,0 +1,8 @@
+---
+source: crates/yuck/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, \"(lol😄 1)\".to_string()))"
+
+---
+Ok(
+    (lol😄 "1"),
+)

crates/yuck/src/parser/snapshots/yuck__parser__test-11.snap

@@ -0,0 +1,8 @@
+---
+source: crates/yuck/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, r#\"(test \"hi\")\"#.to_string()))"
+
+---
+Ok(
+    (test "hi"),
+)

crates/yuck/src/parser/snapshots/yuck__parser__test-12.snap

@@ -0,0 +1,8 @@
+---
+source: crates/yuck/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, r#\"(test \"h\\\"i\")\"#.to_string()))"
+
+---
+Ok(
+    (test "h"i"),
+)

crates/yuck/src/parser/snapshots/yuck__parser__test-13.snap

@@ -0,0 +1,8 @@
+---
+source: crates/yuck/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, r#\"(test \" hi \")\"#.to_string()))"
+
+---
+Ok(
+    (test " hi "),
+)

crates/yuck/src/parser/snapshots/yuck__parser__test-14.snap

@@ -0,0 +1,8 @@
+---
+source: crates/yuck/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, \"(+ (1 2 (* 2 5)))\".to_string()))"
+
+---
+Ok(
+    (+ ("1" "2" (* "2" "5"))),
+)

crates/yuck/src/parser/snapshots/yuck__parser__test-15.snap

@@ -0,0 +1,8 @@
+---
+source: crates/yuck/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, r#\"foo ; test\"#.to_string()))"
+
+---
+Ok(
+    foo,
+)

crates/yuck/src/parser/snapshots/yuck__parser__test-16.snap

@@ -0,0 +1,8 @@
+---
+source: crates/yuck/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, r#\"(f arg ; test\n        arg2)\"#.to_string()))"
+
+---
+Ok(
+    (f arg arg2),
+)

crates/yuck/src/parser/snapshots/yuck__parser__test-17.snap

@@ -0,0 +1,8 @@
+---
+source: crates/yuck/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, \"\\\"h\\\\\\\"i\\\"\".to_string()))"
+
+---
+Ok(
+    "h"i",
+)

crates/yuck/src/parser/snapshots/yuck__parser__test-2.snap

@@ -0,0 +1,8 @@
+---
+source: crates/yuck/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, \"(12)\".to_string()))"
+
+---
+Ok(
+    ("12"),
+)

crates/yuck/src/parser/snapshots/yuck__parser__test-3.snap

@@ -0,0 +1,8 @@
+---
+source: crates/yuck/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, \"1.2\".to_string()))"
+
+---
+Ok(
+    "1.2",
+)

crates/yuck/src/parser/snapshots/yuck__parser__test-4.snap

@@ -0,0 +1,8 @@
+---
+source: crates/yuck/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, \"-1.2\".to_string()))"
+
+---
+Ok(
+    "-1.2",
+)

crates/yuck/src/parser/snapshots/yuck__parser__test-5.snap

@@ -0,0 +1,8 @@
+---
+source: crates/yuck/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, \"(1 2)\".to_string()))"
+
+---
+Ok(
+    ("1" "2"),
+)

crates/yuck/src/parser/snapshots/yuck__parser__test-6.snap

@@ -0,0 +1,8 @@
+---
+source: crates/yuck/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, \"(1 :foo 1)\".to_string()))"
+
+---
+Ok(
+    ("1" :foo "1"),
+)

crates/yuck/src/parser/snapshots/yuck__parser__test-7.snap

@@ -0,0 +1,8 @@
+---
+source: crates/yuck/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, \"(:foo 1)\".to_string()))"
+
+---
+Ok(
+    (:foo "1"),
+)

crates/yuck/src/parser/snapshots/yuck__parser__test-8.snap

@@ -0,0 +1,8 @@
+---
+source: crates/yuck/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, \"(:foo->: 1)\".to_string()))"
+
+---
+Ok(
+    (:foo->: "1"),
+)

crates/yuck/src/parser/snapshots/yuck__parser__test-9.snap

@@ -0,0 +1,8 @@
+---
+source: crates/yuck/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, \"(foo 1)\".to_string()))"
+
+---
+Ok(
+    (foo "1"),
+)

crates/yuck/src/parser/snapshots/yuck__parser__test.snap

@@ -0,0 +1,8 @@
+---
+source: crates/yuck/src/parser/mod.rs
+expression: "p.parse(0, Lexer::new(0, \"1\".to_string()))"
+
+---
+Ok(
+    "1",
+)