Add jq function (#695)

CHANGELOG.md

@@ -11,6 +11,7 @@ All notable changes to eww will be listed here, starting at changes since versio
 - Add support for safe access with index (`?.[n]`) (By: ModProg)
 - Made `and`, `or` and `?:` lazily evaluated in simplexpr (By: ModProg)
 - Add Vanilla CSS support (By: Ezequiel Ramis)
+- Add `jq` function, offering jq-style json processing
 
 ## [0.4.0] (04.09.2022)
 

Cargo.lock

@@ -2,6 +2,15 @@
 # It is not intended for manual editing.
 version = 3
 
+[[package]]
+name = "ahash"
+version = "0.3.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217"
+dependencies = [
+ "const-random",
+]
+
 [[package]]
 name = "ahash"
 version = "0.7.6"
@@ -37,6 +46,23 @@ dependencies = [
  "term",
 ]
 
+[[package]]
+name = "async-trait"
+version = "0.1.64"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "async_once"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2ce4f10ea3abcd6617873bae9f91d1c5332b4a778bd9ce34d0cd517474c1de82"
+
 [[package]]
 name = "atk"
 version = "0.15.1"
@@ -120,6 +146,44 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db"
 
+[[package]]
+name = "cached"
+version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e5877db5d1af7fae60d06b5db9430b68056a69b3582a0be8e3691e87654aeb6"
+dependencies = [
+ "async-trait",
+ "async_once",
+ "cached_proc_macro",
+ "cached_proc_macro_types",
+ "futures",
+ "hashbrown 0.13.2",
+ "instant",
+ "lazy_static",
+ "once_cell",
+ "thiserror",
+ "tokio",
+]
+
+[[package]]
+name = "cached_proc_macro"
+version = "0.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e10ca87c81aaa3a949dbbe2b5e6c2c45dbc94ba4897e45ea31ff9ec5087be3dc"
+dependencies = [
+ "cached_proc_macro_types",
+ "darling",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "cached_proc_macro_types"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a4f925191b4367301851c6d99b09890311d74b0d43f274c0b34c86d308a3663"
+
 [[package]]
 name = "cairo-rs"
 version = "0.15.12"
@@ -165,6 +229,15 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "chumsky"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8d02796e4586c6c41aeb68eae9bfb4558a522c35f1430c14b40136c3706e09e4"
+dependencies = [
+ "ahash 0.3.8",
+]
+
 [[package]]
 name = "clap"
 version = "4.0.27"
@@ -231,6 +304,28 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "const-random"
+version = "0.1.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "368a7a772ead6ce7e1de82bfb04c485f3db8ec744f72925af5735e29a22cc18e"
+dependencies = [
+ "const-random-macro",
+ "proc-macro-hack",
+]
+
+[[package]]
+name = "const-random-macro"
+version = "0.1.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d7d6ab3c3a2282db210df5f02c4dab6e0a7057af0fb7ebd4070f30fe05c0ddb"
+dependencies = [
+ "getrandom",
+ "once_cell",
+ "proc-macro-hack",
+ "tiny-keccak",
+]
+
 [[package]]
 name = "convert_case"
 version = "0.4.0"
@@ -302,6 +397,41 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "darling"
+version = "0.14.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0808e1bd8671fb44a113a14e13497557533369847788fa2ae912b6ebfce9fa8"
+dependencies = [
+ "darling_core",
+ "darling_macro",
+]
+
+[[package]]
+name = "darling_core"
+version = "0.14.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "001d80444f28e193f30c2f293455da62dcf9a6b29918a4253152ae2b1de592cb"
+dependencies = [
+ "fnv",
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim",
+ "syn",
+]
+
+[[package]]
+name = "darling_macro"
+version = "0.14.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b36230598a2d5de7ec1c6f51f72d8a99a9208daff41de2084d06e3fd3ea56685"
+dependencies = [
+ "darling_core",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "derive_more"
 version = "0.99.17"
@@ -342,6 +472,12 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "dyn-clone"
+version = "1.0.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c9b0705efd4599c15a38151f4721f7bc388306f61084d3bfd50bd07fbca5cb60"
+
 [[package]]
 name = "either"
 version = "1.8.0"
@@ -490,6 +626,12 @@ version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
 
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
 [[package]]
 name = "fsevent-sys"
 version = "4.1.0"
@@ -499,6 +641,20 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "futures"
+version = "0.3.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-io",
+ "futures-sink",
+ "futures-task",
+ "futures-util",
+]
+
 [[package]]
 name = "futures-channel"
 version = "0.3.25"
@@ -506,6 +662,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed"
 dependencies = [
  "futures-core",
+ "futures-sink",
 ]
 
 [[package]]
@@ -562,6 +719,7 @@ checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6"
 dependencies = [
  "futures-core",
  "futures-macro",
+ "futures-sink",
  "futures-task",
  "pin-project-lite",
  "pin-utils",
@@ -867,7 +1025,7 @@ version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
 dependencies = [
- "ahash",
+ "ahash 0.7.6",
 ]
 
 [[package]]
@@ -876,6 +1034,12 @@ version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
 
+[[package]]
+name = "hashbrown"
+version = "0.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e"
+
 [[package]]
 name = "heck"
 version = "0.4.0"
@@ -909,6 +1073,12 @@ dependencies = [
  "quick-error",
 ]
 
+[[package]]
+name = "ident_case"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
+
 [[package]]
 name = "indexmap"
 version = "1.9.1"
@@ -952,6 +1122,15 @@ dependencies = [
  "yaml-rust",
 ]
 
+[[package]]
+name = "instant"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "io-lifetimes"
 version = "1.0.2"
@@ -989,6 +1168,42 @@ version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc"
 
+[[package]]
+name = "jaq-core"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1452b4acc3a7f49bd8dd516e90ed0c4f688bada805857275f85957aca2c0e7eb"
+dependencies = [
+ "ahash 0.3.8",
+ "dyn-clone",
+ "indexmap",
+ "itertools",
+ "jaq-parse",
+ "log",
+ "once_cell",
+ "serde_json",
+]
+
+[[package]]
+name = "jaq-parse"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0a2130a59d64a5476f6feeb6b7e48cbe52ef05d8bc1b9174f50baa93e49052fd"
+dependencies = [
+ "chumsky",
+ "serde",
+]
+
+[[package]]
+name = "jaq-std"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "36ab73d2079279e784a52dbbf5f3a5e0d792c89b41fd2c857de87cf698a4e24a"
+dependencies = [
+ "bincode",
+ "jaq-parse",
+]
+
 [[package]]
 name = "kqueue"
 version = "1.0.7"
@@ -1239,9 +1454,9 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.15.0"
+version = "1.17.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1"
+checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
 
 [[package]]
 name = "os_str_bytes"
@@ -1480,9 +1695,9 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.47"
+version = "1.0.51"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725"
+checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6"
 dependencies = [
  "unicode-ident",
 ]
@@ -1756,15 +1971,19 @@ dependencies = [
 name = "simplexpr"
 version = "0.1.0"
 dependencies = [
+ "cached",
  "eww_shared_util",
  "insta",
  "itertools",
+ "jaq-core",
+ "jaq-std",
  "lalrpop",
  "lalrpop-util",
  "once_cell",
  "regex",
  "serde",
  "serde_json",
+ "static_assertions",
  "strsim",
  "strum",
  "thiserror",

crates/eww_shared_util/src/span.rs

@@ -1,3 +1,7 @@
+/// A span is made up of
+/// - the start location
+/// - the end location
+/// - the file id
 #[derive(Eq, PartialEq, Clone, Copy, serde::Serialize, serde::Deserialize)]
 pub struct Span(pub usize, pub usize, pub usize);
 
@@ -39,6 +43,12 @@ impl Span {
         self
     }
 
+    pub fn new_relative(mut self, other_start: usize, other_end: usize) -> Self {
+        self.0 += other_start;
+        self.1 += other_end;
+        self
+    }
+
     pub fn is_dummy(&self) -> bool {
         *self == Self::DUMMY
     }

crates/simplexpr/Cargo.toml

@@ -21,6 +21,10 @@ once_cell = "1.8.0"
 serde = {version = "1.0", features = ["derive"]}
 serde_json = "1.0"
 strsim = "0.10"
+jaq-core = "0.9.0"
+jaq-std = {version = "0.9.0", features = ["bincode"]}
+static_assertions = "1.1.0"
+cached = "0.42.0"
 
 strum = { version = "0.24", features = ["derive"] }
 

crates/simplexpr/src/dynval.rs

@@ -132,6 +132,13 @@ impl DynVal {
         self
     }
 
+    pub fn at_if_dummy(mut self, span: Span) -> Self {
+        if self.1.is_dummy() {
+            self.1 = span;
+        }
+        self
+    }
+
     pub fn from_string(s: String) -> Self {
         DynVal(s, Span::DUMMY)
     }

crates/simplexpr/src/eval.rs

@@ -1,3 +1,4 @@
+use cached::proc_macro::cached;
 use itertools::Itertools;
 
 use crate::{
@@ -8,8 +9,20 @@ use eww_shared_util::{Span, Spanned, VarName};
 use std::{
     collections::HashMap,
     convert::{TryFrom, TryInto},
+    sync::Arc,
 };
 
+#[derive(Debug, thiserror::Error)]
+pub struct JaqParseError(pub Option<jaq_core::parse::Error>);
+impl std::fmt::Display for JaqParseError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match &self.0 {
+            Some(x) => write!(f, "Error parsing jq filter: {x}"),
+            None => write!(f, "Error parsing jq filter"),
+        }
+    }
+}
+
 #[derive(Debug, thiserror::Error)]
 pub enum EvalError {
     #[error("Tried to reference variable `{0}`, but we cannot access variables here")]
@@ -36,13 +49,24 @@ pub enum EvalError {
     #[error("Json operation failed: {0}")]
     SerdeError(#[from] serde_json::error::Error),
 
+    #[error("Error in jq function: {0}")]
+    JaqError(String),
+
+    #[error(transparent)]
+    JaqParseError(JaqParseError),
+
     #[error("{1}")]
     Spanned(Span, Box<EvalError>),
 }
 
+static_assertions::assert_impl_all!(EvalError: Send, Sync);
+
 impl EvalError {
     pub fn at(self, span: Span) -> Self {
-        Self::Spanned(span, Box::new(self))
+        match self {
+            EvalError::Spanned(..) => self,
+            _ => EvalError::Spanned(span, Box::new(self)),
+        }
     }
 
     pub fn map_in_span(self, f: impl FnOnce(Self) -> Self) -> Self {
@@ -113,8 +137,7 @@ impl SimplExpr {
         self.try_map_var_refs(|span, name| match variables.get(&name) {
             Some(value) => Ok(Literal(value.clone())),
             None => {
-                let similar_ish =
+                let similar_ish = variables.keys().filter(|key| strsim::levenshtein(&key.0, &name.0) < 3).cloned().collect_vec();
-                    variables.keys().filter(|key| strsim::levenshtein(&key.0, &name.0) < 3).cloned().collect_vec();
                 Err(EvalError::UnknownVariable(name.clone(), similar_ish).at(span))
             }
         })
@@ -169,8 +192,7 @@ impl SimplExpr {
                 Ok(DynVal(output, *span))
             }
             SimplExpr::VarRef(span, ref name) => {
-                let similar_ish =
+                let similar_ish = values.keys().filter(|keys| strsim::levenshtein(&keys.0, &name.0) < 3).cloned().collect_vec();
-                    values.keys().filter(|keys| strsim::levenshtein(&keys.0, &name.0) < 3).cloned().collect_vec();
                 Ok(values
                     .get(name)
                     .cloned()
@@ -349,11 +371,48 @@ fn call_expr_function(name: &str, args: Vec<DynVal>) -> Result<DynVal, EvalError
             [json] => Ok(DynVal::from(json.as_json_object()?.len() as i32)),
             _ => Err(EvalError::WrongArgCount(name.to_string())),
         },
+        "jq" => match args.as_slice() {
+            [json, code] => run_jaq_function(json.as_json_value()?, code.as_string()?)
+                .map_err(|e| EvalError::Spanned(code.span(), Box::new(e))),
+            _ => Err(EvalError::WrongArgCount(name.to_string())),
+        },
 
         _ => Err(EvalError::UnknownFunction(name.to_string())),
     }
 }
 
+#[cached(size = 10, result = true, sync_writes = true)]
+fn prepare_jaq_filter(code: String) -> Result<Arc<jaq_core::Filter>, EvalError> {
+    let (filter, mut errors) = jaq_core::parse::parse(&code, jaq_core::parse::main());
+    let filter = match filter {
+        Some(x) => x,
+        None => return Err(EvalError::JaqParseError(JaqParseError(errors.pop()))),
+    };
+    let mut defs = jaq_core::Definitions::core();
+    for def in jaq_std::std() {
+        defs.insert(def, &mut errors);
+    }
+
+    let filter = defs.finish(filter, Vec::new(), &mut errors);
+
+    if let Some(error) = errors.pop() {
+        return Err(EvalError::JaqParseError(JaqParseError(Some(error))));
+    }
+    Ok(Arc::new(filter))
+}
+
+fn run_jaq_function(json: serde_json::Value, code: String) -> Result<DynVal, EvalError> {
+    let filter = prepare_jaq_filter(code)?;
+    let inputs = jaq_core::RcIter::new(std::iter::empty());
+    let out = filter
+        .run(jaq_core::Ctx::new([], &inputs), jaq_core::Val::from(json))
+        .map(|x| x.map(Into::<serde_json::Value>::into))
+        .map(|x| x.map(|x| DynVal::from_string(serde_json::to_string(&x).unwrap())))
+        .collect::<Result<_, _>>()
+        .map_err(|e| EvalError::JaqError(e.to_string()))?;
+    Ok(out)
+}
+
 #[cfg(test)]
 mod tests {
     use crate::dynval::DynVal;

crates/yuck/src/format_diagnostic.rs

@@ -187,10 +187,10 @@ impl ToDiagnostic for simplexpr::parser::lexer::LexicalError {
 
 impl ToDiagnostic for simplexpr::eval::EvalError {
     fn to_diagnostic(&self) -> Diagnostic<usize> {
-        use simplexpr::eval::EvalError::*;
+        use simplexpr::eval::EvalError;
         match self {
-            NoVariablesAllowed(name) => gen_diagnostic!(self),
+            EvalError::NoVariablesAllowed(name) => gen_diagnostic!(self),
-            UnknownVariable(name, similar) => {
+            EvalError::UnknownVariable(name, similar) => {
                 let mut notes = Vec::new();
                 if similar.len() == 1 {
                     notes.push(format!("Did you mean `{}`?", similar.first().unwrap()))
@@ -202,7 +202,22 @@ impl ToDiagnostic for simplexpr::eval::EvalError {
                 notes.push(format!("Hint: If you meant to use the literal value \"{}\", surround the value in quotes", name));
                 gen_diagnostic!(self).with_notes(notes)
             }
-            Spanned(span, error) => error.as_ref().to_diagnostic().with_label(span_to_primary_label(*span)),
+            EvalError::Spanned(span, box EvalError::JaqParseError(simplexpr::eval::JaqParseError(Some(err)))) => {
+                let span = span.new_relative(err.span().start, err.span().end).shifted(1);
+                let mut diag = gen_diagnostic!(self, span);
+
+                if let Some(label) = err.label() {
+                    diag = diag.with_label(span_to_secondary_label(span).with_message(label));
+                }
+
+                let expected: Vec<_> = err.expected().filter_map(|x| x.clone()).sorted().collect();
+                if !expected.is_empty() {
+                    let label = format!("Expected one of {} here", expected.join(", "));
+                    diag = diag.with_label(span_to_primary_label(span).with_message(label));
+                }
+                diag
+            }
+            EvalError::Spanned(span, error) => error.as_ref().to_diagnostic().with_label(span_to_primary_label(*span)),
             _ => gen_diagnostic!(self, self.span()),
         }
     }

crates/yuck/src/lib.rs

@@ -1,6 +1,7 @@
 #![allow(unused_imports)]
 #![allow(unused)]
-#![feature(try_blocks)]
+#![allow(clippy::comparison_chain)]
+#![feature(try_blocks, box_patterns)]
 
 pub mod ast_error;
 pub mod config;

docs/src/expression_language.md

@@ -45,4 +45,4 @@ Supported currently are the following features:
 	- `strlength(value)`: Gets the length of the string
 	- `arraylength(value)`: Gets the length of the array
 	- `objectlength(value)`: Gets the amount of entries in the object
-
+	- `jq(value, jq_filter_string)`: run a [jq](https://stedolan.github.io/jq/manual/) style command on a json value. (Uses [jaq](https://crates.io/crates/jaq) internally).