From 7290b9bb8df59e05fad7ec3d0bb3a12f637b4fe2 Mon Sep 17 00:00:00 2001 From: ElKowar <5300871+elkowar@users.noreply.github.com> Date: Sat, 25 Feb 2023 16:27:32 +0100 Subject: [PATCH] Add jq function (#695) --- CHANGELOG.md | 1 + Cargo.lock | 229 ++++++++++++++++++++++++++- crates/eww_shared_util/src/span.rs | 10 ++ crates/simplexpr/Cargo.toml | 4 + crates/simplexpr/src/dynval.rs | 7 + crates/simplexpr/src/eval.rs | 69 +++++++- crates/yuck/src/format_diagnostic.rs | 23 ++- crates/yuck/src/lib.rs | 3 +- docs/src/expression_language.md | 2 +- 9 files changed, 332 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 340fd98..628c693 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ All notable changes to eww will be listed here, starting at changes since versio - Add support for safe access with index (`?.[n]`) (By: ModProg) - Made `and`, `or` and `?:` lazily evaluated in simplexpr (By: ModProg) - Add Vanilla CSS support (By: Ezequiel Ramis) +- Add `jq` function, offering jq-style json processing ## [0.4.0] (04.09.2022) diff --git a/Cargo.lock b/Cargo.lock index d3de724..4c13358 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "ahash" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217" +dependencies = [ + "const-random", +] + [[package]] name = "ahash" version = "0.7.6" @@ -37,6 +46,23 @@ dependencies = [ "term", ] +[[package]] +name = "async-trait" +version = "0.1.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async_once" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ce4f10ea3abcd6617873bae9f91d1c5332b4a778bd9ce34d0cd517474c1de82" + [[package]] name = "atk" version = "0.15.1" @@ -120,6 +146,44 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" +[[package]] +name = "cached" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5877db5d1af7fae60d06b5db9430b68056a69b3582a0be8e3691e87654aeb6" +dependencies = [ + "async-trait", + "async_once", + "cached_proc_macro", + "cached_proc_macro_types", + "futures", + "hashbrown 0.13.2", + "instant", + "lazy_static", + "once_cell", + "thiserror", + "tokio", +] + +[[package]] +name = "cached_proc_macro" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10ca87c81aaa3a949dbbe2b5e6c2c45dbc94ba4897e45ea31ff9ec5087be3dc" +dependencies = [ + "cached_proc_macro_types", + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "cached_proc_macro_types" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a4f925191b4367301851c6d99b09890311d74b0d43f274c0b34c86d308a3663" + [[package]] name = "cairo-rs" version = "0.15.12" @@ -165,6 +229,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chumsky" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d02796e4586c6c41aeb68eae9bfb4558a522c35f1430c14b40136c3706e09e4" +dependencies = [ + "ahash 0.3.8", +] + [[package]] name = "clap" version = "4.0.27" @@ -231,6 +304,28 @@ dependencies = [ "winapi", ] +[[package]] +name = "const-random" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368a7a772ead6ce7e1de82bfb04c485f3db8ec744f72925af5735e29a22cc18e" +dependencies = [ + "const-random-macro", + "proc-macro-hack", +] + +[[package]] +name = "const-random-macro" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d7d6ab3c3a2282db210df5f02c4dab6e0a7057af0fb7ebd4070f30fe05c0ddb" +dependencies = [ + "getrandom", + "once_cell", + "proc-macro-hack", + "tiny-keccak", +] + [[package]] name = "convert_case" version = "0.4.0" @@ -302,6 +397,41 @@ dependencies = [ "syn", ] +[[package]] +name = "darling" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0808e1bd8671fb44a113a14e13497557533369847788fa2ae912b6ebfce9fa8" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "001d80444f28e193f30c2f293455da62dcf9a6b29918a4253152ae2b1de592cb" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b36230598a2d5de7ec1c6f51f72d8a99a9208daff41de2084d06e3fd3ea56685" +dependencies = [ + "darling_core", + "quote", + "syn", +] + [[package]] name = "derive_more" version = "0.99.17" @@ -342,6 +472,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "dyn-clone" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9b0705efd4599c15a38151f4721f7bc388306f61084d3bfd50bd07fbca5cb60" + [[package]] name = "either" version = "1.8.0" @@ -490,6 +626,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "fsevent-sys" version = "4.1.0" @@ -499,6 +641,20 @@ dependencies = [ "libc", ] +[[package]] +name = "futures" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.25" @@ -506,6 +662,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -562,6 +719,7 @@ checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" dependencies = [ "futures-core", "futures-macro", + "futures-sink", "futures-task", "pin-project-lite", "pin-utils", @@ -867,7 +1025,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" dependencies = [ - "ahash", + "ahash 0.7.6", ] [[package]] @@ -876,6 +1034,12 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" + [[package]] name = "heck" version = "0.4.0" @@ -909,6 +1073,12 @@ dependencies = [ "quick-error", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "indexmap" version = "1.9.1" @@ -952,6 +1122,15 @@ dependencies = [ "yaml-rust", ] +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + [[package]] name = "io-lifetimes" version = "1.0.2" @@ -989,6 +1168,42 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" +[[package]] +name = "jaq-core" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1452b4acc3a7f49bd8dd516e90ed0c4f688bada805857275f85957aca2c0e7eb" +dependencies = [ + "ahash 0.3.8", + "dyn-clone", + "indexmap", + "itertools", + "jaq-parse", + "log", + "once_cell", + "serde_json", +] + +[[package]] +name = "jaq-parse" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a2130a59d64a5476f6feeb6b7e48cbe52ef05d8bc1b9174f50baa93e49052fd" +dependencies = [ + "chumsky", + "serde", +] + +[[package]] +name = "jaq-std" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36ab73d2079279e784a52dbbf5f3a5e0d792c89b41fd2c857de87cf698a4e24a" +dependencies = [ + "bincode", + "jaq-parse", +] + [[package]] name = "kqueue" version = "1.0.7" @@ -1239,9 +1454,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.15.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "os_str_bytes" @@ -1480,9 +1695,9 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro2" -version = "1.0.47" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" dependencies = [ "unicode-ident", ] @@ -1756,15 +1971,19 @@ dependencies = [ name = "simplexpr" version = "0.1.0" dependencies = [ + "cached", "eww_shared_util", "insta", "itertools", + "jaq-core", + "jaq-std", "lalrpop", "lalrpop-util", "once_cell", "regex", "serde", "serde_json", + "static_assertions", "strsim", "strum", "thiserror", diff --git a/crates/eww_shared_util/src/span.rs b/crates/eww_shared_util/src/span.rs index fcf6e69..9ee4b67 100644 --- a/crates/eww_shared_util/src/span.rs +++ b/crates/eww_shared_util/src/span.rs @@ -1,3 +1,7 @@ +/// A span is made up of +/// - the start location +/// - the end location +/// - the file id #[derive(Eq, PartialEq, Clone, Copy, serde::Serialize, serde::Deserialize)] pub struct Span(pub usize, pub usize, pub usize); @@ -39,6 +43,12 @@ impl Span { self } + pub fn new_relative(mut self, other_start: usize, other_end: usize) -> Self { + self.0 += other_start; + self.1 += other_end; + self + } + pub fn is_dummy(&self) -> bool { *self == Self::DUMMY } diff --git a/crates/simplexpr/Cargo.toml b/crates/simplexpr/Cargo.toml index 03748b1..edcb5ce 100644 --- a/crates/simplexpr/Cargo.toml +++ b/crates/simplexpr/Cargo.toml @@ -21,6 +21,10 @@ once_cell = "1.8.0" serde = {version = "1.0", features = ["derive"]} serde_json = "1.0" strsim = "0.10" +jaq-core = "0.9.0" +jaq-std = {version = "0.9.0", features = ["bincode"]} +static_assertions = "1.1.0" +cached = "0.42.0" strum = { version = "0.24", features = ["derive"] } diff --git a/crates/simplexpr/src/dynval.rs b/crates/simplexpr/src/dynval.rs index e730d0a..c35b9a9 100644 --- a/crates/simplexpr/src/dynval.rs +++ b/crates/simplexpr/src/dynval.rs @@ -132,6 +132,13 @@ impl DynVal { self } + pub fn at_if_dummy(mut self, span: Span) -> Self { + if self.1.is_dummy() { + self.1 = span; + } + self + } + pub fn from_string(s: String) -> Self { DynVal(s, Span::DUMMY) } diff --git a/crates/simplexpr/src/eval.rs b/crates/simplexpr/src/eval.rs index 3b75048..d0b4811 100644 --- a/crates/simplexpr/src/eval.rs +++ b/crates/simplexpr/src/eval.rs @@ -1,3 +1,4 @@ +use cached::proc_macro::cached; use itertools::Itertools; use crate::{ @@ -8,8 +9,20 @@ use eww_shared_util::{Span, Spanned, VarName}; use std::{ collections::HashMap, convert::{TryFrom, TryInto}, + sync::Arc, }; +#[derive(Debug, thiserror::Error)] +pub struct JaqParseError(pub Option); +impl std::fmt::Display for JaqParseError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self.0 { + Some(x) => write!(f, "Error parsing jq filter: {x}"), + None => write!(f, "Error parsing jq filter"), + } + } +} + #[derive(Debug, thiserror::Error)] pub enum EvalError { #[error("Tried to reference variable `{0}`, but we cannot access variables here")] @@ -36,13 +49,24 @@ pub enum EvalError { #[error("Json operation failed: {0}")] SerdeError(#[from] serde_json::error::Error), + #[error("Error in jq function: {0}")] + JaqError(String), + + #[error(transparent)] + JaqParseError(JaqParseError), + #[error("{1}")] Spanned(Span, Box), } +static_assertions::assert_impl_all!(EvalError: Send, Sync); + impl EvalError { pub fn at(self, span: Span) -> Self { - Self::Spanned(span, Box::new(self)) + match self { + EvalError::Spanned(..) => self, + _ => EvalError::Spanned(span, Box::new(self)), + } } pub fn map_in_span(self, f: impl FnOnce(Self) -> Self) -> Self { @@ -113,8 +137,7 @@ impl SimplExpr { self.try_map_var_refs(|span, name| match variables.get(&name) { Some(value) => Ok(Literal(value.clone())), None => { - let similar_ish = - variables.keys().filter(|key| strsim::levenshtein(&key.0, &name.0) < 3).cloned().collect_vec(); + let similar_ish = variables.keys().filter(|key| strsim::levenshtein(&key.0, &name.0) < 3).cloned().collect_vec(); Err(EvalError::UnknownVariable(name.clone(), similar_ish).at(span)) } }) @@ -169,8 +192,7 @@ impl SimplExpr { Ok(DynVal(output, *span)) } SimplExpr::VarRef(span, ref name) => { - let similar_ish = - values.keys().filter(|keys| strsim::levenshtein(&keys.0, &name.0) < 3).cloned().collect_vec(); + let similar_ish = values.keys().filter(|keys| strsim::levenshtein(&keys.0, &name.0) < 3).cloned().collect_vec(); Ok(values .get(name) .cloned() @@ -349,11 +371,48 @@ fn call_expr_function(name: &str, args: Vec) -> Result Ok(DynVal::from(json.as_json_object()?.len() as i32)), _ => Err(EvalError::WrongArgCount(name.to_string())), }, + "jq" => match args.as_slice() { + [json, code] => run_jaq_function(json.as_json_value()?, code.as_string()?) + .map_err(|e| EvalError::Spanned(code.span(), Box::new(e))), + _ => Err(EvalError::WrongArgCount(name.to_string())), + }, _ => Err(EvalError::UnknownFunction(name.to_string())), } } +#[cached(size = 10, result = true, sync_writes = true)] +fn prepare_jaq_filter(code: String) -> Result, EvalError> { + let (filter, mut errors) = jaq_core::parse::parse(&code, jaq_core::parse::main()); + let filter = match filter { + Some(x) => x, + None => return Err(EvalError::JaqParseError(JaqParseError(errors.pop()))), + }; + let mut defs = jaq_core::Definitions::core(); + for def in jaq_std::std() { + defs.insert(def, &mut errors); + } + + let filter = defs.finish(filter, Vec::new(), &mut errors); + + if let Some(error) = errors.pop() { + return Err(EvalError::JaqParseError(JaqParseError(Some(error)))); + } + Ok(Arc::new(filter)) +} + +fn run_jaq_function(json: serde_json::Value, code: String) -> Result { + let filter = prepare_jaq_filter(code)?; + let inputs = jaq_core::RcIter::new(std::iter::empty()); + let out = filter + .run(jaq_core::Ctx::new([], &inputs), jaq_core::Val::from(json)) + .map(|x| x.map(Into::::into)) + .map(|x| x.map(|x| DynVal::from_string(serde_json::to_string(&x).unwrap()))) + .collect::>() + .map_err(|e| EvalError::JaqError(e.to_string()))?; + Ok(out) +} + #[cfg(test)] mod tests { use crate::dynval::DynVal; diff --git a/crates/yuck/src/format_diagnostic.rs b/crates/yuck/src/format_diagnostic.rs index cdbac1d..25cad79 100644 --- a/crates/yuck/src/format_diagnostic.rs +++ b/crates/yuck/src/format_diagnostic.rs @@ -187,10 +187,10 @@ impl ToDiagnostic for simplexpr::parser::lexer::LexicalError { impl ToDiagnostic for simplexpr::eval::EvalError { fn to_diagnostic(&self) -> Diagnostic { - use simplexpr::eval::EvalError::*; + use simplexpr::eval::EvalError; match self { - NoVariablesAllowed(name) => gen_diagnostic!(self), - UnknownVariable(name, similar) => { + EvalError::NoVariablesAllowed(name) => gen_diagnostic!(self), + EvalError::UnknownVariable(name, similar) => { let mut notes = Vec::new(); if similar.len() == 1 { notes.push(format!("Did you mean `{}`?", similar.first().unwrap())) @@ -202,7 +202,22 @@ impl ToDiagnostic for simplexpr::eval::EvalError { notes.push(format!("Hint: If you meant to use the literal value \"{}\", surround the value in quotes", name)); gen_diagnostic!(self).with_notes(notes) } - Spanned(span, error) => error.as_ref().to_diagnostic().with_label(span_to_primary_label(*span)), + EvalError::Spanned(span, box EvalError::JaqParseError(simplexpr::eval::JaqParseError(Some(err)))) => { + let span = span.new_relative(err.span().start, err.span().end).shifted(1); + let mut diag = gen_diagnostic!(self, span); + + if let Some(label) = err.label() { + diag = diag.with_label(span_to_secondary_label(span).with_message(label)); + } + + let expected: Vec<_> = err.expected().filter_map(|x| x.clone()).sorted().collect(); + if !expected.is_empty() { + let label = format!("Expected one of {} here", expected.join(", ")); + diag = diag.with_label(span_to_primary_label(span).with_message(label)); + } + diag + } + EvalError::Spanned(span, error) => error.as_ref().to_diagnostic().with_label(span_to_primary_label(*span)), _ => gen_diagnostic!(self, self.span()), } } diff --git a/crates/yuck/src/lib.rs b/crates/yuck/src/lib.rs index ff73491..0c6bf82 100644 --- a/crates/yuck/src/lib.rs +++ b/crates/yuck/src/lib.rs @@ -1,6 +1,7 @@ #![allow(unused_imports)] #![allow(unused)] -#![feature(try_blocks)] +#![allow(clippy::comparison_chain)] +#![feature(try_blocks, box_patterns)] pub mod ast_error; pub mod config; diff --git a/docs/src/expression_language.md b/docs/src/expression_language.md index f11c8ee..3cb4b52 100644 --- a/docs/src/expression_language.md +++ b/docs/src/expression_language.md @@ -45,4 +45,4 @@ Supported currently are the following features: - `strlength(value)`: Gets the length of the string - `arraylength(value)`: Gets the length of the array - `objectlength(value)`: Gets the amount of entries in the object - + - `jq(value, jq_filter_string)`: run a [jq](https://stedolan.github.io/jq/manual/) style command on a json value. (Uses [jaq](https://crates.io/crates/jaq) internally).