From 36d77a5c3ae23b8af3951351d0ec16145aed23df Mon Sep 17 00:00:00 2001 From: matt rice Date: Wed, 15 Jul 2026 23:10:18 -0700 Subject: [PATCH 1/3] Fix typo, and test in cttests --- lrpar/cttests/src/calc_actiontype.test | 2 +- lrpar/cttests/src/cgen_helper.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lrpar/cttests/src/calc_actiontype.test b/lrpar/cttests/src/calc_actiontype.test index fb97267e5..2e1ff5e7f 100644 --- a/lrpar/cttests/src/calc_actiontype.test +++ b/lrpar/cttests/src/calc_actiontype.test @@ -1,6 +1,6 @@ name: Test basic user actions using the calculator grammar (Original yacckind) yacckind: Original(YaccOriginalActionKind::UserAction) -recoverer: RecoveryKind::None +recoverer: RecoveryKind::CPCTPlus grammar: | %start Expr %actiontype Result diff --git a/lrpar/cttests/src/cgen_helper.rs b/lrpar/cttests/src/cgen_helper.rs index c4e4e5140..42c0e920e 100644 --- a/lrpar/cttests/src/cgen_helper.rs +++ b/lrpar/cttests/src/cgen_helper.rs @@ -31,7 +31,7 @@ pub(crate) fn run_test_path>(path: P) -> Result<(), Box panic!("YaccKind '{}' not supported", s), None => None, }; - let recoverer = match docs[0]["revoverer"].as_str() { + let recoverer = match docs[0]["recoverer"].as_str() { Some("RecoveryKind::CPCTPlus") => Some(RecoveryKind::CPCTPlus), Some("RecoveryKind::None") => Some(RecoveryKind::None), _ => None, From 0109e964ad9ed5a7ae66dc2a7a758d4a79876905 Mon Sep 17 00:00:00 2001 From: matt rice Date: Wed, 15 Jul 2026 21:18:03 -0700 Subject: [PATCH 2/3] Since bincode is unmaintained use wincode instead. Ports the bincode parser data serialistion to wincode, bumps the depdencencies, and renames the bincode feature to wincode. --- Cargo.toml | 8 +-- cfgrammar/Cargo.toml | 4 +- cfgrammar/src/lib/idxnewtype.rs | 6 +- cfgrammar/src/lib/mod.rs | 6 +- cfgrammar/src/lib/span.rs | 6 +- cfgrammar/src/lib/yacc/grammar.rs | 99 ++--------------------------- cfgrammar/src/lib/yacc/parser.rs | 8 +-- lrlex/Cargo.toml | 2 +- lrlex/src/lib/ctbuilder.rs | 24 +++++-- lrpar/Cargo.toml | 6 +- lrpar/src/lib/ctbuilder.rs | 102 +++++++++++++++++++++++++++--- lrtable/Cargo.toml | 4 +- lrtable/src/lib/mod.rs | 6 +- lrtable/src/lib/statetable.rs | 8 +-- 14 files changed, 151 insertions(+), 138 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 219eb7126..371a74f81 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ codegen-units = 1 panic = 'unwind' [workspace.dependencies] -bincode = "2.0" +wincode = "0.5.5" cactus = "1.0" filetime = "0.2" fnv = "1.0" @@ -36,14 +36,14 @@ getopts = "0.2" glob = "0.3" indexmap = "2" num-traits = "0.2" -packedvec = "1.2" +packedvec = "2.0" quote = "1.0" regex = "1.3" regex-syntax = "0.8" serde = "1.0" -sparsevec = "0.2.2" +sparsevec = "0.3.0" unicode-width = "0.1.11" -vob = "3.0.4" +vob = "4.0.0" proc-macro2 = "1.0" prettyplease = "0.2.31" syn = "2.0" diff --git a/cfgrammar/Cargo.toml b/cfgrammar/Cargo.toml index e0a856a41..db35d3fa2 100644 --- a/cfgrammar/Cargo.toml +++ b/cfgrammar/Cargo.toml @@ -11,14 +11,14 @@ keywords = ["yacc", "grammar"] [features] serde = ["dep:serde", "serde/derive", "vob/serde"] -bincode = ["dep:bincode", "vob/bincode"] +wincode = ["dep:wincode", "vob/wincode"] [lib] name = "cfgrammar" path = "src/lib/mod.rs" [dependencies] -bincode = { workspace = true, optional = true, features = ["derive"] } +wincode = { workspace = true, optional = true, features = ["derive"] } indexmap.workspace = true num-traits.workspace = true regex.workspace = true diff --git a/cfgrammar/src/lib/idxnewtype.rs b/cfgrammar/src/lib/idxnewtype.rs index 7c5082d29..c6ed6158a 100644 --- a/cfgrammar/src/lib/idxnewtype.rs +++ b/cfgrammar/src/lib/idxnewtype.rs @@ -3,18 +3,18 @@ use std::mem::size_of; -#[cfg(feature = "bincode")] -use bincode::{Decode, Encode}; use num_traits::{PrimInt, Unsigned}; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; +#[cfg(feature = "wincode")] +use wincode::{SchemaRead, SchemaWrite}; macro_rules! IdxNewtype { ($(#[$attr:meta])* $n: ident) => { $(#[$attr])* #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] #[cfg_attr(feature="serde", derive(Serialize, Deserialize))] - #[cfg_attr(feature="bincode", derive(Encode, Decode))] + #[cfg_attr(feature="wincode", derive(SchemaRead, SchemaWrite))] pub struct $n(pub T); impl From<$n> for usize { diff --git a/cfgrammar/src/lib/mod.rs b/cfgrammar/src/lib/mod.rs index 3d1664606..fb68ceed2 100644 --- a/cfgrammar/src/lib/mod.rs +++ b/cfgrammar/src/lib/mod.rs @@ -51,10 +51,10 @@ //! [`YaccGrammar::new_with_storaget()`](yacc/grammar/struct.YaccGrammar.html#method.new_with_storaget) //! which take as input a Yacc grammar. -#[cfg(feature = "bincode")] -use bincode::{Decode, Encode}; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; +#[cfg(feature = "wincode")] +use wincode::{SchemaRead, SchemaWrite}; #[doc(hidden)] pub mod header; @@ -73,7 +73,7 @@ pub use crate::idxnewtype::{PIdx, RIdx, SIdx, TIdx}; #[derive(Clone, Copy, Debug, Hash, Eq, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] +#[cfg_attr(feature = "wincode", derive(SchemaRead, SchemaWrite))] pub enum Symbol { Rule(RIdx), Token(TIdx), diff --git a/cfgrammar/src/lib/span.rs b/cfgrammar/src/lib/span.rs index 1491d2cc4..2c2d5366a 100644 --- a/cfgrammar/src/lib/span.rs +++ b/cfgrammar/src/lib/span.rs @@ -1,15 +1,15 @@ -#[cfg(feature = "bincode")] -use bincode::{Decode, Encode}; use proc_macro2::TokenStream; use quote::{ToTokens, TokenStreamExt, quote}; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; +#[cfg(feature = "wincode")] +use wincode::{SchemaRead, SchemaWrite}; /// A `Span` records what portion of the user's input something (e.g. a lexeme or production) /// references (i.e. the `Span` doesn't hold a reference / copy of the actual input). #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] +#[cfg_attr(feature = "wincode", derive(SchemaRead, SchemaWrite))] pub struct Span { start: usize, end: usize, diff --git a/cfgrammar/src/lib/yacc/grammar.rs b/cfgrammar/src/lib/yacc/grammar.rs index f317085f2..2bfae2efc 100644 --- a/cfgrammar/src/lib/yacc/grammar.rs +++ b/cfgrammar/src/lib/yacc/grammar.rs @@ -1,12 +1,12 @@ #![allow(clippy::derive_partial_eq_without_eq)] use std::{cell::RefCell, collections::HashMap, fmt::Write, str::FromStr}; -#[cfg(feature = "bincode")] -use bincode::{Decode, Encode}; use num_traits::{AsPrimitive, PrimInt, Unsigned}; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; use vob::Vob; +#[cfg(feature = "wincode")] +use wincode::{SchemaRead, SchemaWrite}; use super::{ YaccKind, ast, @@ -23,7 +23,7 @@ const IMPLICIT_START_RULE: &str = "^~"; pub type PrecedenceLevel = u64; #[derive(Clone, Copy, Debug, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] +#[cfg_attr(feature = "wincode", derive(SchemaRead, SchemaWrite))] pub struct Precedence { pub level: PrecedenceLevel, pub kind: AssocKind, @@ -31,7 +31,7 @@ pub struct Precedence { #[derive(Clone, Copy, Debug, PartialEq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] +#[cfg_attr(feature = "wincode", derive(SchemaRead, SchemaWrite))] pub enum AssocKind { Left, Right, @@ -41,7 +41,7 @@ pub enum AssocKind { /// Representation of a `YaccGrammar`. See the [top-level documentation](../../index.html) for the /// guarantees this struct makes about rules, tokens, productions, and symbols. #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[cfg_attr(feature = "bincode", derive(Encode))] +#[cfg_attr(feature = "wincode", derive(SchemaRead, SchemaWrite))] pub struct YaccGrammar { /// How many rules does this grammar have? rules_len: RIdx, @@ -103,97 +103,8 @@ pub struct YaccGrammar { expectrr: Option, } -// The implementations of `Decode` and `BorrowDecode` -// Contain a modified copy of the output of `cargo expand` of the derivation: -// #[cfg_attr(feature = "bincode", derive(Decode))] -// -// The current version of bincode has bugs in it's derive macro implementation -// https://github.com/bincode-org/bincode/issues/763 -// https://github.com/bincode-org/bincode/issues/646 -// -// Once those are fixed, we can replace this with derive. - -#[cfg(feature = "bincode")] -impl Decode<__Context> for YaccGrammar -where - StorageT: Decode<__Context> + 'static, -{ - fn decode<__D: bincode::de::Decoder>( - decoder: &mut __D, - ) -> Result { - Ok(Self { - rules_len: Decode::decode(decoder)?, - rule_names: Decode::decode(decoder)?, - token_names: Decode::decode(decoder)?, - token_precs: Decode::decode(decoder)?, - token_epp: Decode::decode(decoder)?, - tokens_len: Decode::decode(decoder)?, - eof_token_idx: Decode::decode(decoder)?, - prods_len: Decode::decode(decoder)?, - start_prod: Decode::decode(decoder)?, - prods: Decode::decode(decoder)?, - rules_prods: Decode::decode(decoder)?, - prods_rules: Decode::decode(decoder)?, - prod_precs: Decode::decode(decoder)?, - prod_spans: Decode::decode(decoder)?, - implicit_rule: Decode::decode(decoder)?, - actions: Decode::decode(decoder)?, - action_spans: Decode::decode(decoder)?, - parse_param: Decode::decode(decoder)?, - parse_generics: Decode::decode(decoder)?, - programs: Decode::decode(decoder)?, - actiontypes: Decode::decode(decoder)?, - avoid_insert: Decode::decode(decoder)?, - expect: Decode::decode(decoder)?, - expectrr: Decode::decode(decoder)?, - }) - } -} - -// Eventually this should just be provided through: -// #[cfg_attr(feature = "bincode", derive(Decode))] -// -// See comment at the corresponding bincode::Decode impl. -#[cfg(feature = "bincode")] -impl<'__de, StorageT, __Context> bincode::BorrowDecode<'__de, __Context> for YaccGrammar -where - StorageT: bincode::de::BorrowDecode<'__de, __Context> + '__de, -{ - fn borrow_decode<__D: ::bincode::de::BorrowDecoder<'__de, Context = __Context>>( - decoder: &mut __D, - ) -> Result { - Ok(Self { - rules_len: bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?, - rule_names: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?, - token_names: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?, - token_precs: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?, - token_epp: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?, - tokens_len: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?, - eof_token_idx: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?, - prods_len: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?, - start_prod: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?, - prods: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?, - rules_prods: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?, - prods_rules: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?, - prod_precs: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?, - prod_spans: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?, - implicit_rule: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?, - actions: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?, - action_spans: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?, - parse_param: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?, - parse_generics: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?, - programs: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?, - actiontypes: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?, - avoid_insert: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?, - expect: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?, - expectrr: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?, - }) - } -} - // Internally, we assume that a grammar's start rule has a single production. Since we manually // create the start rule ourselves (without relying on user input), this is a safe assumption. - impl YaccGrammar { pub fn new(yk: YaccKind, s: &str) -> YaccGrammarResult { YaccGrammar::new_with_storaget(yk, s) diff --git a/cfgrammar/src/lib/yacc/parser.rs b/cfgrammar/src/lib/yacc/parser.rs index a55b64e66..c452d4ae9 100644 --- a/cfgrammar/src/lib/yacc/parser.rs +++ b/cfgrammar/src/lib/yacc/parser.rs @@ -1,7 +1,5 @@ // Note: this is the parser for both YaccKind::Original(YaccOriginalActionKind::GenericParseTree) and YaccKind::Eco yacc kinds. -#[cfg(feature = "bincode")] -use bincode::{Decode, Encode}; use num_traits::PrimInt; use regex::Regex; #[cfg(feature = "serde")] @@ -13,6 +11,8 @@ use std::{ str::FromStr, sync::LazyLock, }; +#[cfg(feature = "wincode")] +use wincode::{SchemaRead, SchemaWrite}; use crate::{ Span, Spanned, @@ -161,7 +161,7 @@ impl fmt::Display for YaccGrammarErrorKind { /// The various different possible Yacc parser errors. #[derive(Debug, PartialEq, Eq, Clone)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] +#[cfg_attr(feature = "wincode", derive(SchemaRead, SchemaWrite))] #[non_exhaustive] pub enum YaccGrammarWarningKind { UnusedRule, @@ -171,7 +171,7 @@ pub enum YaccGrammarWarningKind { /// Any Warning from the Yacc parser returns an instance of this struct. #[derive(Debug, PartialEq, Eq, Clone)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] +#[cfg_attr(feature = "wincode", derive(SchemaRead, SchemaWrite))] pub struct YaccGrammarWarning { /// The specific kind of warning. pub(crate) kind: YaccGrammarWarningKind, diff --git a/lrlex/Cargo.toml b/lrlex/Cargo.toml index 18e70b4a1..d9c043ec6 100644 --- a/lrlex/Cargo.toml +++ b/lrlex/Cargo.toml @@ -33,7 +33,7 @@ regex-syntax.workspace = true num-traits.workspace = true proc-macro2.workspace = true quote.workspace = true -bincode.workspace = true +wincode.workspace = true serde = { workspace = true, optional = true } prettyplease.workspace = true syn.workspace = true diff --git a/lrlex/src/lib/ctbuilder.rs b/lrlex/src/lib/ctbuilder.rs index 99cb99d8b..c37540a88 100644 --- a/lrlex/src/lib/ctbuilder.rs +++ b/lrlex/src/lib/ctbuilder.rs @@ -1,6 +1,5 @@ //! Build grammars at run-time. -use bincode::Encode; use cfgrammar::{ header::{ GrmtoolsSectionParser, Header, HeaderError, HeaderErrorKind, HeaderValue, Namespaced, @@ -32,6 +31,7 @@ use std::{ path::{Path, PathBuf}, sync::{LazyLock, Mutex}, }; +use wincode::SchemaWrite; use crate::{DefaultLexerTypes, LRNonStreamingLexer, LRNonStreamingLexerDef, LexFlags, LexerDef}; @@ -257,12 +257,28 @@ impl CTLexerBuilder<'_, DefaultLexerTypes> { CTLexerBuilder::>::new_with_lexemet() } } - +type FixIntConfig = wincode::config::Configuration; + +type VarIntConfig = wincode::config::Configuration< + true, + 4194304, + wincode::len::BincodeLen, + wincode::int_encoding::LittleEndian, + wincode::int_encoding::VarInt, +>; impl<'a, LexerTypesT: LexerTypes + 'static> CTLexerBuilder<'a, LexerTypesT> where - LexerTypesT::StorageT: - 'static + Debug + Eq + Hash + PrimInt + Encode + TryFrom + Unsigned + ToTokens, + LexerTypesT::StorageT: 'static + + Debug + + Eq + + Hash + + PrimInt + + SchemaWrite + + SchemaWrite + + TryFrom + + Unsigned + + ToTokens, usize: AsPrimitive, { /// Create a new [CTLexerBuilder]. diff --git a/lrpar/Cargo.toml b/lrpar/Cargo.toml index 12dc9fcac..e17460b9d 100644 --- a/lrpar/Cargo.toml +++ b/lrpar/Cargo.toml @@ -23,10 +23,10 @@ _unsealed_unstable_traits = ["_unstable_api"] vergen = { version = "8", default-features = false, features = ["build"] } [dependencies] -cfgrammar = { path="../cfgrammar", version = "0.14", features = ["bincode"] } -lrtable = { path="../lrtable", version = "0.14", features = ["bincode"] } +cfgrammar = { path="../cfgrammar", version = "0.14", features = ["wincode"] } +lrtable = { path="../lrtable", version = "0.14", features = ["wincode"] } -bincode = { workspace = true, features = ["derive"] } +wincode = { workspace = true, features = ["derive"] } cactus.workspace = true filetime.workspace = true indexmap.workspace = true diff --git a/lrpar/src/lib/ctbuilder.rs b/lrpar/src/lib/ctbuilder.rs index 1ba0926c5..6790d7222 100644 --- a/lrpar/src/lib/ctbuilder.rs +++ b/lrpar/src/lib/ctbuilder.rs @@ -22,7 +22,6 @@ use crate::{ #[cfg(feature = "_unstable_api")] use crate::unstable_api::UnstableApi; -use bincode::{Decode, Encode, decode_from_slice, encode_to_vec}; use cfgrammar::{ Location, RIdx, Span, Symbol, header::{GrmtoolsSectionParser, Header, HeaderValue, Value}, @@ -35,6 +34,7 @@ use num_traits::{AsPrimitive, PrimInt, Unsigned}; use proc_macro2::{Literal, TokenStream}; use quote::{ToTokens, TokenStreamExt, format_ident, quote}; use syn::{Generics, parse_quote}; +use wincode::{SchemaRead, SchemaReadOwned, SchemaWrite}; const ACTION_PREFIX: &str = "__gt_"; const GLOBAL_PREFIX: &str = "__GT_"; @@ -221,6 +221,33 @@ impl Visibility { } } +/// Sets the underlying encoding algorithm for serialising the `ParserData` into the generated source files. +/// +/// This correlates to a specific `Configuration` of [wincode::config](https://docs.rs/wincode/latest/wincode/config/index.html). +#[non_exhaustive] +#[derive(SchemaRead, SchemaWrite, Debug, Clone, Copy)] +pub enum SerialisationFormat { + /// See [wincode::FixedInt](https://docs.rs/wincode/latest/wincode/int_encoding/struct.FixedInt.html) + FixedSizeInteger, + /// See [wincode::VarInt](https://docs.rs/wincode/latest/wincode/int_encoding/struct.VarInt.html) + VariableSizedInteger, +} +// We export this for generated code to refer to. +#[doc(hidden)] +pub use wincode; +impl ToTokens for SerialisationFormat { + fn to_tokens(&self, tokens: &mut TokenStream) { + tokens.extend(match self { + SerialisationFormat::FixedSizeInteger => { + quote! {::lrpar::ctbuilder::SerialisationFormat::FixedSizeInteger} + } + SerialisationFormat::VariableSizedInteger => { + quote! {::lrpar::ctbuilder::SerialisationFormat::VariableSizedInteger} + } + }) + } +} + /// A `CTParserBuilder` allows one to specify the criteria for building a statically generated /// parser. pub struct CTParserBuilder<'a, LexerTypesT: LexerTypes> @@ -255,15 +282,33 @@ where ) -> Result<(), Box>, >, >, + serialisation_format: Option, // test function for inspecting private state #[cfg(test)] inspect_callback: Option Result<(), Box>>>, phantom: PhantomData, } +/// Defaults to `wincode::int_encoding::VarInt`. +type FixIntConfig = wincode::config::Configuration; +/// The default config with the last parameter set to `VarInt` +type VarIntConfig = wincode::config::Configuration< + true, + 4194304, + wincode::len::BincodeLen, + wincode::int_encoding::LittleEndian, + wincode::int_encoding::VarInt, +>; + impl< 'a, - StorageT: 'static + Debug + Hash + PrimInt + Encode + Unsigned, + StorageT: 'static + + Debug + + Hash + + PrimInt + + SchemaWrite + + SchemaWrite + + Unsigned, LexerTypesT: LexerTypes, > CTParserBuilder<'a, LexerTypesT> where @@ -305,6 +350,7 @@ where visibility: Visibility::Private, rust_edition: RustEdition::Rust2021, inspect_rt: None, + serialisation_format: Some(SerialisationFormat::VariableSizedInteger), #[cfg(test)] inspect_callback: None, phantom: PhantomData, @@ -924,6 +970,7 @@ where visibility: self.visibility.clone(), rust_edition: self.rust_edition, inspect_rt: None, + serialisation_format: Some(SerialisationFormat::VariableSizedInteger), #[cfg(test)] inspect_callback: None, phantom: PhantomData, @@ -1043,6 +1090,7 @@ where show_warnings, visibility, rust_edition, + serialisation_format, inspect_rt: _, #[cfg(test)] inspect_callback: _, @@ -1066,6 +1114,7 @@ where let cache_info = quote! { BUILD_TIME = #build_time DERIVED_MOD_NAME = #derived_mod_name + ENCODING_CONFIG = #serialisation_format GRAMMAR_PATH = #grammar_path MOD_NAME = #mod_name RECOVERER = #recoverer @@ -1076,6 +1125,7 @@ where RUST_EDITION = #rust_edition RULE_IDS_MAP = [#(#rule_map,)*] VISIBILITY = #visibility + }; let cache_info_str = cache_info.to_string(); quote!(#cache_info_str) @@ -1199,17 +1249,49 @@ where _ => unreachable!(), }; - let grm_data = encode_to_vec(grm, bincode::config::standard())?; - let stable_data = encode_to_vec(stable, bincode::config::standard())?; + let serialisation_format = self + .serialisation_format + .expect("Should already have a default value"); + // Note that the configuration types use associated consts, and thus these configurations represent distinct types. + let (grm_data, stable_data): (Vec, Vec) = match serialisation_format { + SerialisationFormat::FixedSizeInteger => { + let config = wincode::config::Configuration::default().with_fixint_encoding(); + let grm = wincode::config::serialize(grm, config)?; + let stable = wincode::config::serialize(stable, config)?; + (grm, stable) + } + SerialisationFormat::VariableSizedInteger => { + let config = wincode::config::Configuration::default().with_varint_encoding(); + let grm = wincode::config::serialize(grm, config)?; + let stable = wincode::config::serialize(stable, config)?; + (grm, stable) + } + }; + let serialisation_format_str = quote!(serialisation_format).to_string(); Ok(quote! { const __GRM_DATA: &[u8] = &[#(#grm_data,)*]; const __STABLE_DATA: &[u8] = &[#(#stable_data,)*]; + const __SERIALISATION_FORMAT: ::lrpar::ctbuilder::SerialisationFormat = #serialisation_format; fn __lrpar_parser_data() -> &'static ::lrpar::ParserData<#storaget> { static DATA: ::std::sync::OnceLock<::lrpar::ParserData<#storaget>> = ::std::sync::OnceLock::new(); DATA.get_or_init( - || ::lrpar::ctbuilder::_reconstitute(__GRM_DATA, __STABLE_DATA) + || { + // We have to call reconstitute like this because the config parameter takes a trait + // which uses const generics. Thus the two config parameters here are not actually of the same type. + match __SERIALISATION_FORMAT { + ::lrpar::ctbuilder::SerialisationFormat::FixedSizeInteger => { + ::lrpar::ctbuilder::_reconstitute(__GRM_DATA, __STABLE_DATA, ::lrpar::ctbuilder::wincode::config::Configuration::default().with_fixint_encoding()) + } + ::lrpar::ctbuilder::SerialisationFormat::VariableSizedInteger => { + ::lrpar::ctbuilder::_reconstitute(__GRM_DATA, __STABLE_DATA, ::lrpar::ctbuilder::wincode::config::Configuration::default().with_varint_encoding()) + } + _ => { + panic!("Parser source was generated using unknown `SerialisationFormat`: {:?}", #serialisation_format_str) + } + } + } ) } @@ -1632,12 +1714,16 @@ impl ParserData { /// This function is called by generated files; it exists so that generated files don't require a /// direct dependency on bincode. #[doc(hidden)] -pub fn _reconstitute + Eq + Hash + PrimInt + Unsigned + 'static>( +pub fn _reconstitute< + C: wincode::config::Config + Clone + Copy, + StorageT: SchemaReadOwned + Eq + Hash + PrimInt + Unsigned + 'static, +>( grm_buf: &[u8], stable_buf: &[u8], + config: C, ) -> ParserData { - let (grm, _) = decode_from_slice(grm_buf, bincode::config::standard()).unwrap(); - let (stable, _) = decode_from_slice(stable_buf, bincode::config::standard()).unwrap(); + let grm: YaccGrammar = wincode::config::deserialize_from(grm_buf, config).unwrap(); + let stable = wincode::config::deserialize_from(stable_buf, config).unwrap(); ParserData { grm, stable } } diff --git a/lrtable/Cargo.toml b/lrtable/Cargo.toml index 18b095cfa..7a6a31b6d 100644 --- a/lrtable/Cargo.toml +++ b/lrtable/Cargo.toml @@ -9,7 +9,7 @@ license = "Apache-2.0/MIT" categories = ["parsing"] [features] -bincode = ["dep:bincode", "sparsevec/bincode", "cfgrammar/bincode"] +wincode = ["dep:wincode", "sparsevec/wincode", "cfgrammar/wincode"] serde = ["dep:serde", "sparsevec/serde", "cfgrammar/serde"] [lib] @@ -19,7 +19,7 @@ path = "src/lib/mod.rs" [dependencies] cfgrammar = { path="../cfgrammar", version = "0.14" } -bincode = { workspace = true, features = ["derive"], optional = true } +wincode = { workspace = true, features = ["derive"], optional = true } fnv.workspace = true num-traits.workspace = true serde = { workspace = true, features = ["derive"], optional = true } diff --git a/lrtable/src/lib/mod.rs b/lrtable/src/lib/mod.rs index 0cafb464c..a9cde69f2 100644 --- a/lrtable/src/lib/mod.rs +++ b/lrtable/src/lib/mod.rs @@ -6,11 +6,11 @@ use std::{hash::Hash, mem::size_of}; -#[cfg(feature = "bincode")] -use bincode::{Decode, Encode}; use num_traits::{AsPrimitive, PrimInt, Unsigned}; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; +#[cfg(feature = "wincode")] +use wincode::{SchemaRead, SchemaWrite}; mod itemset; mod pager; @@ -28,7 +28,7 @@ macro_rules! IdxNewtype { $(#[$attr])* #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] #[cfg_attr(feature="serde", derive(Serialize, Deserialize))] - #[cfg_attr(feature="bincode", derive(Encode, Decode))] + #[cfg_attr(feature="wincode", derive(SchemaRead, SchemaWrite))] pub struct $n(pub T); impl From<$n> for usize { diff --git a/lrtable/src/lib/statetable.rs b/lrtable/src/lib/statetable.rs index a0689f369..2652644e9 100644 --- a/lrtable/src/lib/statetable.rs +++ b/lrtable/src/lib/statetable.rs @@ -8,8 +8,6 @@ use std::{ marker::PhantomData, }; -#[cfg(feature = "bincode")] -use bincode::{Decode, Encode}; use cfgrammar::{ PIdx, RIdx, Symbol, TIdx, yacc::{AssocKind, YaccGrammar}, @@ -19,11 +17,13 @@ use num_traits::{AsPrimitive, PrimInt, Unsigned}; use serde::{Deserialize, Serialize}; use sparsevec::SparseVec; use vob::{IterSetBits, Vob}; +#[cfg(feature = "wincode")] +use wincode::{SchemaRead, SchemaWrite}; use crate::{StIdx, stategraph::StateGraph}; #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] +#[cfg_attr(feature = "wincode", derive(SchemaRead, SchemaWrite))] #[derive(Debug)] pub struct Conflicts { reduce_reduce: Vec<( @@ -148,7 +148,7 @@ impl fmt::Display for StateTableError { /// A representation of a `StateTable` for a grammar. `actions` and `gotos` are split into two /// separate hashmaps, rather than a single table, due to the different types of their values. #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] +#[cfg_attr(feature = "wincode", derive(SchemaRead, SchemaWrite))] pub struct StateTable { actions: SparseVec, state_actions: Vob, From 2a6440083c0c71d91b80b4b2e685523174f5d414 Mon Sep 17 00:00:00 2001 From: matt rice Date: Fri, 17 Jul 2026 04:22:37 -0700 Subject: [PATCH 3/3] Make serialisation format configurable --- cfgrammar/src/lib/header.rs | 59 ++++----- doc/src/yaccextensions.md | 2 + lrpar/cttests/src/calc_encoding_fixed.test | 27 +++++ lrpar/cttests/src/calc_encoding_fixed2.test | 27 +++++ lrpar/cttests/src/calc_encoding_variable.test | 27 +++++ .../cttests/src/calc_encoding_variable2.test | 27 +++++ lrpar/cttests/src/cgen_helper.rs | 15 ++- lrpar/src/lib/ctbuilder.rs | 114 +++++++++++++++++- lrpar/src/lib/mod.rs | 4 +- lrpar/src/lib/parser.rs | 41 +------ 10 files changed, 271 insertions(+), 72 deletions(-) create mode 100644 lrpar/cttests/src/calc_encoding_fixed.test create mode 100644 lrpar/cttests/src/calc_encoding_fixed2.test create mode 100644 lrpar/cttests/src/calc_encoding_variable.test create mode 100644 lrpar/cttests/src/calc_encoding_variable2.test diff --git a/cfgrammar/src/lib/header.rs b/cfgrammar/src/lib/header.rs index 8b2e0f266..458f3d864 100644 --- a/cfgrammar/src/lib/header.rs +++ b/cfgrammar/src/lib/header.rs @@ -210,6 +210,32 @@ impl From> for Value { } } +impl Value { + pub fn primary_location(&self) -> &T { + match self { + Value::Flag(_, loc) => loc, + Value::Setting(setting) => setting.primary_location(), + } + } +} + +impl Setting { + fn primary_location(&self) -> &T { + match self { + Self::Constructor { arg, .. } => arg.primary_location(), + Self::Unitary(ns) => ns.primary_location(), + Self::Array(_, start_loc, _) => start_loc, + Self::Num(_, loc) | Self::String(_, loc) => loc, + } + } +} + +impl Namespaced { + fn primary_location(&self) -> &T { + &self.member.1 + } +} + static RE_LEADING_WS: LazyLock = LazyLock::new(|| Regex::new(r"^[\p{Pattern_White_Space}]*").unwrap()); static RE_NAME: LazyLock = LazyLock::new(|| { @@ -561,35 +587,6 @@ impl TryFrom<&Value> for YaccKind { fn try_from(value: &Value) -> Result> { let mut err_locs = Vec::new(); match value { - Value::Flag(_, loc) => Err(HeaderError { - kind: HeaderErrorKind::ConversionError( - "From", - "Cannot convert boolean to YaccKind", - ), - locations: vec![loc.clone()], - }), - Value::Setting(Setting::Num(_, loc)) => Err(HeaderError { - kind: HeaderErrorKind::ConversionError( - "From", - "Cannot convert number to YaccKind", - ), - locations: vec![loc.clone()], - }), - Value::Setting(Setting::Array(_, loc, _)) => Err(HeaderError { - kind: HeaderErrorKind::ConversionError( - "From", - "Cannot convert array to YaccKind", - ), - locations: vec![loc.clone()], - }), - Value::Setting(Setting::String(_, loc)) => Err(HeaderError { - kind: HeaderErrorKind::ConversionError( - "From", - "Cannot convert string to YaccKind", - ), - locations: vec![loc.clone()], - }), - Value::Setting(Setting::Unitary(Namespaced { namespace, member: (yk_value, yk_value_loc), @@ -676,6 +673,10 @@ impl TryFrom<&Value> for YaccKind { }) } } + val => Err(HeaderError { + kind: HeaderErrorKind::InvalidEntry("yacckind"), + locations: vec![val.primary_location().clone()], + }), } } } diff --git a/doc/src/yaccextensions.md b/doc/src/yaccextensions.md index 07dbd432d..59cc1c201 100644 --- a/doc/src/yaccextensions.md +++ b/doc/src/yaccextensions.md @@ -8,9 +8,11 @@ But a default can be set or forced by using a `YaccKindResolver`. | `yacckind` | [YaccKind](yacccompatibility.md#yacckinds) | ✓ | | `recoverykind` | [RecoveryKind](errorrecovery.md#recoverykinds) | ✗ | | `test_files`[^†] | Array of string values | ✗ | +| `serialisation_format`[^⹋] | `lrpar::SerialisationFormat` | ✗ | [^†]: Strings containing globs are resolved relative to the yacc `.y` source file. `test_files` is currently experimental. +[^⹋]: Modifies the encoding algorithm used to serialise parser data in the generated code. ## Example diff --git a/lrpar/cttests/src/calc_encoding_fixed.test b/lrpar/cttests/src/calc_encoding_fixed.test new file mode 100644 index 000000000..2986c1b4a --- /dev/null +++ b/lrpar/cttests/src/calc_encoding_fixed.test @@ -0,0 +1,27 @@ +name: Test fixed serialisation format from builder. +yacckind: Original(YaccOriginalActionKind::NoAction) +serialisation_format: SerialisationFormat::FixedSizeInteger +grammar: | + %start Expr + %avoid_insert 'INT' + %% + Expr: Expr '+' Term + | Term + ; + + Term: Term '*' Factor + | Factor + ; + + Factor: '(' Expr ')' + | 'INT' + ; + +lexer: | + %% + [0-9]+ "INT" + \+ "+" + \* "*" + \( "(" + \) ")" + [\t ]+ ; diff --git a/lrpar/cttests/src/calc_encoding_fixed2.test b/lrpar/cttests/src/calc_encoding_fixed2.test new file mode 100644 index 000000000..e52294040 --- /dev/null +++ b/lrpar/cttests/src/calc_encoding_fixed2.test @@ -0,0 +1,27 @@ +name: Test fixed serialisation format from grmtools section. +yacckind: Original(YaccOriginalActionKind::NoAction) +grammar: | + %grmtools{serialisation_format: SerialisationFormat::FixedSizeInteger} + %start Expr + %avoid_insert 'INT' + %% + Expr: Expr '+' Term + | Term + ; + + Term: Term '*' Factor + | Factor + ; + + Factor: '(' Expr ')' + | 'INT' + ; + +lexer: | + %% + [0-9]+ "INT" + \+ "+" + \* "*" + \( "(" + \) ")" + [\t ]+ ; diff --git a/lrpar/cttests/src/calc_encoding_variable.test b/lrpar/cttests/src/calc_encoding_variable.test new file mode 100644 index 000000000..e56d039b5 --- /dev/null +++ b/lrpar/cttests/src/calc_encoding_variable.test @@ -0,0 +1,27 @@ +name: Test fixed serialisation format from builder. +yacckind: Original(YaccOriginalActionKind::NoAction) +serialisation_format: SerialisationFormat::VariableSizedInteger +grammar: | + %start Expr + %avoid_insert 'INT' + %% + Expr: Expr '+' Term + | Term + ; + + Term: Term '*' Factor + | Factor + ; + + Factor: '(' Expr ')' + | 'INT' + ; + +lexer: | + %% + [0-9]+ "INT" + \+ "+" + \* "*" + \( "(" + \) ")" + [\t ]+ ; diff --git a/lrpar/cttests/src/calc_encoding_variable2.test b/lrpar/cttests/src/calc_encoding_variable2.test new file mode 100644 index 000000000..59039f05c --- /dev/null +++ b/lrpar/cttests/src/calc_encoding_variable2.test @@ -0,0 +1,27 @@ +name: Test fixed serialisation format from builder. +yacckind: Original(YaccOriginalActionKind::NoAction) +grammar: | + %grmtools{serialisation_format: SerialisationFormat::VariableSizedInteger} + %start Expr + %avoid_insert 'INT' + %% + Expr: Expr '+' Term + | Term + ; + + Term: Term '*' Factor + | Factor + ; + + Factor: '(' Expr ')' + | 'INT' + ; + +lexer: | + %% + [0-9]+ "INT" + \+ "+" + \* "*" + \( "(" + \) ")" + [\t ]+ ; diff --git a/lrpar/cttests/src/cgen_helper.rs b/lrpar/cttests/src/cgen_helper.rs index 42c0e920e..6d0b35957 100644 --- a/lrpar/cttests/src/cgen_helper.rs +++ b/lrpar/cttests/src/cgen_helper.rs @@ -1,6 +1,6 @@ use cfgrammar::yacc::{YaccKind, YaccOriginalActionKind}; use lrlex::CTLexerBuilder; -use lrpar::RecoveryKind; +use lrpar::{RecoveryKind, SerialisationFormat}; use std::{ env, fs, path::{Path, PathBuf}, @@ -36,6 +36,16 @@ pub(crate) fn run_test_path>(path: P) -> Result<(), Box Some(RecoveryKind::None), _ => None, }; + let encoding = match docs[0]["serialisation_format"].as_str() { + Some("SerialisationFormat::FixedSizeInteger") => { + Some(SerialisationFormat::FixedSizeInteger) + } + Some("SerialisationFormat::VariableSizedInteger") => { + Some(SerialisationFormat::VariableSizedInteger) + } + Some(encoding) => panic!("Unknown SerialisationFormat '{encoding}'"), + _ => None, + }; let (negative_lex_flags, positive_lex_flags) = &docs[0]["lex_flags"] .as_vec() .map(|flags_vec| { @@ -114,6 +124,9 @@ pub(crate) fn run_test_path>(path: P) -> Result<(), Box for Value { + type Error = cfgrammar::header::HeaderError; + fn try_from(kind: SerialisationFormat) -> Result, HeaderError> { + let from_loc = Location::Other("From".to_string()); + Ok(match kind { + SerialisationFormat::FixedSizeInteger => Value::Setting(Setting::Unitary(Namespaced { + namespace: Some(("serialisationformat".to_string(), from_loc.clone())), + member: ("fixedsizeinteger".to_string(), from_loc), + })), + SerialisationFormat::VariableSizedInteger => { + Value::Setting(Setting::Unitary(Namespaced { + namespace: Some(("serialisationformat".to_string(), from_loc.clone())), + member: ("variablesizedinteger".to_string(), from_loc), + })) + } + }) + } +} + +impl TryFrom<&Value> for SerialisationFormat { + type Error = HeaderError; + fn try_from(value: &Value) -> Result> { + let mut err_locs = Vec::new(); + match value { + // Finally handle enum values. + Value::Setting(Setting::Unitary(Namespaced { + namespace, + member: (enc_value, enc_value_loc), + })) => { + if let Some((ns, ns_loc)) = namespace + && ns != "serialisationformat" + { + err_locs.push(ns_loc.clone()); + } + let encodings = [ + ( + "fixedsizeinteger".to_string(), + SerialisationFormat::FixedSizeInteger, + ), + ( + "variablesizedinteger".to_string(), + SerialisationFormat::VariableSizedInteger, + ), + ]; + let enc_found = encodings + .iter() + .find_map(|(enc_str, enc)| (enc_str == enc_value).then_some(enc)); + if let Some(enc) = enc_found { + if err_locs.is_empty() { + Ok(*enc) + } else { + Err(HeaderError { + kind: HeaderErrorKind::InvalidEntry("serialisation_format"), + locations: err_locs, + }) + } + } else { + err_locs.push(enc_value_loc.clone()); + Err(HeaderError { + kind: HeaderErrorKind::InvalidEntry("serialisation_format"), + locations: err_locs, + }) + } + } + val => { + err_locs.push(val.primary_location().clone()); + Err(HeaderError { + kind: HeaderErrorKind::InvalidEntry("serialisation_format"), + locations: err_locs, + }) + } + } + } +} + // We export this for generated code to refer to. #[doc(hidden)] pub use wincode; @@ -350,7 +429,7 @@ where visibility: Visibility::Private, rust_edition: RustEdition::Rust2021, inspect_rt: None, - serialisation_format: Some(SerialisationFormat::VariableSizedInteger), + serialisation_format: None, #[cfg(test)] inspect_callback: None, phantom: PhantomData, @@ -497,6 +576,11 @@ where self } + pub fn serialisation_format(mut self, serialisation_format: SerialisationFormat) -> Self { + self.serialisation_format = Some(serialisation_format); + self + } + #[cfg(test)] pub fn inspect_recoverer( mut self, @@ -616,6 +700,20 @@ where } } + if let Some(encoding) = self.serialisation_format { + match header.entry("serialisation_format".to_string()) { + Entry::Occupied(_) => unreachable!(), + Entry::Vacant(v) => { + let rk_value: Value = Value::try_from(encoding)?; + let mut o = v.insert_entry(HeaderValue( + Location::Other("CTParserBuilder".to_string()), + rk_value, + )); + o.set_merge_behavior(MergeBehavior::Ours); + } + } + } + { let mut lk = GENERATED_PATHS.lock().unwrap(); if lk.contains(outp.as_path()) { @@ -669,6 +767,16 @@ where // Fallback to the default recoverykind. self.recoverer = Some(RecoveryKind::CPCTPlus); } + header.mark_used(&"serialisation_format".to_string()); + if let Some(ec_val) = header + .get("serialisation_format") + .map(|HeaderValue(_, ec_val)| ec_val) + { + self.serialisation_format = Some(SerialisationFormat::try_from(ec_val)?); + } else { + self.serialisation_format = Some(SerialisationFormat::VariableSizedInteger); + } + self.yacckind = Some(ast_validation.yacc_kind()); let warnings = ast_validation.ast().warnings(); let res = YaccGrammar::::new_from_ast_with_validity_info(&ast_validation); @@ -970,7 +1078,7 @@ where visibility: self.visibility.clone(), rust_edition: self.rust_edition, inspect_rt: None, - serialisation_format: Some(SerialisationFormat::VariableSizedInteger), + serialisation_format: self.serialisation_format, #[cfg(test)] inspect_callback: None, phantom: PhantomData, diff --git a/lrpar/src/lib/mod.rs b/lrpar/src/lib/mod.rs index ef85fedbd..00533978e 100644 --- a/lrpar/src/lib/mod.rs +++ b/lrpar/src/lib/mod.rs @@ -206,7 +206,9 @@ pub mod parser; pub mod test_utils; pub use crate::{ - ctbuilder::{CTParser, CTParserBuilder, ParserData, RustEdition, Visibility}, + ctbuilder::{ + CTParser, CTParserBuilder, ParserData, RustEdition, SerialisationFormat, Visibility, + }, lex_api::{LexError, Lexeme, Lexer, LexerTypes, NonStreamingLexer}, parser::{LexParseError, ParseError, ParseRepair, RTParserBuilder, RecoveryKind}, }; diff --git a/lrpar/src/lib/parser.rs b/lrpar/src/lib/parser.rs index 8479eabb1..959021343 100644 --- a/lrpar/src/lib/parser.rs +++ b/lrpar/src/lib/parser.rs @@ -662,27 +662,6 @@ impl TryFrom<&Value> for RecoveryKind { use cfgrammar::header::{HeaderError, HeaderErrorKind, Namespaced, Setting}; match rk { - Value::Flag(_, loc) => Err(HeaderError { - kind: HeaderErrorKind::ConversionError( - "RecoveryKind", - "Cannot convert boolean to RecoveryKind", - ), - locations: vec![loc.clone()], - }), - Value::Setting(Setting::Num(_, loc)) => Err(HeaderError { - kind: HeaderErrorKind::ConversionError( - "RecoveryKind", - "Cannot convert number to RecoveryKind", - ), - locations: vec![loc.clone()], - }), - Value::Setting(Setting::String(_, loc)) => Err(HeaderError { - kind: HeaderErrorKind::ConversionError( - "RecoveryKind", - "Cannot convert string to RecoveryKind", - ), - locations: vec![loc.clone()], - }), Value::Setting(Setting::Unitary(Namespaced { namespace, member: (kind, kind_loc), @@ -708,26 +687,12 @@ impl TryFrom<&Value> for RecoveryKind { }), } } - Value::Setting(Setting::Constructor { - ctor: _, - arg: - Namespaced { - namespace: _, - member: (_, arg_loc), - }, - }) => Err(HeaderError { - kind: HeaderErrorKind::ConversionError( - "RecoveryKind", - "Cannot convert constructor to RecoveryKind", - ), - locations: vec![arg_loc.clone()], - }), - Value::Setting(Setting::Array(_, arr_loc, _)) => Err(HeaderError { + value => Err(HeaderError { kind: HeaderErrorKind::ConversionError( "RecoveryKind", - "Cannot convert array to RecoveryKind", + "Cannot convert to RecoveryKind", ), - locations: vec![arr_loc.clone()], + locations: vec![value.primary_location().clone()], }), } }